From f9c0a299c1e5c1eaf5c28e20a1aacdec982f9f58 Mon Sep 17 00:00:00 2001 From: Veselin Nikolov Date: Wed, 31 May 2023 11:49:10 +0100 Subject: [PATCH 001/273] Implement toMap for WriteConfig.ArrowConnectionInfo Co-authored-by: Yuval Rotenberg --- .../main/java/org/neo4j/gds/config/WriteConfig.java | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/core/src/main/java/org/neo4j/gds/config/WriteConfig.java b/core/src/main/java/org/neo4j/gds/config/WriteConfig.java index 71f08f74c5..f316169aa5 100644 --- a/core/src/main/java/org/neo4j/gds/config/WriteConfig.java +++ b/core/src/main/java/org/neo4j/gds/config/WriteConfig.java @@ -30,6 +30,7 @@ import org.neo4j.gds.core.CypherMapWrapper; import java.util.Collection; +import java.util.HashMap; import java.util.Map; import java.util.Optional; @@ -61,6 +62,7 @@ default void validateWriteConcurrency() { * export builders. */ @Configuration.ConvertWith(method = "org.neo4j.gds.config.WriteConfig.ArrowConnectionInfo#parse") + @Configuration.ToMapValue(value="org.neo4j.gds.config.WriteConfig.ArrowConnectionInfo#toMap") Optional arrowConnectionInfo(); @Configuration.GraphStoreValidationCheck @@ -114,5 +116,13 @@ default boolean useEncryption() { input.getClass().getSimpleName() )); } + + static Map toMap(ArrowConnectionInfo info) { + Map map = new HashMap<>(); + map.put("hostname", info.hostname()); + map.put("port", info.port()); + map.put("useEncryption", info.useEncryption()); + return map; + } } } From 8059cfcce676ae92f36f6662d9430f1cbc3a34b9 Mon Sep 17 00:00:00 2001 From: yuval Date: Fri, 2 Jun 2023 16:14:15 +0200 Subject: [PATCH 002/273] Use RC Neo4j version instad of Dev Co-authored-by: Mats Rydberg --- .../main/java/org/neo4j/gds/compat/ProxyUtil.java | 1 - .../main/java/org/neo4j/gds/compat/Neo4jVersion.java | 12 ++++++------ .../java/org/neo4j/gds/compat/Neo4jVersionTest.java | 2 +- .../src/test/java/org/neo4j/gds/SysInfoProcTest.java | 8 +++----- 4 files changed, 10 insertions(+), 13 deletions(-) diff --git a/compatibility/api/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/ProxyUtil.java b/compatibility/api/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/ProxyUtil.java index bdc9ce856f..527a20a779 100644 --- a/compatibility/api/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/ProxyUtil.java +++ b/compatibility/api/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/ProxyUtil.java @@ -191,7 +191,6 @@ private static Neo4jVersionInfo loadNeo4jVersion() { .reason(e) .build() ) - .neo4jVersion(Neo4jVersion.V_Dev) .build(); } } diff --git a/neo4j-adapter/src/main/java/org/neo4j/gds/compat/Neo4jVersion.java b/neo4j-adapter/src/main/java/org/neo4j/gds/compat/Neo4jVersion.java index 0cb402662e..9beb328163 100644 --- a/neo4j-adapter/src/main/java/org/neo4j/gds/compat/Neo4jVersion.java +++ b/neo4j-adapter/src/main/java/org/neo4j/gds/compat/Neo4jVersion.java @@ -36,7 +36,7 @@ public enum Neo4jVersion { V_5_6, V_5_7, V_5_8, - V_Dev; + V_RC; @Override public String toString() { @@ -59,15 +59,15 @@ public String toString() { return "5.7"; case V_5_8: return "5.8"; - case V_Dev: - return "dev"; + case V_RC: + return "rc"; default: throw new IllegalArgumentException("Unexpected value: " + this.name() + " (sad java 😞)"); } } public MajorMinorVersion semanticVersion() { - if (this == V_Dev) { + if (this == V_RC) { return ImmutableMajorMinorVersion.of(5, 9); } @@ -147,8 +147,8 @@ static Neo4jVersion parse(String version) { return Neo4jVersion.V_5_7; } else if (minorVersion == 8) { return Neo4jVersion.V_5_8; - } else if (minorVersion > 8) { - return Neo4jVersion.V_Dev; + } else if (minorVersion == 9) { + return Neo4jVersion.V_RC; } } diff --git a/neo4j-adapter/src/test/java/org/neo4j/gds/compat/Neo4jVersionTest.java b/neo4j-adapter/src/test/java/org/neo4j/gds/compat/Neo4jVersionTest.java index 295f61e719..999179b681 100644 --- a/neo4j-adapter/src/test/java/org/neo4j/gds/compat/Neo4jVersionTest.java +++ b/neo4j-adapter/src/test/java/org/neo4j/gds/compat/Neo4jVersionTest.java @@ -49,7 +49,7 @@ class Neo4jVersionTest { "5.6.0, V_5_6", "5.7.0, V_5_7", "5.8.0, V_5_8", - "5.9.0, V_Dev", + "5.9.0, V_RC", }) void testParse(String input, Neo4jVersion expected) { assertEquals(expected.name(), Neo4jVersion.parse(input).name()); diff --git a/proc/sysinfo/src/test/java/org/neo4j/gds/SysInfoProcTest.java b/proc/sysinfo/src/test/java/org/neo4j/gds/SysInfoProcTest.java index 0fd69cf176..45f37d74c3 100644 --- a/proc/sysinfo/src/test/java/org/neo4j/gds/SysInfoProcTest.java +++ b/proc/sysinfo/src/test/java/org/neo4j/gds/SysInfoProcTest.java @@ -216,14 +216,12 @@ void testSysInfoProc() throws IOException { "Neo4j 5.8" ); break; - case V_Dev: + case V_RC: expectedCompatibilities = Set.of( "Neo4j Settings RC", - "Neo4j Settings DEV (placeholder)", - "Neo4j Settings DEV", + "Neo4j Settings RC (placeholder)", "Neo4j RC", - "Neo4j DEV (placeholder)", - "Neo4j DEV" + "Neo4j RC (placeholder)" ); break; default: From 6d7fbabc42fe30366b59df998315cc27beaf0056 Mon Sep 17 00:00:00 2001 From: yuval Date: Fri, 2 Jun 2023 16:40:04 +0200 Subject: [PATCH 003/273] Prepare branch for maintenance * Add Aura part to version * Update Pregel API links to new version number * No docs publishing Co-authored-by: Mats Rydberg --- .../ROOT/pages/algorithms/pregel-api.adoc | 6 +- doc/package.json | 2 - doc/publish.yml | 60 ------------------- gradle/version.gradle | 7 ++- 4 files changed, 8 insertions(+), 67 deletions(-) delete mode 100644 doc/publish.yml diff --git a/doc/modules/ROOT/pages/algorithms/pregel-api.adoc b/doc/modules/ROOT/pages/algorithms/pregel-api.adoc index 7189e189d7..2452a266f4 100644 --- a/doc/modules/ROOT/pages/algorithms/pregel-api.adoc +++ b/doc/modules/ROOT/pages/algorithms/pregel-api.adoc @@ -29,7 +29,7 @@ The introduction of a new Pregel algorithm can be separated in two main steps. First, we need to implement the algorithm using the Pregel Java API. Second, we need to expose the algorithm via a Cypher procedure to make use of it. -For an example on how to expose a custom Pregel computation via a Neo4j procedure, have a look at the https://github.com/neo4j/graph-data-science/tree/master/examples/pregel-example/src/main/java/org/neo4j/gds/beta/pregel[Pregel examples]. +For an example on how to expose a custom Pregel computation via a Neo4j procedure, have a look at the https://github.com/neo4j/graph-data-science/tree/2.4/examples/pregel-example/src/main/java/org/neo4j/gds/beta/pregel[Pregel examples]. [[algorithms-pregel-api-java]] == Pregel Java API @@ -549,7 +549,7 @@ For more details, please refer to the xref:algorithms/pregel-api.adoc#algorithms === Building and installing a Neo4j plugin In order to use a Pregel algorithm in Neo4j via a procedure, we need to package it as Neo4j plugin. -The https://github.com/neo4j/graph-data-science/tree/master/examples/pregel-bootstrap[pregel-bootstrap] project is a good starting point. +The https://github.com/neo4j/graph-data-science/tree/2.4/examples/pregel-bootstrap[pregel-bootstrap] project is a good starting point. The `build.gradle` file within the project contains all the dependencies necessary to implement a Pregel algorithm and to generate corresponding procedures. Make sure to change the `gdsVersion` and `neo4jVersion` according to your setup. @@ -578,7 +578,7 @@ dbms.security.procedures.allowlist=custom.pregel.proc.* [[algorithms-pregel-api-example]] == Examples -The https://github.com/neo4j/graph-data-science/tree/master/examples/pregel-example[pregel-examples] module contains a set of examples for Pregel algorithms. +The https://github.com/neo4j/graph-data-science/tree/2.4/examples/pregel-example[pregel-examples] module contains a set of examples for Pregel algorithms. The algorithm implementations demonstrate the usage of the Pregel API. Along with each example, we provide test classes that can be used as a guideline on how to write tests for custom algorithms. To play around, we recommend copying one of the algorithms into the `pregel-bootstrap` project, build it and setup the plugin in Neo4j. diff --git a/doc/package.json b/doc/package.json index 0923bc0302..ac9c910e0e 100644 --- a/doc/package.json +++ b/doc/package.json @@ -9,9 +9,7 @@ "start:publish": "nodemon -e adoc --exec \"npm run build:publish && npm run serve\"", "serve": "node server.js", "build": "antora preview.yml --stacktrace --log-format=pretty", - "build:publish": "antora publish.yml --stacktrace --log-format=pretty", "build-verify": "antora --stacktrace --fetch preview.yml --log-format=json --log-level=info --log-file ./build/log/log.json", - "publish-verify": "antora --stacktrace --fetch publish.yml --log-format=json --log-file ./build/log/log.json" }, "keywords": [ "antora", diff --git a/doc/publish.yml b/doc/publish.yml deleted file mode 100644 index d31048ec23..0000000000 --- a/doc/publish.yml +++ /dev/null @@ -1,60 +0,0 @@ -site: - title: Neo4j Graph Data Science - url: https://neo4j.com/docs - start_page: graph-data-science:ROOT:index.adoc - -content: - sources: - - url: https://github.com/neo-technology/graph-analytics.git - branches: ['2.3', 'master'] - start_path: public/doc - include: public/doc/ - exclude: - - '!**/_includes/*' - - '!**/readme.adoc' - - '!**/README.adoc' -ui: - bundle: - url: https://s3-eu-west-1.amazonaws.com/static-content.neo4j.com/build/ui-bundle-latest.zip - snapshot: true - output_dir: /assets - -urls: - html_extension_style: indexify - -antora: - extensions: - - require: "@neo4j-antora/antora-modify-sitemaps" - sitemap_version: '2.3' - sitemap_loc_version: current - move_sitemaps_to_components: 'true' - -asciidoc: - extensions: - - "@neo4j-documentation/remote-include" - - "@neo4j-documentation/macros" - - "@neo4j-antora/antora-page-roles" - - "@neo4j-antora/antora-table-footnotes" - attributes: - page-theme: docs - page-type: Docs - page-search-type: Docs - page-search-site: Reference Docs - page-canonical-root: /docs - page-pagination: true - page-no-canonical: true - page-origin-private: true - page-hide-toc: false - page-mixpanel: 4bfb2414ab973c741b6f067bf06d5575 - # page-cdn: /static/assets - includePDF: false - sectnums: true - sectnumlevel: 3 - doctype: book - nonhtmloutput: "" - # sectnums: true, removed so they are off by default - # sectnumlevel: 3, - experimental: '' - copyright: 2023 - common-license-page-uri: https://neo4j.com/docs/license/ - operations-manual-base-uri: https://neo4j.com/docs/operations-manual/ diff --git a/gradle/version.gradle b/gradle/version.gradle index d2069cb539..72a10ecf87 100644 --- a/gradle/version.gradle +++ b/gradle/version.gradle @@ -1,3 +1,6 @@ ext { - gdsVersion = '2.4.0' -} + gdsBaseVersion = '2.4.0' + gdsAuraVersion = '21' + + gdsVersion = gdsBaseVersion + (rootProject.hasProperty('aurads') ? "+${gdsAuraVersion}" : "") +} \ No newline at end of file From 4c458c113b3bf216bc4bb98dedd80fad34b82515 Mon Sep 17 00:00:00 2001 From: Mats Rydberg Date: Mon, 5 Jun 2023 13:53:32 +0200 Subject: [PATCH 004/273] Fix version-parsing in test --- .../src/test/java/org/neo4j/gds/BuildInfoPropertiesTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/proc/sysinfo/src/test/java/org/neo4j/gds/BuildInfoPropertiesTest.java b/proc/sysinfo/src/test/java/org/neo4j/gds/BuildInfoPropertiesTest.java index 93521804ba..a4bce7fd3d 100644 --- a/proc/sysinfo/src/test/java/org/neo4j/gds/BuildInfoPropertiesTest.java +++ b/proc/sysinfo/src/test/java/org/neo4j/gds/BuildInfoPropertiesTest.java @@ -106,7 +106,7 @@ void loadFromPropertiesRequiresVersion() { } private Optional findVersion(Path file) throws IOException { - Pattern pattern = Pattern.compile(".*gdsVersion = '(\\d\\.\\d\\.\\d+(-alpha\\d+|-beta\\d+)?)'.*"); + Pattern pattern = Pattern.compile(".*gdsBaseVersion = '(\\d\\.\\d\\.\\d+(-alpha\\d+|-beta\\d+)?)'.*"); return Files.lines(file, StandardCharsets.UTF_8) .flatMap(line -> { var matcher = pattern.matcher(line); From ebbc833248eebb5d8d49047a44338a15fd15970e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20Kie=C3=9Fling?= Date: Mon, 5 Jun 2023 16:25:47 +0200 Subject: [PATCH 005/273] Remove @Internal annotation from ArrowDebugProc --- .../operations-reference/additional-operation-references.adoc | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/modules/ROOT/pages/operations-reference/additional-operation-references.adoc b/doc/modules/ROOT/pages/operations-reference/additional-operation-references.adoc index 5753bcc627..fac60faafc 100644 --- a/doc/modules/ROOT/pages/operations-reference/additional-operation-references.adoc +++ b/doc/modules/ROOT/pages/operations-reference/additional-operation-references.adoc @@ -21,6 +21,7 @@ | xref:graph-catalog-node-ops.adoc#utility-functions-catalog[Accessing a node property in a named graph] | `_gds.util.nodeProperty_` | xref:alpha-algorithms/one-hot-encoding.adoc[One Hot Encoding] | `_gds.alpha.ml.oneHotEncoding_` | xref:common-usage/debug-sysinfo.adoc[Status of the system] | `gds.debug.sysInfo` +| xref:installation/configure-apache-arrow-server.adoc[Monitoring] | `gds.debug.arrow` | xref:management-ops/create-cypher-db.adoc[Create an impermanent database backed by a projected graph] | `gds.alpha.create.cypherdb` | xref:management-ops/create-cypher-db.adoc#drop-cypher-db[Drop an impermanent database backed by a projected graph] | `gds.alpha.drop.cypherdb` | xref:common-usage/monitoring-system.adoc[Get an overview of the system's workload and available resources] | `gds.alpha.systemMonitor` From cd76deae67f1637e1341ef0979e72da8be666d91 Mon Sep 17 00:00:00 2001 From: Veselin Nikolov Date: Tue, 6 Jun 2023 06:44:35 +0100 Subject: [PATCH 006/273] Point to analytical cluster setup docs --- doc/modules/ROOT/pages/production-deployment/neo4j-cluster.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/modules/ROOT/pages/production-deployment/neo4j-cluster.adoc b/doc/modules/ROOT/pages/production-deployment/neo4j-cluster.adoc index 27db2bca5b..157e4a5042 100644 --- a/doc/modules/ROOT/pages/production-deployment/neo4j-cluster.adoc +++ b/doc/modules/ROOT/pages/production-deployment/neo4j-cluster.adoc @@ -29,7 +29,7 @@ GDS has compute-intensive OLAP workloads that may disrupt the cluster operations [NOTE] ====== -Please refer to the https://neo4j.com/docs/operations-manual/current/clustering/[official Neo4j documentation] for details on how to setup Neo4j cluster. +Please refer to the https://neo4j.com/docs/operations-manual/current/clustering/setup/analytics-cluster/[official Neo4j documentation] for details on how to setup Neo4j analytical cluster. Note that the link points to the latest Neo4j version documentation and the configuration settings may differ from earlier versions. ====== From c532ab970f2d77b336cc601a33d4eb39ef35e55a Mon Sep 17 00:00:00 2001 From: Paul Horn Date: Fri, 2 Jun 2023 10:01:52 +0200 Subject: [PATCH 007/273] =?UTF-8?q?Release=20ALL=20the=20docs=20?= =?UTF-8?q?=F0=9F=8E=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Mats Rydberg --- doc/antora.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/antora.yml b/doc/antora.yml index 9066ee038f..e8371acbe2 100644 --- a/doc/antora.yml +++ b/doc/antora.yml @@ -1,6 +1,6 @@ name: graph-data-science title: Neo4j Graph Data Science -version: '2.4-preview' +version: '2.4' start_page: ROOT:index.adoc nav: - modules/ROOT/content-nav.adoc From 9dae11e8547f4aa03c55159b760e10d2b7085720 Mon Sep 17 00:00:00 2001 From: Paul Horn Date: Tue, 6 Jun 2023 12:08:57 +0200 Subject: [PATCH 008/273] docs: Rename Cypher projection to Legacy Cypher projection --- .../pages/common-usage/memory-estimation.adoc | 4 ++-- doc/modules/ROOT/pages/graph-list.adoc | 2 +- .../installation/System-requirements.adoc | 2 +- .../management-ops/graph-catalog-ops.adoc | 4 ++-- .../projections/graph-project-cypher.adoc | 22 +++++++++---------- 5 files changed, 17 insertions(+), 17 deletions(-) diff --git a/doc/modules/ROOT/pages/common-usage/memory-estimation.adoc b/doc/modules/ROOT/pages/common-usage/memory-estimation.adoc index fbef5767d9..7896b52e38 100644 --- a/doc/modules/ROOT/pages/common-usage/memory-estimation.adoc +++ b/doc/modules/ROOT/pages/common-usage/memory-estimation.adoc @@ -82,8 +82,8 @@ With this it is possible to measure the memory consumption of projecting a graph | Name | Type | Default | Optional | Description | xref:management-ops/projections/graph-project.adoc#node-projection-syntax[node projection] | String, List of String or Map | null | yes | The node projection used for anonymous graph creation via a Native projection. | xref:management-ops/projections/graph-project.adoc#relationship-projection-syntax[relationship projection] | String, List of String or Map | null | yes | The relationship projection used for anonymous graph creation a Native projection. -| nodeQuery | String | null | yes | The Cypher query used to select the nodes for anonymous graph creation via a Cypher projection. -| relationshipQuery | String | null | yes | The Cypher query used to select the relationships for anonymous graph creation via a Cypher projection. +| nodeQuery | String | null | yes | The Cypher query used to select the nodes for anonymous graph creation via a Legacy Cypher projection. +| relationshipQuery | String | null | yes | The Cypher query used to select the relationships for anonymous graph creation via a Legacy Cypher projection. | nodeProperties | String, List of String or Map | null | yes | The node properties to project during anonymous graph creation. | relationshipProperties | String, List of String or Map | null | yes | The relationship properties to project during anonymous graph creation. | xref:common-usage/running-algos.adoc#common-configuration-concurrency[concurrency] | Integer | 4 | yes | The number of concurrent threads used for running the algorithm. Also provides the default value for 'readConcurrency' and 'writeConcurrency'. diff --git a/doc/modules/ROOT/pages/graph-list.adoc b/doc/modules/ROOT/pages/graph-list.adoc index a78de46433..da9cc1aa07 100644 --- a/doc/modules/ROOT/pages/graph-list.adoc +++ b/doc/modules/ROOT/pages/graph-list.adoc @@ -95,7 +95,7 @@ CREATE (florentin)-[:KNOWS { since: 2018 }]->(veselin) ---- -Additionally, we will project a few graphs to the graph catalog, for more details see xref:management-ops/projections/graph-project.adoc[native projections] and xref:management-ops/projections/graph-project-cypher.adoc[Cypher projections]. +Additionally, we will project a few graphs to the graph catalog, for more details see xref:management-ops/projections/graph-project.adoc[native projections] and xref:management-ops/projections/graph-project-cypher.adoc[Legacy Cypher projections]. .Project `Person` nodes and `KNOWS` relationships using native projections: [source, cypher, role=noplay graph-project-query] diff --git a/doc/modules/ROOT/pages/installation/System-requirements.adoc b/doc/modules/ROOT/pages/installation/System-requirements.adoc index f224350145..c9c034c76c 100644 --- a/doc/modules/ROOT/pages/installation/System-requirements.adoc +++ b/doc/modules/ROOT/pages/installation/System-requirements.adoc @@ -60,7 +60,7 @@ https://neo4j.com/docs/operations-manual/4.4/reference/configuration-settings/#c However, setting a minimum page cache size is still important when projecting graphs: * For xref:management-ops/projections/graph-project.adoc[native projections], the minimum page cache size for projecting a graph can be roughly estimated by `8KB * 100 * readConcurrency`. -* For xref:management-ops/projections/graph-project-cypher.adoc[Cypher projections], a higher page cache is required depending on the query complexity. +* For xref:management-ops/projections/graph-project-cypher.adoc[Legacy Cypher projections], a higher page cache is required depending on the query complexity. * For xref:management-ops/projections/graph-project-cypher-aggregation.adoc[Cypher aggregation], a higher page cache is required depending on the query complexity. * For xref:graph-project-apache-arrow.adoc[projections through Apache Arrow], the page cache is irrelevant. diff --git a/doc/modules/ROOT/pages/management-ops/graph-catalog-ops.adoc b/doc/modules/ROOT/pages/management-ops/graph-catalog-ops.adoc index 3e182fea6b..ca6284fe25 100644 --- a/doc/modules/ROOT/pages/management-ops/graph-catalog-ops.adoc +++ b/doc/modules/ROOT/pages/management-ops/graph-catalog-ops.adoc @@ -40,7 +40,7 @@ Read more about that in xref:management-ops/administration.adoc[]. Named graphs can be projected from a Neo4j database by using either of - a xref:management-ops/projections/graph-project.adoc[Native projection] -- a xref:management-ops/projections/graph-project-cypher.adoc[Cypher projection] +- a xref:management-ops/projections/graph-project-cypher.adoc[Legacy Cypher projection] - or a xref:management-ops/projections/graph-project-cypher-aggregation.adoc[Cypher Aggregation] But graphs can also be projected into the graph catalog from other sources. @@ -52,7 +52,7 @@ xref:management-ops/projections/graph-generation.adoc[Randomised graphs can be g |=== | Name | Description | xref:management-ops/projections/graph-project.adoc[gds.graph.project] | Adds a graph to the catalog using Native projection. -| xref:management-ops/projections/graph-project-cypher.adoc[gds.graph.project.cypher] | Adds a graph to the catalog using Cypher projection. +| xref:management-ops/projections/graph-project-cypher.adoc[gds.graph.project.cypher] | Adds a graph to the catalog using Legacy Cypher projection. | xref:management-ops/projections/graph-project-cypher-aggregation.adoc[gds.graph.project] | Adds a graph to the catalog using Cypher Aggregation. | xref:management-ops/projections/graph-project-subgraph.adoc[gds.beta.graph.project.subgraph] | Adds a graph to the catalog by filtering an existing graph using node and relationship predicates. | xref:management-ops/projections/rwr.adoc[gds.graph.sample.rwr] | Adds a graph to the catalog by sampling an existing graph using random walk with restarts. diff --git a/doc/modules/ROOT/pages/management-ops/projections/graph-project-cypher.adoc b/doc/modules/ROOT/pages/management-ops/projections/graph-project-cypher.adoc index afad607fb4..ca7954a9f7 100644 --- a/doc/modules/ROOT/pages/management-ops/projections/graph-project-cypher.adoc +++ b/doc/modules/ROOT/pages/management-ops/projections/graph-project-cypher.adoc @@ -1,9 +1,9 @@ [[catalog-graph-project-cypher]] = Projecting graphs using Cypher -:description: This section details projecting GDS graphs using `Cypher` projections. +:description: This section details projecting GDS graphs using legacy `Cypher` projections. -Using Cypher projections is a more flexible and expressive approach with diminished focus on performance compared to the xref:management-ops/projections/graph-project.adoc[native projections]. -Cypher projections are primarily recommended for the development phase (see xref:common-usage/index.adoc[Common usage]). +Using Legacy Cypher projections is a more flexible and expressive approach with diminished focus on performance compared to the xref:management-ops/projections/graph-project.adoc[native projections]. +Legacy Cypher projections are primarily recommended for the development phase (see xref:common-usage/index.adoc[Common usage]). == Considerations @@ -22,15 +22,15 @@ The projected graphs will reside in the catalog until either: === Node property support -Cypher projections can only project a limited set of node property types from a Cypher query. +Legacy Cypher projections can only project a limited set of node property types from a Cypher query. The xref:management-ops/node-properties.adoc#node-properties-supported[Node Properties page] details which node property types are supported. -Other types of node properties have to be transformed or encoded into one of the supported types in order to be projected using a Cypher projection. +Other types of node properties have to be transformed or encoded into one of the supported types in order to be projected using a Legacy Cypher projection. [[graph-project-cypher-syntax]] == Syntax -A Cypher projection takes three mandatory arguments: `graphName`, `nodeQuery` and `relationshipQuery`. +A Legacy Cypher projection takes three mandatory arguments: `graphName`, `nodeQuery` and `relationshipQuery`. In addition, the optional `configuration` parameter allows us to further configure graph creation. [.graph-project-cypher-syntax] @@ -58,7 +58,7 @@ CALL gds.graph.project.cypher( | graphName | no | The name under which the graph is stored in the catalog. | nodeQuery | no | Cypher query to project nodes. The query result must contain an `id` column. Optionally, a `labels` column can be specified to represent node labels. Additional columns are interpreted as properties. | relationshipQuery | no | Cypher query to project relationships. The query result must contain `source` and `target` columns. Optionally, a `type` column can be specified to represent relationship type. Additional columns are interpreted as properties. -| configuration | yes | Additional parameters to configure the Cypher projection. +| configuration | yes | Additional parameters to configure the Legacy Cypher projection. |=== .Configuration @@ -173,15 +173,15 @@ YIELD === Relationship orientation The native projection supports specifying an orientation per relationship type. -The Cypher projection treats every relationship returned by the relationship query as if it were in `NATURAL` orientation and creates a directed relationship from the first provided id (source) to the second (target). +The Legacy Cypher projection treats every relationship returned by the relationship query as if it were in `NATURAL` orientation and creates a directed relationship from the first provided id (source) to the second (target). Projecting in `REVERSE` orientation can be achieved by switching the order of ids in the RETURN clause such as `MATCH (n)-[r:KNOWS]->(m) RETURN id(m) AS source, id(n) AS target, type(r) AS type`. -It not possible to project graphs in `UNDIRECTED` orientation when Cypher projections are used. +It not possible to project graphs in `UNDIRECTED` orientation when Legacy Cypher projections are used. [NOTE] -- Some algorithms require that the graph was loaded with `UNDIRECTED` orientation. -These algorithms can not be used with a graph projected by a Cypher projection. +These algorithms can not be used with a graph projected by a Legacy Cypher projection. -- @@ -222,7 +222,7 @@ RETURN graphName, nodes, rels -- The projected `graphWithProperties` graph contains five nodes and six relationships. -In a Cypher projection every node from the `nodeQuery` gets the same node properties, which means you can't have label-specific properties. +In a Legacy Cypher projection every node from the `nodeQuery` gets the same node properties, which means you can't have label-specific properties. For instance in the example above the `Person` nodes will also get `ratings` and `price` properties, while `Book` nodes get the `age` property. Further, the `price` property has a default value of `5.0`. From 467a1d0e46f4055b738d9c16e2e523a6b2925eb6 Mon Sep 17 00:00:00 2001 From: Paul Horn Date: Tue, 6 Jun 2023 12:00:24 +0200 Subject: [PATCH 009/273] docs: Rename Cypher aggregation to Cypher projection --- .../alpha-algorithms/all-pairs-shortest-path.adoc | 4 ++-- .../pages/installation/System-requirements.adoc | 2 +- .../pages/management-ops/graph-catalog-ops.adoc | 4 ++-- .../graph-project-cypher-aggregation.adoc | 14 +++++++------- .../pages/production-deployment/composite.adoc | 2 +- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/doc/modules/ROOT/pages/alpha-algorithms/all-pairs-shortest-path.adoc b/doc/modules/ROOT/pages/alpha-algorithms/all-pairs-shortest-path.adoc index a93754049b..a596d8111b 100644 --- a/doc/modules/ROOT/pages/alpha-algorithms/all-pairs-shortest-path.adoc +++ b/doc/modules/ROOT/pages/alpha-algorithms/all-pairs-shortest-path.adoc @@ -147,9 +147,9 @@ LIMIT 10 This query returned the top 10 pairs of nodes that are the furthest away from each other. F and E appear to be quite distant from the others. -=== Using Cypher aggregation +=== Using Cypher projection -.The following will project and store an undirected graph using cypher aggregation: +.The following will project and store an undirected graph using cypher projection: [source, cypher, role=noplay graph-project-query, group=cypher] ---- MATCH (src:Loc)-[r:ROAD]->(trg:Loc) diff --git a/doc/modules/ROOT/pages/installation/System-requirements.adoc b/doc/modules/ROOT/pages/installation/System-requirements.adoc index c9c034c76c..e3f3fcb1a7 100644 --- a/doc/modules/ROOT/pages/installation/System-requirements.adoc +++ b/doc/modules/ROOT/pages/installation/System-requirements.adoc @@ -61,7 +61,7 @@ However, setting a minimum page cache size is still important when projecting gr * For xref:management-ops/projections/graph-project.adoc[native projections], the minimum page cache size for projecting a graph can be roughly estimated by `8KB * 100 * readConcurrency`. * For xref:management-ops/projections/graph-project-cypher.adoc[Legacy Cypher projections], a higher page cache is required depending on the query complexity. -* For xref:management-ops/projections/graph-project-cypher-aggregation.adoc[Cypher aggregation], a higher page cache is required depending on the query complexity. +* For xref:management-ops/projections/graph-project-cypher-aggregation.adoc[Cypher projections], a higher page cache is required depending on the query complexity. * For xref:graph-project-apache-arrow.adoc[projections through Apache Arrow], the page cache is irrelevant. However, if it is required to write algorithm results back to Neo4j, the write performance is highly depended on store fragmentation as well as the number of properties and relationships to write. diff --git a/doc/modules/ROOT/pages/management-ops/graph-catalog-ops.adoc b/doc/modules/ROOT/pages/management-ops/graph-catalog-ops.adoc index ca6284fe25..8496f111b6 100644 --- a/doc/modules/ROOT/pages/management-ops/graph-catalog-ops.adoc +++ b/doc/modules/ROOT/pages/management-ops/graph-catalog-ops.adoc @@ -41,7 +41,7 @@ Named graphs can be projected from a Neo4j database by using either of - a xref:management-ops/projections/graph-project.adoc[Native projection] - a xref:management-ops/projections/graph-project-cypher.adoc[Legacy Cypher projection] -- or a xref:management-ops/projections/graph-project-cypher-aggregation.adoc[Cypher Aggregation] +- or a xref:management-ops/projections/graph-project-cypher-aggregation.adoc[Cypher projection] But graphs can also be projected into the graph catalog from other sources. xref:management-ops/projections/graph-project-subgraph.adoc[Subgraph] and xref:management-ops/projections/rwr.adoc[Graph Sampling] projections allow projecting a new graph based off of an existing graph projection. @@ -53,7 +53,7 @@ xref:management-ops/projections/graph-generation.adoc[Randomised graphs can be g | Name | Description | xref:management-ops/projections/graph-project.adoc[gds.graph.project] | Adds a graph to the catalog using Native projection. | xref:management-ops/projections/graph-project-cypher.adoc[gds.graph.project.cypher] | Adds a graph to the catalog using Legacy Cypher projection. -| xref:management-ops/projections/graph-project-cypher-aggregation.adoc[gds.graph.project] | Adds a graph to the catalog using Cypher Aggregation. +| xref:management-ops/projections/graph-project-cypher-aggregation.adoc[gds.graph.project] | Adds a graph to the catalog using Cypher projection. | xref:management-ops/projections/graph-project-subgraph.adoc[gds.beta.graph.project.subgraph] | Adds a graph to the catalog by filtering an existing graph using node and relationship predicates. | xref:management-ops/projections/rwr.adoc[gds.graph.sample.rwr] | Adds a graph to the catalog by sampling an existing graph using random walk with restarts. | xref:management-ops/projections/cnarw.adoc[gds.graph.sample.cnarw] | Adds a graph to the catalog by sampling an existing graph using Common Neighbour Aware Random Walk algorithm. diff --git a/doc/modules/ROOT/pages/management-ops/projections/graph-project-cypher-aggregation.adoc b/doc/modules/ROOT/pages/management-ops/projections/graph-project-cypher-aggregation.adoc index d14584dd6c..b832d9b7df 100644 --- a/doc/modules/ROOT/pages/management-ops/projections/graph-project-cypher-aggregation.adoc +++ b/doc/modules/ROOT/pages/management-ops/projections/graph-project-cypher-aggregation.adoc @@ -1,10 +1,10 @@ [[catalog-graph-project-cypher-aggregation]] -= Projecting graphs using Cypher Aggregation -:description: This section details projecting GDS graphs using `Cypher` aggregations. += Projecting graphs using Cypher +:description: This section details projecting GDS graphs using `Cypher` projections. -Using Cypher aggregations is a more flexible and expressive approach with diminished focus on performance compared to the xref:management-ops/projections/graph-project.adoc[native projections]. -Cypher aggregations are primarily recommended for the development phase (see xref:common-usage/index.adoc[Common usage]). +Using Cypher projection is a more flexible and expressive approach with diminished focus on performance compared to the xref:management-ops/projections/graph-project.adoc[native projections]. +Cypher projections are primarily recommended for the development phase (see xref:common-usage/index.adoc[Common usage]). == Considerations @@ -23,9 +23,9 @@ The projected graphs will reside in the catalog until either: === Node property support -Cypher aggregations can only project a limited set of node property types from a Cypher query. +Cypher projections can only project a limited set of node property types from a Cypher query. The xref:management-ops/node-properties.adoc#node-properties-supported[Node Properties page] details which node property types are supported. -Other types of node properties have to be transformed or encoded into one of the supported types in order to be projected using a Cypher aggregation. +Other types of node properties have to be transformed or encoded into one of the supported types in order to be projected using a Cypher projection. === Selection of node properties and labels @@ -37,7 +37,7 @@ Relevant configuration options are `sourceNodeProperties`, `targetNodeProperties [[graph-project-cypher-aggregation-syntax]] == Syntax -A Cypher aggregation is used in a query as an aggregation over the relationships that are being projected. +A Cypher projection is used in a query as an aggregation over the relationships that are being projected. It takes two mandatory arguments: `graphName`, and `sourceNode`. The third parameter is `targetNode` and is usually provided. The parameter is optional and can be `null` to project an unconnected node. diff --git a/doc/modules/ROOT/pages/production-deployment/composite.adoc b/doc/modules/ROOT/pages/production-deployment/composite.adoc index f6587c5424..e6a5c653e2 100644 --- a/doc/modules/ROOT/pages/production-deployment/composite.adoc +++ b/doc/modules/ROOT/pages/production-deployment/composite.adoc @@ -105,7 +105,7 @@ For every row returned by the first subquery, the operational database is then q In this mode of using GDS in a composite environment, the GDS operations are executed on the Fabric proxy server. The graph projections are then using the data stored on the shards to construct the in-memory graph. -NOTE: Currently only xref:management-ops/projections/graph-project-cypher-aggregation.adoc[Cypher Aggregation] is supported for projecting in-memory graphs on a Composite database. +NOTE: Currently only xref:management-ops/projections/graph-project-cypher-aggregation.adoc[Cypher projection] is supported for projecting in-memory graphs on a Composite database. Graph algorithms can then be executed on the composite database, similar to a single machine setup. This scenario is useful, if a graph that logically represents a single graph is distributed to different Composite shards. From a20b23170157423384743223edaf3b0431acb939 Mon Sep 17 00:00:00 2001 From: Paul Horn Date: Tue, 6 Jun 2023 12:10:11 +0200 Subject: [PATCH 010/273] docs: Declare LCP as deprecated --- .../management-ops/projections/graph-project-cypher.adoc | 8 +++++++- .../operations-reference/graph-operation-references.adoc | 4 ++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/doc/modules/ROOT/pages/management-ops/projections/graph-project-cypher.adoc b/doc/modules/ROOT/pages/management-ops/projections/graph-project-cypher.adoc index ca7954a9f7..d4555ae09c 100644 --- a/doc/modules/ROOT/pages/management-ops/projections/graph-project-cypher.adoc +++ b/doc/modules/ROOT/pages/management-ops/projections/graph-project-cypher.adoc @@ -1,7 +1,13 @@ [[catalog-graph-project-cypher]] -= Projecting graphs using Cypher += Projecting graphs using Cypher (deprecated) :description: This section details projecting GDS graphs using legacy `Cypher` projections. +[NOTE] +-- +This page describes the Legacy Cypher projection, which is deprecated. +The replacement is to use the new Cypher projection, which is described in xref:management-ops/projections/graph-project-cypher-aggregation.adoc[Projecting graphs using Cypher]. +-- + Using Legacy Cypher projections is a more flexible and expressive approach with diminished focus on performance compared to the xref:management-ops/projections/graph-project.adoc[native projections]. Legacy Cypher projections are primarily recommended for the development phase (see xref:common-usage/index.adoc[Common usage]). diff --git a/doc/modules/ROOT/pages/operations-reference/graph-operation-references.adoc b/doc/modules/ROOT/pages/operations-reference/graph-operation-references.adoc index bfda78154d..5eda06b775 100644 --- a/doc/modules/ROOT/pages/operations-reference/graph-operation-references.adoc +++ b/doc/modules/ROOT/pages/operations-reference/graph-operation-references.adoc @@ -11,8 +11,8 @@ .5+<.^|xref:management-ops/projections/graph-project.adoc[Project Graph] | `gds.graph.project` | `gds.graph.project.estimate` -| `gds.graph.project.cypher` -| `gds.graph.project.cypher.estimate` +| `gds.graph.project.cypher` (deprecated) +| `gds.graph.project.cypher.estimate` (deprecated) | `gds.graph.project` (aggregation function) .2+<.^|xref:graph-exists.adoc[Check if a graph exists] | `gds.graph.exists` From 7ab616c5de4a179b1ea186551f7d41ece9867015 Mon Sep 17 00:00:00 2001 From: Paul Horn Date: Tue, 6 Jun 2023 12:11:36 +0200 Subject: [PATCH 011/273] docs: Refer to the proper Cypher projection --- .../partials/algorithms/shared/examples-named-native-note.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/modules/ROOT/partials/algorithms/shared/examples-named-native-note.adoc b/doc/modules/ROOT/partials/algorithms/shared/examples-named-native-note.adoc index a96646d716..f65b037b7b 100644 --- a/doc/modules/ROOT/partials/algorithms/shared/examples-named-native-note.adoc +++ b/doc/modules/ROOT/partials/algorithms/shared/examples-named-native-note.adoc @@ -1,5 +1,5 @@ [NOTE] ==== In the examples below we will use named graphs and native projections as the norm. -However, xref:management-ops/projections/graph-project-cypher.adoc[Cypher projections] can also be used. +However, xref:management-ops/projections/graph-project-cypher-aggregation.adoc[Cypher projections] can also be used. ==== From d4c393e72a014ceae6ed1793cd5e0b02f02eb14a Mon Sep 17 00:00:00 2001 From: Paul Horn Date: Tue, 6 Jun 2023 12:21:11 +0200 Subject: [PATCH 012/273] docs: Rename Cypher projection to Legacy Cypher projection --- doc/modules/ROOT/content-nav.adoc | 2 +- .../ROOT/pages/common-usage/memory-estimation.adoc | 2 +- doc/modules/ROOT/pages/graph-list.adoc | 2 +- .../ROOT/pages/installation/System-requirements.adoc | 2 +- .../ROOT/pages/management-ops/graph-catalog-ops.adoc | 4 ++-- ...oject-cypher.adoc => graph-project-cypher-legacy.adoc} | 8 ++++---- .../pages/management-ops/projections/graph-project.adoc | 4 ++-- 7 files changed, 12 insertions(+), 12 deletions(-) rename doc/modules/ROOT/pages/management-ops/projections/{graph-project-cypher.adoc => graph-project-cypher-legacy.adoc} (98%) diff --git a/doc/modules/ROOT/content-nav.adoc b/doc/modules/ROOT/content-nav.adoc index 676f5ea2e1..da33df3255 100644 --- a/doc/modules/ROOT/content-nav.adoc +++ b/doc/modules/ROOT/content-nav.adoc @@ -18,7 +18,7 @@ * xref:management-ops/index.adoc[] ** xref:management-ops/graph-catalog-ops.adoc[] *** xref:management-ops/projections/graph-project.adoc[] -*** xref:management-ops/projections/graph-project-cypher.adoc[] +*** xref:management-ops/projections/graph-project-cypher-legacy.adoc[] *** xref:management-ops/projections/graph-project-cypher-aggregation.adoc[] *** xref:graph-project-apache-arrow.adoc[] *** xref:management-ops/projections/graph-project-subgraph.adoc[] diff --git a/doc/modules/ROOT/pages/common-usage/memory-estimation.adoc b/doc/modules/ROOT/pages/common-usage/memory-estimation.adoc index 7896b52e38..6d38e76252 100644 --- a/doc/modules/ROOT/pages/common-usage/memory-estimation.adoc +++ b/doc/modules/ROOT/pages/common-usage/memory-estimation.adoc @@ -153,7 +153,7 @@ YIELD requiredMemory, treeView, mapView, bytesMin, bytesMax, nodeCount, relation | "593 KiB" | 607576 | 607576 | 100 | 1000 |=== -The xref:management-ops/projections/graph-project-cypher.adoc[`gds.graph.project.cypher`] procedure has to execute both, the `nodeQuery` and `relationshipQuery`, in order to count the number of nodes and relationships of the graph. +The xref:management-ops/projections/graph-project-cypher-legacy.adoc[`gds.graph.project.cypher`] procedure has to execute both, the `nodeQuery` and `relationshipQuery`, in order to count the number of nodes and relationships of the graph. .Syntax [source, cypher, role=noplay] diff --git a/doc/modules/ROOT/pages/graph-list.adoc b/doc/modules/ROOT/pages/graph-list.adoc index da9cc1aa07..af32030f9e 100644 --- a/doc/modules/ROOT/pages/graph-list.adoc +++ b/doc/modules/ROOT/pages/graph-list.adoc @@ -95,7 +95,7 @@ CREATE (florentin)-[:KNOWS { since: 2018 }]->(veselin) ---- -Additionally, we will project a few graphs to the graph catalog, for more details see xref:management-ops/projections/graph-project.adoc[native projections] and xref:management-ops/projections/graph-project-cypher.adoc[Legacy Cypher projections]. +Additionally, we will project a few graphs to the graph catalog, for more details see xref:management-ops/projections/graph-project.adoc[native projections] and xref:management-ops/projections/graph-project-cypher-legacy.adoc[Legacy Cypher projections]. .Project `Person` nodes and `KNOWS` relationships using native projections: [source, cypher, role=noplay graph-project-query] diff --git a/doc/modules/ROOT/pages/installation/System-requirements.adoc b/doc/modules/ROOT/pages/installation/System-requirements.adoc index e3f3fcb1a7..22a896abd6 100644 --- a/doc/modules/ROOT/pages/installation/System-requirements.adoc +++ b/doc/modules/ROOT/pages/installation/System-requirements.adoc @@ -60,7 +60,7 @@ https://neo4j.com/docs/operations-manual/4.4/reference/configuration-settings/#c However, setting a minimum page cache size is still important when projecting graphs: * For xref:management-ops/projections/graph-project.adoc[native projections], the minimum page cache size for projecting a graph can be roughly estimated by `8KB * 100 * readConcurrency`. -* For xref:management-ops/projections/graph-project-cypher.adoc[Legacy Cypher projections], a higher page cache is required depending on the query complexity. +* For xref:management-ops/projections/graph-project-cypher-legacy.adoc[Legacy Cypher projections], a higher page cache is required depending on the query complexity. * For xref:management-ops/projections/graph-project-cypher-aggregation.adoc[Cypher projections], a higher page cache is required depending on the query complexity. * For xref:graph-project-apache-arrow.adoc[projections through Apache Arrow], the page cache is irrelevant. diff --git a/doc/modules/ROOT/pages/management-ops/graph-catalog-ops.adoc b/doc/modules/ROOT/pages/management-ops/graph-catalog-ops.adoc index 8496f111b6..a707edd743 100644 --- a/doc/modules/ROOT/pages/management-ops/graph-catalog-ops.adoc +++ b/doc/modules/ROOT/pages/management-ops/graph-catalog-ops.adoc @@ -40,7 +40,7 @@ Read more about that in xref:management-ops/administration.adoc[]. Named graphs can be projected from a Neo4j database by using either of - a xref:management-ops/projections/graph-project.adoc[Native projection] -- a xref:management-ops/projections/graph-project-cypher.adoc[Legacy Cypher projection] +- a xref:management-ops/projections/graph-project-cypher-legacy.adoc[Legacy Cypher projection] - or a xref:management-ops/projections/graph-project-cypher-aggregation.adoc[Cypher projection] But graphs can also be projected into the graph catalog from other sources. @@ -52,7 +52,7 @@ xref:management-ops/projections/graph-generation.adoc[Randomised graphs can be g |=== | Name | Description | xref:management-ops/projections/graph-project.adoc[gds.graph.project] | Adds a graph to the catalog using Native projection. -| xref:management-ops/projections/graph-project-cypher.adoc[gds.graph.project.cypher] | Adds a graph to the catalog using Legacy Cypher projection. +| xref:management-ops/projections/graph-project-cypher-legacy.adoc[gds.graph.project.cypher] | Adds a graph to the catalog using Legacy Cypher projection. | xref:management-ops/projections/graph-project-cypher-aggregation.adoc[gds.graph.project] | Adds a graph to the catalog using Cypher projection. | xref:management-ops/projections/graph-project-subgraph.adoc[gds.beta.graph.project.subgraph] | Adds a graph to the catalog by filtering an existing graph using node and relationship predicates. | xref:management-ops/projections/rwr.adoc[gds.graph.sample.rwr] | Adds a graph to the catalog by sampling an existing graph using random walk with restarts. diff --git a/doc/modules/ROOT/pages/management-ops/projections/graph-project-cypher.adoc b/doc/modules/ROOT/pages/management-ops/projections/graph-project-cypher-legacy.adoc similarity index 98% rename from doc/modules/ROOT/pages/management-ops/projections/graph-project-cypher.adoc rename to doc/modules/ROOT/pages/management-ops/projections/graph-project-cypher-legacy.adoc index d4555ae09c..5d4d2df86a 100644 --- a/doc/modules/ROOT/pages/management-ops/projections/graph-project-cypher.adoc +++ b/doc/modules/ROOT/pages/management-ops/projections/graph-project-cypher-legacy.adoc @@ -1,4 +1,4 @@ -[[catalog-graph-project-cypher]] +[[catalog-graph-project-cypher-legacy]] = Projecting graphs using Cypher (deprecated) :description: This section details projecting GDS graphs using legacy `Cypher` projections. @@ -33,13 +33,13 @@ The xref:management-ops/node-properties.adoc#node-properties-supported[Node Prop Other types of node properties have to be transformed or encoded into one of the supported types in order to be projected using a Legacy Cypher projection. -[[graph-project-cypher-syntax]] +[[graph-project-cypher-legacy-syntax]] == Syntax A Legacy Cypher projection takes three mandatory arguments: `graphName`, `nodeQuery` and `relationshipQuery`. In addition, the optional `configuration` parameter allows us to further configure graph creation. -[.graph-project-cypher-syntax] +[.graph-project-cypher-legacy-syntax] -- [source, cypher, role=noplay] ---- @@ -487,7 +487,7 @@ ORDER BY person ASC, numberOfPages DESC |=== -- -If we compare the results to the ones from xref:management-ops/projections/graph-project-cypher.adoc#cypher-relationship-properties[Relationship properties], we can see that using `IS NOT NULL` is filtering out the relationship from Veselin to the book Frankenstein. +If we compare the results to the ones from xref:management-ops/projections/graph-project-cypher-legacy.adoc#cypher-relationship-properties[Relationship properties], we can see that using `IS NOT NULL` is filtering out the relationship from Veselin to the book Frankenstein. This functionality is only expressible with xref:management-ops/projections/graph-project.adoc[native projections] by projecting a xref:management-ops/projections/graph-project-subgraph.adoc[subgraph]. [[cypher-projection-parameters]] diff --git a/doc/modules/ROOT/pages/management-ops/projections/graph-project.adoc b/doc/modules/ROOT/pages/management-ops/projections/graph-project.adoc index 0aabac0150..248a110ae9 100644 --- a/doc/modules/ROOT/pages/management-ops/projections/graph-project.adoc +++ b/doc/modules/ROOT/pages/management-ops/projections/graph-project.adoc @@ -493,7 +493,7 @@ YIELD To specify the orientation, we need to write the `relationshipProjection` with the extended Map-syntax. Projecting the `KNOWS` relationships `UNDIRECTED`, loads each relationship in both directions. -Thus, the `undirectedKnows` graph contains four relationships, twice as many as the `persons` graph in xref:management-ops/projections/graph-project-cypher.adoc#graph-project-example-single-label-type[Simple graph]. +Thus, the `undirectedKnows` graph contains four relationships, twice as many as the `persons` graph in xref:management-ops/projections/graph-project-cypher-legacy.adoc#graph-project-example-single-label-type[Simple graph]. [[node-properties-example]] @@ -778,7 +778,7 @@ We can see, that the two READ relationships between Florentin and the Hobbit sum As mentioned in the xref:management-ops/projections/graph-project.adoc#graph-project-native-syntax[syntax section], the `validateRelationships` flag controls whether an error will be raised when attempting to project a relationship where either the source or target node is not present in the xref:management-ops/projections/graph-project.adoc#node-projection-syntax[node projection]. Note that even if the flag is set to `false` such a relationship will still not be projected but the loading process will not be aborted. -We can simulate such a case with the xref:management-ops/projections/graph-project-cypher.adoc#graph-project-examples[graph present in the Neo4j database]: +We can simulate such a case with the xref:management-ops/projections/graph-project-cypher-legacy.adoc#graph-project-examples[graph present in the Neo4j database]: -- .Project `READ` and `KNOWS` relationships but only `Person` nodes, with `validateRelationships` set to true: From 92a97cde2ea0835bfbd938a975b46f288386f45e Mon Sep 17 00:00:00 2001 From: Paul Horn Date: Tue, 6 Jun 2023 12:22:33 +0200 Subject: [PATCH 013/273] docs: Rename Cypher aggregation to Cypher projection --- doc/modules/ROOT/content-nav.adoc | 2 +- .../installation/System-requirements.adoc | 2 +- .../management-ops/graph-catalog-ops.adoc | 4 ++-- .../graph-project-cypher-legacy.adoc | 4 ++-- ...c => graph-project-cypher-projection.adoc} | 22 +++++++++---------- .../projections/graph-project.adoc | 2 +- .../production-deployment/composite.adoc | 2 +- .../shared/examples-named-native-note.adoc | 2 +- 8 files changed, 20 insertions(+), 20 deletions(-) rename doc/modules/ROOT/pages/management-ops/projections/{graph-project-cypher-aggregation.adoc => graph-project-cypher-projection.adoc} (94%) diff --git a/doc/modules/ROOT/content-nav.adoc b/doc/modules/ROOT/content-nav.adoc index da33df3255..223d0e8ca0 100644 --- a/doc/modules/ROOT/content-nav.adoc +++ b/doc/modules/ROOT/content-nav.adoc @@ -19,7 +19,7 @@ ** xref:management-ops/graph-catalog-ops.adoc[] *** xref:management-ops/projections/graph-project.adoc[] *** xref:management-ops/projections/graph-project-cypher-legacy.adoc[] -*** xref:management-ops/projections/graph-project-cypher-aggregation.adoc[] +*** xref:management-ops/projections/graph-project-cypher-projection.adoc[] *** xref:graph-project-apache-arrow.adoc[] *** xref:management-ops/projections/graph-project-subgraph.adoc[] *** xref:management-ops/projections/rwr.adoc[] diff --git a/doc/modules/ROOT/pages/installation/System-requirements.adoc b/doc/modules/ROOT/pages/installation/System-requirements.adoc index 22a896abd6..91a1e09f70 100644 --- a/doc/modules/ROOT/pages/installation/System-requirements.adoc +++ b/doc/modules/ROOT/pages/installation/System-requirements.adoc @@ -61,7 +61,7 @@ However, setting a minimum page cache size is still important when projecting gr * For xref:management-ops/projections/graph-project.adoc[native projections], the minimum page cache size for projecting a graph can be roughly estimated by `8KB * 100 * readConcurrency`. * For xref:management-ops/projections/graph-project-cypher-legacy.adoc[Legacy Cypher projections], a higher page cache is required depending on the query complexity. -* For xref:management-ops/projections/graph-project-cypher-aggregation.adoc[Cypher projections], a higher page cache is required depending on the query complexity. +* For xref:management-ops/projections/graph-project-cypher-projection.adoc[Cypher projections], a higher page cache is required depending on the query complexity. * For xref:graph-project-apache-arrow.adoc[projections through Apache Arrow], the page cache is irrelevant. However, if it is required to write algorithm results back to Neo4j, the write performance is highly depended on store fragmentation as well as the number of properties and relationships to write. diff --git a/doc/modules/ROOT/pages/management-ops/graph-catalog-ops.adoc b/doc/modules/ROOT/pages/management-ops/graph-catalog-ops.adoc index a707edd743..79ce33f746 100644 --- a/doc/modules/ROOT/pages/management-ops/graph-catalog-ops.adoc +++ b/doc/modules/ROOT/pages/management-ops/graph-catalog-ops.adoc @@ -41,7 +41,7 @@ Named graphs can be projected from a Neo4j database by using either of - a xref:management-ops/projections/graph-project.adoc[Native projection] - a xref:management-ops/projections/graph-project-cypher-legacy.adoc[Legacy Cypher projection] -- or a xref:management-ops/projections/graph-project-cypher-aggregation.adoc[Cypher projection] +- or a xref:management-ops/projections/graph-project-cypher-projection.adoc[Cypher projection] But graphs can also be projected into the graph catalog from other sources. xref:management-ops/projections/graph-project-subgraph.adoc[Subgraph] and xref:management-ops/projections/rwr.adoc[Graph Sampling] projections allow projecting a new graph based off of an existing graph projection. @@ -53,7 +53,7 @@ xref:management-ops/projections/graph-generation.adoc[Randomised graphs can be g | Name | Description | xref:management-ops/projections/graph-project.adoc[gds.graph.project] | Adds a graph to the catalog using Native projection. | xref:management-ops/projections/graph-project-cypher-legacy.adoc[gds.graph.project.cypher] | Adds a graph to the catalog using Legacy Cypher projection. -| xref:management-ops/projections/graph-project-cypher-aggregation.adoc[gds.graph.project] | Adds a graph to the catalog using Cypher projection. +| xref:management-ops/projections/graph-project-cypher-projection.adoc[gds.graph.project] | Adds a graph to the catalog using Cypher projection. | xref:management-ops/projections/graph-project-subgraph.adoc[gds.beta.graph.project.subgraph] | Adds a graph to the catalog by filtering an existing graph using node and relationship predicates. | xref:management-ops/projections/rwr.adoc[gds.graph.sample.rwr] | Adds a graph to the catalog by sampling an existing graph using random walk with restarts. | xref:management-ops/projections/cnarw.adoc[gds.graph.sample.cnarw] | Adds a graph to the catalog by sampling an existing graph using Common Neighbour Aware Random Walk algorithm. diff --git a/doc/modules/ROOT/pages/management-ops/projections/graph-project-cypher-legacy.adoc b/doc/modules/ROOT/pages/management-ops/projections/graph-project-cypher-legacy.adoc index 5d4d2df86a..61f9a620e3 100644 --- a/doc/modules/ROOT/pages/management-ops/projections/graph-project-cypher-legacy.adoc +++ b/doc/modules/ROOT/pages/management-ops/projections/graph-project-cypher-legacy.adoc @@ -5,7 +5,7 @@ [NOTE] -- This page describes the Legacy Cypher projection, which is deprecated. -The replacement is to use the new Cypher projection, which is described in xref:management-ops/projections/graph-project-cypher-aggregation.adoc[Projecting graphs using Cypher]. +The replacement is to use the new Cypher projection, which is described in xref:management-ops/projections/graph-project-cypher-projection.adoc[Projecting graphs using Cypher]. -- Using Legacy Cypher projections is a more flexible and expressive approach with diminished focus on performance compared to the xref:management-ops/projections/graph-project.adoc[native projections]. @@ -312,7 +312,7 @@ ORDER BY person ASC, numberOfPages DESC |=== -- -We can see, that the `numberOfPages` are loaded. The default property value is `Double.Nan` and can be changed as in the previous example xref:management-ops/projections/graph-project-cypher-aggregation.adoc#node-properties-example[Node properties] by using the Cypher function https://neo4j.com/docs/cypher-manual/current/functions/scalar/#functions-coalesce[_coalesce()_]. +We can see, that the `numberOfPages` are loaded. The default property value is `Double.Nan` and can be changed as in the previous example xref:management-ops/projections/graph-project-cypher-projection.adoc#node-properties-example[Node properties] by using the Cypher function https://neo4j.com/docs/cypher-manual/current/functions/scalar/#functions-coalesce[_coalesce()_]. === Parallel relationships diff --git a/doc/modules/ROOT/pages/management-ops/projections/graph-project-cypher-aggregation.adoc b/doc/modules/ROOT/pages/management-ops/projections/graph-project-cypher-projection.adoc similarity index 94% rename from doc/modules/ROOT/pages/management-ops/projections/graph-project-cypher-aggregation.adoc rename to doc/modules/ROOT/pages/management-ops/projections/graph-project-cypher-projection.adoc index b832d9b7df..e8d2e62202 100644 --- a/doc/modules/ROOT/pages/management-ops/projections/graph-project-cypher-aggregation.adoc +++ b/doc/modules/ROOT/pages/management-ops/projections/graph-project-cypher-projection.adoc @@ -1,4 +1,4 @@ -[[catalog-graph-project-cypher-aggregation]] +[[catalog-graph-project-cypher-projection]] = Projecting graphs using Cypher :description: This section details projecting GDS graphs using `Cypher` projections. @@ -34,7 +34,7 @@ This is important when a node can be a source node as well as a target node and Relevant configuration options are `sourceNodeProperties`, `targetNodeProperties`, `sourceNodeLabels` and `targetNodeLabels`. -[[graph-project-cypher-aggregation-syntax]] +[[graph-project-cypher-projection-syntax]] == Syntax A Cypher projection is used in a query as an aggregation over the relationships that are being projected. @@ -44,7 +44,7 @@ The parameter is optional and can be `null` to project an unconnected node. The next and fourth optional `dataConfig` parameter can be used to project node properties and labels as well as relationship properties and type. The last and fifth optional `configuration` parameter can be used for general configuration of the projection such as `readConcurrency`. -[.graph-project-cypher-aggregation-syntax] +[.graph-project-cypher-projection-syntax] -- [source, cypher, role=noplay] ---- @@ -69,11 +69,11 @@ RETURN gds.graph.project( | graphName | no | The name under which the graph is stored in the catalog. | sourceNode | no | The source node of the relationship. Must not be null. | targetNode | yes | The target node of the relationship. The targetNode can be null (for example due to an `OPTIONAL MATCH`), in which case the source node is projected as an unconnected node. -| <> | yes | Properties and labels configuration for the source and target nodes as well as properties and type configuration for the relationship. -| <> | yes | Additional parameters to configure the projection. +| <> | yes | Properties and labels configuration for the source and target nodes as well as properties and type configuration for the relationship. +| <> | yes | Additional parameters to configure the projection. |=== -[[graph-project-cypher-aggregation-syntax-dataConfig]] +[[graph-project-cypher-projection-syntax-dataConfig]] .Nodes configuration [opts="header",cols="1,1,1,4"] |=== @@ -86,7 +86,7 @@ RETURN gds.graph.project( | relationshipType | String | '*' | The type of the relationship. |=== -[[graph-project-cypher-aggregation-syntax-configuration]] +[[graph-project-cypher-projection-syntax-configuration]] .Configuration [opts="header",cols="1,1,1,4"] |=== @@ -112,7 +112,7 @@ RETURN gds.graph.project( NOTE: To get information about a stored graph, such as its schema, one can use xref:graph-list.adoc[gds.graph.list]. -[[graph-project-cypher-aggregation-examples]] +[[graph-project-cypher-projection-examples]] == Examples In order to demonstrate the GDS Cypher Aggregation we are going to create a small social network graph in Neo4j. @@ -139,7 +139,7 @@ CREATE ---- -[[graph-project-cypher-aggregation-example-single-label-type]] +[[graph-project-cypher-projection-example-single-label-type]] === Simple graph A simple graph is a graph with only one node label and relationship type, i.e., a monopartite graph. @@ -465,7 +465,7 @@ ORDER BY person ASC, numberOfPages DESC |=== -- -We can see, that the `numberOfPages` are loaded. The default property value is `Double.Nan` and can be changed as in the previous example xref:management-ops/projections/graph-project-cypher-aggregation.adoc#node-properties-example[Node properties] by using the Cypher function https://neo4j.com/docs/cypher-manual/current/functions/scalar/#functions-coalesce[_coalesce()_]. +We can see, that the `numberOfPages` are loaded. The default property value is `Double.Nan` and can be changed as in the previous example xref:management-ops/projections/graph-project-cypher-projection.adoc#node-properties-example[Node properties] by using the Cypher function https://neo4j.com/docs/cypher-manual/current/functions/scalar/#functions-coalesce[_coalesce()_]. === Parallel relationships @@ -630,5 +630,5 @@ ORDER BY person ASC, numberOfPages DESC |=== -- -If we compare the results to the ones from xref:management-ops/projections/graph-project-cypher-aggregation.adoc#cypher-aggregation-relationship-properties[Relationship properties], we can see that using `IS NOT NULL` is filtering out the relationship from Veselin to the book Frankenstein. +If we compare the results to the ones from xref:management-ops/projections/graph-project-cypher-projection.adoc#cypher-aggregation-relationship-properties[Relationship properties], we can see that using `IS NOT NULL` is filtering out the relationship from Veselin to the book Frankenstein. This functionality is only expressible with xref:management-ops/projections/graph-project.adoc[native projections] by projecting a xref:management-ops/projections/graph-project-subgraph.adoc[subgraph]. diff --git a/doc/modules/ROOT/pages/management-ops/projections/graph-project.adoc b/doc/modules/ROOT/pages/management-ops/projections/graph-project.adoc index 248a110ae9..f80ecebc9b 100644 --- a/doc/modules/ROOT/pages/management-ops/projections/graph-project.adoc +++ b/doc/modules/ROOT/pages/management-ops/projections/graph-project.adoc @@ -627,7 +627,7 @@ ORDER BY person ASC, numberOfPages DESC |=== -- -We can see, that the `numberOfPages` property is loaded. The default property value is `Double.NaN` and could be changed using the Map-Syntax the same as for node properties in xref:management-ops/projections/graph-project-cypher-aggregation.adoc#node-properties-example[Node properties]. +We can see, that the `numberOfPages` property is loaded. The default property value is `Double.NaN` and could be changed using the Map-Syntax the same as for node properties in xref:management-ops/projections/graph-project-cypher-projection.adoc#node-properties-example[Node properties]. === Parallel relationships diff --git a/doc/modules/ROOT/pages/production-deployment/composite.adoc b/doc/modules/ROOT/pages/production-deployment/composite.adoc index e6a5c653e2..887670fa6b 100644 --- a/doc/modules/ROOT/pages/production-deployment/composite.adoc +++ b/doc/modules/ROOT/pages/production-deployment/composite.adoc @@ -105,7 +105,7 @@ For every row returned by the first subquery, the operational database is then q In this mode of using GDS in a composite environment, the GDS operations are executed on the Fabric proxy server. The graph projections are then using the data stored on the shards to construct the in-memory graph. -NOTE: Currently only xref:management-ops/projections/graph-project-cypher-aggregation.adoc[Cypher projection] is supported for projecting in-memory graphs on a Composite database. +NOTE: Currently only xref:management-ops/projections/graph-project-cypher-projection.adoc[Cypher projection] is supported for projecting in-memory graphs on a Composite database. Graph algorithms can then be executed on the composite database, similar to a single machine setup. This scenario is useful, if a graph that logically represents a single graph is distributed to different Composite shards. diff --git a/doc/modules/ROOT/partials/algorithms/shared/examples-named-native-note.adoc b/doc/modules/ROOT/partials/algorithms/shared/examples-named-native-note.adoc index f65b037b7b..30a72a0ef8 100644 --- a/doc/modules/ROOT/partials/algorithms/shared/examples-named-native-note.adoc +++ b/doc/modules/ROOT/partials/algorithms/shared/examples-named-native-note.adoc @@ -1,5 +1,5 @@ [NOTE] ==== In the examples below we will use named graphs and native projections as the norm. -However, xref:management-ops/projections/graph-project-cypher-aggregation.adoc[Cypher projections] can also be used. +However, xref:management-ops/projections/graph-project-cypher-projection.adoc[Cypher projections] can also be used. ==== From b3d374d9d421df8d3c8e8e97802804b2f4c57674 Mon Sep 17 00:00:00 2001 From: Paul Horn Date: Tue, 6 Jun 2023 17:49:50 +0200 Subject: [PATCH 014/273] Change doc tests to new file names --- .../src/main/java/org/neo4j/gds/doc/syntax/SyntaxMode.java | 2 +- .../org/neo4j/gds/doc/GraphProjectCypherAggregationDocTest.java | 2 +- .../test/java/org/neo4j/gds/doc/GraphProjectCypherDocTest.java | 2 +- .../gds/doc/syntax/GraphProjectCypherAggregationSyntaxTest.java | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/doc-test-tools/src/main/java/org/neo4j/gds/doc/syntax/SyntaxMode.java b/doc-test-tools/src/main/java/org/neo4j/gds/doc/syntax/SyntaxMode.java index 89d2f2614f..4eb3d38ad0 100644 --- a/doc-test-tools/src/main/java/org/neo4j/gds/doc/syntax/SyntaxMode.java +++ b/doc-test-tools/src/main/java/org/neo4j/gds/doc/syntax/SyntaxMode.java @@ -33,7 +33,7 @@ public enum SyntaxMode { GRAPH_PROJECT("graph-project-syntax"), GRAPH_PROJECT_CYPHER("graph-project-cypher-syntax"), GRAPH_PROJECT_CYPHER_AGGREGATION( - "graph-project-cypher-aggregation-syntax", + "graph-project-cypher-projection-syntax", true, CustomProcedure.Namespace.AGGREGATION_FUNCTION ), diff --git a/doc-test/src/test/java/org/neo4j/gds/doc/GraphProjectCypherAggregationDocTest.java b/doc-test/src/test/java/org/neo4j/gds/doc/GraphProjectCypherAggregationDocTest.java index 0a920868b5..769e41fb28 100644 --- a/doc-test/src/test/java/org/neo4j/gds/doc/GraphProjectCypherAggregationDocTest.java +++ b/doc-test/src/test/java/org/neo4j/gds/doc/GraphProjectCypherAggregationDocTest.java @@ -45,6 +45,6 @@ protected List> procedures() { @Override protected String adocFile() { - return "pages/management-ops/projections/graph-project-cypher-aggregation.adoc"; + return "pages/management-ops/projections/graph-project-cypher-projection.adoc"; } } diff --git a/doc-test/src/test/java/org/neo4j/gds/doc/GraphProjectCypherDocTest.java b/doc-test/src/test/java/org/neo4j/gds/doc/GraphProjectCypherDocTest.java index 020f99e39e..40fe8c4663 100644 --- a/doc-test/src/test/java/org/neo4j/gds/doc/GraphProjectCypherDocTest.java +++ b/doc-test/src/test/java/org/neo4j/gds/doc/GraphProjectCypherDocTest.java @@ -39,6 +39,6 @@ protected List> procedures() { @Override protected String adocFile() { - return "pages/management-ops/projections/graph-project-cypher.adoc"; + return "pages/management-ops/projections/graph-project-cypher-legacy.adoc"; } } diff --git a/doc-test/src/test/java/org/neo4j/gds/doc/syntax/GraphProjectCypherAggregationSyntaxTest.java b/doc-test/src/test/java/org/neo4j/gds/doc/syntax/GraphProjectCypherAggregationSyntaxTest.java index b51b9968bc..3c60ba825e 100644 --- a/doc-test/src/test/java/org/neo4j/gds/doc/syntax/GraphProjectCypherAggregationSyntaxTest.java +++ b/doc-test/src/test/java/org/neo4j/gds/doc/syntax/GraphProjectCypherAggregationSyntaxTest.java @@ -32,6 +32,6 @@ protected Iterable syntaxModes() { @Override protected String adocFile() { - return "pages/management-ops/projections/graph-project-cypher-aggregation.adoc"; + return "pages/management-ops/projections/graph-project-cypher-projection.adoc"; } } From fe690e5b2017dadf086ec5af4a4c6fb50e0112c5 Mon Sep 17 00:00:00 2001 From: Paul Horn Date: Wed, 7 Jun 2023 09:37:22 +0200 Subject: [PATCH 015/273] Change doc tests to new file names --- .../src/main/java/org/neo4j/gds/doc/syntax/SyntaxMode.java | 2 +- .../org/neo4j/gds/doc/syntax/GraphProjectCypherSyntaxTest.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc-test-tools/src/main/java/org/neo4j/gds/doc/syntax/SyntaxMode.java b/doc-test-tools/src/main/java/org/neo4j/gds/doc/syntax/SyntaxMode.java index 4eb3d38ad0..07582e3b9d 100644 --- a/doc-test-tools/src/main/java/org/neo4j/gds/doc/syntax/SyntaxMode.java +++ b/doc-test-tools/src/main/java/org/neo4j/gds/doc/syntax/SyntaxMode.java @@ -31,7 +31,7 @@ public enum SyntaxMode { GRAPH_DROP("graph-drop-syntax"), MODEL_DROP("model-drop-syntax"), GRAPH_PROJECT("graph-project-syntax"), - GRAPH_PROJECT_CYPHER("graph-project-cypher-syntax"), + GRAPH_PROJECT_CYPHER("graph-project-cypher-legacy-syntax"), GRAPH_PROJECT_CYPHER_AGGREGATION( "graph-project-cypher-projection-syntax", true, diff --git a/doc-test/src/test/java/org/neo4j/gds/doc/syntax/GraphProjectCypherSyntaxTest.java b/doc-test/src/test/java/org/neo4j/gds/doc/syntax/GraphProjectCypherSyntaxTest.java index 3314b16b71..c93cb56f32 100644 --- a/doc-test/src/test/java/org/neo4j/gds/doc/syntax/GraphProjectCypherSyntaxTest.java +++ b/doc-test/src/test/java/org/neo4j/gds/doc/syntax/GraphProjectCypherSyntaxTest.java @@ -32,6 +32,6 @@ protected Iterable syntaxModes() { @Override protected String adocFile() { - return "pages/management-ops/projections/graph-project-cypher.adoc"; + return "pages/management-ops/projections/graph-project-cypher-legacy.adoc"; } } From b0d7fca6efc7ba33c4fe304bb62f103c26372fb2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20Kie=C3=9Fling?= Date: Wed, 7 Jun 2023 10:03:21 +0200 Subject: [PATCH 016/273] Use common exception for checking negative node ids --- .../gds/core/loading/LazyIdMapBuilder.java | 25 ++++++------------- .../gds/core/loading/LoadingExceptions.java | 17 ++++++++++++- 2 files changed, 24 insertions(+), 18 deletions(-) diff --git a/core/src/main/java/org/neo4j/gds/core/loading/LazyIdMapBuilder.java b/core/src/main/java/org/neo4j/gds/core/loading/LazyIdMapBuilder.java index e3d3e44218..f73203b478 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/LazyIdMapBuilder.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/LazyIdMapBuilder.java @@ -30,7 +30,6 @@ import org.neo4j.gds.core.loading.construction.PropertyValues; import org.neo4j.gds.core.utils.paged.ShardedLongLongMap; -import java.util.Locale; import java.util.OptionalLong; import java.util.concurrent.atomic.AtomicBoolean; @@ -40,7 +39,12 @@ public final class LazyIdMapBuilder implements PartialIdMap { private final NodesBuilder nodesBuilder; - public LazyIdMapBuilder(int concurrency, boolean hasLabelInformation, boolean hasProperties, PropertyState propertyState) { + public LazyIdMapBuilder( + int concurrency, + boolean hasLabelInformation, + boolean hasProperties, + PropertyState propertyState + ) { this.intermediateIdMapBuilder = ShardedLongLongMap.builder(concurrency); this.nodesBuilder = GraphFactory.initNodesBuilder() .concurrency(concurrency) @@ -57,7 +61,7 @@ public void prepareForFlush() { } public long addNode(long nodeId, NodeLabelToken nodeLabels) { - checkPositiveId(nodeId); + LoadingExceptions.checkPositiveId(nodeId); long intermediateId = this.intermediateIdMapBuilder.addNode(nodeId); @@ -71,19 +75,6 @@ public long addNode(long nodeId, NodeLabelToken nodeLabels) { return intermediateId; } - /** - * GDS has the general assumption of non-negative original node ids. - */ - private static void checkPositiveId(long nodeId) { - if (nodeId < 0) { - throw new IllegalArgumentException(String.format( - Locale.US, - "GDS expects node ids to be positive. But got a negative id of `%d`.", - nodeId - )); - } - } - public long addNodeWithProperties( long nodeId, PropertyValues properties, @@ -91,7 +82,7 @@ public long addNodeWithProperties( ) { long intermediateId = this.intermediateIdMapBuilder.addNode(nodeId); - checkPositiveId(nodeId); + LoadingExceptions.checkPositiveId(nodeId); // deduplication if (intermediateId < 0) { diff --git a/core/src/main/java/org/neo4j/gds/core/loading/LoadingExceptions.java b/core/src/main/java/org/neo4j/gds/core/loading/LoadingExceptions.java index e24aabf139..48b1d0e56d 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/LoadingExceptions.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/LoadingExceptions.java @@ -32,13 +32,28 @@ public static void validateSourceNodeIsLoaded(long mappedId, long neoId) { validateNodeIsLoaded(mappedId, neoId, "source"); } + /** + * GDS has the general assumption of non-negative original node ids. + */ + public static void checkPositiveId(long nodeId) { + if (nodeId < 0) { + throw new IllegalArgumentException( + String.format( + Locale.US, + "GDS expects node ids to be positive. But got a negative id of `%d`.", + nodeId + ) + ); + } + } + private static void validateNodeIsLoaded(long mappedId, long neoId, String side) { if (mappedId == -1) { throw new IllegalArgumentException( String.format( Locale.US, "Failed to load a relationship because its %s-node with id %s is not part of the node query or projection. " + - "To ignore the relationship, set the configuration parameter `validateRelationships` to false.", + "To ignore the relationship, set the configuration parameter `validateRelationships` to false.", side, neoId ) From 93108c1ffa0e79a9406695ee6178b7783b4e06a4 Mon Sep 17 00:00:00 2001 From: Paul Horn Date: Wed, 7 Jun 2023 10:26:29 +0200 Subject: [PATCH 017/273] Link to new CP instead of LCP MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Florentin Dörre --- doc/modules/ROOT/pages/graph-list.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/modules/ROOT/pages/graph-list.adoc b/doc/modules/ROOT/pages/graph-list.adoc index af32030f9e..3f16115903 100644 --- a/doc/modules/ROOT/pages/graph-list.adoc +++ b/doc/modules/ROOT/pages/graph-list.adoc @@ -95,7 +95,7 @@ CREATE (florentin)-[:KNOWS { since: 2018 }]->(veselin) ---- -Additionally, we will project a few graphs to the graph catalog, for more details see xref:management-ops/projections/graph-project.adoc[native projections] and xref:management-ops/projections/graph-project-cypher-legacy.adoc[Legacy Cypher projections]. +Additionally, we will project a few graphs to the graph catalog, for more details see xref:management-ops/projections/graph-project.adoc[native projections] and xref:management-ops/projections/graph-project-cypher-projection.adoc[Cypher projections]. .Project `Person` nodes and `KNOWS` relationships using native projections: [source, cypher, role=noplay graph-project-query] From c5afd2468b64ee34d9e53b14502c7d5cff48dba6 Mon Sep 17 00:00:00 2001 From: Paul Horn Date: Wed, 7 Jun 2023 10:29:17 +0200 Subject: [PATCH 018/273] Move LCP to the last location in listings of projections MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Florentin Dörre --- .../ROOT/pages/installation/System-requirements.adoc | 2 +- .../ROOT/pages/management-ops/graph-catalog-ops.adoc | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/modules/ROOT/pages/installation/System-requirements.adoc b/doc/modules/ROOT/pages/installation/System-requirements.adoc index 91a1e09f70..6ccd526065 100644 --- a/doc/modules/ROOT/pages/installation/System-requirements.adoc +++ b/doc/modules/ROOT/pages/installation/System-requirements.adoc @@ -60,9 +60,9 @@ https://neo4j.com/docs/operations-manual/4.4/reference/configuration-settings/#c However, setting a minimum page cache size is still important when projecting graphs: * For xref:management-ops/projections/graph-project.adoc[native projections], the minimum page cache size for projecting a graph can be roughly estimated by `8KB * 100 * readConcurrency`. -* For xref:management-ops/projections/graph-project-cypher-legacy.adoc[Legacy Cypher projections], a higher page cache is required depending on the query complexity. * For xref:management-ops/projections/graph-project-cypher-projection.adoc[Cypher projections], a higher page cache is required depending on the query complexity. * For xref:graph-project-apache-arrow.adoc[projections through Apache Arrow], the page cache is irrelevant. +* For xref:management-ops/projections/graph-project-cypher-legacy.adoc[Legacy Cypher projections], a higher page cache is required depending on the query complexity. However, if it is required to write algorithm results back to Neo4j, the write performance is highly depended on store fragmentation as well as the number of properties and relationships to write. We recommend starting with a page cache size of roughly `250MB * writeConcurrency` and evaluate write performance and adapt accordingly. diff --git a/doc/modules/ROOT/pages/management-ops/graph-catalog-ops.adoc b/doc/modules/ROOT/pages/management-ops/graph-catalog-ops.adoc index 79ce33f746..c699bfcddf 100644 --- a/doc/modules/ROOT/pages/management-ops/graph-catalog-ops.adoc +++ b/doc/modules/ROOT/pages/management-ops/graph-catalog-ops.adoc @@ -40,8 +40,8 @@ Read more about that in xref:management-ops/administration.adoc[]. Named graphs can be projected from a Neo4j database by using either of - a xref:management-ops/projections/graph-project.adoc[Native projection] -- a xref:management-ops/projections/graph-project-cypher-legacy.adoc[Legacy Cypher projection] -- or a xref:management-ops/projections/graph-project-cypher-projection.adoc[Cypher projection] +- a xref:management-ops/projections/graph-project-cypher-projection.adoc[Cypher projection] +- or a xref:management-ops/projections/graph-project-cypher-legacy.adoc[Legacy Cypher projection] But graphs can also be projected into the graph catalog from other sources. xref:management-ops/projections/graph-project-subgraph.adoc[Subgraph] and xref:management-ops/projections/rwr.adoc[Graph Sampling] projections allow projecting a new graph based off of an existing graph projection. @@ -52,12 +52,12 @@ xref:management-ops/projections/graph-generation.adoc[Randomised graphs can be g |=== | Name | Description | xref:management-ops/projections/graph-project.adoc[gds.graph.project] | Adds a graph to the catalog using Native projection. -| xref:management-ops/projections/graph-project-cypher-legacy.adoc[gds.graph.project.cypher] | Adds a graph to the catalog using Legacy Cypher projection. -| xref:management-ops/projections/graph-project-cypher-projection.adoc[gds.graph.project] | Adds a graph to the catalog using Cypher projection. +| xref:management-ops/projections/graph-project-cypher-projection.adoc[gds.graph.project] | Adds a graph to the catalog using Cypher projection. | xref:management-ops/projections/graph-project-subgraph.adoc[gds.beta.graph.project.subgraph] | Adds a graph to the catalog by filtering an existing graph using node and relationship predicates. | xref:management-ops/projections/rwr.adoc[gds.graph.sample.rwr] | Adds a graph to the catalog by sampling an existing graph using random walk with restarts. | xref:management-ops/projections/cnarw.adoc[gds.graph.sample.cnarw] | Adds a graph to the catalog by sampling an existing graph using Common Neighbour Aware Random Walk algorithm. | xref:management-ops/projections/graph-generation.adoc[gds.beta.graph.generate] | Creates a new random graph projection of the user-defined properties and dimensions. +| xref:management-ops/projections/graph-project-cypher-legacy.adoc[gds.graph.project.cypher] | Adds a graph to the catalog using Legacy Cypher projection. |=== From 22993077e89e2640d4ac53288d77d31ffbe0b1fa Mon Sep 17 00:00:00 2001 From: Paul Horn Date: Wed, 7 Jun 2023 10:30:26 +0200 Subject: [PATCH 019/273] Turn deprecation note into a deprecation warning MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Florentin Dörre --- .../management-ops/projections/graph-project-cypher-legacy.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/modules/ROOT/pages/management-ops/projections/graph-project-cypher-legacy.adoc b/doc/modules/ROOT/pages/management-ops/projections/graph-project-cypher-legacy.adoc index 61f9a620e3..8eda3ffb70 100644 --- a/doc/modules/ROOT/pages/management-ops/projections/graph-project-cypher-legacy.adoc +++ b/doc/modules/ROOT/pages/management-ops/projections/graph-project-cypher-legacy.adoc @@ -2,7 +2,7 @@ = Projecting graphs using Cypher (deprecated) :description: This section details projecting GDS graphs using legacy `Cypher` projections. -[NOTE] +[WARNING] -- This page describes the Legacy Cypher projection, which is deprecated. The replacement is to use the new Cypher projection, which is described in xref:management-ops/projections/graph-project-cypher-projection.adoc[Projecting graphs using Cypher]. From 4117c11f9ce77250ae0bb53e08b8ec59428bf1ae Mon Sep 17 00:00:00 2001 From: Paul Horn Date: Wed, 7 Jun 2023 10:33:07 +0200 Subject: [PATCH 020/273] Add page redirects for the Cypher projections MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Florentin Dörre --- .../management-ops/projections/graph-project-cypher-legacy.adoc | 1 + .../projections/graph-project-cypher-projection.adoc | 1 + 2 files changed, 2 insertions(+) diff --git a/doc/modules/ROOT/pages/management-ops/projections/graph-project-cypher-legacy.adoc b/doc/modules/ROOT/pages/management-ops/projections/graph-project-cypher-legacy.adoc index 8eda3ffb70..ac0a499cf1 100644 --- a/doc/modules/ROOT/pages/management-ops/projections/graph-project-cypher-legacy.adoc +++ b/doc/modules/ROOT/pages/management-ops/projections/graph-project-cypher-legacy.adoc @@ -1,6 +1,7 @@ [[catalog-graph-project-cypher-legacy]] = Projecting graphs using Cypher (deprecated) :description: This section details projecting GDS graphs using legacy `Cypher` projections. +:page-aliases: managments-ops/projections/graph-project-cypher.adoc [WARNING] -- diff --git a/doc/modules/ROOT/pages/management-ops/projections/graph-project-cypher-projection.adoc b/doc/modules/ROOT/pages/management-ops/projections/graph-project-cypher-projection.adoc index e8d2e62202..beeee7671c 100644 --- a/doc/modules/ROOT/pages/management-ops/projections/graph-project-cypher-projection.adoc +++ b/doc/modules/ROOT/pages/management-ops/projections/graph-project-cypher-projection.adoc @@ -1,6 +1,7 @@ [[catalog-graph-project-cypher-projection]] = Projecting graphs using Cypher :description: This section details projecting GDS graphs using `Cypher` projections. +:page-aliases: managments-ops/projections/graph-project-cypher-aggregation.adoc Using Cypher projection is a more flexible and expressive approach with diminished focus on performance compared to the xref:management-ops/projections/graph-project.adoc[native projections]. From 9c02ea2b9556cf299338af5812a6331eac2ba7be Mon Sep 17 00:00:00 2001 From: Paul Horn Date: Wed, 7 Jun 2023 10:35:56 +0200 Subject: [PATCH 021/273] Link native projection links to itself, not to Cypher projection MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Florentin Dörre --- .../pages/management-ops/projections/graph-project.adoc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/modules/ROOT/pages/management-ops/projections/graph-project.adoc b/doc/modules/ROOT/pages/management-ops/projections/graph-project.adoc index f80ecebc9b..a41eceb4df 100644 --- a/doc/modules/ROOT/pages/management-ops/projections/graph-project.adoc +++ b/doc/modules/ROOT/pages/management-ops/projections/graph-project.adoc @@ -493,7 +493,7 @@ YIELD To specify the orientation, we need to write the `relationshipProjection` with the extended Map-syntax. Projecting the `KNOWS` relationships `UNDIRECTED`, loads each relationship in both directions. -Thus, the `undirectedKnows` graph contains four relationships, twice as many as the `persons` graph in xref:management-ops/projections/graph-project-cypher-legacy.adoc#graph-project-example-single-label-type[Simple graph]. +Thus, the `undirectedKnows` graph contains four relationships, twice as many as the `persons` graph in xref:management-ops/projections/graph-project.adoc#graph-project-example-single-label-type[Simple graph]. [[node-properties-example]] @@ -627,7 +627,7 @@ ORDER BY person ASC, numberOfPages DESC |=== -- -We can see, that the `numberOfPages` property is loaded. The default property value is `Double.NaN` and could be changed using the Map-Syntax the same as for node properties in xref:management-ops/projections/graph-project-cypher-projection.adoc#node-properties-example[Node properties]. +We can see, that the `numberOfPages` property is loaded. The default property value is `Double.NaN` and could be changed using the Map-Syntax the same as for node properties in xref:management-ops/projections/graph-project.adoc#node-properties-example[Node properties]. === Parallel relationships @@ -778,7 +778,7 @@ We can see, that the two READ relationships between Florentin and the Hobbit sum As mentioned in the xref:management-ops/projections/graph-project.adoc#graph-project-native-syntax[syntax section], the `validateRelationships` flag controls whether an error will be raised when attempting to project a relationship where either the source or target node is not present in the xref:management-ops/projections/graph-project.adoc#node-projection-syntax[node projection]. Note that even if the flag is set to `false` such a relationship will still not be projected but the loading process will not be aborted. -We can simulate such a case with the xref:management-ops/projections/graph-project-cypher-legacy.adoc#graph-project-examples[graph present in the Neo4j database]: +We can simulate such a case with the xref:management-ops/projections/graph-project.adoc#graph-project-examples[graph present in the Neo4j database]: -- .Project `READ` and `KNOWS` relationships but only `Person` nodes, with `validateRelationships` set to true: From c10d78755e8e21da672b2e803d271c02863d6a55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Mon, 22 May 2023 14:55:38 +0200 Subject: [PATCH 022/273] Add metric counting models created --- .../java/org/neo4j/gds/core/model/ModelCatalog.java | 12 +++++++++++- .../org/neo4j/gds/core/model/OpenModelCatalog.java | 11 +---------- .../neo4j/gds/core/model/OpenModelCatalogTest.java | 2 +- 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/model-catalog-api/src/main/java/org/neo4j/gds/core/model/ModelCatalog.java b/model-catalog-api/src/main/java/org/neo4j/gds/core/model/ModelCatalog.java index afd111f4e0..6f92e5767b 100644 --- a/model-catalog-api/src/main/java/org/neo4j/gds/core/model/ModelCatalog.java +++ b/model-catalog-api/src/main/java/org/neo4j/gds/core/model/ModelCatalog.java @@ -26,11 +26,21 @@ import java.nio.file.Path; import java.util.Collection; +import java.util.HashSet; +import java.util.Set; import java.util.stream.Stream; public interface ModelCatalog { - void registerListener(ModelCatalogListener listener); + Set LISTENERS = new HashSet<>(); + + static void registerListener(ModelCatalogListener listener) { + LISTENERS.add(listener); + } + + static void unregisterListener(ModelCatalogListener listener) { + LISTENERS.remove(listener); + } void set(Model model); diff --git a/open-model-catalog/src/main/java/org/neo4j/gds/core/model/OpenModelCatalog.java b/open-model-catalog/src/main/java/org/neo4j/gds/core/model/OpenModelCatalog.java index cf95b92bbb..1d9ccbe3fd 100644 --- a/open-model-catalog/src/main/java/org/neo4j/gds/core/model/OpenModelCatalog.java +++ b/open-model-catalog/src/main/java/org/neo4j/gds/core/model/OpenModelCatalog.java @@ -28,7 +28,6 @@ import java.nio.file.Path; import java.util.ArrayList; import java.util.Collection; -import java.util.List; import java.util.Locale; import java.util.Map; import java.util.NoSuchElementException; @@ -41,16 +40,8 @@ public final class OpenModelCatalog implements ModelCatalog { private final Map userCatalogs; - private final List listeners; - public OpenModelCatalog() { this.userCatalogs = new ConcurrentHashMap<>(); - this.listeners = new ArrayList<>(); - } - - @Override - public void registerListener(ModelCatalogListener listener) { - listeners.add(listener); } @Override @@ -63,7 +54,7 @@ public void set(Model model) { return userCatalog; }); - listeners.forEach(listener -> listener.onInsert(model)); + LISTENERS.forEach(listener -> listener.onInsert(model)); } @Override diff --git a/open-model-catalog/src/test/java/org/neo4j/gds/core/model/OpenModelCatalogTest.java b/open-model-catalog/src/test/java/org/neo4j/gds/core/model/OpenModelCatalogTest.java index 053c5b5373..c034f14503 100644 --- a/open-model-catalog/src/test/java/org/neo4j/gds/core/model/OpenModelCatalogTest.java +++ b/open-model-catalog/src/test/java/org/neo4j/gds/core/model/OpenModelCatalogTest.java @@ -121,7 +121,7 @@ void shouldStoreModelsPerType() { @Test void shouldNotifyListeners() { var counter = new Counter(); - modelCatalog.registerListener(model -> counter.increment()); + ModelCatalog.registerListener(model -> counter.increment()); var model = Model.of( "testAlgo", From 7a790fb46e540ffdf8928ffc8f5a497fa7e858b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Thu, 25 May 2023 14:27:22 +0200 Subject: [PATCH 023/273] Reduce static state in metrics based on review comments by Mats Co-authored-by: Mats Rydberg --- .../org/neo4j/gds/core/model/ModelCatalog.java | 12 ++---------- .../neo4j/gds/core/model/OpenModelCatalog.java | 17 ++++++++++++++++- .../gds/core/model/OpenModelCatalogTest.java | 2 +- 3 files changed, 19 insertions(+), 12 deletions(-) diff --git a/model-catalog-api/src/main/java/org/neo4j/gds/core/model/ModelCatalog.java b/model-catalog-api/src/main/java/org/neo4j/gds/core/model/ModelCatalog.java index 6f92e5767b..c9618edc83 100644 --- a/model-catalog-api/src/main/java/org/neo4j/gds/core/model/ModelCatalog.java +++ b/model-catalog-api/src/main/java/org/neo4j/gds/core/model/ModelCatalog.java @@ -26,21 +26,13 @@ import java.nio.file.Path; import java.util.Collection; -import java.util.HashSet; -import java.util.Set; import java.util.stream.Stream; public interface ModelCatalog { - Set LISTENERS = new HashSet<>(); + void registerListener(ModelCatalogListener listener); - static void registerListener(ModelCatalogListener listener) { - LISTENERS.add(listener); - } - - static void unregisterListener(ModelCatalogListener listener) { - LISTENERS.remove(listener); - } + void unregisterListener(ModelCatalogListener listener); void set(Model model); diff --git a/open-model-catalog/src/main/java/org/neo4j/gds/core/model/OpenModelCatalog.java b/open-model-catalog/src/main/java/org/neo4j/gds/core/model/OpenModelCatalog.java index 1d9ccbe3fd..958556e1ef 100644 --- a/open-model-catalog/src/main/java/org/neo4j/gds/core/model/OpenModelCatalog.java +++ b/open-model-catalog/src/main/java/org/neo4j/gds/core/model/OpenModelCatalog.java @@ -28,9 +28,11 @@ import java.nio.file.Path; import java.util.ArrayList; import java.util.Collection; +import java.util.HashSet; import java.util.Locale; import java.util.Map; import java.util.NoSuchElementException; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Stream; @@ -40,8 +42,21 @@ public final class OpenModelCatalog implements ModelCatalog { private final Map userCatalogs; + private final Set listeners; + public OpenModelCatalog() { this.userCatalogs = new ConcurrentHashMap<>(); + this.listeners = new HashSet<>(); + } + + @Override + public void registerListener(ModelCatalogListener listener) { + listeners.add(listener); + } + + @Override + public void unregisterListener(ModelCatalogListener listener) { + listeners.remove(listener); } @Override @@ -54,7 +69,7 @@ public void set(Model model) { return userCatalog; }); - LISTENERS.forEach(listener -> listener.onInsert(model)); + listeners.forEach(listener -> listener.onInsert(model)); } @Override diff --git a/open-model-catalog/src/test/java/org/neo4j/gds/core/model/OpenModelCatalogTest.java b/open-model-catalog/src/test/java/org/neo4j/gds/core/model/OpenModelCatalogTest.java index c034f14503..053c5b5373 100644 --- a/open-model-catalog/src/test/java/org/neo4j/gds/core/model/OpenModelCatalogTest.java +++ b/open-model-catalog/src/test/java/org/neo4j/gds/core/model/OpenModelCatalogTest.java @@ -121,7 +121,7 @@ void shouldStoreModelsPerType() { @Test void shouldNotifyListeners() { var counter = new Counter(); - ModelCatalog.registerListener(model -> counter.increment()); + modelCatalog.registerListener(model -> counter.increment()); var model = Model.of( "testAlgo", From 65074393f083d75362c9e7d38c4aeeb1ab65a551 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Wed, 7 Jun 2023 10:46:48 +0200 Subject: [PATCH 024/273] Fix cherry-pick --- .../src/main/java/org/neo4j/gds/core/model/ModelCatalog.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/model-catalog-api/src/main/java/org/neo4j/gds/core/model/ModelCatalog.java b/model-catalog-api/src/main/java/org/neo4j/gds/core/model/ModelCatalog.java index c9618edc83..bc893371c5 100644 --- a/model-catalog-api/src/main/java/org/neo4j/gds/core/model/ModelCatalog.java +++ b/model-catalog-api/src/main/java/org/neo4j/gds/core/model/ModelCatalog.java @@ -80,6 +80,11 @@ public void registerListener(ModelCatalogListener listener) { } + @Override + public void unregisterListener(ModelCatalogListener listener) { + + } + @Override public void set(Model model) { From 3e4e77ab941883ad152b2be2889464e864f9fc3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Wed, 7 Jun 2023 11:29:35 +0200 Subject: [PATCH 025/273] Use native memory instead off off-heap --- .../ROOT/pages/installation/System-requirements.adoc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/modules/ROOT/pages/installation/System-requirements.adoc b/doc/modules/ROOT/pages/installation/System-requirements.adoc index f224350145..eb7034a9c7 100644 --- a/doc/modules/ROOT/pages/installation/System-requirements.adoc +++ b/doc/modules/ROOT/pages/installation/System-requirements.adoc @@ -74,11 +74,11 @@ Decreasing the page cache size in favor of heap size is *not* recommended if the See https://neo4j.com/docs/operations-manual/4.4/performance/memory-configuration/[Neo4j memory configuration] for general information about page cache sizing. ==== -=== Off-heap memory +=== Native memory -The off-heap space is used by the xref:installation/configure-apache-arrow-server.adoc[Apache Arrow server] to store received data. +Native memory is used by the xref:installation/configure-apache-arrow-server.adoc[Apache Arrow server] to store received data. -If you have the xref:installation/configure-apache-arrow-server.adoc[Apache Arrow server] enabled, we also recommend to reserve some off-heap memory. +If you have the xref:installation/configure-apache-arrow-server.adoc[Apache Arrow server] enabled, we also recommend to reserve some native memory. The amount of memory required depends on the batch size used by the client. Data received through Arrow is temporarily stored in direct memory before being converted and loaded into an on-heap graph. From 2b10356a85ee0d63b6dd4a786f8ddf07cdd22acf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20Kie=C3=9Fling?= Date: Wed, 7 Jun 2023 10:59:40 +0200 Subject: [PATCH 026/273] Throw when the bad collector encounters an error --- .../core/io/db/GdsParallelBatchImporter.java | 31 ++++++++++--------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/io/core/src/main/java/org/neo4j/gds/core/io/db/GdsParallelBatchImporter.java b/io/core/src/main/java/org/neo4j/gds/core/io/db/GdsParallelBatchImporter.java index 2b1c43d8c0..e1b7e89a68 100644 --- a/io/core/src/main/java/org/neo4j/gds/core/io/db/GdsParallelBatchImporter.java +++ b/io/core/src/main/java/org/neo4j/gds/core/io/db/GdsParallelBatchImporter.java @@ -51,7 +51,6 @@ import static org.neo4j.configuration.GraphDatabaseSettings.SYSTEM_DATABASE_NAME; import static org.neo4j.gds.core.io.GraphStoreExporter.DIRECTORY_IS_WRITABLE; import static org.neo4j.gds.utils.StringFormatting.formatWithLocale; -import static org.neo4j.internal.batchimport.input.BadCollector.UNLIMITED_TOLERANCE; public final class GdsParallelBatchImporter { @@ -177,10 +176,12 @@ public void writeDatabase(CompatInput compatInput, boolean startDatabase) { if (startDatabase) { var dbStartTimer = ProgressTimer.start(); if (createAndStartDatabase()) { - log.info(formatWithLocale( - "Database created and started after %s ms", - dbStartTimer.stop().getDuration() - )); + log.info( + formatWithLocale( + "Database created and started after %s ms", + dbStartTimer.stop().getDuration() + ) + ); } else { log.error("Unable to start database " + config.dbName()); } @@ -201,22 +202,22 @@ private void validateDatabaseDoesNotExist(GdsDatabaseLayout databaseLayout) { var metaDataPath = databaseLayout.metadataStore(); var dbExists = Files.exists(metaDataPath) && Files.isReadable(metaDataPath); if (dbExists && !config.force()) { - throw new IllegalArgumentException(formatWithLocale( - "The database [%s] already exists. The graph export procedure can only create new databases.", - config.dbName() - )); + throw new IllegalArgumentException( + formatWithLocale( + "The database [%s] already exists. The graph export procedure can only create new databases.", + config.dbName() + ) + ); } } - private LogService getLogService() { - return config.enableDebugLog() - ? logService - : NullLogService.getInstance(); - } + private LogService getLogService() { return config.enableDebugLog() + ? logService + : NullLogService.getInstance(); } private Collector getCollector() { return config.useBadCollector() - ? Collectors.badCollector(new LoggingOutputStream(log), UNLIMITED_TOLERANCE) + ? Collectors.badCollector(new LoggingOutputStream(log), 0) : Collector.EMPTY; } From adbbb2ab066c56c67c72e76889d2650746ea6827 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20Kie=C3=9Fling?= Date: Wed, 7 Jun 2023 14:43:36 +0200 Subject: [PATCH 027/273] Add documentation for partitioning arrow get streams --- .../pages/graph-catalog-apache-arrow-ops.adoc | 57 ++++++++++++++++++- 1 file changed, 56 insertions(+), 1 deletion(-) diff --git a/doc/modules/ROOT/pages/graph-catalog-apache-arrow-ops.adoc b/doc/modules/ROOT/pages/graph-catalog-apache-arrow-ops.adoc index eb9603ddb3..4023f7a5b5 100644 --- a/doc/modules/ROOT/pages/graph-catalog-apache-arrow-ops.adoc +++ b/doc/modules/ROOT/pages/graph-catalog-apache-arrow-ops.adoc @@ -215,7 +215,7 @@ To stream the topology of one or more relationship types, the client needs to en { graph_name: "my_graph", database_name: "database_name", - procedure_name: "gds.graph.relationshipProperties.stream", + procedure_name: "gds.beta.graph.relationships.stream", configuration: { relationship_types: "REL" } @@ -244,3 +244,58 @@ The schema of the result records is identical to the corresponding procedure: Note, that the relationship type column stores the relationship type encoded as an integer. The corresponding string value needs to be retrieved from the corresponding dictionary value vector. That vector can be loaded from the dictionary provider using the encoding id of the type field. + + +== Partitioning the data streams + +Some use-cases require the data streams to be partitioned. +For example, if the data streams are consumed by a distributed system, the data streams need to be evenly distributed to the members of the distributed system. +To support this use-case, the client can request the data streams to be partitioned by sending the stream request to the `FlightInfo` endpoint of the GDS Flight Server. +The server will then return a number of endpoints, where each endpoint and it's accompanying ticket can be used to stream a partition of the data. +The `concurrency` settings of the ticket can be used to control the number of partitions. + +For example, to stream the topology of one or more relationship types, the client needs to encode that information in the ticket as follows: + +---- +{ + graph_name: "my_graph", + database_name: "database_name", + procedure_name: "gds.beta.graph.relationships.stream", + concurrency: 2, + configuration: { + relationship_types: "REL" + } +} +---- + +This will create at most 2 partitions of the data streams. +The server will answer with 2 tickets: + +---- +[ + { + graph_name: "my_graph", + database_name: "database_name", + procedure_name: "gds.beta.graph.relationships.stream", + concurrency: 4, + partition_offset: 0, + partition_size: 100, + configuration: { + relationship_types: "REL" + } + }, + { + graph_name: "my_graph", + database_name: "database_name", + procedure_name: "gds.beta.graph.relationships.stream", + partition_offset: 100, + partition_size: 100, + concurrency: 4, + configuration: { + relationship_types: "REL" + } + } +] +---- + +Each of the tickets can now be used to request a partition data via the `GET` endpoint of the GDS Flight Server. From 71f2d12d7e3603754e951675043ba8065e6e4112 Mon Sep 17 00:00:00 2001 From: ioannispan Date: Wed, 7 Jun 2023 13:12:17 +0200 Subject: [PATCH 028/273] Add missing syntaxt testing --- .../org/neo4j/gds/doc/syntax/SyntaxMode.java | 5 ++- .../doc/syntax/GraphGenerationSyntaxTest.java | 37 +++++++++++++++++++ .../projections/graph-generation.adoc | 14 ++++--- 3 files changed, 50 insertions(+), 6 deletions(-) create mode 100644 doc-test/src/test/java/org/neo4j/gds/doc/syntax/GraphGenerationSyntaxTest.java diff --git a/doc-test-tools/src/main/java/org/neo4j/gds/doc/syntax/SyntaxMode.java b/doc-test-tools/src/main/java/org/neo4j/gds/doc/syntax/SyntaxMode.java index 89d2f2614f..74202b6d5f 100644 --- a/doc-test-tools/src/main/java/org/neo4j/gds/doc/syntax/SyntaxMode.java +++ b/doc-test-tools/src/main/java/org/neo4j/gds/doc/syntax/SyntaxMode.java @@ -73,7 +73,10 @@ public enum SyntaxMode { SYSTEM_MONITOR("system-monitor-syntax", false), SYS_INFO("debug-sysinfo-syntax", false), WRITE_NODE_LABEL("include-with-write-node-label", false), - MUTATE_NODE_LABEL("include-with-mutate-node-label", false),; + MUTATE_NODE_LABEL("include-with-mutate-node-label", false), + GRAPH_GENERATE("include-with-graph-generate"), + ; + private final String mode; public final boolean hasParameters; diff --git a/doc-test/src/test/java/org/neo4j/gds/doc/syntax/GraphGenerationSyntaxTest.java b/doc-test/src/test/java/org/neo4j/gds/doc/syntax/GraphGenerationSyntaxTest.java new file mode 100644 index 0000000000..20ca0ed6b0 --- /dev/null +++ b/doc-test/src/test/java/org/neo4j/gds/doc/syntax/GraphGenerationSyntaxTest.java @@ -0,0 +1,37 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.doc.syntax; + +import java.util.List; + +import static org.neo4j.gds.doc.syntax.SyntaxMode.GRAPH_GENERATE; + +class GraphGenerationSyntaxTest extends SyntaxTestBase { + + @Override + protected Iterable syntaxModes() { + return List.of(SyntaxModeMeta.of(GRAPH_GENERATE)); + } + + @Override + protected String adocFile() { + return "pages/management-ops/projections/graph-generation.adoc"; + } +} diff --git a/doc/modules/ROOT/pages/management-ops/projections/graph-generation.adoc b/doc/modules/ROOT/pages/management-ops/projections/graph-generation.adoc index c484061a1c..49a396ceff 100644 --- a/doc/modules/ROOT/pages/management-ops/projections/graph-generation.adoc +++ b/doc/modules/ROOT/pages/management-ops/projections/graph-generation.adoc @@ -26,13 +26,16 @@ The graph generation is parameterized by three dimensions: [[graph-generation-syntax]] == Syntax - -.The following describes the API for running the algorithm +[.include-with-graph-generate] +==== +.The following describes the API for running the graph generation procedure [source, cypher, role=noplay] ---- -CALL gds.beta.graph.generate(graphName: String, nodeCount: Integer, averageDegree: Integer, { - relationshipDistribution: String, - relationshipProperty: Map +CALL gds.beta.graph.generate( + graphName: String, + nodeCount: Integer, + averageDegree: Integer, + configuration: Map }) YIELD name, nodes, relationships, generateMillis, relationshipSeed, averageDegree, relationshipDistribution, relationshipProperty ---- @@ -72,6 +75,7 @@ YIELD name, nodes, relationships, generateMillis, relationshipSeed, averageDegre | relationshipDistribution | String | The probability distribution method used to connect generated nodes. | relationshipProperty | String | The configuration of the generated relationship property. |=== +==== [[graph-generation-distribution]] == Relationship Distribution From d91277d383b539e2424f18662e4457c3d69abf10 Mon Sep 17 00:00:00 2001 From: ioannispan Date: Wed, 7 Jun 2023 14:44:04 +0200 Subject: [PATCH 029/273] Add example --- .../neo4j/gds/doc/GraphGenerationDocTest.java | 47 ++++++++ .../projections/graph-generation.adoc | 112 ++++++++++++++++++ 2 files changed, 159 insertions(+) create mode 100644 doc-test/src/test/java/org/neo4j/gds/doc/GraphGenerationDocTest.java diff --git a/doc-test/src/test/java/org/neo4j/gds/doc/GraphGenerationDocTest.java b/doc-test/src/test/java/org/neo4j/gds/doc/GraphGenerationDocTest.java new file mode 100644 index 0000000000..18b3212ebc --- /dev/null +++ b/doc-test/src/test/java/org/neo4j/gds/doc/GraphGenerationDocTest.java @@ -0,0 +1,47 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.doc; + +import org.neo4j.gds.beta.generator.GraphGenerateProc; +import org.neo4j.gds.catalog.GraphStreamRelationshipPropertiesProc; +import org.neo4j.gds.catalog.GraphStreamRelationshipsProc; +import org.neo4j.gds.functions.AsNodeFunc; + +import java.util.List; + +class GraphGenerationDocTest extends SingleFileDocTestBase { + + @Override + protected List> functions() { + return List.of(AsNodeFunc.class); + } + + @Override + protected List> procedures() { + return List.of(GraphGenerateProc.class, GraphStreamRelationshipsProc.class, + GraphStreamRelationshipPropertiesProc.class + ); + } + + @Override + protected String adocFile() { + return "pages/management-ops/projections/graph-generation.adoc"; + } +} diff --git a/doc/modules/ROOT/pages/management-ops/projections/graph-generation.adoc b/doc/modules/ROOT/pages/management-ops/projections/graph-generation.adoc index 49a396ceff..8c1b7d2d7c 100644 --- a/doc/modules/ROOT/pages/management-ops/projections/graph-generation.adoc +++ b/doc/modules/ROOT/pages/management-ops/projections/graph-generation.adoc @@ -116,3 +116,115 @@ Currently, there are two supported methods to generate relationship properties: * `FIXED` - Assigns a fixed value to every relationship. The `value` parameter must be set. * `RANDOM` - Assigns a random value between the lower (`min`) and upper (`max`) bound. + +[[graph-generation-example]] +== Examples + +In the following we will demonstrate the usage of the random graph generation procedure. + +[[graph-generation-unweighted]] +=== Generating unweighted graphs + +[role=query-example,group=unweighted] +-- +.The following will produce a graph with unweighted relationships +[source,cypher,role=noplay] +---- +CALL gds.beta.graph.generate('graph',5,2, {relationshipSeed:19}) +YIELD name, nodes, relationships, relationshipDistribution +---- + +.Results +[opts="header"] +|=== +| name | nodes | relationships | relationshipDistribution +| "graph"| 5 | 10 | "UNIFORM" +|=== +-- + +A new in-memory graph called `graph` with `5` nodes and `10` relationships has been created and added to the graph catalog. +We can examine its topology with the `gds.beta.graph.relationships` procedure. + +[role=query-example,group=unweighted] +-- +.The following will show the produced relationships +[source,cypher,role=noplay] +---- +CALL gds.beta.graph.relationships.stream('graph') +YIELD sourceNodeId,targetNodeId +RETURN sourceNodeId as source, targetNodeId as target +ORDER BY source ASC,target ASC +---- + +.Results +[opts="header"] +|=== +| source |target +| 0 | 1 +| 0 | 2 +| 1 | 0 +| 1 | 4 +| 2 | 1 +| 2 | 4 +| 3 | 0 +| 3 | 1 +| 4 | 0 +| 4 | 3 +|=== +-- + +[[graph-generation-weighted]] +=== Generating weighted graphs + +To generated graphs with weighted relationships we must specify the `relationshipProperty` parameter as discussed xref:graph-generation-relationship-property[above]. + +[role=query-example,group=weighted] +-- +.The following will produce a graph with weighted relationships +[source,cypher,role=noplay] +---- +CALL gds.beta.graph.generate('weightedGraph',5,2, {relationshipSeed:19, + relationshipProperty: {type: 'RANDOM', min: 5.0, max: 10.0, name: 'score'}}) +YIELD name, nodes, relationships, relationshipDistribution +---- + +.Results +[opts="header"] +|=== +|name| nodes | relationships | relationshipDistribution +| "weightedGraph"| 5 | 10 | "UNIFORM" +|=== +-- + +The produced graph, `weightedGraph`, has a property named `score` containing a random value between 5.0 and 10.0 for each relationship. +We can use `gds.graph.relationshipProperty.stream` to stream the relationships of the graph along with their score values. + +[role=query-example,group=weighted] +-- +.The following will show the produced relationships +[source,cypher,role=noplay] +---- +CALL gds.graph.relationshipProperty.stream('weightedGraph','score') +YIELD sourceNodeId, targetNodeId, propertyValue +RETURN sourceNodeId as source, targetNodeId as target, propertyValue as score +ORDER BY source ASC,target ASC, score +---- + +.Results +[opts="header"] +|=== +| source |target | score +| 0 | 1 | 6.258381821615686 +| 0 | 2 | 6.791408433596591 +| 1 | 4 | 8.747179900968224 +| 1 | 4 | 9.469695236791349 +| 2 | 0 | 7.061710127800056 +| 2 | 1 | 5.060444167785128 +| 3 | 1 | 6.308266834622538 +| 3 | 4 | 9.040323743901354 +| 4 | 1 | 7.939688205556302 +| 4 | 1 | 7.988277646384441 +|=== +-- + +Notice that despite `graph` and `weightedGraph` having the same `relationshipSeed`, their actual topology differs. From 3de4421a9426221ff00e61ffdcf5c2d3d20cfac5 Mon Sep 17 00:00:00 2001 From: Ioannis Pan <74839024+IoannisPanagiotas@users.noreply.github.com> Date: Wed, 7 Jun 2023 15:07:29 +0200 Subject: [PATCH 030/273] Update SyntaxMode.java --- .../src/main/java/org/neo4j/gds/doc/syntax/SyntaxMode.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/doc-test-tools/src/main/java/org/neo4j/gds/doc/syntax/SyntaxMode.java b/doc-test-tools/src/main/java/org/neo4j/gds/doc/syntax/SyntaxMode.java index 74202b6d5f..ad9d048087 100644 --- a/doc-test-tools/src/main/java/org/neo4j/gds/doc/syntax/SyntaxMode.java +++ b/doc-test-tools/src/main/java/org/neo4j/gds/doc/syntax/SyntaxMode.java @@ -74,10 +74,8 @@ public enum SyntaxMode { SYS_INFO("debug-sysinfo-syntax", false), WRITE_NODE_LABEL("include-with-write-node-label", false), MUTATE_NODE_LABEL("include-with-mutate-node-label", false), - GRAPH_GENERATE("include-with-graph-generate"), - ; - - + GRAPH_GENERATE("include-with-graph-generate"),; + private final String mode; public final boolean hasParameters; public final CustomProcedure.Namespace namespace; From 0a8e762f8424c6fc350b8bf8230d4e04170d4386 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Wed, 7 Jun 2023 17:11:11 +0200 Subject: [PATCH 031/273] Use correct relationshipCount for degree partitioning --- .../gds/beta/filter/RelationshipsFilter.java | 34 +++++++++++++------ 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/subgraph-filtering/src/main/java/org/neo4j/gds/beta/filter/RelationshipsFilter.java b/subgraph-filtering/src/main/java/org/neo4j/gds/beta/filter/RelationshipsFilter.java index 53de58f7b1..cfdfbbaa81 100644 --- a/subgraph-filtering/src/main/java/org/neo4j/gds/beta/filter/RelationshipsFilter.java +++ b/subgraph-filtering/src/main/java/org/neo4j/gds/beta/filter/RelationshipsFilter.java @@ -26,6 +26,7 @@ import org.neo4j.gds.beta.filter.expression.EvaluationContext; import org.neo4j.gds.beta.filter.expression.Expression; import org.neo4j.gds.core.Aggregation; +import org.neo4j.gds.core.concurrency.ParallelUtil; import org.neo4j.gds.core.concurrency.RunWithConcurrency; import org.neo4j.gds.core.loading.SingleTypeRelationships; import org.neo4j.gds.core.loading.construction.GraphFactory; @@ -42,6 +43,7 @@ import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.IntStream; +import java.util.stream.LongStream; import static org.neo4j.gds.api.AdjacencyCursor.NOT_FOUND; @@ -106,11 +108,13 @@ private static SingleTypeRelationships filterRelationshipType( var propertyConfigs = propertyKeys .stream() - .map(propertyKey -> GraphFactory.PropertyConfig.of( - propertyKey, - Aggregation.NONE, - graphStore.relationshipPropertyValues(relType, propertyKey).defaultValue() - )) + .map( + propertyKey -> GraphFactory.PropertyConfig.of( + propertyKey, + Aggregation.NONE, + graphStore.relationshipPropertyValues(relType, propertyKey).defaultValue() + ) + ) .collect(Collectors.toList()); var relationshipsBuilder = GraphFactory.initRelationshipsBuilder() @@ -128,13 +132,23 @@ private static SingleTypeRelationships filterRelationshipType( .boxed() .collect(Collectors.toMap(propertyKeys::get, Function.identity())); + // computing the count is expected to be a lot cheaper than bad partitioning + var relevantRelationshipsCount = concurrency > 1 + ? ParallelUtil.parallelStream( + LongStream.range(0, outputNodes.nodeCount()), + concurrency, + stream -> stream.map( + id -> compositeIterator.degree(inputNodes.toMappedNodeId(outputNodes.toOriginalNodeId(id))) + ).sum() + ) + : graphStore.relationshipCount(relType); + var relationshipFilterTasks = PartitionUtils.degreePartition( outputNodes.nodeCount(), - graphStore.relationshipCount(relType), - compositeIterator::degree, + relevantRelationshipsCount, + nodeId -> compositeIterator.degree(inputNodes.toMappedNodeId(outputNodes.toOriginalNodeId(nodeId))), concurrency, - partition -> - new RelationshipFilterTask( + partition -> new RelationshipFilterTask( partition, relationshipExpr, compositeIterator.concurrentCopy(), @@ -155,7 +169,7 @@ private static SingleTypeRelationships filterRelationshipType( .executor(executorService) .run(); - return relationshipsBuilder.build(); + return relationshipsBuilder.build(); } private RelationshipsFilter() {} From c7d840cd647adfb0dd036401112bfab68a3d3ba8 Mon Sep 17 00:00:00 2001 From: Paul Horn Date: Wed, 7 Jun 2023 12:23:50 +0200 Subject: [PATCH 032/273] Rename appendix-b to migration-gds-1-to-gds-2 --- doc/modules/ROOT/content-nav.adoc | 16 ++++++++-------- doc/modules/ROOT/pages/index.adoc | 2 +- .../index.adoc | 15 ++++++++------- .../migration-algorithms.adoc | 1 + .../migration-algos-common.adoc | 1 + .../migration-graph-drop.adoc | 1 + .../migration-graph-listing.adoc | 1 + .../migration-graph-projection.adoc | 1 + .../migration-memory-estimation.adoc | 1 + .../migration-ml.adoc | 1 + 10 files changed, 24 insertions(+), 16 deletions(-) rename doc/modules/ROOT/pages/{appendix-b => migration-gds-1-to-gds-2}/index.adoc (60%) rename doc/modules/ROOT/pages/{appendix-b => migration-gds-1-to-gds-2}/migration-algorithms.adoc (92%) rename doc/modules/ROOT/pages/{appendix-b => migration-gds-1-to-gds-2}/migration-algos-common.adoc (91%) rename doc/modules/ROOT/pages/{appendix-b => migration-gds-1-to-gds-2}/migration-graph-drop.adoc (91%) rename doc/modules/ROOT/pages/{appendix-b => migration-gds-1-to-gds-2}/migration-graph-listing.adoc (91%) rename doc/modules/ROOT/pages/{appendix-b => migration-gds-1-to-gds-2}/migration-graph-projection.adoc (96%) rename doc/modules/ROOT/pages/{appendix-b => migration-gds-1-to-gds-2}/migration-memory-estimation.adoc (95%) rename doc/modules/ROOT/pages/{appendix-b => migration-gds-1-to-gds-2}/migration-ml.adoc (88%) diff --git a/doc/modules/ROOT/content-nav.adoc b/doc/modules/ROOT/content-nav.adoc index 223d0e8ca0..238254e6a2 100644 --- a/doc/modules/ROOT/content-nav.adoc +++ b/doc/modules/ROOT/content-nav.adoc @@ -160,11 +160,11 @@ *** xref:operations-reference/machine-learning-references.adoc[] *** xref:operations-reference/additional-operation-references.adoc[] *** xref:operations-reference/configuration-settings.adoc[] -** xref:appendix-b/index.adoc[] -*** xref:appendix-b/migration-algos-common.adoc[] -*** xref:appendix-b/migration-graph-projection.adoc[] -*** xref:appendix-b/migration-graph-listing.adoc[] -*** xref:appendix-b/migration-graph-drop.adoc[] -*** xref:appendix-b/migration-memory-estimation.adoc[] -*** xref:appendix-b/migration-algorithms.adoc[] -*** xref:appendix-b/migration-ml.adoc[] +** xref:migration-gds-1-to-gds-2/index.adoc[] +*** xref:migration-gds-1-to-gds-2/migration-algos-common.adoc[] +*** xref:migration-gds-1-to-gds-2/migration-graph-projection.adoc[] +*** xref:migration-gds-1-to-gds-2/migration-graph-listing.adoc[] +*** xref:migration-gds-1-to-gds-2/migration-graph-drop.adoc[] +*** xref:migration-gds-1-to-gds-2/migration-memory-estimation.adoc[] +*** xref:migration-gds-1-to-gds-2/migration-algorithms.adoc[] +*** xref:migration-gds-1-to-gds-2/migration-ml.adoc[] diff --git a/doc/modules/ROOT/pages/index.adoc b/doc/modules/ROOT/pages/index.adoc index 042b05a932..811f238db2 100644 --- a/doc/modules/ROOT/pages/index.adoc +++ b/doc/modules/ROOT/pages/index.adoc @@ -32,7 +32,7 @@ The manual covers the following areas: * xref:production-deployment/index.adoc[Production deployment] -- This chapter explains advanced details with regards to common Neo4j components. * xref:python-client/index.adoc[Python client] -- Documentation of the Graph Data Science client for Python users. * xref:operations-reference/appendix-a.adoc[Operations reference] -- Reference of all procedures contained in the Neo4j Graph Data Science library. -* xref:appendix-b/index.adoc[Migration from Graph Data Science library Version 1.x] -- Additional resources - migration guide, books, etc - to help using the Neo4j Graph Data Science library. +* xref:migration-gds-1-to-gds-2/index.adoc[Migration from Graph Data Science library Version 1.x] -- Additional resources - migration guide, books, etc - to help using the Neo4j Graph Data Science library. The source code of the library is available at https://github.com/neo4j/graph-data-science[GitHub]. If you have a suggestion on how we can improve the library or want to report a problem, you can create a https://github.com/neo4j/graph-data-science/issues/new[new issue]. diff --git a/doc/modules/ROOT/pages/appendix-b/index.adoc b/doc/modules/ROOT/pages/migration-gds-1-to-gds-2/index.adoc similarity index 60% rename from doc/modules/ROOT/pages/appendix-b/index.adoc rename to doc/modules/ROOT/pages/migration-gds-1-to-gds-2/index.adoc index f67aa65675..07b4ae1415 100644 --- a/doc/modules/ROOT/pages/appendix-b/index.adoc +++ b/doc/modules/ROOT/pages/migration-gds-1-to-gds-2/index.adoc @@ -2,6 +2,7 @@ [[appendix-b]] = Migration from Graph Data Science library Version 1.x :description: If you have previously used Graph Data Science library version 1.x, you can find the information you will need to migrate to using version 2.x in this section. +:page-aliases: appendix-b/index.adoc == Who should read this guide @@ -17,10 +18,10 @@ In this section we will focus on side-by-side examples of operations using the s This section is divided into the following sub-sections: -* xref:appendix-b/migration-algos-common.adoc[Common Changes] -* xref:appendix-b/migration-graph-projection.adoc[Graph Projection] -* xref:appendix-b/migration-graph-listing.adoc[Graph Listing] -* xref:appendix-b/migration-graph-drop.adoc[Graph Drop] -* xref:appendix-b/migration-memory-estimation.adoc[Memory Estimation] -* xref:appendix-b/migration-algorithms.adoc[Algorithms] -* xref:appendix-b/migration-ml.adoc[Machine Learning] +* xref:migration-gds-1-to-gds-2/migration-algos-common.adoc[Common Changes] +* xref:migration-gds-1-to-gds-2/migration-graph-projection.adoc[Graph Projection] +* xref:migration-gds-1-to-gds-2/migration-graph-listing.adoc[Graph Listing] +* xref:migration-gds-1-to-gds-2/migration-graph-drop.adoc[Graph Drop] +* xref:migration-gds-1-to-gds-2/migration-memory-estimation.adoc[Memory Estimation] +* xref:migration-gds-1-to-gds-2/migration-algorithms.adoc[Algorithms] +* xref:migration-gds-1-to-gds-2/migration-ml.adoc[Machine Learning] diff --git a/doc/modules/ROOT/pages/appendix-b/migration-algorithms.adoc b/doc/modules/ROOT/pages/migration-gds-1-to-gds-2/migration-algorithms.adoc similarity index 92% rename from doc/modules/ROOT/pages/appendix-b/migration-algorithms.adoc rename to doc/modules/ROOT/pages/migration-gds-1-to-gds-2/migration-algorithms.adoc index 9b8f6e2114..5a71604b20 100644 --- a/doc/modules/ROOT/pages/appendix-b/migration-algorithms.adoc +++ b/doc/modules/ROOT/pages/migration-gds-1-to-gds-2/migration-algorithms.adoc @@ -1,6 +1,7 @@ [[migration-algorithms]] = Algorithms :description: This section covers migration for all algorithms in the Neo4j Graph Data Science library. +:page-aliases: appendix-b/migration-algorithms.adoc include::partial$/migration/migration-algorithms-betweenness-centrality.adoc[leveloffset=+1] diff --git a/doc/modules/ROOT/pages/appendix-b/migration-algos-common.adoc b/doc/modules/ROOT/pages/migration-gds-1-to-gds-2/migration-algos-common.adoc similarity index 91% rename from doc/modules/ROOT/pages/appendix-b/migration-algos-common.adoc rename to doc/modules/ROOT/pages/migration-gds-1-to-gds-2/migration-algos-common.adoc index 2b1bcf2ba1..3ff37a6d5d 100644 --- a/doc/modules/ROOT/pages/appendix-b/migration-algos-common.adoc +++ b/doc/modules/ROOT/pages/migration-gds-1-to-gds-2/migration-algos-common.adoc @@ -1,5 +1,6 @@ [[migration-algos-common]] = Common changes +:page-aliases: appendix-b/migration-algos-common.adoc This section describes changes between version 1.x and 2.x that are common to all procedures. diff --git a/doc/modules/ROOT/pages/appendix-b/migration-graph-drop.adoc b/doc/modules/ROOT/pages/migration-gds-1-to-gds-2/migration-graph-drop.adoc similarity index 91% rename from doc/modules/ROOT/pages/appendix-b/migration-graph-drop.adoc rename to doc/modules/ROOT/pages/migration-gds-1-to-gds-2/migration-graph-drop.adoc index fd3d085560..e9a3425a8d 100644 --- a/doc/modules/ROOT/pages/appendix-b/migration-graph-drop.adoc +++ b/doc/modules/ROOT/pages/migration-gds-1-to-gds-2/migration-graph-drop.adoc @@ -1,5 +1,6 @@ [[migration-graph-drop]] = Graph drop +:page-aliases: appendix-b/migration-graph-drop.adoc .Changes in the YIELD fields [opts=header, cols="1,1"] diff --git a/doc/modules/ROOT/pages/appendix-b/migration-graph-listing.adoc b/doc/modules/ROOT/pages/migration-gds-1-to-gds-2/migration-graph-listing.adoc similarity index 91% rename from doc/modules/ROOT/pages/appendix-b/migration-graph-listing.adoc rename to doc/modules/ROOT/pages/migration-gds-1-to-gds-2/migration-graph-listing.adoc index af96427f8b..db5e7a6bd9 100644 --- a/doc/modules/ROOT/pages/appendix-b/migration-graph-listing.adoc +++ b/doc/modules/ROOT/pages/migration-gds-1-to-gds-2/migration-graph-listing.adoc @@ -1,5 +1,6 @@ [[migration-graph-listing]] = Graph listing +:page-aliases: appendix-b/migration-graph-listing.adoc .Changes in the YIELD fields [opts=header, cols="1,1"] diff --git a/doc/modules/ROOT/pages/appendix-b/migration-graph-projection.adoc b/doc/modules/ROOT/pages/migration-gds-1-to-gds-2/migration-graph-projection.adoc similarity index 96% rename from doc/modules/ROOT/pages/appendix-b/migration-graph-projection.adoc rename to doc/modules/ROOT/pages/migration-gds-1-to-gds-2/migration-graph-projection.adoc index e185458e27..876b02bc4d 100644 --- a/doc/modules/ROOT/pages/appendix-b/migration-graph-projection.adoc +++ b/doc/modules/ROOT/pages/migration-gds-1-to-gds-2/migration-graph-projection.adoc @@ -1,5 +1,6 @@ [[migration-graph-projection]] = Graph projection +:page-aliases: appendix-b/migration-graph-projection.adoc .Changes in the YIELD fields [opts=header, cols="1,1"] diff --git a/doc/modules/ROOT/pages/appendix-b/migration-memory-estimation.adoc b/doc/modules/ROOT/pages/migration-gds-1-to-gds-2/migration-memory-estimation.adoc similarity index 95% rename from doc/modules/ROOT/pages/appendix-b/migration-memory-estimation.adoc rename to doc/modules/ROOT/pages/migration-gds-1-to-gds-2/migration-memory-estimation.adoc index 5e7f01ee2c..1068b6bc92 100644 --- a/doc/modules/ROOT/pages/appendix-b/migration-memory-estimation.adoc +++ b/doc/modules/ROOT/pages/migration-gds-1-to-gds-2/migration-memory-estimation.adoc @@ -1,5 +1,6 @@ [[migration-memory-estimation]] = Memory estimation +:page-aliases: appendix-b/migration-memory-estimation.adoc .Estimating memory for algorithms without loading the graph: [opts=header,cols="1a,1a"] diff --git a/doc/modules/ROOT/pages/appendix-b/migration-ml.adoc b/doc/modules/ROOT/pages/migration-gds-1-to-gds-2/migration-ml.adoc similarity index 88% rename from doc/modules/ROOT/pages/appendix-b/migration-ml.adoc rename to doc/modules/ROOT/pages/migration-gds-1-to-gds-2/migration-ml.adoc index 3550996e42..4b03f4681d 100644 --- a/doc/modules/ROOT/pages/appendix-b/migration-ml.adoc +++ b/doc/modules/ROOT/pages/migration-gds-1-to-gds-2/migration-ml.adoc @@ -1,6 +1,7 @@ [[migration-ml]] = Machine Learning :description: This section covers migration for Machine Learning algorithms in the Neo4j Graph Data Science library. +:page-aliases: appendix-b/migration-ml.adoc include::partial$/migration/migration-algorithms-node-classification.adoc[leveloffset=+1] From e4d06652f18d759eb172a3452276bda0f7002e97 Mon Sep 17 00:00:00 2001 From: Paul Horn Date: Wed, 7 Jun 2023 12:24:16 +0200 Subject: [PATCH 033/273] Add migration guide for LCP to CPv2 --- doc/modules/ROOT/content-nav.adoc | 1 + .../pages/common-usage/memory-estimation.adoc | 1 + doc/modules/ROOT/pages/index.adoc | 1 + .../graph-project-cypher-projection.adoc | 11 +- .../pages/migration-lcp-to-cpv2/index.adoc | 465 ++++++++++++++++++ 5 files changed, 476 insertions(+), 3 deletions(-) create mode 100644 doc/modules/ROOT/pages/migration-lcp-to-cpv2/index.adoc diff --git a/doc/modules/ROOT/content-nav.adoc b/doc/modules/ROOT/content-nav.adoc index 238254e6a2..61dd42e6c5 100644 --- a/doc/modules/ROOT/content-nav.adoc +++ b/doc/modules/ROOT/content-nav.adoc @@ -168,3 +168,4 @@ *** xref:migration-gds-1-to-gds-2/migration-memory-estimation.adoc[] *** xref:migration-gds-1-to-gds-2/migration-algorithms.adoc[] *** xref:migration-gds-1-to-gds-2/migration-ml.adoc[] +** xref:migration-lcp-to-cpv2/index.adoc[] diff --git a/doc/modules/ROOT/pages/common-usage/memory-estimation.adoc b/doc/modules/ROOT/pages/common-usage/memory-estimation.adoc index 6d38e76252..03ab54dcdf 100644 --- a/doc/modules/ROOT/pages/common-usage/memory-estimation.adoc +++ b/doc/modules/ROOT/pages/common-usage/memory-estimation.adoc @@ -116,6 +116,7 @@ The `nodeProjection` and `relationshipProjection` parameters follow the same syn The result of running `gds.graph.project.estimate` has the same form as the algorithm memory estimation results above. +[[estimate-procedure-fictive-graph]] It is also possible to estimate the memory of a fictive graph, by explicitly specifying its node and relationship count. Using this feature, one can estimate the memory consumption of an arbitrarily sized graph. diff --git a/doc/modules/ROOT/pages/index.adoc b/doc/modules/ROOT/pages/index.adoc index 811f238db2..4f9cce6c48 100644 --- a/doc/modules/ROOT/pages/index.adoc +++ b/doc/modules/ROOT/pages/index.adoc @@ -33,6 +33,7 @@ The manual covers the following areas: * xref:python-client/index.adoc[Python client] -- Documentation of the Graph Data Science client for Python users. * xref:operations-reference/appendix-a.adoc[Operations reference] -- Reference of all procedures contained in the Neo4j Graph Data Science library. * xref:migration-gds-1-to-gds-2/index.adoc[Migration from Graph Data Science library Version 1.x] -- Additional resources - migration guide, books, etc - to help using the Neo4j Graph Data Science library. +* xref:migration-lcp-to-cpv2/index.adoc[Migration from Legacy to new Cypher projections] -- Migration guide to help migration from the Legacy Cypher projections to the new Cypher projections. The source code of the library is available at https://github.com/neo4j/graph-data-science[GitHub]. If you have a suggestion on how we can improve the library or want to report a problem, you can create a https://github.com/neo4j/graph-data-science/issues/new[new issue]. diff --git a/doc/modules/ROOT/pages/management-ops/projections/graph-project-cypher-projection.adoc b/doc/modules/ROOT/pages/management-ops/projections/graph-project-cypher-projection.adoc index beeee7671c..960bc78a27 100644 --- a/doc/modules/ROOT/pages/management-ops/projections/graph-project-cypher-projection.adoc +++ b/doc/modules/ROOT/pages/management-ops/projections/graph-project-cypher-projection.adoc @@ -158,7 +158,7 @@ RETURN ---- .Results -[opts="header", cols="1,1,1m"] +[opts="header", cols="0,1,1m"] |=== | graph | nodes | rels | "persons" | 3 | 2 @@ -182,7 +182,7 @@ RETURN ---- .Results -[opts="header", cols="1,1,1m"] +[opts="header",cols="1,1,1m"] |=== | graph | nodes | rels | "persons" | 5 | 2 @@ -190,6 +190,7 @@ RETURN -- +[[graph-project-cypher-projection-arbitrary-source-and-target-id-values]] === Arbitrary source and target ID values So far, the examples showed how to project a graph based on existing nodes. @@ -198,7 +199,7 @@ It is also possible to pass INTEGER values directly. [role=query-example] -- .Project arbitrary id values: -[source, cypher, role=noplay] +[source,cypher,role=noplay] ---- UNWIND [ [42, 84], [13, 37], [19, 84] ] AS sourceAndTarget WITH sourceAndTarget[0] AS source, sourceAndTarget[1] AS target @@ -222,6 +223,7 @@ As such, `.write` procedures cannot be executed on this graph. -- +[[graph-project-cypher-projection-multi-graph]] === Multi-graph A multi-graph is a graph with multiple node labels and relationship types. @@ -314,6 +316,7 @@ RETURN g.graphName AS graph, g.nodeCount AS nodes, g.relationshipCount AS rels |=== -- +[[graph-project-cypher-projection-undirected-relationships]] ==== Undirected relationships Relationships can be projected as undirected by specifying the `undirectedRelationshipTypes` parameter. @@ -469,6 +472,7 @@ ORDER BY person ASC, numberOfPages DESC We can see, that the `numberOfPages` are loaded. The default property value is `Double.Nan` and can be changed as in the previous example xref:management-ops/projections/graph-project-cypher-projection.adoc#node-properties-example[Node properties] by using the Cypher function https://neo4j.com/docs/cypher-manual/current/functions/scalar/#functions-coalesce[_coalesce()_]. +[[graph-project-cypher-projection-parallel-relationships]] === Parallel relationships The Property Graph Model in Neo4j supports parallel relationships, i.e., multiple relationships between two nodes. @@ -580,6 +584,7 @@ ORDER BY numberOfPages DESC, person We can see, that the two `READ` relationships between Florentin and the Hobbit sum up to `46` numberOfPages. +[[graph-project-cypher-projection-filtered]] === Projecting filtered Neo4j graphs Cypher-projections allow us to specify the graph to project in a more fine-grained way. diff --git a/doc/modules/ROOT/pages/migration-lcp-to-cpv2/index.adoc b/doc/modules/ROOT/pages/migration-lcp-to-cpv2/index.adoc new file mode 100644 index 0000000000..aeedc1f3e9 --- /dev/null +++ b/doc/modules/ROOT/pages/migration-lcp-to-cpv2/index.adoc @@ -0,0 +1,465 @@ +[appendix] +[[appendix-c]] += Migration from Legacy to new Cypher projection +:description: If you have been using `gds.graph.project.cypher` projections, you can find the information you will need to migrate to using the new Cypher projection. + + +== Who should read this guide + +This guide is intended for users who have been using the legacy Cypher projection `gds.graph.project.cypher`. +Cypher projections are now done using the `gds.graph.project` aggregation function. +We assume that most of the mentioned operations and concepts can be understood with little explanation. +Thus we are intentionally brief in the examples and comparisons. +Please see xref:management-ops/projections/graph-project-cypher-projection.adoc[the documentation for the Cypher projection] for more details. + +== Structural Changes + +The Legacy Cypher projection is a standalone procedure call where Cypher queries are passed as string arguments and executed by GDS. +The new Cypher projection is an aggregation function that is called as part of a Cypher query. +GDS is no longer responsible or in control of the execution of the Cypher queries. +Migrating to the new Cypher projection requires changes of how the Cypher query is written as a whole. + +There are no longer separate queries for nodes and relationships. +Instead, write one query that produces the source- and target node pairs and use `gds.graph.project` to aggregate into the graph catalog. +Since the relationship query from the Legacy Cypher projection already required you to return the source- and target node pairs, it is a good starting point for the new query. +Roughly speaking, the query has to be rewritten as follows: + +.Structural changes between the two Cypher projections: +[opts=header,cols="1a,1a"] +|=== +| Legacy | New +| +[source, cypher, role=noplay] +---- +CALL gds.graph.project.cypher( + $graphName, + $nodeQuery, + $relationshipQuery, + $configuration +) +---- +| +[source, cypher, role=noplay] +---- +$relationshipQuery +RETURN gds.graph.project( + $graphName, + sourceNode, + targetNode, + $dataConfig, + $configuration +) +---- +|=== + +The query no longer needs to adhere to a certain structure and you can use any Cypher query that produces the source- and target node pairs. + +== Semantic Changes + +The Legacy Cypher projections has separate queries for nodes and relationships. +The nodes query is executed first and defines all the nodes in the graph. +The relationships query is executed second and the previously imported nodes act as a filter for the relationships. +Only relationships between the previously imported nodes are imported into the graph. +Any node that was imported as part of the node query, but does not appear in any of the relationships, results in a disconnected node in the graph. +By default, all nodes are disconnected unless they also appear in a relationship. + +The new Cypher projection does not have separate queries for nodes and relationships. +The node query is no longer needed and nodes are implicitly created from the source- and target node pairs. +Disconnected nodes have to be explicitly created in the query by providing `NULL` in place of the target node. +By default, all nodes are connected unless they are explicitly disconnected. + +Since the new Cypher projection is no longer in charge of executing the Cypher queries, the graph configuration can no longer return the node- and relationship queries. + +== Examples + +The following examples are based on the examples listed in the documentation for xref:management-ops/projections/graph-project-cypher-legacy.adoc[the Legacy Cypher projection] and xref:management-ops/projections/graph-project-cypher-projection.adoc[the new Cypher projection]. + +=== Simple graph + +.Side-by-side comparison of the two Cypher projections: +[opts=header,cols="1a,1a"] +|=== +| Legacy | New +2+| : Simple graph projection with potentially disconnected nodes +| +[source, cypher, role=noplay] +---- +CALL gds.graph.project.cypher( + 'persons', + 'MATCH (n:Person) RETURN id(n) AS id', + 'MATCH (n:Person)-[r:KNOWS]->(m:Person) RETURN id(n) AS source, id(m) AS target') +YIELD + graphName AS graph, nodeCount AS nodes, relationshipCount AS rels +---- +| +[source, cypher, role=noplay] +---- +MATCH (n:Person) +OPTIONAL MATCH (n)-[r:KNOWS]->(m:Person) +WITH gds.graph.project('persons', n, m) AS g +RETURN + g.graphName AS graph, g.nodeCount AS node, g.relationshipCount AS rels +---- +2+| : Simple graph projection without disconnected nodes +| +Not applicable, Legacy Cypher projection cannot guarantee connected nodes. +| +[source, cypher, role=noplay] +---- +MATCH (n:Person)-[r:KNOWS]->(m:Person) +WITH gds.graph.project('persons', n, m) AS g +RETURN + g.graphName AS graph, g.nodeCount AS node, g.relationshipCount AS rels +---- +|=== + +The direct translation requires the use of an `OPTIONAL MATCH` clause to create disconnected nodes in order to create an identical graph. +This may not have been what you wanted originally, but was required since the Legacy Cypher projection could not guarantee connected nodes. +By using what is equivalent to the `$relationshipQuery`, we now also get only connected nodes in the new Cypher projection. + +Another difference is that we pass the nodes directly to the new Cypher projection. +The Legacy Cypher projection required us to pass the node ids. +By passing the nodes directly, the Cypher projection knows that the source for the projection is a neo4j database and it enables the use of `.write` procedures. +It is also possible to pass node ids instead of nodes `... gds.graph.project('persons', id(n), id(m))`, but this is only recommended if the source for the projection is not a neo4j database. +See xref:management-ops/projections/graph-project-cypher-projection.adoc#graph-project-cypher-projection-arbitrary-source-and-target-id-values[Arbitrary source and target id values] for more details. + +=== Multi-graph + +.Side-by-side comparison of the two Cypher projections: +[opts=header,cols="1a,1a"] +|=== +| Legacy | New +2+| : Multi-graph projection +| +[source, cypher, role=noplay] +---- +CALL gds.graph.project.cypher( + 'personsAndBooks', + 'MATCH (n) WHERE n:Person OR n:Book RETURN id(n) AS id, labels(n) AS labels', + 'MATCH (n)-[r:KNOWS\|READ]->(m) RETURN id(n) AS source, id(m) AS target, type(r) AS type') +YIELD + graphName AS graph, nodeQuery, nodeCount AS nodes, relationshipCount AS rels +---- +| +[source, cypher, role=noplay] +---- +MATCH (n) +WHERE n:Person OR n:Book +OPTIONAL MATCH (n)-[r:KNOWS\|READ]->(m) +WHERE m:Person OR m:Book +WITH gds.graph.project( + 'personsAndBooks', + n, + m, + { + sourceNodeLabels: labels(n), + targetNodeLabels: labels(m), + relationshipType: type(r) + } +) AS g +RETURN + g.graphName AS graph, g.nodeCount AS node, g.relationshipCount AS rels +---- +|=== + +Similar to the previous example, we have to use an `OPTIONAL MATCH` clause to create disconnected nodes in order to create an identical graph. +The query can also look different depending on the actual graph schema and whether disconnected nodes are desired. + +Node labels and relationship types are passed as an additional configuration map to the new Cypher projection. +Node labels need to be passed as `sourceNodeLabels` and `targetNodeLabels` and relationship types need to be passed as `relationshipType`. +See xref:management-ops/projections/graph-project-cypher-projection.adoc#graph-project-cypher-projection-multi-graph[Multi-graph] for more details. + +=== Node properties + +.Side-by-side comparison of the two Cypher projections: +[opts=header,cols="1a,1a"] +|=== +| Legacy | New +2+| : Graph projection with node properties +| +[source, cypher, role=noplay] +---- +CALL gds.graph.project.cypher( + 'graphWithProperties', + 'MATCH (n:Person) + RETURN + id(n) AS id, + labels(n) AS labels, + n.age AS age', + 'MATCH (n)-[r:KNOWS]->(m) RETURN id(n) AS source, id(m) AS target, type(r) AS type' +) +YIELD + graphName, nodeCount AS nodes, relationshipCount AS rels +RETURN graphName, nodes, rels +---- +| +[source, cypher, role=noplay] +---- +MATCH (n:Person)-[r:KNOWS]->(m:Person) +WITH gds.graph.project( + 'graphWithProperties', + n, + m, + { + sourceNodeLabels: labels(n), + targetNodeLabels: labels(m), + sourceNodeProperties: n { .age }, + targetNodeProperties: m { .age }, + relationshipType: type(r) + } +) AS g +RETURN + g.graphName AS graph, g.nodeCount AS node, g.relationshipCount AS rels +---- +2+| : Graph projection with optional node properties +| +[source, cypher, role=noplay] +---- +CALL gds.graph.project.cypher( + 'graphWithProperties', + 'MATCH (n) + WHERE n:Book OR n:Person + RETURN + id(n) AS id, + labels(n) AS labels, + coalesce(n.age, 18) AS age', + coalesce(n.price, 5.0) as price, + n.ratings as ratings', + 'MATCH (n)-[r:KNOWS\|READ]->(m) RETURN id(n) AS source, id(m) AS target, type(r) AS type' +) +YIELD + graphName, nodeCount AS nodes, relationshipCount AS rels +RETURN graphName, nodes, rels +---- +| +[source, cypher, role=noplay] +---- +MATCH (n:Person)-[r:KNOWS\|READ]->(m) +WITH gds.graph.project( + 'graphWithProperties', + n, + m, + { + sourceNodeLabels: labels(n), + targetNodeLabels: labels(m), + sourceNodeProperties: n { age: coalesce(n.age, 18), price: coalesce(n.price, 5.0), .ratings }, + targetNodeProperties: n { age: coalesce(n.age, 18), price: coalesce(n.price, 5.0), .ratings }, + relationshipType: type(r) + } +) AS g +RETURN + g.graphName AS graph, g.nodeCount AS node, g.relationshipCount AS rels +---- +|=== + +Similar to the previous example, we pass the labels and properties in an additional map. +We can use map projections as well as any other Cypher expression to create the properties. +See xref:management-ops/projections/graph-project-cypher-projection.adoc#node-properties-example[Node properties] for more details. + + +=== Relationship properties + +.Side-by-side comparison of the two Cypher projections: +[opts=header,cols="1a,1a"] +|=== +| Legacy | New +2+| : Graph projection with relationship properties +| +[source, cypher, role=noplay] +---- +CALL gds.graph.project.cypher( + 'readWithProperties', + 'MATCH (n) RETURN id(n) AS id', + 'MATCH (n)-[r:READ]->(m) + RETURN id(n) AS source, id(m) AS target, r.numberOfPages AS numberOfPages' +) +YIELD + graphName AS graph, nodeCount AS nodes, relationshipCount AS rels +---- +| +[source, cypher, role=noplay] +---- +MATCH (n)-[r:READ]->(m) +WITH gds.graph.project( + 'readWithProperties', + n, + m, + { relationshipProperties: r { .numberOfPages } } +) AS g +RETURN + g.graphName AS graph, g.nodeCount AS nodes, g.relationshipCount AS rels + +---- +|=== + +Similar to the previous example, we pass properties in an additional map, here using the `relationshipProperties` key. +We can use map projections as well as any other Cypher expression to create the properties. +See xref:management-ops/projections/graph-project-cypher-projection.adoc#cypher-aggregation-relationship-properties[Relationship properties] for more details. + + +=== Parallel Relationship + +.Side-by-side comparison of the two Cypher projections: +[opts=header,cols="1a,1a"] +|=== +| Legacy | New +2+| : Graph projection with parallel relationships +| +[source, cypher, role=noplay] +---- +CALL gds.graph.project.cypher( + 'readCount', + 'MATCH (n) RETURN id(n) AS id', + 'MATCH (n)-[r:READ]->(m) + RETURN id(n) AS source, id(m) AS target, type(r) AS type, count(r) AS numberOfReads' +) +YIELD + graphName AS graph, nodeCount AS nodes, relationshipCount AS rels +---- +| +[source, cypher, role=noplay] +---- +MATCH (n)-[r:READ]->(m) +WITH n, m, count(r) AS numberOfReads +WITH gds.graph.project( + 'readCount', + n, + m, + { + relationshipProperties: { numberOfReads: numberOfReads } + } +) AS g +RETURN + g.graphName AS graph, g.nodeCount AS nodes, g.relationshipCount AS rels +---- +2+| : Graph projection with parallel relationship and relationship properties +| +[source, cypher, role=noplay] +---- +CALL gds.graph.project.cypher( + 'readSums', + 'MATCH (n) RETURN id(n) AS id', + 'MATCH (n)-[r:READ]->(m) + RETURN id(n) AS source, id(m) AS target, sum(r.numberOfPages) AS numberOfPages' +) +YIELD + graphName AS graph, nodeCount AS nodes, relationshipCount AS rels +---- +| +[source, cypher, role=noplay] +---- +MATCH (n)-[r:READ]->(m) +WITH n, m, sum(r.numberOfPages) AS numberOfPages +WITH gds.graph.project( + 'readSums', + n, + m, + { + relationshipProperties: { numberOfPages: numberOfPages } + } +) AS g +RETURN + g.graphName AS graph, g.nodeCount AS nodes, g.relationshipCount AS rels +---- +|=== + +Similar to Legacy Cypher projections, there is no mechanism to let GDS aggregate parallel relationships. +Aggregations over parallel relationships are done in the query by any means that are appropriate for the graph schema and data. +See xref:management-ops/projections/graph-project-cypher-projection.adoc#graph-project-cypher-projection-parallel-relationships[Parallel relationship] for more details. + + +=== Projecting filtered graphs + +.Side-by-side comparison of the two Cypher projections: +[opts=header,cols="1a,1a"] +|=== +| Legacy | New +2+| : Graph projection with filtered graphs +| +[source, cypher, role=noplay] +---- +CALL gds.graph.project.cypher( + 'existingNumberOfPages', + 'MATCH (n) RETURN id(n) AS id', + 'MATCH (n)-[r:READ]->(m) + WHERE r.numberOfPages IS NOT NULL + RETURN id(n) AS source, id(m) AS target, r.numberOfPages AS numberOfPages' +) +YIELD + graphName AS graph, nodeCount AS nodes, relationshipCount AS rels +---- +| +[source, cypher, role=noplay] +---- +MATCH (n) OPTIONAL MATCH (n)-[r:READ]->(m) +WHERE r.numberOfPages IS NOT NULL +WITH gds.graph.project('existingNumberOfPages', n, m, { relationshipProperties: r { .numberOfPages } }) AS g +RETURN + g.graphName AS graph, g.nodeCount AS nodes, g.relationshipCount AS rels + +---- +|=== + +Similar to Legacy Cypher projections, we can apply any Cypher method of filtering the data before passing it on to the Cypher projection. +See xref:management-ops/projections/graph-project-cypher-projection.adoc#graph-project-cypher-projection-filtered[Projecting filtered Neo4j graphs] for more details. + + +=== Projecting undirected graphs + +.Side-by-side comparison of the two Cypher projections: +[opts=header,cols="1a,1a"] +|=== +| Legacy | New +2+| : Graph projection with undirected graphs +| +Not applicable, Legacy Cypher projection cannot project undirected graphs. +| +[source, cypher, role=noplay] +---- +MATCH (n)-[r:KNOWS\|READ]->(m) +WHERE n:Book OR n:Person +WITH gds.graph.project( + 'graphWithUndirectedRelationships', + source, + target, + {}, + {undirectedRelationshipTypes: ['*']} +) as g +RETURN g.graphName AS graph, g.nodeCount AS nodes, g.relationshipCount AS rels +---- +|=== + +The new Cypher projection can project undirected graphs. +See xref:management-ops/projections/graph-project-cypher-projection.adoc#graph-project-cypher-projection-undirected-relationships[Undirected relationships] for more details. + + +=== Memory estimation + +.Side-by-side comparison of the two Cypher projections: +[opts=header,cols="1a,1a"] +|=== +| Legacy | New +2+| : Memory estimation of projected graphs +| +[source, cypher, role=noplay] +---- +CALL gds.graph.project.cypher.estimate( + 'MATCH (n:Person) RETURN id(n) AS id', + 'MATCH (n:Person)-[r:KNOWS]->(m:Person) RETURN id(n) AS source, id(m) AS target' +) YIELD requiredMemory, bytesMin, bytesMax +---- +| +[source, cypher, role=noplay] +---- +MATCH (n:Person)-[r:KNOWS]-(m) +WITH count(n) as nodeCount, count(r) as relationshipCount +CALL gds.graph.project.estimate('*', '*', { + nodeCount: nodeCount, + relationshipCount: relationshipCount, +}) +YIELD requiredMemory, bytesMin, bytesMax +---- +|=== + +Since the new Cypher projection is no longer a procedure, there is also no `.estimate` method. +Instead, we can use xref:common-usage/memory-estimation.adoc#estimate-procedure-fictive-graph[the `gds.graph.project.estimate` procedure] to estimate the memory requirements of the graph projection. From 848e0e760827948311525b2ec8b98f09fc656b90 Mon Sep 17 00:00:00 2001 From: Paul Horn Date: Wed, 7 Jun 2023 18:14:05 +0200 Subject: [PATCH 034/273] Link to migration guide from the deprecation warning MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Florentin Dörre --- .../management-ops/projections/graph-project-cypher-legacy.adoc | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/modules/ROOT/pages/management-ops/projections/graph-project-cypher-legacy.adoc b/doc/modules/ROOT/pages/management-ops/projections/graph-project-cypher-legacy.adoc index ac0a499cf1..bc9d603409 100644 --- a/doc/modules/ROOT/pages/management-ops/projections/graph-project-cypher-legacy.adoc +++ b/doc/modules/ROOT/pages/management-ops/projections/graph-project-cypher-legacy.adoc @@ -7,6 +7,7 @@ -- This page describes the Legacy Cypher projection, which is deprecated. The replacement is to use the new Cypher projection, which is described in xref:management-ops/projections/graph-project-cypher-projection.adoc[Projecting graphs using Cypher]. +A migration guide is available at xref:migration-lcp-to-cpv2/index.adoc[Appendix C, Migration from Legacy to new Cypher projection]. -- Using Legacy Cypher projections is a more flexible and expressive approach with diminished focus on performance compared to the xref:management-ops/projections/graph-project.adoc[native projections]. From 4c549c0723d41573fb677f507f3931c9941c9a36 Mon Sep 17 00:00:00 2001 From: Paul Horn Date: Wed, 7 Jun 2023 18:28:48 +0200 Subject: [PATCH 035/273] Link to the LCP docs from the migration guide --- doc/modules/ROOT/pages/migration-lcp-to-cpv2/index.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/modules/ROOT/pages/migration-lcp-to-cpv2/index.adoc b/doc/modules/ROOT/pages/migration-lcp-to-cpv2/index.adoc index aeedc1f3e9..bd8cbe0cf9 100644 --- a/doc/modules/ROOT/pages/migration-lcp-to-cpv2/index.adoc +++ b/doc/modules/ROOT/pages/migration-lcp-to-cpv2/index.adoc @@ -6,7 +6,7 @@ == Who should read this guide -This guide is intended for users who have been using the legacy Cypher projection `gds.graph.project.cypher`. +This guide is intended for users who have been using the Legacy Cypher projection xref:management-ops/projections/graph-project-cypher-legacy.adoc[`gds.graph.project.cypher`]. Cypher projections are now done using the `gds.graph.project` aggregation function. We assume that most of the mentioned operations and concepts can be understood with little explanation. Thus we are intentionally brief in the examples and comparisons. From 0a06e7fa120687c23d9e312ab3b6d72b8085b2b1 Mon Sep 17 00:00:00 2001 From: Paul Horn Date: Wed, 7 Jun 2023 18:28:58 +0200 Subject: [PATCH 036/273] Capitalize Neo4j --- doc/modules/ROOT/pages/migration-lcp-to-cpv2/index.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/modules/ROOT/pages/migration-lcp-to-cpv2/index.adoc b/doc/modules/ROOT/pages/migration-lcp-to-cpv2/index.adoc index bd8cbe0cf9..a8e11b81be 100644 --- a/doc/modules/ROOT/pages/migration-lcp-to-cpv2/index.adoc +++ b/doc/modules/ROOT/pages/migration-lcp-to-cpv2/index.adoc @@ -119,7 +119,7 @@ By using what is equivalent to the `$relationshipQuery`, we now also get only co Another difference is that we pass the nodes directly to the new Cypher projection. The Legacy Cypher projection required us to pass the node ids. -By passing the nodes directly, the Cypher projection knows that the source for the projection is a neo4j database and it enables the use of `.write` procedures. +By passing the nodes directly, the Cypher projection knows that the source for the projection is a Neo4j database and it enables the use of `.write` procedures. It is also possible to pass node ids instead of nodes `... gds.graph.project('persons', id(n), id(m))`, but this is only recommended if the source for the projection is not a neo4j database. See xref:management-ops/projections/graph-project-cypher-projection.adoc#graph-project-cypher-projection-arbitrary-source-and-target-id-values[Arbitrary source and target id values] for more details. From 5a7853e7217b479556326deb5f8d9a91513c16b9 Mon Sep 17 00:00:00 2001 From: Paul Horn Date: Wed, 7 Jun 2023 18:29:32 +0200 Subject: [PATCH 037/273] Use OPTIONAL MATCH to have equivalent queries --- doc/modules/ROOT/pages/migration-lcp-to-cpv2/index.adoc | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/doc/modules/ROOT/pages/migration-lcp-to-cpv2/index.adoc b/doc/modules/ROOT/pages/migration-lcp-to-cpv2/index.adoc index a8e11b81be..6ef3850fbb 100644 --- a/doc/modules/ROOT/pages/migration-lcp-to-cpv2/index.adoc +++ b/doc/modules/ROOT/pages/migration-lcp-to-cpv2/index.adoc @@ -195,7 +195,8 @@ RETURN graphName, nodes, rels | [source, cypher, role=noplay] ---- -MATCH (n:Person)-[r:KNOWS]->(m:Person) +MATCH (n:Person) +OPTIONAL MATCH (n)-[r:KNOWS]->(m:Person) WITH gds.graph.project( 'graphWithProperties', n, @@ -234,7 +235,10 @@ RETURN graphName, nodes, rels | [source, cypher, role=noplay] ---- -MATCH (n:Person)-[r:KNOWS\|READ]->(m) +MATCH (n) +WHERE n:Person OR n:Book +OPTIONAL MATCH (n)-[r:KNOWS\|READ]->(m) +WHERE m:Person OR m:Book WITH gds.graph.project( 'graphWithProperties', n, From cbf8dca303287796bc06607ec810d9437140234e Mon Sep 17 00:00:00 2001 From: Paul Horn Date: Wed, 7 Jun 2023 18:29:46 +0200 Subject: [PATCH 038/273] Fully uppercase AS in queries --- doc/modules/ROOT/pages/migration-lcp-to-cpv2/index.adoc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/modules/ROOT/pages/migration-lcp-to-cpv2/index.adoc b/doc/modules/ROOT/pages/migration-lcp-to-cpv2/index.adoc index 6ef3850fbb..d35f3acf38 100644 --- a/doc/modules/ROOT/pages/migration-lcp-to-cpv2/index.adoc +++ b/doc/modules/ROOT/pages/migration-lcp-to-cpv2/index.adoc @@ -224,8 +224,8 @@ CALL gds.graph.project.cypher( id(n) AS id, labels(n) AS labels, coalesce(n.age, 18) AS age', - coalesce(n.price, 5.0) as price, - n.ratings as ratings', + coalesce(n.price, 5.0) AS price, + n.ratings AS ratings', 'MATCH (n)-[r:KNOWS\|READ]->(m) RETURN id(n) AS source, id(m) AS target, type(r) AS type' ) YIELD @@ -428,7 +428,7 @@ WITH gds.graph.project( target, {}, {undirectedRelationshipTypes: ['*']} -) as g +) AS g RETURN g.graphName AS graph, g.nodeCount AS nodes, g.relationshipCount AS rels ---- |=== @@ -456,7 +456,7 @@ CALL gds.graph.project.cypher.estimate( [source, cypher, role=noplay] ---- MATCH (n:Person)-[r:KNOWS]-(m) -WITH count(n) as nodeCount, count(r) as relationshipCount +WITH count(n) AS nodeCount, count(r) AS relationshipCount CALL gds.graph.project.estimate('*', '*', { nodeCount: nodeCount, relationshipCount: relationshipCount, From 34efc251d8c1aec5359fe1799c8e2199a2864347 Mon Sep 17 00:00:00 2001 From: Veselin Nikolov Date: Thu, 8 Jun 2023 07:15:36 +0100 Subject: [PATCH 039/273] Improve wording Co-authored-by: Jessica Wright <49636617+AlexicaWright@users.noreply.github.com> --- doc/modules/ROOT/pages/production-deployment/neo4j-cluster.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/modules/ROOT/pages/production-deployment/neo4j-cluster.adoc b/doc/modules/ROOT/pages/production-deployment/neo4j-cluster.adoc index 157e4a5042..9df8805c17 100644 --- a/doc/modules/ROOT/pages/production-deployment/neo4j-cluster.adoc +++ b/doc/modules/ROOT/pages/production-deployment/neo4j-cluster.adoc @@ -29,7 +29,7 @@ GDS has compute-intensive OLAP workloads that may disrupt the cluster operations [NOTE] ====== -Please refer to the https://neo4j.com/docs/operations-manual/current/clustering/setup/analytics-cluster/[official Neo4j documentation] for details on how to setup Neo4j analytical cluster. +Please refer to the https://neo4j.com/docs/operations-manual/current/clustering/setup/analytics-cluster/[official Neo4j documentation] for details on how to set up a Neo4j analytics cluster. Note that the link points to the latest Neo4j version documentation and the configuration settings may differ from earlier versions. ====== From 34fd391358eaab4c6eb1e4c55012c5b098c6b359 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20Kie=C3=9Fling?= Date: Wed, 7 Jun 2023 17:01:10 +0200 Subject: [PATCH 040/273] Simplify logging for graph store to database export processes --- .../db/ProgressTrackerExecutionMonitor.java | 103 ++++-------------- .../gds/catalog/GraphStoreExportProc.java | 5 +- 2 files changed, 20 insertions(+), 88 deletions(-) diff --git a/io/core/src/main/java/org/neo4j/gds/core/io/db/ProgressTrackerExecutionMonitor.java b/io/core/src/main/java/org/neo4j/gds/core/io/db/ProgressTrackerExecutionMonitor.java index 56fbfccccf..67277dd9b7 100644 --- a/io/core/src/main/java/org/neo4j/gds/core/io/db/ProgressTrackerExecutionMonitor.java +++ b/io/core/src/main/java/org/neo4j/gds/core/io/db/ProgressTrackerExecutionMonitor.java @@ -25,63 +25,27 @@ import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.core.utils.progress.tasks.Task; import org.neo4j.gds.core.utils.progress.tasks.Tasks; -import org.neo4j.internal.batchimport.CountGroupsStage; -import org.neo4j.internal.batchimport.DataImporter; -import org.neo4j.internal.batchimport.NodeCountsAndLabelIndexBuildStage; -import org.neo4j.internal.batchimport.NodeDegreeCountStage; -import org.neo4j.internal.batchimport.NodeFirstGroupStage; -import org.neo4j.internal.batchimport.RelationshipCountsAndTypeIndexBuildStage; -import org.neo4j.internal.batchimport.RelationshipGroupStage; -import org.neo4j.internal.batchimport.RelationshipLinkbackStage; -import org.neo4j.internal.batchimport.RelationshipLinkforwardStage; -import org.neo4j.internal.batchimport.ScanAndCacheGroupsStage; -import org.neo4j.internal.batchimport.SparseNodeFirstRelationshipStage; -import org.neo4j.internal.batchimport.WriteGroupsStage; import org.neo4j.internal.batchimport.staging.ExecutionMonitor; import org.neo4j.internal.batchimport.staging.StageExecution; -import org.neo4j.internal.batchimport.stats.Keys; -import org.neo4j.internal.batchimport.stats.Stat; -import org.neo4j.internal.batchimport.store.BatchingNeoStores; -import org.neo4j.internal.helpers.collection.Iterables; import java.time.Clock; -import java.util.Objects; +import java.util.List; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicLong; -import java.util.stream.StreamSupport; + +import static org.neo4j.gds.utils.StringFormatting.formatWithLocale; public final class ProgressTrackerExecutionMonitor implements CompatExecutionMonitor { private final Clock clock; private final long intervalMillis; - // The total task volume per stage. - // Set to 0 at the beginning of a stage. - private final AtomicLong stageProgressTotal; - // The progressed task volume per stage. - // Set to 0 at the beginning of a stage. - private final AtomicLong stageProgressCurrent; - private final ProgressTracker progressTracker; - private DependencyResolver dependencyResolver; - public static Task progressTask(long nodeCount, long relationshipCount) { + public static Task progressTask() { return Tasks.task( GraphStoreToDatabaseExporter.class.getSimpleName(), - Tasks.leaf(DataImporter.NODE_IMPORT_NAME, nodeCount), - Tasks.leaf(DataImporter.RELATIONSHIP_IMPORT_NAME, relationshipCount), - Tasks.leaf(NodeDegreeCountStage.NAME), - Tasks.leaf(RelationshipLinkforwardStage.NAME, 2 * relationshipCount), - Tasks.leaf(RelationshipGroupStage.NAME), - Tasks.leaf(SparseNodeFirstRelationshipStage.NAME), - Tasks.leaf(RelationshipLinkbackStage.NAME, 2 * relationshipCount), - Tasks.leaf(CountGroupsStage.NAME), - Tasks.leaf(ScanAndCacheGroupsStage.NAME), - Tasks.leaf(WriteGroupsStage.NAME), - Tasks.leaf(NodeFirstGroupStage.NAME), - Tasks.leaf(NodeCountsAndLabelIndexBuildStage.NAME, nodeCount), - Tasks.leaf(RelationshipCountsAndTypeIndexBuildStage.NAME, relationshipCount) + List.of() ); } @@ -98,46 +62,33 @@ private ProgressTrackerExecutionMonitor(ProgressTracker progressTracker, Clock c this.clock = clock; this.intervalMillis = unit.toMillis(time); this.progressTracker = progressTracker; - this.stageProgressTotal = new AtomicLong(0); - this.stageProgressCurrent = new AtomicLong(0); } @Override public void initialize(DependencyResolver dependencyResolver) { - this.dependencyResolver = dependencyResolver; this.progressTracker.beginSubTask(); } @Override public void start(StageExecution execution) { - this.stageProgressTotal.set(0); - this.stageProgressCurrent.set(0); - - var neoStores = this.dependencyResolver.resolveDependency(BatchingNeoStores.class); - var relationshipRecordIdCount = Neo4jProxy.getHighId(neoStores.getRelationshipStore()); - var groupCount = Neo4jProxy.getHighId(neoStores.getTemporaryRelationshipGroupStore()); - - switch (execution.getStageName()) { - case NodeDegreeCountStage.NAME: - this.progressTracker.beginSubTask(execution.getStageName(), relationshipRecordIdCount); - break; - case CountGroupsStage.NAME: - case WriteGroupsStage.NAME: - case NodeFirstGroupStage.NAME: - this.progressTracker.beginSubTask(execution.getStageName(), groupCount); - break; - default: - this.progressTracker.beginSubTask(execution.getStageName()); - } - - if (this.progressTracker.currentVolume() != Task.UNKNOWN_VOLUME) { - this.stageProgressTotal.set(this.progressTracker.currentVolume()); - } + progressTracker.logInfo( + formatWithLocale( + "%s :: %s :: Start", + GraphStoreToDatabaseExporter.class.getSimpleName(), + execution.getStageName() + ) + ); } @Override public void end(StageExecution execution, long totalTimeMillis) { - this.progressTracker.endSubTask(execution.getStageName()); + progressTracker.logInfo( + formatWithLocale( + "%s :: %s :: Finished", + GraphStoreToDatabaseExporter.class.getSimpleName(), + execution.getStageName() + ) + ); } @Override @@ -148,23 +99,7 @@ public void done(boolean successful, long totalTimeMillis, String additionalInfo @Override public void check(StageExecution execution) { - // Cap the total to not produce percentages > 100. - var progress = Math.min(progress(execution), this.stageProgressTotal.get()); - this.progressTracker.logProgress(progress - this.stageProgressCurrent.getAndSet(progress)); - } - private static long progress(StageExecution execution) { - return StreamSupport - .stream(execution.steps().spliterator(), false) - .map(step -> step.stats().stat(Keys.progress)) - .filter(Objects::nonNull) - .map(Stat::asLong) - .findFirst() - .orElseGet(() -> { - var doneBatches = Iterables.last(execution.steps()).stats().stat(Keys.done_batches).asLong(); - var batchSize = execution.getConfig().batchSize(); - return doneBatches * batchSize; - }); } @Override diff --git a/proc/catalog/src/main/java/org/neo4j/gds/catalog/GraphStoreExportProc.java b/proc/catalog/src/main/java/org/neo4j/gds/catalog/GraphStoreExportProc.java index 336f738250..ba8250e135 100644 --- a/proc/catalog/src/main/java/org/neo4j/gds/catalog/GraphStoreExportProc.java +++ b/proc/catalog/src/main/java/org/neo4j/gds/catalog/GraphStoreExportProc.java @@ -75,10 +75,7 @@ public Stream database( validateGraphStore(graphStore, exportConfig); var progressTracker = new TaskProgressTracker( - ProgressTrackerExecutionMonitor.progressTask( - graphStore.nodeCount(), - graphStore.relationshipCount() - ), + ProgressTrackerExecutionMonitor.progressTask(), executionContext().log(), exportConfig.writeConcurrency(), exportConfig.jobId(), From d2b06c6a28fff453e53ef284edd638c5ca8abeef Mon Sep 17 00:00:00 2001 From: Max Kiessling Date: Thu, 8 Jun 2023 09:33:58 +0200 Subject: [PATCH 041/273] Fix the log output and adapt tests --- .../gds/core/io/db/ProgressTrackerExecutionMonitor.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/io/core/src/main/java/org/neo4j/gds/core/io/db/ProgressTrackerExecutionMonitor.java b/io/core/src/main/java/org/neo4j/gds/core/io/db/ProgressTrackerExecutionMonitor.java index 67277dd9b7..e3ab725380 100644 --- a/io/core/src/main/java/org/neo4j/gds/core/io/db/ProgressTrackerExecutionMonitor.java +++ b/io/core/src/main/java/org/neo4j/gds/core/io/db/ProgressTrackerExecutionMonitor.java @@ -73,8 +73,7 @@ public void initialize(DependencyResolver dependencyResolver) { public void start(StageExecution execution) { progressTracker.logInfo( formatWithLocale( - "%s :: %s :: Start", - GraphStoreToDatabaseExporter.class.getSimpleName(), + "%s :: Start", execution.getStageName() ) ); @@ -84,8 +83,7 @@ public void start(StageExecution execution) { public void end(StageExecution execution, long totalTimeMillis) { progressTracker.logInfo( formatWithLocale( - "%s :: %s :: Finished", - GraphStoreToDatabaseExporter.class.getSimpleName(), + "%s :: Finished", execution.getStageName() ) ); From 6da5841200a67743c00a897bd1b4932631a89652 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonatan=20J=C3=A4derberg?= Date: Wed, 7 Jun 2023 12:33:18 +0200 Subject: [PATCH 042/273] Remove unnecessary singleton --- .../org/neo4j/gds/executor/ComputationResultConsumer.java | 5 ----- 1 file changed, 5 deletions(-) diff --git a/executor/src/main/java/org/neo4j/gds/executor/ComputationResultConsumer.java b/executor/src/main/java/org/neo4j/gds/executor/ComputationResultConsumer.java index aef3533876..5916236c81 100644 --- a/executor/src/main/java/org/neo4j/gds/executor/ComputationResultConsumer.java +++ b/executor/src/main/java/org/neo4j/gds/executor/ComputationResultConsumer.java @@ -25,9 +25,4 @@ @FunctionalInterface public interface ComputationResultConsumer, ALGO_RESULT, CONFIG extends AlgoBaseConfig, RESULT> { RESULT consume(ComputationResult computationResult, ExecutionContext executionContext); - - static , ALGO_RESULT, CONFIG extends AlgoBaseConfig> - ComputationResultConsumer> identity() { - return (result, executionContext) -> result; - } } From ee0fcf50893becbc3477b735745043654eba0dab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonatan=20J=C3=A4derberg?= Date: Wed, 7 Jun 2023 14:52:55 +0200 Subject: [PATCH 043/273] Go over centrality stream result consumers --- .../neo4j/gds/influenceMaximization/CELF.java | 10 ----- .../ClosenessCentralityStreamSpec.java | 40 +++++++++---------- .../DegreeCentralityNodePropertyValues.java | 5 +-- .../DegreeCentralityStreamSpecification.java | 11 +++-- .../influenceMaximization/CELFStreamProc.java | 2 +- .../influenceMaximization/CELFStreamSpec.java | 31 ++++++++------ .../influenceMaximization/StreamResult.java | 4 +- .../kcore/KCoreDecompositionStreamSpec.java | 4 +- .../gds/pagerank/PageRankStreamSpec.java | 30 +++++++------- 9 files changed, 63 insertions(+), 74 deletions(-) rename algo/src/main/java/org/neo4j/gds/influenceMaximization/InfluenceMaximizationResult.java => proc/centrality/src/main/java/org/neo4j/gds/influenceMaximization/StreamResult.java (89%) diff --git a/algo/src/main/java/org/neo4j/gds/influenceMaximization/CELF.java b/algo/src/main/java/org/neo4j/gds/influenceMaximization/CELF.java index ffccd82a49..4cedaee5b3 100644 --- a/algo/src/main/java/org/neo4j/gds/influenceMaximization/CELF.java +++ b/algo/src/main/java/org/neo4j/gds/influenceMaximization/CELF.java @@ -31,8 +31,6 @@ import java.util.Optional; import java.util.concurrent.ExecutorService; -import java.util.stream.LongStream; -import java.util.stream.Stream; public class CELF extends Algorithm { @@ -185,12 +183,4 @@ private void lazyForwardPart(long firstSeedNode) { } progressTracker.endSubTask(); } - - public Stream resultStream() { - return LongStream.of(seedSetNodes.keys().toArray()) - .mapToObj(node -> new InfluenceMaximizationResult( - graph.toOriginalNodeId(node), - seedSetNodes.getOrDefault(node, 0) - )); - } } diff --git a/proc/centrality/src/main/java/org/neo4j/gds/closeness/ClosenessCentralityStreamSpec.java b/proc/centrality/src/main/java/org/neo4j/gds/closeness/ClosenessCentralityStreamSpec.java index 3fadaf2c7d..d9385388df 100644 --- a/proc/centrality/src/main/java/org/neo4j/gds/closeness/ClosenessCentralityStreamSpec.java +++ b/proc/centrality/src/main/java/org/neo4j/gds/closeness/ClosenessCentralityStreamSpec.java @@ -21,7 +21,6 @@ import org.neo4j.gds.api.IdMap; import org.neo4j.gds.common.CentralityStreamResult; -import org.neo4j.gds.core.utils.paged.HugeDoubleArray; import org.neo4j.gds.executor.AlgorithmSpec; import org.neo4j.gds.executor.ComputationResultConsumer; import org.neo4j.gds.executor.ExecutionContext; @@ -31,6 +30,7 @@ import java.util.stream.LongStream; import java.util.stream.Stream; +import static org.neo4j.gds.LoggingUtil.runWithExceptionLogging; import static org.neo4j.gds.closeness.ClosenessCentrality.CLOSENESS_DESCRIPTION; import static org.neo4j.gds.executor.ExecutionMode.STREAM; @@ -54,26 +54,22 @@ public NewConfigFunction newConfigFunction() { @Override public ComputationResultConsumer> computationResultConsumer() { - return (computationResult, executionContext) -> { - - if (computationResult.isGraphEmpty()) { - return Stream.empty(); - } - - var nodePropertyValues = computationResult.result() - .map(ClosenessCentralityResult::centralities) - .orElseGet(() -> HugeDoubleArray.newArray(0)) - .asNodeProperties(); - var graph = computationResult.graph(); - return LongStream - .range(IdMap.START_NODE_ID, graph.nodeCount()) - .filter(nodePropertyValues::hasValue) - .mapToObj(nodeId -> - new CentralityStreamResult( - graph.toOriginalNodeId(nodeId), - nodePropertyValues.doubleValue(nodeId) - )); - - }; + return (computationResult, executionContext) -> runWithExceptionLogging( + "Result streaming failed", + executionContext.log(), + () -> computationResult.result() + .map(result -> { + var nodePropertyValues = result.centralities().asNodeProperties(); + var graph = computationResult.graph(); + return LongStream + .range(IdMap.START_NODE_ID, graph.nodeCount()) + .filter(nodePropertyValues::hasValue) + .mapToObj(nodeId -> + new CentralityStreamResult( + graph.toOriginalNodeId(nodeId), + nodePropertyValues.doubleValue(nodeId) + )); + }).orElseGet(Stream::empty) + ); } } diff --git a/proc/centrality/src/main/java/org/neo4j/gds/degree/DegreeCentralityNodePropertyValues.java b/proc/centrality/src/main/java/org/neo4j/gds/degree/DegreeCentralityNodePropertyValues.java index 70e481d40d..2bddf77937 100644 --- a/proc/centrality/src/main/java/org/neo4j/gds/degree/DegreeCentralityNodePropertyValues.java +++ b/proc/centrality/src/main/java/org/neo4j/gds/degree/DegreeCentralityNodePropertyValues.java @@ -33,10 +33,7 @@ static DegreeCentralityNodePropertyValues from(ComputationResult new DegreeCentralityNodePropertyValues(0, (nodeId) -> -1L)); } - private DegreeCentralityNodePropertyValues( - long nodeCount, - DegreeCentrality.DegreeFunction degreeFunction - ) { + DegreeCentralityNodePropertyValues(long nodeCount, DegreeCentrality.DegreeFunction degreeFunction) { this.nodeCount = nodeCount; this.degreeFunction = degreeFunction; } diff --git a/proc/centrality/src/main/java/org/neo4j/gds/degree/DegreeCentralityStreamSpecification.java b/proc/centrality/src/main/java/org/neo4j/gds/degree/DegreeCentralityStreamSpecification.java index 106c6070af..8169d0aef8 100644 --- a/proc/centrality/src/main/java/org/neo4j/gds/degree/DegreeCentralityStreamSpecification.java +++ b/proc/centrality/src/main/java/org/neo4j/gds/degree/DegreeCentralityStreamSpecification.java @@ -27,7 +27,6 @@ import org.neo4j.gds.executor.GdsCallable; import org.neo4j.gds.executor.NewConfigFunction; -import java.util.Optional; import java.util.stream.LongStream; import java.util.stream.Stream; @@ -57,12 +56,11 @@ public ComputationResultConsumer runWithExceptionLogging( "Result streaming failed", executionContext.log(), - () -> Optional.ofNullable(computationResult.result()) + () -> computationResult.result() .map(result -> { - var nodePropertyValues = DegreeCentralityNodePropertyValues.from(computationResult); var graph = computationResult.graph(); - return LongStream - .range(IdMap.START_NODE_ID, graph.nodeCount()) + var nodePropertyValues = new DegreeCentralityNodePropertyValues(graph.nodeCount(), result); + return LongStream.range(IdMap.START_NODE_ID, graph.nodeCount()) .filter(nodePropertyValues::hasValue) .mapToObj(nodeId -> new CentralityStreamResult( @@ -70,5 +68,6 @@ public ComputationResultConsumer stream( + public Stream stream( @Name(value = "graphName") String graphName, @Name(value = "configuration", defaultValue = "{}") Map configuration ) { diff --git a/proc/centrality/src/main/java/org/neo4j/gds/influenceMaximization/CELFStreamSpec.java b/proc/centrality/src/main/java/org/neo4j/gds/influenceMaximization/CELFStreamSpec.java index aa3fb1a7aa..3974f252d7 100644 --- a/proc/centrality/src/main/java/org/neo4j/gds/influenceMaximization/CELFStreamSpec.java +++ b/proc/centrality/src/main/java/org/neo4j/gds/influenceMaximization/CELFStreamSpec.java @@ -26,14 +26,15 @@ import org.neo4j.gds.executor.GdsCallable; import org.neo4j.gds.executor.NewConfigFunction; -import java.util.Optional; +import java.util.stream.LongStream; import java.util.stream.Stream; +import static org.neo4j.gds.LoggingUtil.runWithExceptionLogging; import static org.neo4j.gds.executor.ExecutionMode.STREAM; import static org.neo4j.gds.influenceMaximization.CELFStreamProc.DESCRIPTION; @GdsCallable(name = "gds.beta.influenceMaximization.celf.stream", description = DESCRIPTION, executionMode = STREAM) -public class CELFStreamSpec implements AlgorithmSpec, CELFAlgorithmFactory> { +public class CELFStreamSpec implements AlgorithmSpec, CELFAlgorithmFactory> { @Override public String name() { @@ -51,16 +52,20 @@ public NewConfigFunction newConfigFunction() } @Override - public ComputationResultConsumer> computationResultConsumer() { - return (computationResult, executionContext) -> { - if (computationResult.result().isEmpty()) { - return Stream.empty(); - } - - return Optional.ofNullable(computationResult.algorithm()) - .map(CELF::resultStream) - .orElseGet(Stream::empty); - }; - + public ComputationResultConsumer> computationResultConsumer() { + return (computationResult, executionContext) -> runWithExceptionLogging( + "Result streaming failed", + executionContext.log(), + () -> computationResult.result() + .map(result -> { + var graph = computationResult.graph(); + return LongStream.of(result.keys().toArray()) + .mapToObj(node -> new StreamResult( + graph.toOriginalNodeId(node), + result.getOrDefault(node, 0) + )); + }) + .orElseGet(Stream::empty) + ); } } diff --git a/algo/src/main/java/org/neo4j/gds/influenceMaximization/InfluenceMaximizationResult.java b/proc/centrality/src/main/java/org/neo4j/gds/influenceMaximization/StreamResult.java similarity index 89% rename from algo/src/main/java/org/neo4j/gds/influenceMaximization/InfluenceMaximizationResult.java rename to proc/centrality/src/main/java/org/neo4j/gds/influenceMaximization/StreamResult.java index abaeca0a80..212e4e79fa 100644 --- a/algo/src/main/java/org/neo4j/gds/influenceMaximization/InfluenceMaximizationResult.java +++ b/proc/centrality/src/main/java/org/neo4j/gds/influenceMaximization/StreamResult.java @@ -19,11 +19,11 @@ */ package org.neo4j.gds.influenceMaximization; -public class InfluenceMaximizationResult { +public class StreamResult { public final long nodeId; public final double spread; - InfluenceMaximizationResult(long nodeId, double spread) { + StreamResult(long nodeId, double spread) { this.nodeId = nodeId; this.spread = spread; } diff --git a/proc/centrality/src/main/java/org/neo4j/gds/kcore/KCoreDecompositionStreamSpec.java b/proc/centrality/src/main/java/org/neo4j/gds/kcore/KCoreDecompositionStreamSpec.java index 25f5353a27..5be3aff608 100644 --- a/proc/centrality/src/main/java/org/neo4j/gds/kcore/KCoreDecompositionStreamSpec.java +++ b/proc/centrality/src/main/java/org/neo4j/gds/kcore/KCoreDecompositionStreamSpec.java @@ -55,9 +55,9 @@ public ComputationResultConsumer runWithExceptionLogging( "Result streaming failed", executionContext.log(), - () ->computationResult.result() + () -> computationResult.result() .map(result -> { - var coreValues=result.coreValues(); + var coreValues = result.coreValues(); var graph = computationResult.graph(); return LongStream .range(IdMap.START_NODE_ID, graph.nodeCount()) diff --git a/proc/centrality/src/main/java/org/neo4j/gds/pagerank/PageRankStreamSpec.java b/proc/centrality/src/main/java/org/neo4j/gds/pagerank/PageRankStreamSpec.java index ebe87c17e9..6952afdc4b 100644 --- a/proc/centrality/src/main/java/org/neo4j/gds/pagerank/PageRankStreamSpec.java +++ b/proc/centrality/src/main/java/org/neo4j/gds/pagerank/PageRankStreamSpec.java @@ -19,6 +19,7 @@ */ package org.neo4j.gds.pagerank; +import org.neo4j.gds.api.IdMap; import org.neo4j.gds.common.CentralityStreamResult; import org.neo4j.gds.executor.AlgorithmSpec; import org.neo4j.gds.executor.ComputationResultConsumer; @@ -29,6 +30,7 @@ import java.util.stream.LongStream; import java.util.stream.Stream; +import static org.neo4j.gds.LoggingUtil.runWithExceptionLogging; import static org.neo4j.gds.executor.ExecutionMode.STREAM; import static org.neo4j.gds.pagerank.PageRankProcCompanion.PAGE_RANK_DESCRIPTION; @@ -52,19 +54,19 @@ public NewConfigFunction newConfigFunction() { @Override public ComputationResultConsumer> computationResultConsumer() { - return (computationResult, executionContext) -> { - if (computationResult.result().isEmpty()) { - return Stream.empty(); - } - var graph = computationResult.graph(); - var scores = computationResult.result().get().scores(); - return LongStream.range(0, graph.nodeCount()).mapToObj(nodeId -> { - return new CentralityStreamResult( - graph.toOriginalNodeId(nodeId), - scores.get(nodeId) - ); - }); - }; - + return (computationResult, executionContext) -> runWithExceptionLogging( + "Result streaming failed", + executionContext.log(), + () -> computationResult.result() + .map(result -> { + var graph = computationResult.graph(); + var scores = result.scores(); + return LongStream.range(IdMap.START_NODE_ID, graph.nodeCount()) + .mapToObj(nodeId -> new CentralityStreamResult( + graph.toOriginalNodeId(nodeId), + scores.get(nodeId) + )); + }).orElseGet(Stream::empty) + ); } } From cfe0c8d57da5045152e91032a1e4cbcf63569783 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonatan=20J=C3=A4derberg?= Date: Wed, 7 Jun 2023 17:50:05 +0200 Subject: [PATCH 044/273] Go over stream community result consumers and fix an ID mapping bug in conductance stream spec --- .../ApproxMaxKCutStreamSpec.java | 37 +++++----- .../conductance/ConductanceStreamSpec.java | 29 ++++---- .../K1ColoringStreamSpecification.java | 16 ++--- .../neo4j/gds/kmeans/KmeansStreamSpec.java | 41 ++++++----- .../LabelPropagationStreamSpecification.java | 38 ++++------ .../neo4j/gds/leiden/LeidenStreamSpec.java | 69 +++++++------------ .../neo4j/gds/louvain/LouvainStreamSpec.java | 64 +++++++---------- .../gds/modularity/ModularityStreamSpec.java | 23 ++++--- ...larityOptimizationStreamSpecification.java | 46 +++++-------- .../java/org/neo4j/gds/scc/SccStreamSpec.java | 27 ++++---- .../LocalClusteringCoefficientStreamSpec.java | 28 ++++---- .../gds/triangle/TriangleCountStatsSpec.java | 30 ++++---- .../triangle/TriangleStreamSpecification.java | 19 ++--- .../neo4j/gds/wcc/WccStreamSpecification.java | 41 ++++++----- 14 files changed, 228 insertions(+), 280 deletions(-) diff --git a/proc/community/src/main/java/org/neo4j/gds/approxmaxkcut/ApproxMaxKCutStreamSpec.java b/proc/community/src/main/java/org/neo4j/gds/approxmaxkcut/ApproxMaxKCutStreamSpec.java index 1d6b82cdbd..b906d5d94b 100644 --- a/proc/community/src/main/java/org/neo4j/gds/approxmaxkcut/ApproxMaxKCutStreamSpec.java +++ b/proc/community/src/main/java/org/neo4j/gds/approxmaxkcut/ApproxMaxKCutStreamSpec.java @@ -21,7 +21,7 @@ import org.neo4j.gds.CommunityProcCompanion; -import org.neo4j.gds.api.properties.nodes.EmptyLongNodePropertyValues; +import org.neo4j.gds.api.IdMap; import org.neo4j.gds.approxmaxkcut.config.ApproxMaxKCutStreamConfig; import org.neo4j.gds.executor.AlgorithmSpec; import org.neo4j.gds.executor.ComputationResultConsumer; @@ -32,6 +32,7 @@ import java.util.stream.LongStream; import java.util.stream.Stream; +import static org.neo4j.gds.LoggingUtil.runWithExceptionLogging; import static org.neo4j.gds.approxmaxkcut.ApproxMaxKCut.APPROX_MAX_K_CUT_DESCRIPTION; import static org.neo4j.gds.executor.ExecutionMode.MUTATE_NODE_PROPERTY; @@ -54,21 +55,23 @@ public NewConfigFunction newConfigFunction() { @Override public ComputationResultConsumer> computationResultConsumer() { - return (computationResult, executionContext) ->{ - if (computationResult.isGraphEmpty()){ - return Stream.empty(); - } - var nodeProperties = CommunityProcCompanion.considerSizeFilter( - computationResult.config(), - computationResult.result() - .map(MaxKCutResult::asNodeProperties) - .orElse(EmptyLongNodePropertyValues.INSTANCE) - ); - var graph = computationResult.graph(); - - return LongStream.range(0,graph.nodeCount()) - .filter(nodeProperties::hasValue) - .mapToObj( nodeId -> new StreamResult(graph.toOriginalNodeId(nodeId),nodeProperties.longValue(nodeId))); - }; + return (computationResult, executionContext) -> runWithExceptionLogging( + "Result streaming failed", + executionContext.log(), + () -> computationResult.result() + .map(result -> { + var graph = computationResult.graph(); + var nodeProperties = CommunityProcCompanion.considerSizeFilter( + computationResult.config(), + result.asNodeProperties() + ); + return LongStream.range(IdMap.START_NODE_ID, graph.nodeCount()) + .filter(nodeProperties::hasValue) + .mapToObj(nodeId -> new StreamResult( + graph.toOriginalNodeId(nodeId), + nodeProperties.longValue(nodeId) + )); + }).orElseGet(Stream::empty) + ); } } diff --git a/proc/community/src/main/java/org/neo4j/gds/conductance/ConductanceStreamSpec.java b/proc/community/src/main/java/org/neo4j/gds/conductance/ConductanceStreamSpec.java index 56f0e18111..2b3c0e9557 100644 --- a/proc/community/src/main/java/org/neo4j/gds/conductance/ConductanceStreamSpec.java +++ b/proc/community/src/main/java/org/neo4j/gds/conductance/ConductanceStreamSpec.java @@ -28,6 +28,7 @@ import java.util.stream.LongStream; import java.util.stream.Stream; +import static org.neo4j.gds.LoggingUtil.runWithExceptionLogging; import static org.neo4j.gds.conductance.Conductance.CONDUCTANCE_DESCRIPTION; import static org.neo4j.gds.executor.ExecutionMode.STREAM; @@ -49,22 +50,20 @@ public NewConfigFunction newConfigFunction() { return (username, configuration) -> ConductanceStreamConfig.of(configuration); } - @SuppressWarnings("unchecked") @Override public ComputationResultConsumer> computationResultConsumer() { - return (computationResult, executionContext) -> { - if (computationResult.result().isEmpty()) { - return Stream.empty(); - } - var result=computationResult.result().get(); - var condunctances = result.communityConductances(); - - return LongStream - .range(0, condunctances.capacity()) - .filter(c -> !Double.isNaN(condunctances.get(c))) - .mapToObj(c -> new StreamResult(c, condunctances.get(c))); - }; - + return (computationResult, executionContext) -> runWithExceptionLogging( + "Result streaming failed", + executionContext.log(), + () -> computationResult.result() + .map(result -> { + var graph = computationResult.graph(); + var condunctances = result.communityConductances(); + return LongStream + .range(0, condunctances.capacity()) + .filter(c -> !Double.isNaN(condunctances.get(c))) + .mapToObj(c -> new StreamResult(graph.toOriginalNodeId(c), condunctances.get(c))); + }).orElseGet(Stream::empty) + ); } - } diff --git a/proc/community/src/main/java/org/neo4j/gds/k1coloring/K1ColoringStreamSpecification.java b/proc/community/src/main/java/org/neo4j/gds/k1coloring/K1ColoringStreamSpecification.java index ac89659bd5..f220bc4ca2 100644 --- a/proc/community/src/main/java/org/neo4j/gds/k1coloring/K1ColoringStreamSpecification.java +++ b/proc/community/src/main/java/org/neo4j/gds/k1coloring/K1ColoringStreamSpecification.java @@ -25,7 +25,6 @@ import org.neo4j.gds.api.properties.nodes.NodePropertyValues; import org.neo4j.gds.core.utils.paged.HugeLongArray; import org.neo4j.gds.executor.AlgorithmSpec; -import org.neo4j.gds.executor.ComputationResult; import org.neo4j.gds.executor.ComputationResultConsumer; import org.neo4j.gds.executor.ExecutionContext; import org.neo4j.gds.executor.GdsCallable; @@ -58,12 +57,11 @@ public NewConfigFunction newConfigFunction() { @Override public ComputationResultConsumer> computationResultConsumer() { - return (ComputationResult computationResult, ExecutionContext executionContext) -> - runWithExceptionLogging("Result streaming failed", executionContext.log(), () -> { - if (computationResult.isGraphEmpty()) { - return Stream.empty(); - } - + return (computationResult, executionContext) -> runWithExceptionLogging( + "Result streaming failed", + executionContext.log(), + () -> computationResult.result() + .map(result -> { var graph = computationResult.graph(); var config = computationResult.config(); var nodePropertyValues = (NodePropertyValues) CommunityProcCompanion.considerSizeFilter( @@ -79,7 +77,7 @@ public ComputationResultConsumer newConfigFunction() { @Override public ComputationResultConsumer> computationResultConsumer() { - - return (computationResult, executionContext) -> { - if(computationResult.result().isEmpty()) { - return Stream.empty(); - } - - var result = computationResult.result().get(); - var communities = result.communities(); - var distances = result.distanceFromCenter(); - var silhuette = result.silhouette(); - var graph = computationResult.graph(); - return LongStream - .range(IdMap.START_NODE_ID, graph.nodeCount()) - .mapToObj(nodeId -> new StreamResult( - graph.toOriginalNodeId(nodeId), - communities.get(nodeId), - distances.get(nodeId), - silhuette == null ? -1 : silhuette.get(nodeId) - )); - }; + return (computationResult, executionContext) -> runWithExceptionLogging( + "Result streaming failed", + executionContext.log(), + () -> computationResult.result() + .map(result -> { + var communities = result.communities(); + var distances = result.distanceFromCenter(); + var silhuette = result.silhouette(); + var graph = computationResult.graph(); + return LongStream + .range(IdMap.START_NODE_ID, graph.nodeCount()) + .mapToObj(nodeId -> new StreamResult( + graph.toOriginalNodeId(nodeId), + communities.get(nodeId), + distances.get(nodeId), + silhuette == null ? -1 : silhuette.get(nodeId) + )); + }).orElseGet(Stream::empty) + ); } - } diff --git a/proc/community/src/main/java/org/neo4j/gds/labelpropagation/LabelPropagationStreamSpecification.java b/proc/community/src/main/java/org/neo4j/gds/labelpropagation/LabelPropagationStreamSpecification.java index b4b0a95ffe..5538df24b7 100644 --- a/proc/community/src/main/java/org/neo4j/gds/labelpropagation/LabelPropagationStreamSpecification.java +++ b/proc/community/src/main/java/org/neo4j/gds/labelpropagation/LabelPropagationStreamSpecification.java @@ -21,14 +21,12 @@ import org.neo4j.gds.CommunityProcCompanion; import org.neo4j.gds.api.IdMap; -import org.neo4j.gds.core.utils.paged.HugeLongArray; import org.neo4j.gds.executor.AlgorithmSpec; import org.neo4j.gds.executor.ComputationResultConsumer; import org.neo4j.gds.executor.ExecutionContext; import org.neo4j.gds.executor.GdsCallable; import org.neo4j.gds.executor.NewConfigFunction; -import java.util.Optional; import java.util.stream.LongStream; import java.util.stream.Stream; @@ -58,27 +56,21 @@ public ComputationResultConsumer runWithExceptionLogging( "Result streaming failed", executionContext.log(), - () -> { - - return Optional.ofNullable(computationResult.result()) - .map(result -> { - var graph = computationResult.graph(); - var nodePropertyValues = CommunityProcCompanion.nodeProperties( - computationResult.config(), - computationResult.result() - .map(LabelPropagationResult::labels) - .orElseGet(() -> HugeLongArray.newArray(0)) - .asNodeProperties() - ); - return LongStream - .range(IdMap.START_NODE_ID, graph.nodeCount()) - .filter(nodePropertyValues::hasValue) - .mapToObj(nodeId -> new StreamResult( - graph.toOriginalNodeId(nodeId), - nodePropertyValues.longValue(nodeId) - )); - }).orElseGet(Stream::empty); - } + () -> computationResult.result() + .map(result -> { + var graph = computationResult.graph(); + var nodePropertyValues = CommunityProcCompanion.nodeProperties( + computationResult.config(), + result.labels().asNodeProperties() + ); + return LongStream + .range(IdMap.START_NODE_ID, graph.nodeCount()) + .filter(nodePropertyValues::hasValue) + .mapToObj(nodeId -> new StreamResult( + graph.toOriginalNodeId(nodeId), + nodePropertyValues.longValue(nodeId) + )); + }).orElseGet(Stream::empty) ); } } diff --git a/proc/community/src/main/java/org/neo4j/gds/leiden/LeidenStreamSpec.java b/proc/community/src/main/java/org/neo4j/gds/leiden/LeidenStreamSpec.java index b73eb68da7..3dd26a591a 100644 --- a/proc/community/src/main/java/org/neo4j/gds/leiden/LeidenStreamSpec.java +++ b/proc/community/src/main/java/org/neo4j/gds/leiden/LeidenStreamSpec.java @@ -20,11 +20,8 @@ package org.neo4j.gds.leiden; import org.neo4j.gds.CommunityProcCompanion; -import org.neo4j.gds.api.properties.nodes.EmptyLongNodePropertyValues; -import org.neo4j.gds.api.properties.nodes.NodePropertyValues; -import org.neo4j.gds.core.utils.paged.HugeLongArray; +import org.neo4j.gds.api.IdMap; import org.neo4j.gds.executor.AlgorithmSpec; -import org.neo4j.gds.executor.ComputationResult; import org.neo4j.gds.executor.ComputationResultConsumer; import org.neo4j.gds.executor.ExecutionContext; import org.neo4j.gds.executor.GdsCallable; @@ -33,6 +30,7 @@ import java.util.stream.LongStream; import java.util.stream.Stream; +import static org.neo4j.gds.LoggingUtil.runWithExceptionLogging; import static org.neo4j.gds.executor.ExecutionMode.STREAM; import static org.neo4j.gds.leiden.LeidenStreamProc.DESCRIPTION; @@ -55,48 +53,33 @@ public NewConfigFunction newConfigFunction() { @Override public ComputationResultConsumer> computationResultConsumer() { - return (computationResult, executionContext) -> { - if (computationResult.result().isEmpty()) { - return Stream.empty(); - } - - var leidenResult = computationResult.result().get(); - var graph = computationResult.graph(); - var nodeProperties = nodeProperties(computationResult); - - boolean includeIntermediateCommunities = computationResult.config().includeIntermediateCommunities(); - - return LongStream.range(0, graph.nodeCount()) - .filter(nodeProperties::hasValue) - .mapToObj(nodeId -> { - long[] intermediateCommunityIds = includeIntermediateCommunities - ? leidenResult.getIntermediateCommunities(nodeId) - : null; - long communityId = nodeProperties.longValue(nodeId); - - return new StreamResult( - graph.toOriginalNodeId(nodeId), - intermediateCommunityIds, - communityId + return (computationResult, executionContext) -> runWithExceptionLogging( + "Result streaming failed", + executionContext.log(), + () -> computationResult.result() + .map(result -> { + var graph = computationResult.graph(); + var nodeProperties = CommunityProcCompanion.nodeProperties( + computationResult.config(), + result.dendrogramManager().getCurrent().asNodeProperties() ); - }); - }; - } + var includeIntermediateCommunities = computationResult.config().includeIntermediateCommunities(); - protected NodePropertyValues nodeProperties(ComputationResult computationResult) { - return getCommunities(computationResult); - } + return LongStream.range(IdMap.START_NODE_ID, graph.nodeCount()) + .filter(nodeProperties::hasValue) + .mapToObj(nodeId -> { + long[] intermediateCommunityIds = includeIntermediateCommunities + ? result.getIntermediateCommunities(nodeId) + : null; + long communityId = nodeProperties.longValue(nodeId); - private static NodePropertyValues getCommunities( - ComputationResult computationResult - ) { - return CommunityProcCompanion.nodeProperties( - computationResult.config(), - computationResult.result() - .map(LeidenResult::dendrogramManager) - .map(LeidenDendrogramManager::getCurrent) - .map(HugeLongArray::asNodeProperties) - .orElse(EmptyLongNodePropertyValues.INSTANCE) + return new StreamResult( + graph.toOriginalNodeId(nodeId), + intermediateCommunityIds, + communityId + ); + }); + }).orElseGet(Stream::empty) ); } } diff --git a/proc/community/src/main/java/org/neo4j/gds/louvain/LouvainStreamSpec.java b/proc/community/src/main/java/org/neo4j/gds/louvain/LouvainStreamSpec.java index c579dcc0f7..5c179ccab2 100644 --- a/proc/community/src/main/java/org/neo4j/gds/louvain/LouvainStreamSpec.java +++ b/proc/community/src/main/java/org/neo4j/gds/louvain/LouvainStreamSpec.java @@ -20,11 +20,7 @@ package org.neo4j.gds.louvain; import org.neo4j.gds.CommunityProcCompanion; -import org.neo4j.gds.api.properties.nodes.EmptyLongNodePropertyValues; -import org.neo4j.gds.api.properties.nodes.NodePropertyValues; -import org.neo4j.gds.core.utils.paged.HugeLongArray; import org.neo4j.gds.executor.AlgorithmSpec; -import org.neo4j.gds.executor.ComputationResult; import org.neo4j.gds.executor.ComputationResultConsumer; import org.neo4j.gds.executor.ExecutionContext; import org.neo4j.gds.executor.GdsCallable; @@ -33,6 +29,7 @@ import java.util.stream.LongStream; import java.util.stream.Stream; +import static org.neo4j.gds.LoggingUtil.runWithExceptionLogging; import static org.neo4j.gds.executor.ExecutionMode.STREAM; import static org.neo4j.gds.louvain.LouvainConstants.DESCRIPTION; @@ -55,43 +52,30 @@ public NewConfigFunction newConfigFunction() { @Override public ComputationResultConsumer> computationResultConsumer() { - return (computationResult, executionContext) -> { - if (computationResult.result().isEmpty()) { - return Stream.empty(); - } + return (computationResult, executionContext) -> runWithExceptionLogging( + "Result streaming failed", + executionContext.log(), + () -> computationResult.result() + .map(result -> { + var graph = computationResult.graph(); + var config = computationResult.config(); + var nodePropertyValues = CommunityProcCompanion.nodeProperties( + config, + result.dendrogramManager().getCurrent().asNodeProperties() + ); + var includeIntermediateCommunities = config.includeIntermediateCommunities(); - var graph = computationResult.graph(); - var nodeCount = graph.nodeCount(); - var nodePropertyValues = nodeProperties(computationResult); - var includeIntermediateCommunities = computationResult.config().includeIntermediateCommunities(); - var louvainResult = computationResult.result().get(); - - return LongStream.range(0, nodeCount) - .boxed(). - filter(nodePropertyValues::hasValue) - .map(nodeId -> { - long[] communities = includeIntermediateCommunities ? louvainResult.getIntermediateCommunities( - nodeId) : null; - long communityId = nodePropertyValues.longValue(nodeId); - return new StreamResult(graph.toOriginalNodeId(nodeId), communities, communityId); - }); - }; - } - - protected NodePropertyValues nodeProperties(ComputationResult computationResult) { - return getCommunities(computationResult); - } - - private static NodePropertyValues getCommunities( - ComputationResult computationResult - ) { - return CommunityProcCompanion.nodeProperties( - computationResult.config(), - computationResult.result() - .map(LouvainResult::dendrogramManager) - .map(LouvainDendrogramManager::getCurrent) - .map(HugeLongArray::asNodeProperties) - .orElse(EmptyLongNodePropertyValues.INSTANCE) + return LongStream.range(0, graph.nodeCount()) + .boxed(). + filter(nodePropertyValues::hasValue) + .map(nodeId -> { + var communities = includeIntermediateCommunities + ? result.getIntermediateCommunities(nodeId) + : null; + var communityId = nodePropertyValues.longValue(nodeId); + return new StreamResult(graph.toOriginalNodeId(nodeId), communities, communityId); + }); + }).orElseGet(Stream::empty) ); } } diff --git a/proc/community/src/main/java/org/neo4j/gds/modularity/ModularityStreamSpec.java b/proc/community/src/main/java/org/neo4j/gds/modularity/ModularityStreamSpec.java index fd1f80db7d..c738ef9c7a 100644 --- a/proc/community/src/main/java/org/neo4j/gds/modularity/ModularityStreamSpec.java +++ b/proc/community/src/main/java/org/neo4j/gds/modularity/ModularityStreamSpec.java @@ -28,6 +28,7 @@ import java.util.stream.LongStream; import java.util.stream.Stream; +import static org.neo4j.gds.LoggingUtil.runWithExceptionLogging; import static org.neo4j.gds.executor.ExecutionMode.STREAM; import static org.neo4j.gds.modularity.ModularityStreamProc.DESCRIPTION; @@ -50,15 +51,17 @@ public NewConfigFunction newConfigFunction() { @Override public ComputationResultConsumer> computationResultConsumer() { - return (computationResult, executionContext) -> { - var modularityResult = computationResult.result() - .orElseGet(ModularityResult::empty); - - var communityModularities = modularityResult.modularityScores(); - return LongStream - .range(0, modularityResult.communityCount()) - .mapToObj(communityModularities::get) - .map(StreamResult::from); - }; + return (computationResult, executionContext) -> runWithExceptionLogging( + "Result streaming failed", + executionContext.log(), + () -> computationResult.result() + .map(result -> { + var communityModularities = result.modularityScores(); + return LongStream + .range(0, result.communityCount()) + .mapToObj(communityModularities::get) + .map(StreamResult::from); + }).orElseGet(Stream::empty) + ); } } diff --git a/proc/community/src/main/java/org/neo4j/gds/modularityoptimization/ModularityOptimizationStreamSpecification.java b/proc/community/src/main/java/org/neo4j/gds/modularityoptimization/ModularityOptimizationStreamSpecification.java index f2e09f1fdf..041aaaaa3c 100644 --- a/proc/community/src/main/java/org/neo4j/gds/modularityoptimization/ModularityOptimizationStreamSpecification.java +++ b/proc/community/src/main/java/org/neo4j/gds/modularityoptimization/ModularityOptimizationStreamSpecification.java @@ -21,10 +21,7 @@ import org.neo4j.gds.CommunityProcCompanion; import org.neo4j.gds.api.IdMap; -import org.neo4j.gds.api.properties.nodes.EmptyLongNodePropertyValues; -import org.neo4j.gds.api.properties.nodes.NodePropertyValues; import org.neo4j.gds.executor.AlgorithmSpec; -import org.neo4j.gds.executor.ComputationResult; import org.neo4j.gds.executor.ComputationResultConsumer; import org.neo4j.gds.executor.ExecutionContext; import org.neo4j.gds.executor.GdsCallable; @@ -57,33 +54,22 @@ public NewConfigFunction newConfigFunction() @Override public ComputationResultConsumer> computationResultConsumer() { - return (computationResult, executionContext) -> - runWithExceptionLogging("Result streaming failed", executionContext.log(), () -> { - if (computationResult.isGraphEmpty()) { - return Stream.empty(); - } - - var graph = computationResult.graph(); - var config = computationResult.config(); - var nodePropertyValues = nodeProperties(computationResult); - return LongStream - .range(IdMap.START_NODE_ID, graph.nodeCount()) - .filter(nodePropertyValues::hasValue) - .mapToObj(nodeId -> new ModularityOptimizationStreamResult( - graph.toOriginalNodeId(nodeId), - nodePropertyValues.longValue(nodeId) - )); - }); - } - - private NodePropertyValues nodeProperties( - ComputationResult computationResult - ) { - return CommunityProcCompanion.nodeProperties( - computationResult.config(), - computationResult.result() - .map(ModularityOptimizationResult::asNodeProperties) - .orElse(EmptyLongNodePropertyValues.INSTANCE) + return (computationResult, executionContext) -> runWithExceptionLogging( + "Result streaming failed", + executionContext.log(), + () -> computationResult.result() + .map(result -> { + var graph = computationResult.graph(); + var config = computationResult.config(); + var nodePropertyValues = CommunityProcCompanion.nodeProperties(config, result.asNodeProperties()); + return LongStream + .range(IdMap.START_NODE_ID, graph.nodeCount()) + .filter(nodePropertyValues::hasValue) + .mapToObj(nodeId -> new ModularityOptimizationStreamResult( + graph.toOriginalNodeId(nodeId), + nodePropertyValues.longValue(nodeId) + )); + }).orElseGet(Stream::empty) ); } } diff --git a/proc/community/src/main/java/org/neo4j/gds/scc/SccStreamSpec.java b/proc/community/src/main/java/org/neo4j/gds/scc/SccStreamSpec.java index c538f35bc8..28a8fdb42d 100644 --- a/proc/community/src/main/java/org/neo4j/gds/scc/SccStreamSpec.java +++ b/proc/community/src/main/java/org/neo4j/gds/scc/SccStreamSpec.java @@ -19,7 +19,7 @@ */ package org.neo4j.gds.scc; -import org.neo4j.gds.api.Graph; +import org.neo4j.gds.api.IdMap; import org.neo4j.gds.core.utils.paged.HugeLongArray; import org.neo4j.gds.executor.AlgorithmSpec; import org.neo4j.gds.executor.ComputationResultConsumer; @@ -30,6 +30,7 @@ import java.util.stream.LongStream; import java.util.stream.Stream; +import static org.neo4j.gds.LoggingUtil.runWithExceptionLogging; import static org.neo4j.gds.executor.ExecutionMode.STREAM; import static org.neo4j.gds.scc.Scc.NOT_VALID; import static org.neo4j.gds.scc.Scc.SCC_DESCRIPTION; @@ -53,17 +54,17 @@ public NewConfigFunction newConfigFunction() { @Override public ComputationResultConsumer> computationResultConsumer() { - return (computationResult, executionContext) -> { - Graph graph = computationResult.graph(); - HugeLongArray components = computationResult.result().orElseGet(() -> HugeLongArray.newArray(0)); - - if (graph.isEmpty()) { - return Stream.empty(); - } - - return LongStream.range(0, graph.nodeCount()) - .filter(i -> components.get(i) != NOT_VALID) - .mapToObj(i -> new StreamResult(graph.toOriginalNodeId(i), components.get(i))); - }; + return (computationResult, executionContext) -> runWithExceptionLogging( + "Result streaming failed", + executionContext.log(), + () -> computationResult.result() + .map(result -> { + var graph = computationResult.graph(); + var components = computationResult.result().orElseGet(() -> HugeLongArray.newArray(0)); + return LongStream.range(IdMap.START_NODE_ID, graph.nodeCount()) + .filter(i -> components.get(i) != NOT_VALID) + .mapToObj(i -> new StreamResult(graph.toOriginalNodeId(i), components.get(i))); + }).orElseGet(Stream::empty) + ); } } diff --git a/proc/community/src/main/java/org/neo4j/gds/triangle/LocalClusteringCoefficientStreamSpec.java b/proc/community/src/main/java/org/neo4j/gds/triangle/LocalClusteringCoefficientStreamSpec.java index 51ae2b5764..221209ca1f 100644 --- a/proc/community/src/main/java/org/neo4j/gds/triangle/LocalClusteringCoefficientStreamSpec.java +++ b/proc/community/src/main/java/org/neo4j/gds/triangle/LocalClusteringCoefficientStreamSpec.java @@ -28,6 +28,7 @@ import java.util.stream.LongStream; import java.util.stream.Stream; +import static org.neo4j.gds.LoggingUtil.runWithExceptionLogging; import static org.neo4j.gds.executor.ExecutionMode.STREAM; import static org.neo4j.gds.triangle.LocalClusteringCoefficientCompanion.DESCRIPTION; @@ -50,19 +51,18 @@ public NewConfigFunction newConfigFuncti @Override public ComputationResultConsumer> computationResultConsumer() { - return (computationResult, executionContext) -> { - - if (computationResult.result().isEmpty()) { - return Stream.of(); - } - - var graph = computationResult.graph(); - var result = computationResult.result().get(); - return LongStream.range(0, graph.nodeCount()) - .mapToObj(i -> new LocalClusteringCoefficientStreamResult( - graph.toOriginalNodeId(i), - result.localClusteringCoefficients().get(i) - )); - }; + return (computationResult, executionContext) -> runWithExceptionLogging( + "Result streaming failed", + executionContext.log(), + () -> computationResult.result() + .map(result -> { + var graph = computationResult.graph(); + return LongStream.range(0, graph.nodeCount()) + .mapToObj(i -> new LocalClusteringCoefficientStreamResult( + graph.toOriginalNodeId(i), + result.localClusteringCoefficients().get(i) + )); + }).orElseGet(Stream::empty) + ); } } diff --git a/proc/community/src/main/java/org/neo4j/gds/triangle/TriangleCountStatsSpec.java b/proc/community/src/main/java/org/neo4j/gds/triangle/TriangleCountStatsSpec.java index 17e8372425..ca3b6503de 100644 --- a/proc/community/src/main/java/org/neo4j/gds/triangle/TriangleCountStatsSpec.java +++ b/proc/community/src/main/java/org/neo4j/gds/triangle/TriangleCountStatsSpec.java @@ -28,6 +28,7 @@ import java.util.stream.Stream; import static org.neo4j.gds.BaseProc.STATS_DESCRIPTION; +import static org.neo4j.gds.LoggingUtil.runWithExceptionLogging; import static org.neo4j.gds.executor.ExecutionMode.STATS; @GdsCallable(name = "gds.triangleCount.stats", description = STATS_DESCRIPTION, executionMode = STATS) @@ -50,21 +51,20 @@ public NewConfigFunction newConfigFunction() { @Override public ComputationResultConsumer> computationResultConsumer() { - return (computationResult, executionContext) -> { - var builder = new TriangleCountStatsResult.Builder(); - computationResult.result() - .ifPresent(result -> + return (computationResult, executionContext) -> runWithExceptionLogging( + "Result streaming failed", + executionContext.log(), + () -> computationResult.result() + .map(result -> { + var builder = new TriangleCountStatsResult.Builder(); + builder.withGlobalTriangleCount(result.globalTriangles()); builder - .withGlobalTriangleCount(result.globalTriangles()) - ); - - builder - .withPreProcessingMillis(computationResult.preProcessingMillis()) - .withComputeMillis(computationResult.computeMillis()) - .withNodeCount(computationResult.graph().nodeCount()) - .withConfig(computationResult.config()); - - return Stream.of(builder.build()); - }; + .withPreProcessingMillis(computationResult.preProcessingMillis()) + .withComputeMillis(computationResult.computeMillis()) + .withNodeCount(computationResult.graph().nodeCount()) + .withConfig(computationResult.config()); + return Stream.of(builder.build()); + }).orElseGet(Stream::empty) + ); } } diff --git a/proc/community/src/main/java/org/neo4j/gds/triangle/TriangleStreamSpecification.java b/proc/community/src/main/java/org/neo4j/gds/triangle/TriangleStreamSpecification.java index 451c868098..a469046f79 100644 --- a/proc/community/src/main/java/org/neo4j/gds/triangle/TriangleStreamSpecification.java +++ b/proc/community/src/main/java/org/neo4j/gds/triangle/TriangleStreamSpecification.java @@ -27,6 +27,7 @@ import java.util.stream.Stream; +import static org.neo4j.gds.LoggingUtil.runWithExceptionLogging; import static org.neo4j.gds.executor.ExecutionMode.STREAM; import static org.neo4j.gds.triangle.TriangleProc.DESCRIPTION; @@ -50,14 +51,14 @@ public NewConfigFunction newConfigFunction() { @Override public ComputationResultConsumer, TriangleCountBaseConfig, Stream> computationResultConsumer() { - return (computationResult, executionContext) -> { - if (computationResult.result().isEmpty()) { - return Stream.empty(); - } - - var resultStream = computationResult.result().get(); - executionContext.closeableResourceRegistry().register(resultStream); - return resultStream; - }; + return (computationResult, executionContext) -> runWithExceptionLogging( + "Result streaming failed", + executionContext.log(), + () -> computationResult.result() + .map(result -> { + executionContext.closeableResourceRegistry().register(result); + return result; + }).orElseGet(Stream::empty) + ); } } diff --git a/proc/community/src/main/java/org/neo4j/gds/wcc/WccStreamSpecification.java b/proc/community/src/main/java/org/neo4j/gds/wcc/WccStreamSpecification.java index 6f7640868d..7f3897ff57 100644 --- a/proc/community/src/main/java/org/neo4j/gds/wcc/WccStreamSpecification.java +++ b/proc/community/src/main/java/org/neo4j/gds/wcc/WccStreamSpecification.java @@ -21,7 +21,6 @@ import org.neo4j.gds.CommunityProcCompanion; import org.neo4j.gds.api.IdMap; -import org.neo4j.gds.api.properties.nodes.EmptyLongNodePropertyValues; import org.neo4j.gds.core.utils.paged.dss.DisjointSetStruct; import org.neo4j.gds.executor.AlgorithmSpec; import org.neo4j.gds.executor.ComputationResultConsumer; @@ -33,6 +32,7 @@ import java.util.stream.LongStream; import java.util.stream.Stream; +import static org.neo4j.gds.LoggingUtil.runWithExceptionLogging; import static org.neo4j.gds.wcc.WccSpecification.WCC_DESCRIPTION; @GdsCallable(name = "gds.wcc.stream", description = WCC_DESCRIPTION, executionMode = ExecutionMode.STREAM) @@ -55,26 +55,25 @@ public NewConfigFunction newConfigFunction() { @Override public ComputationResultConsumer> computationResultConsumer() { - return (computationResult, executionContext) -> { - if (computationResult.isGraphEmpty()) { - return Stream.empty(); - } - - var graph = computationResult.graph(); - var nodePropertyValues = CommunityProcCompanion.nodeProperties( - computationResult.config(), - computationResult.result() - .map(DisjointSetStruct::asNodeProperties) - .orElse(EmptyLongNodePropertyValues.INSTANCE) - ); - return LongStream - .range(IdMap.START_NODE_ID, graph.nodeCount()) - .filter(nodePropertyValues::hasValue) - .mapToObj(nodeId -> new StreamResult( - graph.toOriginalNodeId(nodeId), - nodePropertyValues.longValue(nodeId) - )); - }; + return (computationResult, executionContext) -> runWithExceptionLogging( + "Result streaming failed", + executionContext.log(), + () -> computationResult.result() + .map(result -> { + var graph = computationResult.graph(); + var nodePropertyValues = CommunityProcCompanion.nodeProperties( + computationResult.config(), + result.asNodeProperties() + ); + return LongStream + .range(IdMap.START_NODE_ID, graph.nodeCount()) + .filter(nodePropertyValues::hasValue) + .mapToObj(nodeId -> new StreamResult( + graph.toOriginalNodeId(nodeId), + nodePropertyValues.longValue(nodeId) + )); + }).orElseGet(Stream::empty) + ); } @SuppressWarnings("unused") From 9fe16369f8dbcb04ffdb9f7a746f6daeffd18228 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonatan=20J=C3=A4derberg?= Date: Wed, 7 Jun 2023 17:53:14 +0200 Subject: [PATCH 045/273] Document expectation for ComputationResult#result --- .../main/java/org/neo4j/gds/executor/ComputationResult.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/executor/src/main/java/org/neo4j/gds/executor/ComputationResult.java b/executor/src/main/java/org/neo4j/gds/executor/ComputationResult.java index 170acbf4df..fdbd6e5624 100644 --- a/executor/src/main/java/org/neo4j/gds/executor/ComputationResult.java +++ b/executor/src/main/java/org/neo4j/gds/executor/ComputationResult.java @@ -38,6 +38,10 @@ public interface ComputationResult, ALGO_RESULT @Nullable A algorithm(); + /** + * Result is empty if no computation happened, which basically means the graph was empty. + * @return The result if computation happened. + */ Optional result(); Graph graph(); From d20e8be9bd44f430b9f068985058e0b68817c7ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonatan=20J=C3=A4derberg?= Date: Wed, 7 Jun 2023 23:19:09 +0200 Subject: [PATCH 046/273] Go over embedding and ML stream result consumers --- .../embeddings/fastrp/FastRPCompanion.java | 4 +++ .../embeddings/fastrp/FastRPStreamSpec.java | 25 ++++++++-------- .../graphsage/GraphSageStreamSpec.java | 25 +++++++--------- .../embeddings/hashgnn/HashGNNStreamSpec.java | 4 +-- .../node2vec/Node2VecStreamSpec.java | 9 ++++-- .../LinkPredictionPipelineStreamSpec.java | 30 +++++++++++-------- .../NodeClassificationPipelineStreamSpec.java | 25 +++++++--------- .../NodeRegressionPipelineStreamSpec.java | 7 ++--- 8 files changed, 65 insertions(+), 64 deletions(-) diff --git a/proc/embeddings/src/main/java/org/neo4j/gds/embeddings/fastrp/FastRPCompanion.java b/proc/embeddings/src/main/java/org/neo4j/gds/embeddings/fastrp/FastRPCompanion.java index c79e67fdab..28f9f973e5 100644 --- a/proc/embeddings/src/main/java/org/neo4j/gds/embeddings/fastrp/FastRPCompanion.java +++ b/proc/embeddings/src/main/java/org/neo4j/gds/embeddings/fastrp/FastRPCompanion.java @@ -37,6 +37,10 @@ static NodePropertyValues nodeProperties(Compu .orElse(EmptyFloatArrayNodePropertyValues.INSTANCE); } + static NodePropertyValues nodeProperties(FastRP.FastRPResult result) { + return new EmbeddingNodePropertyValues(result.embeddings()); + } + private static class EmbeddingNodePropertyValues implements FloatArrayNodePropertyValues { private final HugeObjectArray embeddings; private final long nodeCount; diff --git a/proc/embeddings/src/main/java/org/neo4j/gds/embeddings/fastrp/FastRPStreamSpec.java b/proc/embeddings/src/main/java/org/neo4j/gds/embeddings/fastrp/FastRPStreamSpec.java index 2eb709448b..cefe623534 100644 --- a/proc/embeddings/src/main/java/org/neo4j/gds/embeddings/fastrp/FastRPStreamSpec.java +++ b/proc/embeddings/src/main/java/org/neo4j/gds/embeddings/fastrp/FastRPStreamSpec.java @@ -54,19 +54,18 @@ public ComputationResultConsumer runWithExceptionLogging( "Result streaming failed", executionContext.log(), - () -> { - - if (computationResult.result().isEmpty()) { - return Stream.empty(); - } - - var graph = computationResult.graph(); - var nodePropertyValues = FastRPCompanion.nodeProperties(computationResult); - return LongStream - .range(IdMap.START_NODE_ID, nodePropertyValues.nodeCount()) - .filter(nodePropertyValues::hasValue) - .mapToObj(nodeId -> new StreamResult(graph.toOriginalNodeId(nodeId), nodePropertyValues.floatArrayValue(nodeId))); - } + () -> computationResult.result() + .map(result -> { + var graph = computationResult.graph(); + var nodePropertyValues = FastRPCompanion.nodeProperties(result); + return LongStream + .range(IdMap.START_NODE_ID, nodePropertyValues.nodeCount()) + .filter(nodePropertyValues::hasValue) + .mapToObj(nodeId -> new StreamResult( + graph.toOriginalNodeId(nodeId), + nodePropertyValues.floatArrayValue(nodeId) + )); + }).orElseGet(Stream::empty) ); } } diff --git a/proc/embeddings/src/main/java/org/neo4j/gds/embeddings/graphsage/GraphSageStreamSpec.java b/proc/embeddings/src/main/java/org/neo4j/gds/embeddings/graphsage/GraphSageStreamSpec.java index 60daa99066..a1b7ce2c1f 100644 --- a/proc/embeddings/src/main/java/org/neo4j/gds/embeddings/graphsage/GraphSageStreamSpec.java +++ b/proc/embeddings/src/main/java/org/neo4j/gds/embeddings/graphsage/GraphSageStreamSpec.java @@ -19,6 +19,7 @@ */ package org.neo4j.gds.embeddings.graphsage; +import org.neo4j.gds.api.IdMap; import org.neo4j.gds.embeddings.graphsage.algo.GraphSage; import org.neo4j.gds.embeddings.graphsage.algo.GraphSageAlgorithmFactory; import org.neo4j.gds.embeddings.graphsage.algo.GraphSageResult; @@ -60,23 +61,19 @@ public ComputationResultConsumer runWithExceptionLogging( "GraphSage streaming failed", executionContext.log(), - () -> { - if (computationResult.result().isEmpty()) { - return Stream.empty(); - } - var graph = computationResult.graph(); - var embeddings = computationResult.result().get().embeddings(); - - return LongStream.range(0, graph.nodeCount()) - .mapToObj(internalNodeId -> new StreamResult( - graph.toOriginalNodeId(internalNodeId), - embeddings.get(internalNodeId) - )); - } + () -> computationResult.result() + .map(result -> { + var graph = computationResult.graph(); + var embeddings = result.embeddings(); + return LongStream.range(IdMap.START_NODE_ID, graph.nodeCount()) + .mapToObj(internalNodeId -> new StreamResult( + graph.toOriginalNodeId(internalNodeId), + embeddings.get(internalNodeId) + )); + }).orElseGet(Stream::empty) ); } - @Override public ValidationConfiguration validationConfig(ExecutionContext executionContext) { return new GraphSageConfigurationValidation<>(executionContext.modelCatalog()); diff --git a/proc/embeddings/src/main/java/org/neo4j/gds/embeddings/hashgnn/HashGNNStreamSpec.java b/proc/embeddings/src/main/java/org/neo4j/gds/embeddings/hashgnn/HashGNNStreamSpec.java index fd7f748ecf..c97adaf3b7 100644 --- a/proc/embeddings/src/main/java/org/neo4j/gds/embeddings/hashgnn/HashGNNStreamSpec.java +++ b/proc/embeddings/src/main/java/org/neo4j/gds/embeddings/hashgnn/HashGNNStreamSpec.java @@ -19,6 +19,7 @@ */ package org.neo4j.gds.embeddings.hashgnn; +import org.neo4j.gds.api.IdMap; import org.neo4j.gds.executor.AlgorithmSpec; import org.neo4j.gds.executor.ComputationResultConsumer; import org.neo4j.gds.executor.ExecutionContext; @@ -52,7 +53,6 @@ public NewConfigFunction newConfigFunction() { @Override public ComputationResultConsumer> computationResultConsumer() { - return (computationResult, executionContext) -> runWithExceptionLogging( "HashGNN streaming failed", executionContext.log(), @@ -60,7 +60,7 @@ public ComputationResultConsumer { var graph = computationResult.graph(); return LongStream - .range(0, graph.nodeCount()) + .range(IdMap.START_NODE_ID, graph.nodeCount()) .mapToObj(i -> new StreamResult( graph.toOriginalNodeId(i), result.embeddings().doubleArrayValue(i) diff --git a/proc/embeddings/src/main/java/org/neo4j/gds/embeddings/node2vec/Node2VecStreamSpec.java b/proc/embeddings/src/main/java/org/neo4j/gds/embeddings/node2vec/Node2VecStreamSpec.java index eb72c55a8d..b8c8afaee0 100644 --- a/proc/embeddings/src/main/java/org/neo4j/gds/embeddings/node2vec/Node2VecStreamSpec.java +++ b/proc/embeddings/src/main/java/org/neo4j/gds/embeddings/node2vec/Node2VecStreamSpec.java @@ -29,6 +29,7 @@ import java.util.stream.LongStream; import java.util.stream.Stream; +import static org.neo4j.gds.LoggingUtil.runWithExceptionLogging; import static org.neo4j.gds.executor.ExecutionMode.STREAM; @GdsCallable(name = "gds.beta.node2vec.stream", description = Node2VecCompanion.DESCRIPTION, executionMode = STREAM) @@ -50,7 +51,10 @@ public NewConfigFunction newConfigFunction() { @Override public ComputationResultConsumer> computationResultConsumer() { - return (computationResult, executionContext) -> computationResult.result() + return (computationResult, executionContext) -> runWithExceptionLogging( + "Node2Vec streaming failed", + executionContext.log(), + () -> computationResult.result() .map(result -> { var graph = computationResult.graph(); var nodePropertyValues = new EmbeddingNodePropertyValues(result.embeddings()); @@ -61,7 +65,6 @@ public ComputationResultConsumer newConfigFun @Override public ComputationResultConsumer> computationResultConsumer() { - return (computationResult, executionContext) -> { - return computationResult.result().map(result -> { - var graphStore = computationResult.graphStore(); - Collection labelFilter = computationResult.algorithm().labelFilter().predictNodeLabels(); - var graph = graphStore.getGraph(labelFilter); + return (computationResult, executionContext) -> runWithExceptionLogging( + "Result streaming failed", + executionContext.log(), + () -> computationResult.result() + .map(result -> { + var graphStore = computationResult.graphStore(); + Collection labelFilter = computationResult.algorithm().labelFilter().predictNodeLabels(); + var graph = graphStore.getGraph(labelFilter); - return result.stream() - .map(predictedLink -> new StreamResult( - graph.toOriginalNodeId(predictedLink.sourceId()), - graph.toOriginalNodeId(predictedLink.targetId()), - predictedLink.probability() - )); - }).orElseGet(Stream::empty); - }; + return result.stream() + .map(predictedLink -> new StreamResult( + graph.toOriginalNodeId(predictedLink.sourceId()), + graph.toOriginalNodeId(predictedLink.targetId()), + predictedLink.probability() + )); + }).orElseGet(Stream::empty) + ); } } diff --git a/proc/machine-learning/src/main/java/org/neo4j/gds/ml/pipeline/node/classification/predict/NodeClassificationPipelineStreamSpec.java b/proc/machine-learning/src/main/java/org/neo4j/gds/ml/pipeline/node/classification/predict/NodeClassificationPipelineStreamSpec.java index 42ea60d46b..ba56c36360 100644 --- a/proc/machine-learning/src/main/java/org/neo4j/gds/ml/pipeline/node/classification/predict/NodeClassificationPipelineStreamSpec.java +++ b/proc/machine-learning/src/main/java/org/neo4j/gds/ml/pipeline/node/classification/predict/NodeClassificationPipelineStreamSpec.java @@ -19,6 +19,7 @@ */ package org.neo4j.gds.ml.pipeline.node.classification.predict; +import org.neo4j.gds.api.IdMap; import org.neo4j.gds.core.model.Model; import org.neo4j.gds.core.utils.paged.HugeObjectArray; import org.neo4j.gds.executor.AlgorithmSpec; @@ -62,33 +63,27 @@ public NewConfigFunction newConfi @Override public ComputationResultConsumer> computationResultConsumer() { - return (computationResult, executionContext) -> - runWithExceptionLogging( - "Result streaming failed", - executionContext.log(), - () -> { - if (computationResult.result().isEmpty()) { - return Stream.empty(); - } - + return (computationResult, executionContext) -> runWithExceptionLogging( + "Result streaming failed", + executionContext.log(), + () -> computationResult.result() + .map(result -> { var pipelineGraphFilter = computationResult.algorithm().nodePropertyStepFilter(); var graph = computationResult.graphStore().getGraph(pipelineGraphFilter.nodeLabels()); - var result = computationResult.result().get(); var predictedClasses = result.predictedClasses(); var predictedProbabilities = result.predictedProbabilities(); return LongStream - .range(0, graph.nodeCount()) - .boxed() - .map(nodeId -> + .range(IdMap.START_NODE_ID, graph.nodeCount()) + .mapToObj(nodeId -> new NodeClassificationStreamResult( graph.toOriginalNodeId(nodeId), predictedClasses.get(nodeId), nodePropertiesAsList(predictedProbabilities, nodeId) ) ); - } - ); + }).orElseGet(Stream::empty) + ); } private static List nodePropertiesAsList( diff --git a/proc/machine-learning/src/main/java/org/neo4j/gds/ml/pipeline/node/regression/predict/NodeRegressionPipelineStreamSpec.java b/proc/machine-learning/src/main/java/org/neo4j/gds/ml/pipeline/node/regression/predict/NodeRegressionPipelineStreamSpec.java index 5d90435431..5772d3641a 100644 --- a/proc/machine-learning/src/main/java/org/neo4j/gds/ml/pipeline/node/regression/predict/NodeRegressionPipelineStreamSpec.java +++ b/proc/machine-learning/src/main/java/org/neo4j/gds/ml/pipeline/node/regression/predict/NodeRegressionPipelineStreamSpec.java @@ -76,8 +76,8 @@ public ComputationResultConsumer { - return computationResult.result().map(result -> { + () -> computationResult.result() + .map(result -> { Graph graph = computationResult.graph(); NodePropertyValues nodePropertyValues = result.asNodeProperties(); return LongStream @@ -87,8 +87,7 @@ public ComputationResultConsumer Date: Thu, 8 Jun 2023 09:23:25 +0200 Subject: [PATCH 047/273] Go over remaining stream result consumers --- .../gds/scaling/ScalePropertiesProc.java | 37 ++++++--- .../scaling/ScalePropertiesStreamSpec.java | 33 ++++---- .../ShortestPathStreamResultConsumer.java | 45 ++++++----- .../randomwalk/RandomWalkStreamSpec.java | 40 ++++------ .../bellmanford/BellmanFordStreamSpec.java | 66 ++++++++-------- .../AllShortestPathsDeltaStreamSpec.java | 3 +- .../spanningtree/SpanningTreeStreamSpec.java | 41 +++++----- .../paths/steiner/SteinerTreeStreamSpec.java | 45 +++++------ .../TopologicalSortStreamSpec.java | 27 ++++--- .../BfsStreamComputationResultConsumer.java | 30 +++---- .../DfsStreamComputationResultConsumer.java | 30 +++---- ...averseStreamComputationResultConsumer.java | 7 +- ...fsStreamComputationResultConsumerTest.java | 2 - ...fsStreamComputationResultConsumerTest.java | 2 - ...seStreamComputationResultConsumerTest.java | 2 - ...PregelStreamComputationResultConsumer.java | 78 +++++++++---------- .../FilteredKnnStreamSpecification.java | 12 ++- .../FilteredNodeSimilarityStreamSpec.java | 27 ++++--- .../knn/KnnStreamSpecification.java | 11 ++- .../NodeSimilarityStreamSpecification.java | 11 ++- 20 files changed, 284 insertions(+), 265 deletions(-) diff --git a/proc/misc/src/main/java/org/neo4j/gds/scaling/ScalePropertiesProc.java b/proc/misc/src/main/java/org/neo4j/gds/scaling/ScalePropertiesProc.java index 37212baae9..5a876d62d8 100644 --- a/proc/misc/src/main/java/org/neo4j/gds/scaling/ScalePropertiesProc.java +++ b/proc/misc/src/main/java/org/neo4j/gds/scaling/ScalePropertiesProc.java @@ -40,17 +40,11 @@ static NodePropertyValues nodeProperties( .map(ScaleProperties.Result::scaledProperties) .orElseGet(() -> HugeObjectArray.newArray(double[].class, 0)); - return new DoubleArrayNodePropertyValues() { - @Override - public long nodeCount() { - return size; - } - - @Override - public double[] doubleArrayValue(long nodeId) { - return scaledProperties.get(nodeId); - } - }; + return new ScaledNodePropertyValues(size, scaledProperties); + } + + static NodePropertyValues nodeProperties(long size, HugeObjectArray scaledProperties) { + return new ScaledNodePropertyValues(size, scaledProperties); } static void validateLegacyScalers(ScalePropertiesBaseConfig config, boolean allowL1L2Scalers) { @@ -59,4 +53,25 @@ static void validateLegacyScalers(ScalePropertiesBaseConfig config, boolean allo ScalerFactory.throwForInvalidScaler(specifiedScaler); } } + + static final class ScaledNodePropertyValues implements DoubleArrayNodePropertyValues { + + private final long size; + private final HugeObjectArray scaledProperties; + + private ScaledNodePropertyValues(long size, HugeObjectArray scaledProperties) { + this.size = size; + this.scaledProperties = scaledProperties; + } + + @Override + public long nodeCount() { + return size; + } + + @Override + public double[] doubleArrayValue(long nodeId) { + return scaledProperties.get(nodeId); + } + } } diff --git a/proc/misc/src/main/java/org/neo4j/gds/scaling/ScalePropertiesStreamSpec.java b/proc/misc/src/main/java/org/neo4j/gds/scaling/ScalePropertiesStreamSpec.java index 54b61ef29e..c3885533f8 100644 --- a/proc/misc/src/main/java/org/neo4j/gds/scaling/ScalePropertiesStreamSpec.java +++ b/proc/misc/src/main/java/org/neo4j/gds/scaling/ScalePropertiesStreamSpec.java @@ -19,7 +19,6 @@ */ package org.neo4j.gds.scaling; -import org.neo4j.gds.api.properties.nodes.NodePropertyValues; import org.neo4j.gds.executor.AlgorithmSpec; import org.neo4j.gds.executor.ComputationResultConsumer; import org.neo4j.gds.executor.ExecutionContext; @@ -32,6 +31,7 @@ import java.util.stream.LongStream; import java.util.stream.Stream; +import static org.neo4j.gds.LoggingUtil.runWithExceptionLogging; import static org.neo4j.gds.executor.ExecutionMode.STREAM; import static org.neo4j.gds.scaling.ScalePropertiesProc.SCALE_PROPERTIES_DESCRIPTION; import static org.neo4j.gds.scaling.ScalePropertiesProc.validateLegacyScalers; @@ -66,21 +66,20 @@ public NewConfigFunction newConfigFunction() { @Override public ComputationResultConsumer> computationResultConsumer() { - return (computationResult, executionContext) -> { - if (computationResult.result().isEmpty()) { - return Stream.empty(); - } - - var graph = computationResult.graph(); - - NodePropertyValues nodeProperties = ScalePropertiesProc.nodeProperties(computationResult); - - return LongStream - .range(0, graph.nodeCount()) - .mapToObj(nodeId -> new ScalePropertiesStreamProc.Result( - graph.toOriginalNodeId(nodeId), - nodeProperties.doubleArrayValue(nodeId) - )); - }; + return (computationResult, executionContext) -> runWithExceptionLogging( + "Result streaming failed", + executionContext.log(), + () -> computationResult.result() + .map(result -> { + var graph = computationResult.graph(); + var nodeProperties = ScalePropertiesProc.nodeProperties(graph.nodeCount(), result.scaledProperties()); + return LongStream + .range(0, graph.nodeCount()) + .mapToObj(nodeId -> new ScalePropertiesStreamProc.Result( + graph.toOriginalNodeId(nodeId), + nodeProperties.doubleArrayValue(nodeId) + )); + }).orElseGet(Stream::empty) + ); } } diff --git a/proc/path-finding/src/main/java/org/neo4j/gds/paths/ShortestPathStreamResultConsumer.java b/proc/path-finding/src/main/java/org/neo4j/gds/paths/ShortestPathStreamResultConsumer.java index fe277bc2b4..4eaa80bda8 100644 --- a/proc/path-finding/src/main/java/org/neo4j/gds/paths/ShortestPathStreamResultConsumer.java +++ b/proc/path-finding/src/main/java/org/neo4j/gds/paths/ShortestPathStreamResultConsumer.java @@ -28,31 +28,34 @@ import java.util.stream.Stream; +import static org.neo4j.gds.LoggingUtil.runWithExceptionLogging; + public final class ShortestPathStreamResultConsumer, CONFIG extends AlgoBaseConfig> implements ComputationResultConsumer> { @Override public Stream consume( - ComputationResult computationResult, ExecutionContext executionContext + ComputationResult computationResult, + ExecutionContext executionContext ) { - - if (computationResult.result().isEmpty()) { - return Stream.empty(); - } - - var graph = computationResult.graph(); - var shouldReturnPath = executionContext - .returnColumns() - .contains("path") && computationResult.graphStore().capabilities().canWriteToDatabase(); - - var resultBuilder = new StreamResult.Builder(graph, executionContext.nodeLookup()); - - var resultStream = computationResult.result().get() - .mapPaths(path -> resultBuilder.build(path, shouldReturnPath)); - - // this is necessary in order to close the result stream which triggers - // the progress tracker to close its root task - executionContext.closeableResourceRegistry().register(resultStream); - - return resultStream; + return runWithExceptionLogging( + "Result streaming failed", + executionContext.log(), + () -> computationResult.result() + .map(result -> { + var graph = computationResult.graph(); + var shouldReturnPath = executionContext.returnColumns().contains("path") + && computationResult.graphStore().capabilities().canWriteToDatabase(); + + var resultBuilder = new StreamResult.Builder(graph, executionContext.nodeLookup()); + + var resultStream = result.mapPaths(path -> resultBuilder.build(path, shouldReturnPath)); + + // this is necessary in order to close the result stream which triggers + // the progress tracker to close its root task + executionContext.closeableResourceRegistry().register(resultStream); + + return resultStream; + }).orElseGet(Stream::empty) + ); } } diff --git a/proc/path-finding/src/main/java/org/neo4j/gds/paths/randomwalk/RandomWalkStreamSpec.java b/proc/path-finding/src/main/java/org/neo4j/gds/paths/randomwalk/RandomWalkStreamSpec.java index 615725fc66..f1dd4c8096 100644 --- a/proc/path-finding/src/main/java/org/neo4j/gds/paths/randomwalk/RandomWalkStreamSpec.java +++ b/proc/path-finding/src/main/java/org/neo4j/gds/paths/randomwalk/RandomWalkStreamSpec.java @@ -37,6 +37,7 @@ import java.util.function.Function; import java.util.stream.Stream; +import static org.neo4j.gds.LoggingUtil.runWithExceptionLogging; import static org.neo4j.gds.executor.ExecutionMode.STREAM; @@ -60,31 +61,24 @@ public NewConfigFunction newConfigFunction() { @Override public ComputationResultConsumer, RandomWalkStreamConfig, Stream> computationResultConsumer() { return (computationResult, executionContext) -> { - var graph = computationResult.graph(); - if (graph.isEmpty()) { - return Stream.empty(); - } - - var returnPath = executionContext - .returnColumns() - .contains("path"); - + var returnPath = executionContext.returnColumns().contains("path"); Function, Path> pathCreator = returnPath - ? (List nodes) -> PathFactory.create( - executionContext.nodeLookup(), - nodes, - RelationshipType.withName("NEXT") - ) + ? (List nodes) -> PathFactory.create(executionContext.nodeLookup(), nodes, RelationshipType.withName("NEXT")) : (List nodes) -> null; - - return computationResult.result() - .orElseGet(Stream::empty) - .map(nodes -> { - var translatedNodes = translateInternalToNeoIds(nodes, graph); - var path = pathCreator.apply(translatedNodes); - - return new StreamResult(translatedNodes, path); - }); + return runWithExceptionLogging( + "Result streaming failed", + executionContext.log(), + () -> computationResult.result() + .map(result -> { + var graph = computationResult.graph(); + return result + .map(nodes -> { + var translatedNodes = translateInternalToNeoIds(nodes, graph); + var path = pathCreator.apply(translatedNodes); + return new StreamResult(translatedNodes, path); + }); + }).orElseGet(Stream::empty) + ); }; } diff --git a/proc/path-finding/src/main/java/org/neo4j/gds/paths/singlesource/bellmanford/BellmanFordStreamSpec.java b/proc/path-finding/src/main/java/org/neo4j/gds/paths/singlesource/bellmanford/BellmanFordStreamSpec.java index 55a6257972..b3d119704e 100644 --- a/proc/path-finding/src/main/java/org/neo4j/gds/paths/singlesource/bellmanford/BellmanFordStreamSpec.java +++ b/proc/path-finding/src/main/java/org/neo4j/gds/paths/singlesource/bellmanford/BellmanFordStreamSpec.java @@ -32,6 +32,7 @@ import java.util.stream.Stream; +import static org.neo4j.gds.LoggingUtil.runWithExceptionLogging; import static org.neo4j.gds.executor.ExecutionMode.STREAM; @GdsCallable(name = "gds.bellmanFord.stream", description = BellmanFord.DESCRIPTION, executionMode = STREAM) @@ -55,40 +56,37 @@ public NewConfigFunction newConfigFunction() { @SuppressWarnings("unchecked") @Override public ComputationResultConsumer> computationResultConsumer() { - return (computationResult, executionContext) -> { - - var graph = computationResult.graph(); - - if (computationResult.result().isEmpty()) { - return Stream.empty(); - } - - var shouldReturnPath = executionContext - .returnColumns() - .contains("route") - && computationResult.graphStore().capabilities().canWriteToDatabase(); - - var result = computationResult.result().get(); - var containsNegativeCycle = result.containsNegativeCycle(); - - var resultBuilder = new StreamResult.Builder(graph, executionContext.nodeLookup()) - .withIsCycle(containsNegativeCycle); - - PathFindingResult algorithmResult; - if (containsNegativeCycle) { - algorithmResult = result.negativeCycles(); - } else { - algorithmResult = result.shortestPaths(); - } - - var resultStream = algorithmResult.mapPaths(path -> resultBuilder.build(path, shouldReturnPath)); - - // this is necessary in order to close the result stream which triggers - // the progress tracker to close its root task - executionContext.closeableResourceRegistry().register(resultStream); - return resultStream; - - }; + return (computationResult, executionContext) -> runWithExceptionLogging( + "Result streaming failed", + executionContext.log(), + () -> computationResult.result() + .map(result -> { + var graph = computationResult.graph(); + var shouldReturnPath = executionContext + .returnColumns() + .contains("route") + && computationResult.graphStore().capabilities().canWriteToDatabase(); + + var containsNegativeCycle = result.containsNegativeCycle(); + + var resultBuilder = new StreamResult.Builder(graph, executionContext.nodeLookup()) + .withIsCycle(containsNegativeCycle); + + PathFindingResult algorithmResult; + if (containsNegativeCycle) { + algorithmResult = result.negativeCycles(); + } else { + algorithmResult = result.shortestPaths(); + } + + var resultStream = algorithmResult.mapPaths(path -> resultBuilder.build(path, shouldReturnPath)); + + // this is necessary in order to close the result stream which triggers + // the progress tracker to close its root task + executionContext.closeableResourceRegistry().register(resultStream); + return resultStream; + }).orElseGet(Stream::empty) + ); } @Override diff --git a/proc/path-finding/src/main/java/org/neo4j/gds/paths/singlesource/delta/AllShortestPathsDeltaStreamSpec.java b/proc/path-finding/src/main/java/org/neo4j/gds/paths/singlesource/delta/AllShortestPathsDeltaStreamSpec.java index f4903e53aa..3e843ba1ed 100644 --- a/proc/path-finding/src/main/java/org/neo4j/gds/paths/singlesource/delta/AllShortestPathsDeltaStreamSpec.java +++ b/proc/path-finding/src/main/java/org/neo4j/gds/paths/singlesource/delta/AllShortestPathsDeltaStreamSpec.java @@ -53,13 +53,12 @@ public NewConfigFunction newConfigFunction() return (username, configuration) -> AllShortestPathsDeltaStreamConfig.of(configuration); } - @SuppressWarnings("unchecked") @Override public ComputationResultConsumer> computationResultConsumer() { return new ShortestPathStreamResultConsumer<>(); } -@Override + @Override public boolean releaseProgressTask() { return false; } diff --git a/proc/path-finding/src/main/java/org/neo4j/gds/paths/spanningtree/SpanningTreeStreamSpec.java b/proc/path-finding/src/main/java/org/neo4j/gds/paths/spanningtree/SpanningTreeStreamSpec.java index bb1f0dcf94..804dc99751 100644 --- a/proc/path-finding/src/main/java/org/neo4j/gds/paths/spanningtree/SpanningTreeStreamSpec.java +++ b/proc/path-finding/src/main/java/org/neo4j/gds/paths/spanningtree/SpanningTreeStreamSpec.java @@ -19,7 +19,7 @@ */ package org.neo4j.gds.paths.spanningtree; -import org.neo4j.gds.api.Graph; +import org.neo4j.gds.api.IdMap; import org.neo4j.gds.executor.AlgorithmSpec; import org.neo4j.gds.executor.ComputationResultConsumer; import org.neo4j.gds.executor.ExecutionContext; @@ -33,6 +33,7 @@ import java.util.stream.LongStream; import java.util.stream.Stream; +import static org.neo4j.gds.LoggingUtil.runWithExceptionLogging; import static org.neo4j.gds.executor.ExecutionMode.STREAM; @GdsCallable(name = "gds.beta.spanningTree.stream", description = SpanningTreeWriteProc.DESCRIPTION, executionMode = STREAM) @@ -55,24 +56,24 @@ public NewConfigFunction newConfigFunction() { } public ComputationResultConsumer> computationResultConsumer() { - - return (computationResult, executionContext) -> { - if (computationResult.result().isEmpty()) { - return Stream.empty(); - } - - var sourceNode = computationResult.config().sourceNode(); - Graph graph = computationResult.graph(); - SpanningTree spanningTree = computationResult.result().get(); - return LongStream.range(0, graph.nodeCount()) - .filter(nodeId -> spanningTree.parent(nodeId) >= 0 || sourceNode == graph.toOriginalNodeId(nodeId)) - .mapToObj(nodeId -> new StreamResult( - graph.toOriginalNodeId(nodeId), - (sourceNode == graph.toOriginalNodeId(nodeId)) ? - sourceNode : - graph.toOriginalNodeId(spanningTree.parent(nodeId)), - spanningTree.costToParent(nodeId) - )); - }; + return (computationResult, executionContext) -> runWithExceptionLogging( + "Result streaming failed", + executionContext.log(), + () -> computationResult.result() + .map(result -> { + var sourceNode = computationResult.config().sourceNode(); + var graph = computationResult.graph(); + return LongStream.range(IdMap.START_NODE_ID, graph.nodeCount()) + .filter(nodeId -> result.parent(nodeId) >= 0 || sourceNode == graph.toOriginalNodeId(nodeId)) + .mapToObj(nodeId -> { + var originalId = graph.toOriginalNodeId(nodeId); + return new StreamResult( + originalId, + (sourceNode == originalId) ? sourceNode : graph.toOriginalNodeId(result.parent(nodeId)), + result.costToParent(nodeId) + ); + }); + }).orElseGet(Stream::empty) + ); } } diff --git a/proc/path-finding/src/main/java/org/neo4j/gds/paths/steiner/SteinerTreeStreamSpec.java b/proc/path-finding/src/main/java/org/neo4j/gds/paths/steiner/SteinerTreeStreamSpec.java index fb228f3d92..0b3c7bcbb7 100644 --- a/proc/path-finding/src/main/java/org/neo4j/gds/paths/steiner/SteinerTreeStreamSpec.java +++ b/proc/path-finding/src/main/java/org/neo4j/gds/paths/steiner/SteinerTreeStreamSpec.java @@ -19,7 +19,7 @@ */ package org.neo4j.gds.paths.steiner; -import org.neo4j.gds.api.Graph; +import org.neo4j.gds.api.IdMap; import org.neo4j.gds.executor.AlgorithmSpec; import org.neo4j.gds.executor.ComputationResultConsumer; import org.neo4j.gds.executor.ExecutionContext; @@ -33,6 +33,7 @@ import java.util.stream.LongStream; import java.util.stream.Stream; +import static org.neo4j.gds.LoggingUtil.runWithExceptionLogging; import static org.neo4j.gds.executor.ExecutionMode.STREAM; @GdsCallable(name = "gds.beta.SteinerTree.stream", description = SteinerTreeStatsProc.DESCRIPTION, executionMode = STREAM) @@ -55,26 +56,26 @@ public NewConfigFunction newConfigFunction() { } public ComputationResultConsumer> computationResultConsumer() { - - return (computationResult, executionContext) -> { - if (computationResult.result().isEmpty()) { - return Stream.empty(); - } - - var sourceNode = computationResult.config().sourceNode(); - Graph graph = computationResult.graph(); - var steinerTreeResult = computationResult.result().get(); - var parentArray = steinerTreeResult.parentArray(); - var costArray = steinerTreeResult.relationshipToParentCost(); - return LongStream.range(0, graph.nodeCount()) - .filter(nodeId -> parentArray.get(nodeId) != ShortestPathsSteinerAlgorithm.PRUNED) - .mapToObj(nodeId -> new StreamResult( - graph.toOriginalNodeId(nodeId), - (sourceNode == graph.toOriginalNodeId(nodeId)) ? - sourceNode : - graph.toOriginalNodeId(parentArray.get(nodeId)), - costArray.get(nodeId) - )); - }; + return (computationResult, executionContext) -> runWithExceptionLogging( + "Result streaming failed", + executionContext.log(), + () -> computationResult.result() + .map(result -> { + var sourceNode = computationResult.config().sourceNode(); + var graph = computationResult.graph(); + var parents = result.parentArray(); + var costs = result.relationshipToParentCost(); + return LongStream.range(IdMap.START_NODE_ID, graph.nodeCount()) + .filter(nodeId -> parents.get(nodeId) != ShortestPathsSteinerAlgorithm.PRUNED) + .mapToObj(nodeId -> { + var originalId = graph.toOriginalNodeId(nodeId); + return new StreamResult( + originalId, + (sourceNode == originalId) ? sourceNode : graph.toOriginalNodeId(parents.get(nodeId)), + costs.get(nodeId) + ); + }); + }).orElseGet(Stream::empty) + ); } } diff --git a/proc/path-finding/src/main/java/org/neo4j/gds/paths/topologicalsort/TopologicalSortStreamSpec.java b/proc/path-finding/src/main/java/org/neo4j/gds/paths/topologicalsort/TopologicalSortStreamSpec.java index 822d1e8465..95861f1ba9 100644 --- a/proc/path-finding/src/main/java/org/neo4j/gds/paths/topologicalsort/TopologicalSortStreamSpec.java +++ b/proc/path-finding/src/main/java/org/neo4j/gds/paths/topologicalsort/TopologicalSortStreamSpec.java @@ -19,7 +19,7 @@ */ package org.neo4j.gds.paths.topologicalsort; -import org.neo4j.gds.api.Graph; +import org.neo4j.gds.api.IdMap; import org.neo4j.gds.executor.AlgorithmSpec; import org.neo4j.gds.executor.ComputationResultConsumer; import org.neo4j.gds.executor.ExecutionContext; @@ -33,6 +33,7 @@ import java.util.stream.LongStream; import java.util.stream.Stream; +import static org.neo4j.gds.LoggingUtil.runWithExceptionLogging; import static org.neo4j.gds.executor.ExecutionMode.STREAM; @GdsCallable(name = "gds.alpha.topologicalSort.stream", description = TopologicalSortStreamProc.TOPOLOGICAL_SORT_DESCRIPTION, executionMode = STREAM) @@ -55,16 +56,18 @@ public NewConfigFunction newConfigFunction() { @Override public ComputationResultConsumer> computationResultConsumer() { - return (computationResult, executionContext) -> - { - if (computationResult.result().isEmpty()) { - return Stream.empty(); - } - - Graph graph = computationResult.graph(); - var topologicallySortedNodes = computationResult.result().get().value(); - return LongStream.range(0, graph.nodeCount()) - .mapToObj(nodeId -> new TopologicalSortStreamResult(graph.toOriginalNodeId(topologicallySortedNodes.get(nodeId)))); - }; + return (computationResult, executionContext) -> runWithExceptionLogging( + "Result streaming failed", + executionContext.log(), + () -> computationResult.result() + .map(result -> { + var graph = computationResult.graph(); + var topologicallySortedNodes = result.value(); + return LongStream.range(IdMap.START_NODE_ID, graph.nodeCount()) + .mapToObj(nodeId -> new TopologicalSortStreamResult( + graph.toOriginalNodeId(topologicallySortedNodes.get(nodeId)) + )); + }).orElseGet(Stream::empty) + ); } } diff --git a/proc/path-finding/src/main/java/org/neo4j/gds/paths/traverse/BfsStreamComputationResultConsumer.java b/proc/path-finding/src/main/java/org/neo4j/gds/paths/traverse/BfsStreamComputationResultConsumer.java index f92ed5b59a..d8c5ccb455 100644 --- a/proc/path-finding/src/main/java/org/neo4j/gds/paths/traverse/BfsStreamComputationResultConsumer.java +++ b/proc/path-finding/src/main/java/org/neo4j/gds/paths/traverse/BfsStreamComputationResultConsumer.java @@ -26,6 +26,8 @@ import java.util.stream.Stream; +import static org.neo4j.gds.LoggingUtil.runWithExceptionLogging; + class BfsStreamComputationResultConsumer implements ComputationResultConsumer> { private final PathFactoryFacade pathFactoryFacade; @@ -39,20 +41,20 @@ public Stream consume( ComputationResult computationResult, ExecutionContext executionContext ) { - if (computationResult.result().isEmpty()) { - return Stream.empty(); - } - - return TraverseStreamComputationResultConsumer.consume( - computationResult.config().sourceNode(), - computationResult.result().get(), - computationResult.graph()::toOriginalNodeId, - computationResult.graph().isEmpty(), - BfsStreamResult::new, - executionContext.returnColumns().contains("path"), - pathFactoryFacade, - BfsStreamProc.NEXT, - executionContext.nodeLookup() + return runWithExceptionLogging( + "Result streaming failed", + executionContext.log(), + () -> computationResult.result() + .map(result -> TraverseStreamComputationResultConsumer.consume( + computationResult.config().sourceNode(), + result, + computationResult.graph()::toOriginalNodeId, + BfsStreamResult::new, + executionContext.returnColumns().contains("path"), + pathFactoryFacade, + BfsStreamProc.NEXT, + executionContext.nodeLookup() + )).orElseGet(Stream::empty) ); } } diff --git a/proc/path-finding/src/main/java/org/neo4j/gds/paths/traverse/DfsStreamComputationResultConsumer.java b/proc/path-finding/src/main/java/org/neo4j/gds/paths/traverse/DfsStreamComputationResultConsumer.java index a3c70f64d6..15c75e7343 100644 --- a/proc/path-finding/src/main/java/org/neo4j/gds/paths/traverse/DfsStreamComputationResultConsumer.java +++ b/proc/path-finding/src/main/java/org/neo4j/gds/paths/traverse/DfsStreamComputationResultConsumer.java @@ -26,6 +26,8 @@ import java.util.stream.Stream; +import static org.neo4j.gds.LoggingUtil.runWithExceptionLogging; + class DfsStreamComputationResultConsumer implements ComputationResultConsumer> { private final PathFactoryFacade pathFactoryFacade; @@ -39,20 +41,20 @@ public Stream consume( ComputationResult computationResult, ExecutionContext executionContext ) { - if (computationResult.result().isEmpty()) { - return Stream.empty(); - } - - return TraverseStreamComputationResultConsumer.consume( - computationResult.config().sourceNode(), - computationResult.result().get(), - computationResult.graph()::toOriginalNodeId, - computationResult.graph().isEmpty(), - DfsStreamResult::new, - executionContext.returnColumns().contains("path"), - pathFactoryFacade, - DfsStreamProc.NEXT, - executionContext.nodeLookup() + return runWithExceptionLogging( + "Result streaming failed", + executionContext.log(), + () -> computationResult.result() + .map(result -> TraverseStreamComputationResultConsumer.consume( + computationResult.config().sourceNode(), + result, + computationResult.graph()::toOriginalNodeId, + DfsStreamResult::new, + executionContext.returnColumns().contains("path"), + pathFactoryFacade, + DfsStreamProc.NEXT, + executionContext.nodeLookup() + )).orElseGet(Stream::empty) ); } } diff --git a/proc/path-finding/src/main/java/org/neo4j/gds/paths/traverse/TraverseStreamComputationResultConsumer.java b/proc/path-finding/src/main/java/org/neo4j/gds/paths/traverse/TraverseStreamComputationResultConsumer.java index acf177cb0e..91528d371b 100644 --- a/proc/path-finding/src/main/java/org/neo4j/gds/paths/traverse/TraverseStreamComputationResultConsumer.java +++ b/proc/path-finding/src/main/java/org/neo4j/gds/paths/traverse/TraverseStreamComputationResultConsumer.java @@ -37,19 +37,14 @@ private TraverseStreamComputationResultConsumer() {} static Stream consume( long sourceNodeId, - @Nullable HugeLongArray nodes, + HugeLongArray nodes, LongUnaryOperator toOriginalNodeId, - boolean graphIsEmpty, ConcreteResultTransformer resultTransformer, boolean shouldReturnPath, PathFactoryFacade pathFactoryFacade, RelationshipType relationshipType, NodeLookup nodeLookup ) { - if (graphIsEmpty || null == nodes) { - return Stream.empty(); - } - var nodesArray = nodes.toArray(); var nodeList = Arrays .stream(nodesArray) diff --git a/proc/path-finding/src/test/java/org/neo4j/gds/paths/traverse/BfsStreamComputationResultConsumerTest.java b/proc/path-finding/src/test/java/org/neo4j/gds/paths/traverse/BfsStreamComputationResultConsumerTest.java index c11f0d4b7b..f47887892a 100644 --- a/proc/path-finding/src/test/java/org/neo4j/gds/paths/traverse/BfsStreamComputationResultConsumerTest.java +++ b/proc/path-finding/src/test/java/org/neo4j/gds/paths/traverse/BfsStreamComputationResultConsumerTest.java @@ -54,7 +54,6 @@ class BfsStreamComputationResultConsumerTest { @Test void shouldNotComputePath() { - when(graphMock.isEmpty()).thenReturn(false); when(graphMock.toOriginalNodeId(anyLong())).then(returnsFirstArg()); when(computationResultMock.graph()).thenReturn(graphMock); @@ -83,7 +82,6 @@ void shouldNotComputePath() { @Test void shouldComputePath() { - when(graphMock.isEmpty()).thenReturn(false); when(graphMock.toOriginalNodeId(anyLong())).then(returnsFirstArg()); when(computationResultMock.graph()).thenReturn(graphMock); diff --git a/proc/path-finding/src/test/java/org/neo4j/gds/paths/traverse/DfsStreamComputationResultConsumerTest.java b/proc/path-finding/src/test/java/org/neo4j/gds/paths/traverse/DfsStreamComputationResultConsumerTest.java index b76ce8c051..aa9543c0d8 100644 --- a/proc/path-finding/src/test/java/org/neo4j/gds/paths/traverse/DfsStreamComputationResultConsumerTest.java +++ b/proc/path-finding/src/test/java/org/neo4j/gds/paths/traverse/DfsStreamComputationResultConsumerTest.java @@ -53,7 +53,6 @@ class DfsStreamComputationResultConsumerTest { @Test void shouldNotComputePath() { - when(graphMock.isEmpty()).thenReturn(false); when(graphMock.toOriginalNodeId(anyLong())).then(returnsFirstArg()); when(computationResultMock.graph()).thenReturn(graphMock); @@ -82,7 +81,6 @@ void shouldNotComputePath() { @Test void shouldComputePath() { - when(graphMock.isEmpty()).thenReturn(false); when(graphMock.toOriginalNodeId(anyLong())).then(returnsFirstArg()); when(computationResultMock.graph()).thenReturn(graphMock); diff --git a/proc/path-finding/src/test/java/org/neo4j/gds/paths/traverse/TraverseStreamComputationResultConsumerTest.java b/proc/path-finding/src/test/java/org/neo4j/gds/paths/traverse/TraverseStreamComputationResultConsumerTest.java index df1ff58f20..eded824696 100644 --- a/proc/path-finding/src/test/java/org/neo4j/gds/paths/traverse/TraverseStreamComputationResultConsumerTest.java +++ b/proc/path-finding/src/test/java/org/neo4j/gds/paths/traverse/TraverseStreamComputationResultConsumerTest.java @@ -43,7 +43,6 @@ void shouldNotComputePath() { 0L, HugeLongArray.of(1L, 2L), l -> l, - false, TestResult::new, false, pathFactoryFacadeMock, @@ -70,7 +69,6 @@ void shouldComputePath() { 0L, HugeLongArray.of(1L, 2L), l -> l, - false, TestResult::new, true, pathFactoryFacadeMock, diff --git a/proc/pregel/src/main/java/org/neo4j/gds/pregel/proc/PregelStreamComputationResultConsumer.java b/proc/pregel/src/main/java/org/neo4j/gds/pregel/proc/PregelStreamComputationResultConsumer.java index 32c9d2a364..628d17e668 100644 --- a/proc/pregel/src/main/java/org/neo4j/gds/pregel/proc/PregelStreamComputationResultConsumer.java +++ b/proc/pregel/src/main/java/org/neo4j/gds/pregel/proc/PregelStreamComputationResultConsumer.java @@ -49,46 +49,46 @@ public Stream consume( return runWithExceptionLogging( "Result streaming failed", executionContext.log(), - () -> { - var result = computationResult.result(); - if (result.isEmpty()) { - return Stream.empty(); - } - var nodeValues = result.get().nodeValues(); - // TODO: reduce the inner iteration to what's visible upfront - // TODO: map the element to a property lookup function upfront - // for every node - return LongStream.range(IdMap.START_NODE_ID, computationResult.graph().nodeCount()).mapToObj(nodeId -> { - // for every schema element - Map values = nodeValues.schema().elements() - .stream() - // if its visible - .filter(element -> element.visibility() == PregelSchema.Visibility.PUBLIC) - // collect a String->Object map - .collect(Collectors.toMap( - // of the element's property key - Element::propertyKey, - // to a value - element -> { - // retrieved based on the elements property type - switch (element.propertyType()) { - case LONG: - return nodeValues.longProperties(element.propertyKey()).get(nodeId); - case DOUBLE: - return nodeValues.doubleProperties(element.propertyKey()).get(nodeId); - case DOUBLE_ARRAY: - return nodeValues.doubleArrayProperties(element.propertyKey()).get(nodeId); - case LONG_ARRAY: - return nodeValues.longArrayProperties(element.propertyKey()).get(nodeId); - default: - throw new IllegalArgumentException("Unsupported property type: " + element.propertyType()); - } - } - )); - return new PregelStreamResult(computationResult.graph().toOriginalNodeId(nodeId), values); - }); + () -> computationResult.result() + .map(result -> { + var nodeValues = result.nodeValues(); + // TODO: reduce the inner iteration to what's visible upfront + // TODO: map the element to a property lookup function upfront + // for every node + return LongStream.range(IdMap.START_NODE_ID, computationResult.graph().nodeCount()) + .mapToObj(nodeId -> { + // for every schema element + Map values = nodeValues.schema().elements() + .stream() + // if its visible + .filter(element -> element.visibility() == PregelSchema.Visibility.PUBLIC) + // collect a String->Object map + .collect(Collectors.toMap( + // of the element's property key + Element::propertyKey, + // to a value + element -> { + // retrieved based on the elements property type + switch (element.propertyType()) { + case LONG: + return nodeValues.longProperties(element.propertyKey()).get(nodeId); + case DOUBLE: + return nodeValues.doubleProperties(element.propertyKey()).get(nodeId); + case DOUBLE_ARRAY: + return nodeValues.doubleArrayProperties(element.propertyKey()).get( + nodeId); + case LONG_ARRAY: + return nodeValues.longArrayProperties(element.propertyKey()) + .get(nodeId); + default: + throw new IllegalArgumentException("Unsupported property type: " + element.propertyType()); + } + } + )); + return new PregelStreamResult(computationResult.graph().toOriginalNodeId(nodeId), values); + }); - } + }).orElseGet(Stream::empty) ); } } diff --git a/proc/similarity/src/main/java/org/neo4j/gds/similarity/filteredknn/FilteredKnnStreamSpecification.java b/proc/similarity/src/main/java/org/neo4j/gds/similarity/filteredknn/FilteredKnnStreamSpecification.java index 4290cc96a4..b9212aa486 100644 --- a/proc/similarity/src/main/java/org/neo4j/gds/similarity/filteredknn/FilteredKnnStreamSpecification.java +++ b/proc/similarity/src/main/java/org/neo4j/gds/similarity/filteredknn/FilteredKnnStreamSpecification.java @@ -29,6 +29,8 @@ import java.util.stream.Stream; +import static org.neo4j.gds.LoggingUtil.runWithExceptionLogging; + @GdsCallable(name = "gds.alpha.knn.filtered.stream", executionMode = ExecutionMode.STREAM) public class FilteredKnnStreamSpecification implements AlgorithmSpec, FilteredKnnFactory> { @@ -49,8 +51,10 @@ public NewConfigFunction newConfigFunction() { @Override public ComputationResultConsumer> computationResultConsumer() { - return (computationResult, executionContext) -> { - return computationResult.result() + return (computationResult, executionContext) -> runWithExceptionLogging( + "Result streaming failed", + executionContext.log(), + () -> computationResult.result() .map(result -> { var graph = computationResult.graph(); return result.similarityResultStream() @@ -60,7 +64,7 @@ public ComputationResultConsumer newConfigFunction() @Override public ComputationResultConsumer> computationResultConsumer() { - return (computationResult, executionContext) -> { - var graph = computationResult.graph(); - var similarityResultStream = computationResult.result() - .map(NodeSimilarityResult::streamResult) - .orElseGet(Stream::empty); + return (computationResult, executionContext) -> runWithExceptionLogging( + "Result streaming failed", + executionContext.log(), + () -> computationResult.result() + .map(result -> { + var graph = computationResult.graph(); + var similarityResultStream = result.streamResult(); + return similarityResultStream.map(internalSimilarityResult -> { + internalSimilarityResult.node1 = graph.toOriginalNodeId(internalSimilarityResult.node1); + internalSimilarityResult.node2 = graph.toOriginalNodeId(internalSimilarityResult.node2); - return similarityResultStream.map(internalSimilarityResult -> { - internalSimilarityResult.node1 = graph.toOriginalNodeId(internalSimilarityResult.node1); - internalSimilarityResult.node2 = graph.toOriginalNodeId(internalSimilarityResult.node2); - - return internalSimilarityResult; - }); - }; + return internalSimilarityResult; + }); + }).orElseGet(Stream::empty) + ); } } diff --git a/proc/similarity/src/main/java/org/neo4j/gds/similarity/knn/KnnStreamSpecification.java b/proc/similarity/src/main/java/org/neo4j/gds/similarity/knn/KnnStreamSpecification.java index 46fb11d55f..e7d8a23140 100644 --- a/proc/similarity/src/main/java/org/neo4j/gds/similarity/knn/KnnStreamSpecification.java +++ b/proc/similarity/src/main/java/org/neo4j/gds/similarity/knn/KnnStreamSpecification.java @@ -29,6 +29,7 @@ import java.util.stream.Stream; +import static org.neo4j.gds.LoggingUtil.runWithExceptionLogging; import static org.neo4j.gds.similarity.knn.KnnProc.KNN_DESCRIPTION; @GdsCallable(name = "gds.knn.stream", description = KNN_DESCRIPTION, executionMode = ExecutionMode.STREAM) @@ -51,8 +52,10 @@ public NewConfigFunction newConfigFunction() { @Override public ComputationResultConsumer> computationResultConsumer() { - return (computationResult, executionContext) -> { - return computationResult.result() + return (computationResult, executionContext) -> runWithExceptionLogging( + "Result streaming failed", + executionContext.log(), + () -> computationResult.result() .map(result -> { var graph = computationResult.graph(); return result.streamSimilarityResult() @@ -62,7 +65,7 @@ public ComputationResultConsumer newConfigFunction() { @Override public ComputationResultConsumer> computationResultConsumer() { - return (computationResult, executionContext) -> { - return computationResult.result() + return (computationResult, executionContext) -> runWithExceptionLogging( + "Result streaming failed", + executionContext.log(), + () -> computationResult.result() .map(result -> { var graph = computationResult.graph(); @@ -62,7 +65,7 @@ public ComputationResultConsumer Date: Thu, 8 Jun 2023 11:59:04 +0100 Subject: [PATCH 048/273] Update Maven coordinates to point to 2.4.0 Co-authored-by: Ioannis Panagiotas --- README.adoc | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/README.adoc b/README.adoc index 642522587a..fc50589906 100644 --- a/README.adoc +++ b/README.adoc @@ -93,7 +93,7 @@ For the most basic set of features, like graph loading and the graph representat org.neo4j.gds core - 2.3.7 + 2.4.0 ---- @@ -105,21 +105,21 @@ The algorithms are located in the `algo-common`, `algo` and `alpha-algo` modules org.neo4j.gds algo-common - 2.3.7 + 2.4.0 org.neo4j.gds algo - 2.3.7 + 2.4.0 org.neo4j.gds alpha-algo - 2.3.7 + 2.4.0 ---- @@ -131,28 +131,28 @@ The procedures are located in the `proc-common`, `proc` and `alpha-proc` modules org.neo4j.gds proc-common - 2.3.7 + 2.4.0 org.neo4j.gds proc - 2.3.7 + 2.4.0 org.neo4j.gds alpha-proc - 2.3.7 + 2.4.0 org.neo4j.gds open-write-services - 2.3.7 + 2.4.0 ---- From 69da1a963244b8d10eae62b3caa54a61a8ed2a31 Mon Sep 17 00:00:00 2001 From: Paul Horn Date: Fri, 9 Jun 2023 10:18:57 +0200 Subject: [PATCH 049/273] Fix table name to data config MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Florentin Dörre --- .../projections/graph-project-cypher-projection.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/modules/ROOT/pages/management-ops/projections/graph-project-cypher-projection.adoc b/doc/modules/ROOT/pages/management-ops/projections/graph-project-cypher-projection.adoc index 960bc78a27..9ae8a056f8 100644 --- a/doc/modules/ROOT/pages/management-ops/projections/graph-project-cypher-projection.adoc +++ b/doc/modules/ROOT/pages/management-ops/projections/graph-project-cypher-projection.adoc @@ -75,7 +75,7 @@ RETURN gds.graph.project( |=== [[graph-project-cypher-projection-syntax-dataConfig]] -.Nodes configuration +.Data configuration [opts="header",cols="1,1,1,4"] |=== | Name | Type | Default | Description From 5de23cad9ee0386d713b9aa36b5088e6297e7255 Mon Sep 17 00:00:00 2001 From: Veselin Nikolov Date: Fri, 9 Jun 2023 09:44:27 +0100 Subject: [PATCH 050/273] Update Aura version for 2.4.0-23 release --- gradle/version.gradle | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gradle/version.gradle b/gradle/version.gradle index 72a10ecf87..611f7d7a2c 100644 --- a/gradle/version.gradle +++ b/gradle/version.gradle @@ -1,6 +1,6 @@ ext { gdsBaseVersion = '2.4.0' - gdsAuraVersion = '21' + gdsAuraVersion = '23' gdsVersion = gdsBaseVersion + (rootProject.hasProperty('aurads') ? "+${gdsAuraVersion}" : "") -} \ No newline at end of file +} From 03863a2f92415ee7bb9e64b389b8b580f74f12d7 Mon Sep 17 00:00:00 2001 From: Adam Schill Collberg Date: Fri, 9 Jun 2023 10:59:51 +0200 Subject: [PATCH 051/273] Make sure pipelines can find scaleProperties on Neo4j 5.X.Y Co-Authored-By: Jacob Sznajdman --- .../java/org/neo4j/gds/ml/pipeline/NodePropertyStepFactory.java | 2 ++ proc/misc/build.gradle | 2 ++ 2 files changed, 4 insertions(+) diff --git a/pipeline/src/main/java/org/neo4j/gds/ml/pipeline/NodePropertyStepFactory.java b/pipeline/src/main/java/org/neo4j/gds/ml/pipeline/NodePropertyStepFactory.java index 1343703a99..ea53bbefe3 100644 --- a/pipeline/src/main/java/org/neo4j/gds/ml/pipeline/NodePropertyStepFactory.java +++ b/pipeline/src/main/java/org/neo4j/gds/ml/pipeline/NodePropertyStepFactory.java @@ -100,6 +100,8 @@ public static GdsCallableFinder.GdsCallableDefinition getGdsCallableDefinition(S var gdsCallableDefinition = GdsCallableFinder .findByName(normalizedName) + // If this is thrown in a production setting, it may be that you just need to add `annotationProcessor project(':procedure-collector')` + // to the `build.gradle` file of the package containing your proc .orElseThrow(() -> new IllegalArgumentException(formatWithLocale( "Could not find a procedure called %s", normalizedName diff --git a/proc/misc/build.gradle b/proc/misc/build.gradle index 3b19237f04..7c04a15cff 100644 --- a/proc/misc/build.gradle +++ b/proc/misc/build.gradle @@ -5,6 +5,8 @@ description = 'Neo4j Graph Data Science :: Procedures :: Misc' group = 'org.neo4j.gds' dependencies { + annotationProcessor project(':procedure-collector') + implementation project(':algo-common') implementation project(':annotations') implementation project(':config-api') From 92049e763a738c4288bf82b8be6da22385c281f8 Mon Sep 17 00:00:00 2001 From: yuval Date: Fri, 9 Jun 2023 11:59:40 +0200 Subject: [PATCH 052/273] 5.9 compat --- .../5.9/neo4j-kernel-adapter/build.gradle | 65 ++ .../gds/compat/_59/Neo4jProxyFactoryImpl.java | 31 + .../compat/_59/SettingProxyFactoryImpl.java | 31 + .../compat/_59/BoltTransactionRunnerImpl.java | 97 ++ .../gds/compat/_59/CallableProcedureImpl.java | 53 + .../CallableUserAggregationFunctionImpl.java | 77 ++ .../gds/compat/_59/CompatAccessModeImpl.java | 44 + .../_59/CompatGraphDatabaseAPIImpl.java | 74 ++ .../gds/compat/_59/CompatIndexQueryImpl.java | 31 + .../_59/CompatUsernameAuthSubjectImpl.java | 35 + .../compat/_59/CompositeNodeCursorImpl.java | 32 + .../gds/compat/_59/GdsDatabaseLayoutImpl.java | 50 + ...sDatabaseManagementServiceBuilderImpl.java | 54 ++ .../gds/compat/_59/Neo4jProxyFactoryImpl.java | 44 + .../neo4j/gds/compat/_59/Neo4jProxyImpl.java | 917 ++++++++++++++++++ .../compat/_59/NodeLabelIndexLookupImpl.java | 68 ++ .../gds/compat/_59/PartitionedStoreScan.java | 58 ++ .../_59/ReferencePropertyReference.java | 49 + .../compat/_59/ScanBasedStoreScanImpl.java | 40 + .../compat/_59/SettingProxyFactoryImpl.java | 44 + .../gds/compat/_59/SettingProxyImpl.java | 87 ++ .../org/neo4j/gds/compat/_59/TestLogImpl.java | 146 +++ .../compat/_59/VirtualRelationshipImpl.java | 41 + .../5.9/storage-engine-adapter/build.gradle | 68 ++ .../_59/InMemoryStorageEngineFactory.java | 268 +++++ .../_59/StorageEngineProxyFactoryImpl.java | 31 + .../InMemoryCommandCreationContextImpl.java | 107 ++ .../compat/_59/InMemoryCountsStoreImpl.java | 112 +++ .../_59/InMemoryMetaDataProviderImpl.java | 201 ++++ .../gds/compat/_59/InMemoryNodeCursor.java | 82 ++ .../_59/InMemoryNodePropertyCursor.java | 45 + .../compat/_59/InMemoryPropertyCursor.java | 71 ++ .../_59/InMemoryPropertySelectionImpl.java | 55 ++ .../InMemoryRelationshipPropertyCursor.java | 60 ++ .../_59/InMemoryRelationshipScanCursor.java | 61 ++ .../InMemoryRelationshipTraversalCursor.java | 47 + .../_59/InMemoryStorageEngineFactory.java | 554 +++++++++++ .../compat/_59/InMemoryStorageEngineImpl.java | 341 +++++++ .../compat/_59/InMemoryStorageLocksImpl.java | 86 ++ .../gds/compat/_59/InMemoryStoreVersion.java | 68 ++ .../_59/InMemoryTransactionIdStoreImpl.java | 117 +++ .../gds/compat/_59/InMemoryVersionCheck.java | 61 ++ .../_59/StorageEngineProxyFactoryImpl.java | 44 + .../compat/_59/StorageEngineProxyImpl.java | 156 +++ .../InMemoryLogVersionRepository59.java | 71 ++ ...InMemoryStorageCommandReaderFactory59.java | 43 + .../InMemoryStorageReader59.java | 332 +++++++ gradle/dependencies.gradle | 1 + 48 files changed, 5250 insertions(+) create mode 100644 compatibility/5.9/neo4j-kernel-adapter/build.gradle create mode 100644 compatibility/5.9/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_59/Neo4jProxyFactoryImpl.java create mode 100644 compatibility/5.9/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_59/SettingProxyFactoryImpl.java create mode 100644 compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/BoltTransactionRunnerImpl.java create mode 100644 compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/CallableProcedureImpl.java create mode 100644 compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/CallableUserAggregationFunctionImpl.java create mode 100644 compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/CompatAccessModeImpl.java create mode 100644 compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/CompatGraphDatabaseAPIImpl.java create mode 100644 compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/CompatIndexQueryImpl.java create mode 100644 compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/CompatUsernameAuthSubjectImpl.java create mode 100644 compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/CompositeNodeCursorImpl.java create mode 100644 compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/GdsDatabaseLayoutImpl.java create mode 100644 compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/GdsDatabaseManagementServiceBuilderImpl.java create mode 100644 compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/Neo4jProxyFactoryImpl.java create mode 100644 compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/Neo4jProxyImpl.java create mode 100644 compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/NodeLabelIndexLookupImpl.java create mode 100644 compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/PartitionedStoreScan.java create mode 100644 compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/ReferencePropertyReference.java create mode 100644 compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/ScanBasedStoreScanImpl.java create mode 100644 compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/SettingProxyFactoryImpl.java create mode 100644 compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/SettingProxyImpl.java create mode 100644 compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/TestLogImpl.java create mode 100644 compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/VirtualRelationshipImpl.java create mode 100644 compatibility/5.9/storage-engine-adapter/build.gradle create mode 100644 compatibility/5.9/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_59/InMemoryStorageEngineFactory.java create mode 100644 compatibility/5.9/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_59/StorageEngineProxyFactoryImpl.java create mode 100644 compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryCommandCreationContextImpl.java create mode 100644 compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryCountsStoreImpl.java create mode 100644 compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryMetaDataProviderImpl.java create mode 100644 compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryNodeCursor.java create mode 100644 compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryNodePropertyCursor.java create mode 100644 compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryPropertyCursor.java create mode 100644 compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryPropertySelectionImpl.java create mode 100644 compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryRelationshipPropertyCursor.java create mode 100644 compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryRelationshipScanCursor.java create mode 100644 compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryRelationshipTraversalCursor.java create mode 100644 compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryStorageEngineFactory.java create mode 100644 compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryStorageEngineImpl.java create mode 100644 compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryStorageLocksImpl.java create mode 100644 compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryStoreVersion.java create mode 100644 compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryTransactionIdStoreImpl.java create mode 100644 compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryVersionCheck.java create mode 100644 compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/StorageEngineProxyFactoryImpl.java create mode 100644 compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/StorageEngineProxyImpl.java create mode 100644 compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository59.java create mode 100644 compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory59.java create mode 100644 compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageReader59.java diff --git a/compatibility/5.9/neo4j-kernel-adapter/build.gradle b/compatibility/5.9/neo4j-kernel-adapter/build.gradle new file mode 100644 index 0000000000..d712c10535 --- /dev/null +++ b/compatibility/5.9/neo4j-kernel-adapter/build.gradle @@ -0,0 +1,65 @@ +apply plugin: 'java-library' +apply plugin: 'me.champeau.mrjar' + +description = 'Neo4j Graph Data Science :: Neo4j Kernel Adapter 5.9' + +group = 'org.neo4j.gds' + +// for all 5.x versions +if (ver.'neo4j'.startsWith('5.')) { + sourceSets { + main { + java { + srcDirs = ['src/main/java17'] + } + } + } + + dependencies { + annotationProcessor project(':annotations') + annotationProcessor group: 'org.immutables', name: 'value', version: ver.'immutables' + annotationProcessor group: 'org.neo4j', name: 'annotations', version: neos.'5.9' + + compileOnly project(':annotations') + compileOnly group: 'com.github.spotbugs', name: 'spotbugs-annotations', version: ver.'spotbugsToolVersion' + compileOnly group: 'org.immutables', name: 'value-annotations', version: ver.'immutables' + compileOnly group: 'org.neo4j', name: 'annotations', version: neos.'5.9' + compileOnly group: 'org.neo4j', name: 'neo4j', version: neos.'5.9' + compileOnly group: 'org.neo4j', name: 'neo4j-record-storage-engine', version: neos.'5.9' + compileOnly group: 'org.neo4j.community', name: 'it-test-support', version: neos.'5.9' + + implementation project(':neo4j-kernel-adapter-api') + } +} else { + multiRelease { + targetVersions 11, 17 + } + + if (!project.hasProperty('no-forbidden-apis')) { + forbiddenApisJava17 { + exclude('**') + } + } + + dependencies { + annotationProcessor group: 'org.neo4j', name: 'annotations', version: ver.'neo4j' + + compileOnly project(':annotations') + compileOnly group: 'org.neo4j', name: 'annotations', version: ver.'neo4j' + + implementation project(':neo4j-kernel-adapter-api') + + java17AnnotationProcessor project(':annotations') + java17AnnotationProcessor group: 'org.immutables', name: 'value', version: ver.'immutables' + java17AnnotationProcessor group: 'org.neo4j', name: 'annotations', version: neos.'5.9' + + java17CompileOnly project(':annotations') + java17CompileOnly group: 'org.immutables', name: 'value-annotations', version: ver.'immutables' + java17CompileOnly group: 'org.neo4j', name: 'neo4j', version: neos.'5.9' + java17CompileOnly group: 'org.neo4j', name: 'neo4j-record-storage-engine', version: neos.'5.9' + java17CompileOnly group: 'org.neo4j.community', name: 'it-test-support', version: neos.'5.9' + java17CompileOnly group: 'com.github.spotbugs', name: 'spotbugs-annotations', version: ver.'spotbugsToolVersion' + + java17Implementation project(':neo4j-kernel-adapter-api') + } +} diff --git a/compatibility/5.9/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_59/Neo4jProxyFactoryImpl.java b/compatibility/5.9/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_59/Neo4jProxyFactoryImpl.java new file mode 100644 index 0000000000..26324fc643 --- /dev/null +++ b/compatibility/5.9/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_59/Neo4jProxyFactoryImpl.java @@ -0,0 +1,31 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * This file contains proprietary code that is only available via a commercial license from Neo4j. + * For more information, see https://neo4j.com/contact-us/ + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jProxyApi; +import org.neo4j.gds.compat.Neo4jProxyFactory; +import org.neo4j.gds.compat.Neo4jVersion; + +@ServiceProvider +public final class Neo4jProxyFactoryImpl implements Neo4jProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return false; + } + + @Override + public Neo4jProxyApi load() { + throw new UnsupportedOperationException("5.9 compatibility requires JDK17"); + } + + @Override + public String description() { + return "Neo4j 5.9 (placeholder)"; + } +} diff --git a/compatibility/5.9/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_59/SettingProxyFactoryImpl.java b/compatibility/5.9/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_59/SettingProxyFactoryImpl.java new file mode 100644 index 0000000000..791ce29673 --- /dev/null +++ b/compatibility/5.9/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_59/SettingProxyFactoryImpl.java @@ -0,0 +1,31 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * This file contains proprietary code that is only available via a commercial license from Neo4j. + * For more information, see https://neo4j.com/contact-us/ + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jVersion; +import org.neo4j.gds.compat.SettingProxyApi; +import org.neo4j.gds.compat.SettingProxyFactory; + +@ServiceProvider +public final class SettingProxyFactoryImpl implements SettingProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return false; + } + + @Override + public SettingProxyApi load() { + throw new UnsupportedOperationException("5.9 compatibility requires JDK17"); + } + + @Override + public String description() { + return "Neo4j Settings 5.9 (placeholder)"; + } +} diff --git a/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/BoltTransactionRunnerImpl.java b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/BoltTransactionRunnerImpl.java new file mode 100644 index 0000000000..479f8dbf17 --- /dev/null +++ b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/BoltTransactionRunnerImpl.java @@ -0,0 +1,97 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.bolt.dbapi.BoltGraphDatabaseServiceSPI; +import org.neo4j.bolt.dbapi.BoltTransaction; +import org.neo4j.bolt.protocol.common.message.AccessMode; +import org.neo4j.bolt.protocol.common.message.request.connection.RoutingContext; +import org.neo4j.bolt.tx.statement.StatementQuerySubscriber; +import org.neo4j.exceptions.KernelException; +import org.neo4j.gds.compat.BoltQuerySubscriber; +import org.neo4j.gds.compat.BoltTransactionRunner; +import org.neo4j.graphdb.QueryStatistics; +import org.neo4j.internal.kernel.api.connectioninfo.ClientConnectionInfo; +import org.neo4j.internal.kernel.api.security.LoginContext; +import org.neo4j.kernel.api.KernelTransaction; +import org.neo4j.kernel.impl.query.QueryExecutionConfiguration; +import org.neo4j.kernel.impl.query.QueryExecutionKernelException; +import org.neo4j.values.virtual.MapValue; + +import java.time.Duration; +import java.util.List; +import java.util.Map; + +public class BoltTransactionRunnerImpl extends BoltTransactionRunner { + + @Override + protected BoltQuerySubscriber boltQuerySubscriber() { + var subscriber = new StatementQuerySubscriber(); + return new BoltQuerySubscriber<>() { + @Override + public void assertSucceeded() throws KernelException { + subscriber.assertSuccess(); + } + + @Override + public QueryStatistics queryStatistics() { + return subscriber.getStatistics(); + } + + @Override + public StatementQuerySubscriber innerSubscriber() { + return subscriber; + } + }; + } + + @Override + protected void executeQuery( + BoltTransaction boltTransaction, + String query, + MapValue parameters, + StatementQuerySubscriber querySubscriber + ) throws QueryExecutionKernelException { + boltTransaction.executeQuery(query, parameters, true, querySubscriber); + } + + @Override + protected BoltTransaction beginBoltWriteTransaction( + BoltGraphDatabaseServiceSPI fabricDb, + LoginContext loginContext, + KernelTransaction.Type kernelTransactionType, + ClientConnectionInfo clientConnectionInfo, + List bookmarks, + Duration txTimeout, + Map txMetadata + ) { + return fabricDb.beginTransaction( + kernelTransactionType, + loginContext, + clientConnectionInfo, + bookmarks, + txTimeout, + AccessMode.WRITE, + txMetadata, + new RoutingContext(true, Map.of()), + QueryExecutionConfiguration.DEFAULT_CONFIG + ); + } +} diff --git a/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/CallableProcedureImpl.java b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/CallableProcedureImpl.java new file mode 100644 index 0000000000..08ff067f96 --- /dev/null +++ b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/CallableProcedureImpl.java @@ -0,0 +1,53 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.collection.RawIterator; +import org.neo4j.gds.annotation.SuppressForbidden; +import org.neo4j.gds.compat.CompatCallableProcedure; +import org.neo4j.internal.kernel.api.exceptions.ProcedureException; +import org.neo4j.internal.kernel.api.procs.ProcedureSignature; +import org.neo4j.kernel.api.ResourceMonitor; +import org.neo4j.kernel.api.procedure.CallableProcedure; +import org.neo4j.kernel.api.procedure.Context; +import org.neo4j.values.AnyValue; + +@SuppressForbidden(reason = "This is the compat API") +public final class CallableProcedureImpl implements CallableProcedure { + private final CompatCallableProcedure procedure; + + CallableProcedureImpl(CompatCallableProcedure procedure) { + this.procedure = procedure; + } + + @Override + public ProcedureSignature signature() { + return this.procedure.signature(); + } + + @Override + public RawIterator apply( + Context ctx, + AnyValue[] input, + ResourceMonitor resourceMonitor + ) throws ProcedureException { + return this.procedure.apply(ctx, input); + } +} diff --git a/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/CallableUserAggregationFunctionImpl.java b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/CallableUserAggregationFunctionImpl.java new file mode 100644 index 0000000000..de837dc4b2 --- /dev/null +++ b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/CallableUserAggregationFunctionImpl.java @@ -0,0 +1,77 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.gds.annotation.SuppressForbidden; +import org.neo4j.gds.compat.CompatUserAggregationFunction; +import org.neo4j.gds.compat.CompatUserAggregator; +import org.neo4j.internal.kernel.api.exceptions.ProcedureException; +import org.neo4j.internal.kernel.api.procs.UserAggregationReducer; +import org.neo4j.internal.kernel.api.procs.UserAggregationUpdater; +import org.neo4j.internal.kernel.api.procs.UserFunctionSignature; +import org.neo4j.kernel.api.procedure.CallableUserAggregationFunction; +import org.neo4j.kernel.api.procedure.Context; +import org.neo4j.values.AnyValue; + +@SuppressForbidden(reason = "This is the compat API") +public final class CallableUserAggregationFunctionImpl implements CallableUserAggregationFunction { + private final CompatUserAggregationFunction function; + + CallableUserAggregationFunctionImpl(CompatUserAggregationFunction function) { + this.function = function; + } + + @Override + public UserFunctionSignature signature() { + return this.function.signature(); + } + + @Override + public UserAggregationReducer createReducer(Context ctx) throws ProcedureException { + return new UserAggregatorImpl(this.function.create(ctx)); + } + + private static final class UserAggregatorImpl implements UserAggregationReducer, UserAggregationUpdater { + private final CompatUserAggregator aggregator; + + private UserAggregatorImpl(CompatUserAggregator aggregator) { + this.aggregator = aggregator; + } + + @Override + public UserAggregationUpdater newUpdater() { + return this; + } + + @Override + public void update(AnyValue[] input) throws ProcedureException { + this.aggregator.update(input); + } + + @Override + public void applyUpdates() { + } + + @Override + public AnyValue result() throws ProcedureException { + return this.aggregator.result(); + } + } +} diff --git a/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/CompatAccessModeImpl.java b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/CompatAccessModeImpl.java new file mode 100644 index 0000000000..49f92fb86e --- /dev/null +++ b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/CompatAccessModeImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.gds.compat.CompatAccessMode; +import org.neo4j.gds.compat.CustomAccessMode; +import org.neo4j.internal.kernel.api.RelTypeSupplier; +import org.neo4j.internal.kernel.api.TokenSet; + +import java.util.function.Supplier; + +public final class CompatAccessModeImpl extends CompatAccessMode { + + CompatAccessModeImpl(CustomAccessMode custom) { + super(custom); + } + + @Override + public boolean allowsReadNodeProperty(Supplier labels, int propertyKey) { + return custom.allowsReadNodeProperty(propertyKey); + } + + @Override + public boolean allowsReadRelationshipProperty(RelTypeSupplier relType, int propertyKey) { + return custom.allowsReadRelationshipProperty(propertyKey); + } +} diff --git a/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/CompatGraphDatabaseAPIImpl.java b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/CompatGraphDatabaseAPIImpl.java new file mode 100644 index 0000000000..4fbf4bafb5 --- /dev/null +++ b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/CompatGraphDatabaseAPIImpl.java @@ -0,0 +1,74 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.dbms.api.DatabaseManagementService; +import org.neo4j.dbms.systemgraph.TopologyGraphDbmsModel; +import org.neo4j.gds.compat.GdsGraphDatabaseAPI; +import org.neo4j.internal.kernel.api.connectioninfo.ClientConnectionInfo; +import org.neo4j.internal.kernel.api.security.LoginContext; +import org.neo4j.kernel.api.KernelTransaction; +import org.neo4j.kernel.api.exceptions.Status; +import org.neo4j.kernel.impl.coreapi.InternalTransaction; +import org.neo4j.kernel.impl.coreapi.TransactionExceptionMapper; + +import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; + +final class CompatGraphDatabaseAPIImpl extends GdsGraphDatabaseAPI { + + CompatGraphDatabaseAPIImpl(DatabaseManagementService dbms) { + super(dbms); + } + + @Override + public boolean isAvailable() { + return api.isAvailable(); + } + + @Override + public TopologyGraphDbmsModel.HostedOnMode mode() { + // NOTE: This means we can never start clusters locally, which is probably fine since: + // 1) We never did this before + // 2) We only use this for tests and benchmarks + return TopologyGraphDbmsModel.HostedOnMode.SINGLE; + } + + @Override + public InternalTransaction beginTransaction( + KernelTransaction.Type type, + LoginContext loginContext, + ClientConnectionInfo clientInfo, + long timeout, + TimeUnit unit, + Consumer terminationCallback, + TransactionExceptionMapper transactionExceptionMapper + ) { + return api.beginTransaction( + type, + loginContext, + clientInfo, + timeout, + unit, + terminationCallback, + transactionExceptionMapper + ); + } +} diff --git a/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/CompatIndexQueryImpl.java b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/CompatIndexQueryImpl.java new file mode 100644 index 0000000000..ccef737c59 --- /dev/null +++ b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/CompatIndexQueryImpl.java @@ -0,0 +1,31 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.gds.compat.CompatIndexQuery; +import org.neo4j.internal.kernel.api.PropertyIndexQuery; + +final class CompatIndexQueryImpl implements CompatIndexQuery { + final PropertyIndexQuery indexQuery; + + CompatIndexQueryImpl(PropertyIndexQuery indexQuery) { + this.indexQuery = indexQuery; + } +} diff --git a/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/CompatUsernameAuthSubjectImpl.java b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/CompatUsernameAuthSubjectImpl.java new file mode 100644 index 0000000000..85c75a5d37 --- /dev/null +++ b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/CompatUsernameAuthSubjectImpl.java @@ -0,0 +1,35 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.gds.compat.CompatUsernameAuthSubject; +import org.neo4j.internal.kernel.api.security.AuthSubject; + +final class CompatUsernameAuthSubjectImpl extends CompatUsernameAuthSubject { + + CompatUsernameAuthSubjectImpl(String username, AuthSubject authSubject) { + super(username, authSubject); + } + + @Override + public String executingUser() { + return username; + } +} diff --git a/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/CompositeNodeCursorImpl.java b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/CompositeNodeCursorImpl.java new file mode 100644 index 0000000000..3fc8fbfca6 --- /dev/null +++ b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/CompositeNodeCursorImpl.java @@ -0,0 +1,32 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.gds.compat.CompositeNodeCursor; +import org.neo4j.internal.kernel.api.NodeLabelIndexCursor; + +import java.util.List; + +public final class CompositeNodeCursorImpl extends CompositeNodeCursor { + + CompositeNodeCursorImpl(List cursors, int[] labelIds) { + super(cursors, labelIds); + } +} diff --git a/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/GdsDatabaseLayoutImpl.java b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/GdsDatabaseLayoutImpl.java new file mode 100644 index 0000000000..01a2134ad1 --- /dev/null +++ b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/GdsDatabaseLayoutImpl.java @@ -0,0 +1,50 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.gds.compat.GdsDatabaseLayout; +import org.neo4j.io.layout.DatabaseLayout; + +import java.nio.file.Path; + +public class GdsDatabaseLayoutImpl implements GdsDatabaseLayout { + private final DatabaseLayout databaseLayout; + + public GdsDatabaseLayoutImpl(DatabaseLayout databaseLayout) {this.databaseLayout = databaseLayout;} + + @Override + public Path databaseDirectory() { + return databaseLayout.databaseDirectory(); + } + + @Override + public Path getTransactionLogsDirectory() { + return databaseLayout.getTransactionLogsDirectory(); + } + + @Override + public Path metadataStore() { + return databaseLayout.metadataStore(); + } + + public DatabaseLayout databaseLayout() { + return databaseLayout; + } +} diff --git a/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/GdsDatabaseManagementServiceBuilderImpl.java b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/GdsDatabaseManagementServiceBuilderImpl.java new file mode 100644 index 0000000000..e8e8185129 --- /dev/null +++ b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/GdsDatabaseManagementServiceBuilderImpl.java @@ -0,0 +1,54 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.dbms.api.DatabaseManagementService; +import org.neo4j.dbms.api.DatabaseManagementServiceBuilderImplementation; +import org.neo4j.gds.compat.GdsDatabaseManagementServiceBuilder; +import org.neo4j.graphdb.config.Setting; + +import java.nio.file.Path; +import java.util.Map; + +public class GdsDatabaseManagementServiceBuilderImpl implements GdsDatabaseManagementServiceBuilder { + + private final DatabaseManagementServiceBuilderImplementation dbmsBuilder; + + GdsDatabaseManagementServiceBuilderImpl(Path storeDir) { + this.dbmsBuilder = new DatabaseManagementServiceBuilderImplementation(storeDir); + } + + @Override + public GdsDatabaseManagementServiceBuilder setConfigRaw(Map configMap) { + dbmsBuilder.setConfigRaw(configMap); + return this; + } + + @Override + public GdsDatabaseManagementServiceBuilder setConfig(Setting setting, S value) { + dbmsBuilder.setConfig(setting, value); + return this; + } + + @Override + public DatabaseManagementService build() { + return dbmsBuilder.build(); + } +} diff --git a/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/Neo4jProxyFactoryImpl.java b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/Neo4jProxyFactoryImpl.java new file mode 100644 index 0000000000..25ad1de6d4 --- /dev/null +++ b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/Neo4jProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jProxyApi; +import org.neo4j.gds.compat.Neo4jProxyFactory; +import org.neo4j.gds.compat.Neo4jVersion; + +@ServiceProvider +public final class Neo4jProxyFactoryImpl implements Neo4jProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return version == Neo4jVersion.V_5_9; + } + + @Override + public Neo4jProxyApi load() { + return new Neo4jProxyImpl(); + } + + @Override + public String description() { + return "Neo4j 5.9"; + } +} diff --git a/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/Neo4jProxyImpl.java b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/Neo4jProxyImpl.java new file mode 100644 index 0000000000..43d25ade58 --- /dev/null +++ b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/Neo4jProxyImpl.java @@ -0,0 +1,917 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import org.neo4j.common.DependencyResolver; +import org.neo4j.common.EntityType; +import org.neo4j.configuration.BootloaderSettings; +import org.neo4j.configuration.Config; +import org.neo4j.configuration.GraphDatabaseSettings; +import org.neo4j.configuration.SettingValueParsers; +import org.neo4j.configuration.connectors.ConnectorPortRegister; +import org.neo4j.configuration.connectors.ConnectorType; +import org.neo4j.configuration.helpers.DatabaseNameValidator; +import org.neo4j.dbms.api.DatabaseManagementService; +import org.neo4j.exceptions.KernelException; +import org.neo4j.fabric.FabricDatabaseManager; +import org.neo4j.gds.annotation.SuppressForbidden; +import org.neo4j.gds.compat.BoltTransactionRunner; +import org.neo4j.gds.compat.CompatCallableProcedure; +import org.neo4j.gds.compat.CompatExecutionMonitor; +import org.neo4j.gds.compat.CompatIndexQuery; +import org.neo4j.gds.compat.CompatInput; +import org.neo4j.gds.compat.CompatUserAggregationFunction; +import org.neo4j.gds.compat.CompositeNodeCursor; +import org.neo4j.gds.compat.CustomAccessMode; +import org.neo4j.gds.compat.GdsDatabaseLayout; +import org.neo4j.gds.compat.GdsDatabaseManagementServiceBuilder; +import org.neo4j.gds.compat.GdsGraphDatabaseAPI; +import org.neo4j.gds.compat.GraphDatabaseApiProxy; +import org.neo4j.gds.compat.InputEntityIdVisitor; +import org.neo4j.gds.compat.Neo4jProxyApi; +import org.neo4j.gds.compat.PropertyReference; +import org.neo4j.gds.compat.StoreScan; +import org.neo4j.gds.compat.TestLog; +import org.neo4j.graphdb.GraphDatabaseService; +import org.neo4j.graphdb.Node; +import org.neo4j.graphdb.Relationship; +import org.neo4j.graphdb.RelationshipType; +import org.neo4j.graphdb.config.Setting; +import org.neo4j.internal.batchimport.AdditionalInitialIds; +import org.neo4j.internal.batchimport.BatchImporter; +import org.neo4j.internal.batchimport.BatchImporterFactory; +import org.neo4j.internal.batchimport.Configuration; +import org.neo4j.internal.batchimport.IndexConfig; +import org.neo4j.internal.batchimport.InputIterable; +import org.neo4j.internal.batchimport.Monitor; +import org.neo4j.internal.batchimport.input.Collector; +import org.neo4j.internal.batchimport.input.IdType; +import org.neo4j.internal.batchimport.input.Input; +import org.neo4j.internal.batchimport.input.InputEntityVisitor; +import org.neo4j.internal.batchimport.input.PropertySizeCalculator; +import org.neo4j.internal.batchimport.input.ReadableGroups; +import org.neo4j.internal.batchimport.staging.ExecutionMonitor; +import org.neo4j.internal.batchimport.staging.StageExecution; +import org.neo4j.internal.helpers.HostnamePort; +import org.neo4j.internal.id.IdGenerator; +import org.neo4j.internal.id.IdGeneratorFactory; +import org.neo4j.internal.kernel.api.Cursor; +import org.neo4j.internal.kernel.api.IndexQueryConstraints; +import org.neo4j.internal.kernel.api.IndexReadSession; +import org.neo4j.internal.kernel.api.NodeCursor; +import org.neo4j.internal.kernel.api.NodeLabelIndexCursor; +import org.neo4j.internal.kernel.api.NodeValueIndexCursor; +import org.neo4j.internal.kernel.api.PropertyCursor; +import org.neo4j.internal.kernel.api.PropertyIndexQuery; +import org.neo4j.internal.kernel.api.QueryContext; +import org.neo4j.internal.kernel.api.Read; +import org.neo4j.internal.kernel.api.RelationshipScanCursor; +import org.neo4j.internal.kernel.api.Scan; +import org.neo4j.internal.kernel.api.TokenPredicate; +import org.neo4j.internal.kernel.api.connectioninfo.ClientConnectionInfo; +import org.neo4j.internal.kernel.api.procs.FieldSignature; +import org.neo4j.internal.kernel.api.procs.Neo4jTypes; +import org.neo4j.internal.kernel.api.procs.ProcedureSignature; +import org.neo4j.internal.kernel.api.procs.QualifiedName; +import org.neo4j.internal.kernel.api.procs.UserFunctionSignature; +import org.neo4j.internal.kernel.api.security.AccessMode; +import org.neo4j.internal.kernel.api.security.AuthSubject; +import org.neo4j.internal.kernel.api.security.SecurityContext; +import org.neo4j.internal.recordstorage.RecordIdType; +import org.neo4j.internal.schema.IndexCapability; +import org.neo4j.internal.schema.IndexDescriptor; +import org.neo4j.internal.schema.IndexOrder; +import org.neo4j.internal.schema.SchemaDescriptors; +import org.neo4j.io.fs.FileSystemAbstraction; +import org.neo4j.io.layout.DatabaseLayout; +import org.neo4j.io.layout.Neo4jLayout; +import org.neo4j.io.layout.recordstorage.RecordDatabaseLayout; +import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.context.CursorContextFactory; +import org.neo4j.io.pagecache.context.EmptyVersionContextSupplier; +import org.neo4j.io.pagecache.tracing.PageCacheTracer; +import org.neo4j.kernel.api.KernelTransaction; +import org.neo4j.kernel.api.KernelTransactionHandle; +import org.neo4j.kernel.api.procedure.CallableProcedure; +import org.neo4j.kernel.api.procedure.CallableUserAggregationFunction; +import org.neo4j.kernel.database.NormalizedDatabaseName; +import org.neo4j.kernel.impl.coreapi.InternalTransaction; +import org.neo4j.kernel.impl.index.schema.IndexImporterFactoryImpl; +import org.neo4j.kernel.impl.query.QueryExecutionConfiguration; +import org.neo4j.kernel.impl.query.TransactionalContext; +import org.neo4j.kernel.impl.query.TransactionalContextFactory; +import org.neo4j.kernel.impl.store.RecordStore; +import org.neo4j.kernel.impl.store.format.RecordFormatSelector; +import org.neo4j.kernel.impl.store.format.RecordFormats; +import org.neo4j.kernel.impl.store.record.AbstractBaseRecord; +import org.neo4j.kernel.impl.transaction.log.EmptyLogTailMetadata; +import org.neo4j.kernel.impl.transaction.log.files.TransactionLogInitializer; +import org.neo4j.logging.Log; +import org.neo4j.logging.internal.LogService; +import org.neo4j.memory.EmptyMemoryTracker; +import org.neo4j.procedure.Mode; +import org.neo4j.scheduler.JobScheduler; +import org.neo4j.ssl.config.SslPolicyLoader; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.StorageEngineFactory; +import org.neo4j.util.Bits; +import org.neo4j.values.storable.TextArray; +import org.neo4j.values.storable.ValueCategory; +import org.neo4j.values.storable.Values; +import org.neo4j.values.virtual.MapValue; +import org.neo4j.values.virtual.NodeValue; +import org.neo4j.values.virtual.VirtualValues; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Locale; +import java.util.Optional; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +import static java.lang.String.format; +import static org.neo4j.gds.compat.InternalReadOps.countByIdGenerator; +import static org.neo4j.io.pagecache.context.EmptyVersionContextSupplier.EMPTY; + +public final class Neo4jProxyImpl implements Neo4jProxyApi { + + @Override + public GdsGraphDatabaseAPI newDb(DatabaseManagementService dbms) { + return new CompatGraphDatabaseAPIImpl(dbms); + } + + @Override + public String validateExternalDatabaseName(String databaseName) { + var normalizedName = new NormalizedDatabaseName(databaseName); + DatabaseNameValidator.validateExternalDatabaseName(normalizedName); + return normalizedName.name(); + } + + @Override + public AccessMode accessMode(CustomAccessMode customAccessMode) { + return new CompatAccessModeImpl(customAccessMode); + } + + @Override + public String username(AuthSubject subject) { + return subject.executingUser(); + } + + @Override + public SecurityContext securityContext( + String username, + AuthSubject authSubject, + AccessMode mode, + String databaseName + ) { + return new SecurityContext( + new CompatUsernameAuthSubjectImpl(username, authSubject), + mode, + // GDS is always operating from an embedded context + ClientConnectionInfo.EMBEDDED_CONNECTION, + databaseName + ); + } + + @Override + public long getHighId(RecordStore recordStore) { + return recordStore.getIdGenerator().getHighId(); + } + + @Override + public List> entityCursorScan( + KernelTransaction transaction, + int[] labelIds, + int batchSize, + boolean allowPartitionedScan + ) { + if (allowPartitionedScan) { + return partitionedNodeLabelIndexScan(transaction, batchSize, labelIds); + } else { + var read = transaction.dataRead(); + return Arrays + .stream(labelIds) + .mapToObj(read::nodeLabelScan) + .map(scan -> scanToStoreScan(scan, batchSize)) + .collect(Collectors.toList()); + } + } + + @Override + public PropertyCursor allocatePropertyCursor(KernelTransaction kernelTransaction) { + return kernelTransaction + .cursors() + .allocatePropertyCursor(kernelTransaction.cursorContext(), kernelTransaction.memoryTracker()); + } + + @Override + public PropertyReference propertyReference(NodeCursor nodeCursor) { + return ReferencePropertyReference.of(nodeCursor.propertiesReference()); + } + + @Override + public PropertyReference propertyReference(RelationshipScanCursor relationshipScanCursor) { + return ReferencePropertyReference.of(relationshipScanCursor.propertiesReference()); + } + + @Override + public PropertyReference noPropertyReference() { + return ReferencePropertyReference.empty(); + } + + @Override + public void nodeProperties( + KernelTransaction kernelTransaction, + long nodeReference, + PropertyReference reference, + PropertyCursor cursor + ) { + var neoReference = ((ReferencePropertyReference) reference).reference; + kernelTransaction + .dataRead() + .nodeProperties(nodeReference, neoReference, PropertySelection.ALL_PROPERTIES, cursor); + } + + @Override + public void relationshipProperties( + KernelTransaction kernelTransaction, + long relationshipReference, + PropertyReference reference, + PropertyCursor cursor + ) { + var neoReference = ((ReferencePropertyReference) reference).reference; + kernelTransaction + .dataRead() + .relationshipProperties(relationshipReference, neoReference, PropertySelection.ALL_PROPERTIES, cursor); + } + + @Override + public NodeCursor allocateNodeCursor(KernelTransaction kernelTransaction) { + return kernelTransaction.cursors().allocateNodeCursor(kernelTransaction.cursorContext()); + } + + @Override + public RelationshipScanCursor allocateRelationshipScanCursor(KernelTransaction kernelTransaction) { + return kernelTransaction.cursors().allocateRelationshipScanCursor(kernelTransaction.cursorContext()); + } + + @Override + public NodeLabelIndexCursor allocateNodeLabelIndexCursor(KernelTransaction kernelTransaction) { + return kernelTransaction.cursors().allocateNodeLabelIndexCursor(kernelTransaction.cursorContext()); + } + + @Override + public NodeValueIndexCursor allocateNodeValueIndexCursor(KernelTransaction kernelTransaction) { + return kernelTransaction + .cursors() + .allocateNodeValueIndexCursor(kernelTransaction.cursorContext(), kernelTransaction.memoryTracker()); + } + + @Override + public boolean hasNodeLabelIndex(KernelTransaction kernelTransaction) { + return NodeLabelIndexLookupImpl.hasNodeLabelIndex(kernelTransaction); + } + + @Override + public StoreScan nodeLabelIndexScan( + KernelTransaction transaction, + int labelId, + int batchSize, + boolean allowPartitionedScan + ) { + if (allowPartitionedScan) { + return partitionedNodeLabelIndexScan(transaction, batchSize, labelId).get(0); + } else { + var read = transaction.dataRead(); + return scanToStoreScan(read.nodeLabelScan(labelId), batchSize); + } + } + + @Override + public StoreScan scanToStoreScan(Scan scan, int batchSize) { + return new ScanBasedStoreScanImpl<>(scan, batchSize); + } + + private List> partitionedNodeLabelIndexScan( + KernelTransaction transaction, + int batchSize, + int... labelIds + ) { + var indexDescriptor = NodeLabelIndexLookupImpl.findUsableMatchingIndex( + transaction, + SchemaDescriptors.forAnyEntityTokens(EntityType.NODE) + ); + + if (indexDescriptor == IndexDescriptor.NO_INDEX) { + throw new IllegalStateException("There is no index that can back a node label scan."); + } + + var read = transaction.dataRead(); + + // Our current strategy is to select the token with the highest count + // and use that one as the driving partitioned index scan. The partitions + // of all other partitioned index scans will be aligned to that one. + int maxToken = labelIds[0]; + long maxCount = read.countsForNodeWithoutTxState(labelIds[0]); + + for (int i = 1; i < labelIds.length; i++) { + long count = read.countsForNodeWithoutTxState(labelIds[i]); + if (count > maxCount) { + maxCount = count; + maxToken = labelIds[i]; + } + } + + int numberOfPartitions = PartitionedStoreScan.getNumberOfPartitions(maxCount, batchSize); + + try { + var session = read.tokenReadSession(indexDescriptor); + + var partitionedScan = read.nodeLabelScan( + session, + numberOfPartitions, + transaction.cursorContext(), + new TokenPredicate(maxToken) + ); + + var scans = new ArrayList>(labelIds.length); + scans.add(new PartitionedStoreScan(partitionedScan)); + + // Initialize the remaining index scans with the partitioning of the first scan. + for (int labelToken : labelIds) { + if (labelToken != maxToken) { + var scan = read.nodeLabelScan(session, partitionedScan, new TokenPredicate(labelToken)); + scans.add(new PartitionedStoreScan(scan)); + } + } + + return scans; + } catch (KernelException e) { + // should not happen, we check for the index existence and applicability + // before reading it + throw new RuntimeException("Unexpected error while initialising reading from node label index", e); + } + } + + @Override + public CompatIndexQuery rangeIndexQuery( + int propertyKeyId, + double from, + boolean fromInclusive, + double to, + boolean toInclusive + ) { + return new CompatIndexQueryImpl(PropertyIndexQuery.range(propertyKeyId, from, fromInclusive, to, toInclusive)); + } + + @Override + public CompatIndexQuery rangeAllIndexQuery(int propertyKeyId) { + var rangePredicate = PropertyIndexQuery.range( + propertyKeyId, + Values.doubleValue(Double.NEGATIVE_INFINITY), + true, + Values.doubleValue(Double.POSITIVE_INFINITY), + true + ); + return new CompatIndexQueryImpl(rangePredicate); + } + + @Override + public void nodeIndexSeek( + Read dataRead, + IndexReadSession index, + NodeValueIndexCursor cursor, + IndexOrder indexOrder, + boolean needsValues, + CompatIndexQuery query + ) throws KernelException { + var indexQueryConstraints = indexOrder == IndexOrder.NONE + ? IndexQueryConstraints.unordered(needsValues) + : IndexQueryConstraints.constrained(indexOrder, needsValues); + + dataRead.nodeIndexSeek( + (QueryContext) dataRead, + index, + cursor, + indexQueryConstraints, + ((CompatIndexQueryImpl) query).indexQuery + ); + } + + @Override + public CompositeNodeCursor compositeNodeCursor(List cursors, int[] labelIds) { + return new CompositeNodeCursorImpl(cursors, labelIds); + } + + @Override + public Configuration batchImporterConfig( + int batchSize, + int writeConcurrency, + Optional pageCacheMemory, + boolean highIO, + IndexConfig indexConfig + ) { + return new org.neo4j.internal.batchimport.Configuration() { + @Override + public int batchSize() { + return batchSize; + } + + @Override + public int maxNumberOfWorkerThreads() { + return writeConcurrency; + } + + @Override + public boolean highIO() { + return highIO; + } + + @Override + public IndexConfig indexConfig() { + return indexConfig; + } + }; + } + + @Override + public int writeConcurrency(Configuration batchImportConfiguration) { + return batchImportConfiguration.maxNumberOfWorkerThreads(); + } + + @Override + public BatchImporter instantiateBatchImporter( + BatchImporterFactory factory, + GdsDatabaseLayout directoryStructure, + FileSystemAbstraction fileSystem, + PageCacheTracer pageCacheTracer, + Configuration configuration, + LogService logService, + ExecutionMonitor executionMonitor, + AdditionalInitialIds additionalInitialIds, + Config dbConfig, + RecordFormats recordFormats, + JobScheduler jobScheduler, + Collector badCollector + ) { + dbConfig.set(GraphDatabaseSettings.db_format, recordFormats.name()); + var databaseLayout = ((GdsDatabaseLayoutImpl) directoryStructure).databaseLayout(); + return factory.instantiate( + databaseLayout, + fileSystem, + pageCacheTracer, + configuration, + logService, + executionMonitor, + additionalInitialIds, + new EmptyLogTailMetadata(dbConfig), + dbConfig, + Monitor.NO_MONITOR, + jobScheduler, + badCollector, + TransactionLogInitializer.getLogFilesInitializer(), + new IndexImporterFactoryImpl(), + EmptyMemoryTracker.INSTANCE, + new CursorContextFactory(PageCacheTracer.NULL, EmptyVersionContextSupplier.EMPTY) + ); + } + + @Override + public Input batchInputFrom(CompatInput compatInput) { + return new InputFromCompatInput(compatInput); + } + + @Override + public InputEntityIdVisitor.Long inputEntityLongIdVisitor(IdType idType, ReadableGroups groups) { + switch (idType) { + case ACTUAL -> { + return new InputEntityIdVisitor.Long() { + @Override + public void visitNodeId(InputEntityVisitor visitor, long id) { + visitor.id(id); + } + + @Override + public void visitSourceId(InputEntityVisitor visitor, long id) { + visitor.startId(id); + } + + @Override + public void visitTargetId(InputEntityVisitor visitor, long id) { + visitor.endId(id); + } + }; + } + case INTEGER -> { + var globalGroup = groups.get(null); + + return new InputEntityIdVisitor.Long() { + @Override + public void visitNodeId(InputEntityVisitor visitor, long id) { + visitor.id(id, globalGroup); + } + + @Override + public void visitSourceId(InputEntityVisitor visitor, long id) { + visitor.startId(id, globalGroup); + } + + @Override + public void visitTargetId(InputEntityVisitor visitor, long id) { + visitor.endId(id, globalGroup); + } + }; + } + default -> throw new IllegalStateException("Unexpected value: " + idType); + } + } + + @Override + public InputEntityIdVisitor.String inputEntityStringIdVisitor(ReadableGroups groups) { + var globalGroup = groups.get(null); + + return new InputEntityIdVisitor.String() { + @Override + public void visitNodeId(InputEntityVisitor visitor, String id) { + visitor.id(id, globalGroup); + } + + @Override + public void visitSourceId(InputEntityVisitor visitor, String id) { + visitor.startId(id, globalGroup); + } + + @Override + public void visitTargetId(InputEntityVisitor visitor, String id) { + visitor.endId(id, globalGroup); + } + }; + } + + @Override + public Setting additionalJvm() { + return BootloaderSettings.additional_jvm; + } + + @Override + public Setting pageCacheMemory() { + return GraphDatabaseSettings.pagecache_memory; + } + + @Override + public Long pageCacheMemoryValue(String value) { + return SettingValueParsers.BYTES.parse(value); + } + + @Override + public ProcedureSignature procedureSignature( + QualifiedName name, + List inputSignature, + List outputSignature, + Mode mode, + boolean admin, + String deprecated, + String description, + String warning, + boolean eager, + boolean caseInsensitive, + boolean systemProcedure, + boolean internal, + boolean allowExpiredCredentials + ) { + return new ProcedureSignature( + name, + inputSignature, + outputSignature, + mode, + admin, + deprecated, + description, + warning, + eager, + caseInsensitive, + systemProcedure, + internal, + allowExpiredCredentials + ); + } + + @Override + public long getHighestPossibleNodeCount( + Read read, IdGeneratorFactory idGeneratorFactory + ) { + return countByIdGenerator(idGeneratorFactory, RecordIdType.NODE).orElseGet(read::nodesGetCount); + } + + @Override + public long getHighestPossibleRelationshipCount( + Read read, IdGeneratorFactory idGeneratorFactory + ) { + return countByIdGenerator(idGeneratorFactory, RecordIdType.RELATIONSHIP).orElseGet(read::relationshipsGetCount); + } + + @Override + public String versionLongToString(long storeVersion) { + // copied from org.neo4j.kernel.impl.store.LegacyMetadataHandler.versionLongToString which is private + if (storeVersion == -1) { + return "Unknown"; + } + Bits bits = Bits.bitsFromLongs(new long[]{storeVersion}); + int length = bits.getShort(8); + if (length == 0 || length > 7) { + throw new IllegalArgumentException(format(Locale.ENGLISH, "The read version string length %d is not proper.", length)); + } + char[] result = new char[length]; + for (int i = 0; i < length; i++) { + result[i] = (char) bits.getShort(8); + } + return new String(result); + } + + private static final class InputFromCompatInput implements Input { + private final CompatInput delegate; + + private InputFromCompatInput(CompatInput delegate) { + this.delegate = delegate; + } + + @Override + public InputIterable nodes(Collector badCollector) { + return delegate.nodes(badCollector); + } + + @Override + public InputIterable relationships(Collector badCollector) { + return delegate.relationships(badCollector); + } + + @Override + public IdType idType() { + return delegate.idType(); + } + + @Override + public ReadableGroups groups() { + return delegate.groups(); + } + + @Override + public Estimates calculateEstimates(PropertySizeCalculator propertySizeCalculator) throws IOException { + return delegate.calculateEstimates((values, kernelTransaction) -> propertySizeCalculator.calculateSize( + values, + kernelTransaction.cursorContext(), + kernelTransaction.memoryTracker() + )); + } + } + + @Override + public TestLog testLog() { + return new TestLogImpl(); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public Log getUserLog(LogService logService, Class loggingClass) { + return logService.getUserLog(loggingClass); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public Log getInternalLog(LogService logService, Class loggingClass) { + return logService.getInternalLog(loggingClass); + } + + @Override + public NodeValue nodeValue(long id, TextArray labels, MapValue properties) { + return VirtualValues.nodeValue(id, String.valueOf(id), labels, properties); + } + + @Override + public Relationship virtualRelationship(long id, Node startNode, Node endNode, RelationshipType type) { + return new VirtualRelationshipImpl(id, startNode, endNode, type); + } + + @Override + public GdsDatabaseManagementServiceBuilder databaseManagementServiceBuilder(Path storeDir) { + return new GdsDatabaseManagementServiceBuilderImpl(storeDir); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public RecordFormats selectRecordFormatForStore( + DatabaseLayout databaseLayout, + FileSystemAbstraction fs, + PageCache pageCache, + LogService logService, + PageCacheTracer pageCacheTracer + ) { + return RecordFormatSelector.selectForStore( + (RecordDatabaseLayout) databaseLayout, + fs, + pageCache, + logService.getInternalLogProvider(), + new CursorContextFactory(pageCacheTracer, EMPTY) + ); + } + + @Override + public boolean isNotNumericIndex(IndexCapability indexCapability) { + return !indexCapability.areValueCategoriesAccepted(ValueCategory.NUMBER); + } + + @Override + public void setAllowUpgrades(Config.Builder configBuilder, boolean value) { + } + + @Override + public String defaultRecordFormatSetting() { + return GraphDatabaseSettings.db_format.defaultValue(); + } + + @Override + public void configureRecordFormat(Config.Builder configBuilder, String recordFormat) { + var databaseRecordFormat = recordFormat.toLowerCase(Locale.ENGLISH); + configBuilder.set(GraphDatabaseSettings.db_format, databaseRecordFormat); + } + + @Override + public GdsDatabaseLayout databaseLayout(Config config, String databaseName) { + var storageEngineFactory = StorageEngineFactory.selectStorageEngine(config); + var dbLayout = neo4jLayout(config).databaseLayout(databaseName); + var databaseLayout = storageEngineFactory.formatSpecificDatabaseLayout(dbLayout); + return new GdsDatabaseLayoutImpl(databaseLayout); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public Neo4jLayout neo4jLayout(Config config) { + return Neo4jLayout.of(config); + } + + @Override + public BoltTransactionRunner boltTransactionRunner() { + return new BoltTransactionRunnerImpl(); + } + + @Override + public HostnamePort getLocalBoltAddress(ConnectorPortRegister connectorPortRegister) { + return connectorPortRegister.getLocalAddress(ConnectorType.BOLT); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public SslPolicyLoader createSllPolicyLoader( + FileSystemAbstraction fileSystem, + Config config, + LogService logService + ) { + return SslPolicyLoader.create(fileSystem, config, logService.getInternalLogProvider()); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public RecordFormats recordFormatSelector( + String databaseName, + Config databaseConfig, + FileSystemAbstraction fs, + LogService logService, + GraphDatabaseService databaseService + ) { + var neo4jLayout = Neo4jLayout.of(databaseConfig); + var recordDatabaseLayout = RecordDatabaseLayout.of(neo4jLayout, databaseName); + return RecordFormatSelector.selectForStoreOrConfigForNewDbs( + databaseConfig, + recordDatabaseLayout, + fs, + GraphDatabaseApiProxy.resolveDependency(databaseService, PageCache.class), + logService.getInternalLogProvider(), + GraphDatabaseApiProxy.resolveDependency(databaseService, CursorContextFactory.class) + ); + } + + @Override + public ExecutionMonitor executionMonitor(CompatExecutionMonitor compatExecutionMonitor) { + return new ExecutionMonitor.Adapter( + compatExecutionMonitor.checkIntervalMillis(), + TimeUnit.MILLISECONDS + ) { + + @Override + public void initialize(DependencyResolver dependencyResolver) { + compatExecutionMonitor.initialize(dependencyResolver); + } + + @Override + public void start(StageExecution execution) { + compatExecutionMonitor.start(execution); + } + + @Override + public void end(StageExecution execution, long totalTimeMillis) { + compatExecutionMonitor.end(execution, totalTimeMillis); + } + + @Override + public void done(boolean successful, long totalTimeMillis, String additionalInformation) { + compatExecutionMonitor.done(successful, totalTimeMillis, additionalInformation); + } + + @Override + public void check(StageExecution execution) { + compatExecutionMonitor.check(execution); + } + }; + } + + @Override + @SuppressFBWarnings("NP_LOAD_OF_KNOWN_NULL_VALUE") // We assign nulls because it makes the code more readable + public UserFunctionSignature userFunctionSignature( + QualifiedName name, + List inputSignature, + Neo4jTypes.AnyType type, + String description, + boolean internal, + boolean threadSafe, + Optional deprecatedBy + ) { + String category = null; // No predefined categpry (like temporal or math) + var caseInsensitive = false; // case sensitive name match + var isBuiltIn = false; // is built in; never true for GDS + + return new UserFunctionSignature( + name, + inputSignature, + type, + deprecatedBy.orElse(null), + description, + category, + caseInsensitive, + isBuiltIn, + internal, + threadSafe + ); + } + + @Override + @SuppressForbidden(reason = "This is the compat API") + public CallableProcedure callableProcedure(CompatCallableProcedure procedure) { + return new CallableProcedureImpl(procedure); + } + + @Override + @SuppressForbidden(reason = "This is the compat API") + public CallableUserAggregationFunction callableUserAggregationFunction(CompatUserAggregationFunction function) { + return new CallableUserAggregationFunctionImpl(function); + } + + @Override + public long transactionId(KernelTransactionHandle kernelTransactionHandle) { + return kernelTransactionHandle.getTransactionSequenceNumber(); + } + + @Override + public void reserveNeo4jIds(IdGeneratorFactory generatorFactory, int size, CursorContext cursorContext) { + IdGenerator idGenerator = generatorFactory.get(RecordIdType.NODE); + + idGenerator.nextConsecutiveIdRange(size, false, cursorContext); + } + + @Override + public TransactionalContext newQueryContext( + TransactionalContextFactory contextFactory, + InternalTransaction tx, + String queryText, + MapValue queryParameters + ) { + return contextFactory.newContext(tx, queryText, queryParameters, QueryExecutionConfiguration.DEFAULT_CONFIG); + } + + @Override + public boolean isCompositeDatabase(GraphDatabaseService databaseService) { + var databaseManager = GraphDatabaseApiProxy.resolveDependency(databaseService, FabricDatabaseManager.class); + return databaseManager.isFabricDatabase(GraphDatabaseApiProxy.databaseId(databaseService)); + } +} diff --git a/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/NodeLabelIndexLookupImpl.java b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/NodeLabelIndexLookupImpl.java new file mode 100644 index 0000000000..849d9a4d5c --- /dev/null +++ b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/NodeLabelIndexLookupImpl.java @@ -0,0 +1,68 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.common.EntityType; +import org.neo4j.internal.kernel.api.InternalIndexState; +import org.neo4j.internal.kernel.api.SchemaRead; +import org.neo4j.internal.kernel.api.exceptions.schema.IndexNotFoundKernelException; +import org.neo4j.internal.schema.IndexDescriptor; +import org.neo4j.internal.schema.IndexType; +import org.neo4j.internal.schema.SchemaDescriptor; +import org.neo4j.internal.schema.SchemaDescriptors; +import org.neo4j.kernel.api.KernelTransaction; + +final class NodeLabelIndexLookupImpl { + + static boolean hasNodeLabelIndex(KernelTransaction transaction) { + return NodeLabelIndexLookupImpl.findUsableMatchingIndex( + transaction, + SchemaDescriptors.forAnyEntityTokens(EntityType.NODE) + ) != IndexDescriptor.NO_INDEX; + } + + static IndexDescriptor findUsableMatchingIndex( + KernelTransaction transaction, + SchemaDescriptor schemaDescriptor + ) { + var schemaRead = transaction.schemaRead(); + var iterator = schemaRead.index(schemaDescriptor); + while (iterator.hasNext()) { + var index = iterator.next(); + if (index.getIndexType() == IndexType.LOOKUP && indexIsOnline(schemaRead, index)) { + return index; + } + } + return IndexDescriptor.NO_INDEX; + } + + private static boolean indexIsOnline(SchemaRead schemaRead, IndexDescriptor index) { + var state = InternalIndexState.FAILED; + try { + state = schemaRead.indexGetState(index); + } catch (IndexNotFoundKernelException e) { + // Well the index should always exist here, but if we didn't find it while checking the state, + // then we obviously don't want to use it. + } + return state == InternalIndexState.ONLINE; + } + + private NodeLabelIndexLookupImpl() {} +} diff --git a/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/PartitionedStoreScan.java b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/PartitionedStoreScan.java new file mode 100644 index 0000000000..2a8da868ae --- /dev/null +++ b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/PartitionedStoreScan.java @@ -0,0 +1,58 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.gds.compat.StoreScan; +import org.neo4j.internal.kernel.api.NodeLabelIndexCursor; +import org.neo4j.internal.kernel.api.PartitionedScan; +import org.neo4j.kernel.api.KernelTransaction; + +final class PartitionedStoreScan implements StoreScan { + private final PartitionedScan scan; + + PartitionedStoreScan(PartitionedScan scan) { + this.scan = scan; + } + + static int getNumberOfPartitions(long nodeCount, int batchSize) { + int numberOfPartitions; + if (nodeCount > 0) { + // ceil div to try to get enough partitions so a single one does + // not include more nodes than batchSize + long partitions = ((nodeCount - 1) / batchSize) + 1; + + // value must be positive + if (partitions < 1) { + partitions = 1; + } + + numberOfPartitions = (int) Long.min(Integer.MAX_VALUE, partitions); + } else { + // we have no partitions to scan, but the value must still be positive + numberOfPartitions = 1; + } + return numberOfPartitions; + } + + @Override + public boolean reserveBatch(NodeLabelIndexCursor cursor, KernelTransaction ktx) { + return scan.reservePartition(cursor, ktx.cursorContext(), ktx.securityContext().mode()); + } +} diff --git a/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/ReferencePropertyReference.java b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/ReferencePropertyReference.java new file mode 100644 index 0000000000..a24847fc2d --- /dev/null +++ b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/ReferencePropertyReference.java @@ -0,0 +1,49 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.gds.compat.PropertyReference; +import org.neo4j.storageengine.api.Reference; + +import java.util.Objects; + +public final class ReferencePropertyReference implements PropertyReference { + + private static final PropertyReference EMPTY = new ReferencePropertyReference(null); + + public final Reference reference; + + private ReferencePropertyReference(Reference reference) { + this.reference = reference; + } + + public static PropertyReference of(Reference reference) { + return new ReferencePropertyReference(Objects.requireNonNull(reference)); + } + + public static PropertyReference empty() { + return EMPTY; + } + + @Override + public boolean isEmpty() { + return reference == null; + } +} diff --git a/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/ScanBasedStoreScanImpl.java b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/ScanBasedStoreScanImpl.java new file mode 100644 index 0000000000..c108e8bec6 --- /dev/null +++ b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/ScanBasedStoreScanImpl.java @@ -0,0 +1,40 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.gds.compat.StoreScan; +import org.neo4j.internal.kernel.api.Cursor; +import org.neo4j.internal.kernel.api.Scan; +import org.neo4j.kernel.api.KernelTransaction; + +public final class ScanBasedStoreScanImpl implements StoreScan { + private final Scan scan; + private final int batchSize; + + public ScanBasedStoreScanImpl(Scan scan, int batchSize) { + this.scan = scan; + this.batchSize = batchSize; + } + + @Override + public boolean reserveBatch(C cursor, KernelTransaction ktx) { + return scan.reserveBatch(cursor, batchSize, ktx.cursorContext(), ktx.securityContext().mode()); + } +} diff --git a/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/SettingProxyFactoryImpl.java b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/SettingProxyFactoryImpl.java new file mode 100644 index 0000000000..c39ad8aa27 --- /dev/null +++ b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/SettingProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jVersion; +import org.neo4j.gds.compat.SettingProxyApi; +import org.neo4j.gds.compat.SettingProxyFactory; + +@ServiceProvider +public final class SettingProxyFactoryImpl implements SettingProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return version == Neo4jVersion.V_5_9; + } + + @Override + public SettingProxyApi load() { + return new SettingProxyImpl(); + } + + @Override + public String description() { + return "Neo4j Settings 5.9"; + } +} diff --git a/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/SettingProxyImpl.java b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/SettingProxyImpl.java new file mode 100644 index 0000000000..662daf64e7 --- /dev/null +++ b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/SettingProxyImpl.java @@ -0,0 +1,87 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.configuration.Config; +import org.neo4j.configuration.SettingBuilder; +import org.neo4j.dbms.systemgraph.TopologyGraphDbmsModel; +import org.neo4j.gds.compat.DatabaseMode; +import org.neo4j.gds.compat.SettingProxyApi; +import org.neo4j.graphdb.GraphDatabaseService; +import org.neo4j.graphdb.config.Setting; +import org.neo4j.kernel.impl.factory.GraphDatabaseFacade; +import org.neo4j.kernel.internal.GraphDatabaseAPI; + +public class SettingProxyImpl implements SettingProxyApi { + + @Override + public Setting setting(org.neo4j.gds.compat.Setting setting) { + var builder = SettingBuilder.newBuilder(setting.name(), setting.parser(), setting.defaultValue()); + if (setting.dynamic()) { + builder = builder.dynamic(); + } + if (setting.immutable()) { + builder = builder.immutable(); + } + setting.dependency().ifPresent(builder::setDependency); + setting.constraints().forEach(builder::addConstraint); + return builder.build(); + } + + @Override + public DatabaseMode databaseMode(Config config, GraphDatabaseService databaseService) { + return switch (((GraphDatabaseAPI) databaseService).mode()) { + case RAFT -> DatabaseMode.CORE; + case REPLICA -> DatabaseMode.READ_REPLICA; + case SINGLE -> DatabaseMode.SINGLE; + case VIRTUAL -> throw new UnsupportedOperationException("What's a virtual database anyway?"); + }; + } + + @Override + public void setDatabaseMode(Config config, DatabaseMode databaseMode, GraphDatabaseService databaseService) { + // super hacky, there is no way to set the mode of a database without restarting it + if (!(databaseService instanceof GraphDatabaseFacade db)) { + throw new IllegalArgumentException( + "Cannot set database mode on a database that is not a GraphDatabaseFacade"); + } + try { + var modeField = GraphDatabaseFacade.class.getDeclaredField("mode"); + modeField.setAccessible(true); + modeField.set(db, switch (databaseMode) { + case CORE -> TopologyGraphDbmsModel.HostedOnMode.RAFT; + case READ_REPLICA -> TopologyGraphDbmsModel.HostedOnMode.REPLICA; + case SINGLE -> TopologyGraphDbmsModel.HostedOnMode.SINGLE; + }); + } catch (NoSuchFieldException e) { + throw new RuntimeException( + "Could not set the mode field because it no longer exists. This compat layer needs to be updated.", + e + ); + } catch (IllegalAccessException e) { + throw new RuntimeException("Could not get the permissions to set the mode field.", e); + } + } + + @Override + public String secondaryModeName() { + return "Secondary"; + } +} diff --git a/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/TestLogImpl.java b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/TestLogImpl.java new file mode 100644 index 0000000000..7d09c9ce75 --- /dev/null +++ b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/TestLogImpl.java @@ -0,0 +1,146 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.gds.annotation.SuppressForbidden; +import org.neo4j.gds.compat.TestLog; + +import java.util.ArrayList; +import java.util.Locale; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.ConcurrentMap; + +public class TestLogImpl implements TestLog { + + private final ConcurrentMap> messages; + + TestLogImpl() { + messages = new ConcurrentHashMap<>(3); + } + + @Override + public void assertContainsMessage(String level, String fragment) { + if (!containsMessage(level, fragment)) { + throw new RuntimeException( + String.format( + Locale.US, + "Expected log output to contain `%s` for log level `%s`%nLog messages:%n%s", + fragment, + level, + String.join("\n", messages.get(level)) + ) + ); + } + } + + @Override + public boolean containsMessage(String level, String fragment) { + ConcurrentLinkedQueue messageList = messages.getOrDefault(level, new ConcurrentLinkedQueue<>()); + return messageList.stream().anyMatch((message) -> message.contains(fragment)); + } + + @Override + public boolean hasMessages(String level) { + return !messages.getOrDefault(level, new ConcurrentLinkedQueue<>()).isEmpty(); + } + + @Override + public ArrayList getMessages(String level) { + return new ArrayList<>(messages.getOrDefault(level, new ConcurrentLinkedQueue<>())); + } + + @SuppressForbidden(reason = "test log can print") + public void printMessages() { + System.out.println("TestLog Messages: " + messages); + } + + @Override + public boolean isDebugEnabled() { + return true; + } + + @Override + public void debug(String message) { + logMessage(DEBUG, message); + } + + @Override + public void debug(String message, Throwable throwable) { + debug(String.format(Locale.US, "%s - %s", message, throwable.getMessage())); + } + + @Override + public void debug(String format, Object... arguments) { + debug(String.format(Locale.US, format, arguments)); + } + + @Override + public void info(String message) { + logMessage(INFO, message); + } + + @Override + public void info(String message, Throwable throwable) { + info(String.format(Locale.US, "%s - %s", message, throwable.getMessage())); + } + + @Override + public void info(String format, Object... arguments) { + info(String.format(Locale.US, format, arguments)); + } + + @Override + public void warn(String message) { + logMessage(WARN, message); + } + + @Override + public void warn(String message, Throwable throwable) { + warn(String.format(Locale.US, "%s - %s", message, throwable.getMessage())); + } + + @Override + public void warn(String format, Object... arguments) { + warn(String.format(Locale.US, format, arguments)); + } + + @Override + public void error(String message) { + logMessage(ERROR, message); + } + + @Override + public void error(String message, Throwable throwable) { + error(String.format(Locale.US, "%s - %s", message, throwable.getMessage())); + } + + @Override + public void error(String format, Object... arguments) { + error(String.format(Locale.US, format, arguments)); + } + + private void logMessage(String level, String message) { + messages.computeIfAbsent( + level, + (ignore) -> new ConcurrentLinkedQueue<>() + ).add(message); + } +} diff --git a/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/VirtualRelationshipImpl.java b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/VirtualRelationshipImpl.java new file mode 100644 index 0000000000..07708b93b2 --- /dev/null +++ b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/VirtualRelationshipImpl.java @@ -0,0 +1,41 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.gds.compat.VirtualRelationship; +import org.neo4j.graphdb.Node; +import org.neo4j.graphdb.RelationshipType; + +public class VirtualRelationshipImpl extends VirtualRelationship { + + VirtualRelationshipImpl( + long id, + Node startNode, + Node endNode, + RelationshipType type + ) { + super(id, startNode, endNode, type); + } + + @Override + public String getElementId() { + return Long.toString(getId()); + } +} diff --git a/compatibility/5.9/storage-engine-adapter/build.gradle b/compatibility/5.9/storage-engine-adapter/build.gradle new file mode 100644 index 0000000000..501dac746d --- /dev/null +++ b/compatibility/5.9/storage-engine-adapter/build.gradle @@ -0,0 +1,68 @@ +apply plugin: 'java-library' +apply plugin: 'me.champeau.mrjar' + +description = 'Neo4j Graph Data Science :: Storage Engine Adapter 5.9' + +group = 'org.neo4j.gds' + +// for all 5.x versions +if (ver.'neo4j'.startsWith('5.')) { + sourceSets { + main { + java { + srcDirs = ['src/main/java17'] + } + } + } + + dependencies { + annotationProcessor project(':annotations') + annotationProcessor group: 'org.immutables', name: 'value', version: ver.'immutables' + annotationProcessor group: 'org.neo4j', name: 'annotations', version: neos.'5.9' + + compileOnly project(':annotations') + compileOnly project(':progress-tracking') + compileOnly group: 'org.immutables', name: 'value-annotations', version: ver.'immutables' + compileOnly group: 'org.neo4j', name: 'neo4j', version: neos.'5.9' + compileOnly group: 'org.neo4j', name: 'neo4j-record-storage-engine', version: neos.'5.9' + + implementation project(':core') + implementation project(':storage-engine-adapter-api') + implementation project(':config-api') + implementation project(':string-formatting') + } +} else { + multiRelease { + targetVersions 11, 17 + } + + if (!project.hasProperty('no-forbidden-apis')) { + forbiddenApisJava17 { + exclude('**') + } + } + + dependencies { + annotationProcessor group: 'org.neo4j', name: 'annotations', version: ver.'neo4j' + compileOnly group: 'org.neo4j', name: 'annotations', version: ver.'neo4j' + compileOnly group: 'org.neo4j', name: 'neo4j-kernel-api', version: ver.'neo4j' + + implementation project(':neo4j-adapter') + implementation project(':storage-engine-adapter-api') + + java17AnnotationProcessor project(':annotations') + java17AnnotationProcessor group: 'org.immutables', name: 'value', version: ver.'immutables' + java17AnnotationProcessor group: 'org.neo4j', name: 'annotations', version: neos.'5.9' + + java17CompileOnly project(':annotations') + java17CompileOnly project(':progress-tracking') + java17CompileOnly group: 'org.immutables', name: 'value-annotations', version: ver.'immutables' + java17CompileOnly group: 'org.neo4j', name: 'neo4j', version: neos.'5.9' + java17CompileOnly group: 'org.neo4j', name: 'neo4j-record-storage-engine', version: neos.'5.9' + + java17Implementation project(':core') + java17Implementation project(':storage-engine-adapter-api') + java17Implementation project(':config-api') + java17Implementation project(':string-formatting') + } +} diff --git a/compatibility/5.9/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_59/InMemoryStorageEngineFactory.java b/compatibility/5.9/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_59/InMemoryStorageEngineFactory.java new file mode 100644 index 0000000000..4bdc7bd36f --- /dev/null +++ b/compatibility/5.9/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_59/InMemoryStorageEngineFactory.java @@ -0,0 +1,268 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.configuration.Config; +import org.neo4j.dbms.database.readonly.DatabaseReadOnlyChecker; +import org.neo4j.index.internal.gbptree.RecoveryCleanupWorkCollector; +import org.neo4j.internal.id.IdController; +import org.neo4j.internal.id.IdGeneratorFactory; +import org.neo4j.internal.schema.IndexConfigCompleter; +import org.neo4j.internal.schema.SchemaRule; +import org.neo4j.internal.schema.SchemaState; +import org.neo4j.io.fs.FileSystemAbstraction; +import org.neo4j.io.layout.DatabaseLayout; +import org.neo4j.io.layout.Neo4jLayout; +import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.tracing.PageCacheTracer; +import org.neo4j.lock.LockService; +import org.neo4j.logging.LogProvider; +import org.neo4j.logging.internal.LogService; +import org.neo4j.memory.MemoryTracker; +import org.neo4j.monitoring.DatabaseHealth; +import org.neo4j.scheduler.JobScheduler; +import org.neo4j.storageengine.api.CommandReaderFactory; +import org.neo4j.storageengine.api.ConstraintRuleAccessor; +import org.neo4j.storageengine.api.LogVersionRepository; +import org.neo4j.storageengine.api.MetadataProvider; +import org.neo4j.storageengine.api.StorageEngine; +import org.neo4j.storageengine.api.StorageEngineFactory; +import org.neo4j.storageengine.api.StorageFilesState; +import org.neo4j.storageengine.api.StoreId; +import org.neo4j.storageengine.api.StoreVersion; +import org.neo4j.storageengine.api.StoreVersionCheck; +import org.neo4j.storageengine.api.TransactionIdStore; +import org.neo4j.storageengine.migration.RollingUpgradeCompatibility; +import org.neo4j.storageengine.migration.SchemaRuleMigrationAccess; +import org.neo4j.storageengine.migration.StoreMigrationParticipant; +import org.neo4j.token.TokenHolders; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.List; +import java.util.Optional; +import java.util.UUID; + +@ServiceProvider +public class InMemoryStorageEngineFactory implements StorageEngineFactory { + + @Override + public String name() { + return "unsupported59"; + } + + @Override + public StoreVersionCheck versionCheck( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + LogService logService, + PageCacheTracer pageCacheTracer + ) { + throw new UnsupportedOperationException("5.9 storage engine requires JDK17"); + } + + @Override + public StoreVersion versionInformation(String storeVersion) { + throw new UnsupportedOperationException("5.9 storage engine requires JDK17"); + } + + @Override + public StoreVersion versionInformation(StoreId storeId) { + throw new UnsupportedOperationException("5.9 storage engine requires JDK17"); + } + + @Override + public RollingUpgradeCompatibility rollingUpgradeCompatibility() { + throw new UnsupportedOperationException("5.9 storage engine requires JDK17"); + } + + @Override + public List migrationParticipants( + FileSystemAbstraction fs, + Config config, + PageCache pageCache, + JobScheduler jobScheduler, + LogService logService, + PageCacheTracer cacheTracer, + MemoryTracker memoryTracker + ) { + throw new UnsupportedOperationException("5.9 storage engine requires JDK17"); + } + + @Override + public StorageEngine instantiate( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + TokenHolders tokenHolders, + SchemaState schemaState, + ConstraintRuleAccessor constraintSemantics, + IndexConfigCompleter indexConfigCompleter, + LockService lockService, + IdGeneratorFactory idGeneratorFactory, + IdController idController, + DatabaseHealth databaseHealth, + LogProvider internalLogProvider, + LogProvider userLogProvider, + RecoveryCleanupWorkCollector recoveryCleanupWorkCollector, + PageCacheTracer cacheTracer, + boolean createStoreIfNotExists, + DatabaseReadOnlyChecker readOnlyChecker, + MemoryTracker memoryTracker + ) { + throw new UnsupportedOperationException("5.9 storage engine requires JDK17"); + } + + @Override + public List listStorageFiles(FileSystemAbstraction fileSystem, DatabaseLayout databaseLayout) throws + IOException { + throw new UnsupportedOperationException("5.9 storage engine requires JDK17"); + } + + @Override + public boolean storageExists(FileSystemAbstraction fileSystem, DatabaseLayout databaseLayout, PageCache pageCache) { + return false; + } + + @Override + public TransactionIdStore readOnlyTransactionIdStore( + FileSystemAbstraction filySystem, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) throws IOException { + throw new UnsupportedOperationException("5.9 storage engine requires JDK17"); + } + + @Override + public LogVersionRepository readOnlyLogVersionRepository( + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) throws IOException { + throw new UnsupportedOperationException("5.9 storage engine requires JDK17"); + } + + @Override + public MetadataProvider transactionMetaDataStore( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + PageCacheTracer cacheTracer, + DatabaseReadOnlyChecker readOnlyChecker + ) throws IOException { + throw new UnsupportedOperationException("5.9 storage engine requires JDK17"); + } + + @Override + public StoreId storeId( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) throws IOException { + throw new UnsupportedOperationException("5.9 storage engine requires JDK17"); + } + + @Override + public void setStoreId( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext, + StoreId storeId, + long upgradeTxChecksum, + long upgradeTxCommitTimestamp + ) throws IOException { + throw new UnsupportedOperationException("5.9 storage engine requires JDK17"); + } + + @Override + public void setExternalStoreUUID( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext, + UUID externalStoreId + ) throws IOException { + throw new UnsupportedOperationException("5.9 storage engine requires JDK17"); + } + + @Override + public Optional databaseIdUuid( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) { + throw new UnsupportedOperationException("5.9 storage engine requires JDK17"); + } + + @Override + public SchemaRuleMigrationAccess schemaRuleMigrationAccess( + FileSystemAbstraction fs, + PageCache pageCache, + Config config, + DatabaseLayout databaseLayout, + LogService logService, + String recordFormats, + PageCacheTracer cacheTracer, + CursorContext cursorContext, + MemoryTracker memoryTracker + ) { + throw new UnsupportedOperationException("5.9 storage engine requires JDK17"); + } + + @Override + public List loadSchemaRules( + FileSystemAbstraction fs, + PageCache pageCache, + Config config, + DatabaseLayout databaseLayout, + CursorContext cursorContext + ) { + throw new UnsupportedOperationException("5.9 storage engine requires JDK17"); + } + + @Override + public StorageFilesState checkStoreFileState( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache + ) { + throw new UnsupportedOperationException("5.9 storage engine requires JDK17"); + } + + @Override + public CommandReaderFactory commandReaderFactory() { + throw new UnsupportedOperationException("5.9 storage engine requires JDK17"); + } + + @Override + public DatabaseLayout databaseLayout(Neo4jLayout neo4jLayout, String databaseName) { + throw new UnsupportedOperationException("5.9 storage engine requires JDK17"); + } +} diff --git a/compatibility/5.9/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_59/StorageEngineProxyFactoryImpl.java b/compatibility/5.9/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_59/StorageEngineProxyFactoryImpl.java new file mode 100644 index 0000000000..c08f9a2716 --- /dev/null +++ b/compatibility/5.9/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_59/StorageEngineProxyFactoryImpl.java @@ -0,0 +1,31 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * This file contains proprietary code that is only available via a commercial license from Neo4j. + * For more information, see https://neo4j.com/contact-us/ + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jVersion; +import org.neo4j.gds.compat.StorageEngineProxyApi; +import org.neo4j.gds.compat.StorageEngineProxyFactory; + +@ServiceProvider +public class StorageEngineProxyFactoryImpl implements StorageEngineProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return false; + } + + @Override + public StorageEngineProxyApi load() { + throw new UnsupportedOperationException("5.9 storage engine requires JDK17"); + } + + @Override + public String description() { + return "Storage Engine 5.9"; + } +} diff --git a/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryCommandCreationContextImpl.java b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryCommandCreationContextImpl.java new file mode 100644 index 0000000000..6bad2296b0 --- /dev/null +++ b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryCommandCreationContextImpl.java @@ -0,0 +1,107 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.configuration.Config; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.kernel.KernelVersion; +import org.neo4j.kernel.KernelVersionProvider; +import org.neo4j.lock.LockTracer; +import org.neo4j.lock.ResourceLocker; +import org.neo4j.storageengine.api.CommandCreationContext; +import org.neo4j.storageengine.api.cursor.StoreCursors; + +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Supplier; + +public class InMemoryCommandCreationContextImpl implements CommandCreationContext { + + private final AtomicLong schemaTokens; + private final AtomicInteger propertyTokens; + private final AtomicInteger labelTokens; + private final AtomicInteger typeTokens; + + InMemoryCommandCreationContextImpl() { + this.schemaTokens = new AtomicLong(0); + this.propertyTokens = new AtomicInteger(0); + this.labelTokens = new AtomicInteger(0); + this.typeTokens = new AtomicInteger(0); + } + + @Override + public long reserveNode() { + throw new UnsupportedOperationException("Creating nodes is not supported"); + } + + @Override + public long reserveRelationship( + long sourceNode, + long targetNode, + int relationshipType, + boolean sourceNodeAddedInTx, + boolean targetNodeAddedInTx + ) { + throw new UnsupportedOperationException("Creating relationships is not supported"); + } + + @Override + public long reserveSchema() { + return schemaTokens.getAndIncrement(); + } + + @Override + public int reserveLabelTokenId() { + return labelTokens.getAndIncrement(); + } + + @Override + public int reservePropertyKeyTokenId() { + return propertyTokens.getAndIncrement(); + } + + @Override + public int reserveRelationshipTypeTokenId() { + return typeTokens.getAndIncrement(); + } + + @Override + public void close() { + + } + + @Override + public void initialize( + KernelVersionProvider kernelVersionProvider, + CursorContext cursorContext, + StoreCursors storeCursors, + Supplier oldestActiveTransactionSequenceNumber, + ResourceLocker locks, + Supplier lockTracer + ) { + + } + + @Override + public KernelVersion kernelVersion() { + // NOTE: Double-check if this is still correct when you copy this into a new compat layer + return KernelVersion.getLatestVersion(Config.newBuilder().build()); + } +} diff --git a/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryCountsStoreImpl.java b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryCountsStoreImpl.java new file mode 100644 index 0000000000..7cf4f836b7 --- /dev/null +++ b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryCountsStoreImpl.java @@ -0,0 +1,112 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.annotations.documented.ReporterFactory; +import org.neo4j.counts.CountsAccessor; +import org.neo4j.counts.CountsStorage; +import org.neo4j.counts.CountsVisitor; +import org.neo4j.gds.NodeLabel; +import org.neo4j.gds.api.GraphStore; +import org.neo4j.internal.helpers.progress.ProgressMonitorFactory; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.context.CursorContextFactory; +import org.neo4j.io.pagecache.tracing.FileFlushEvent; +import org.neo4j.memory.MemoryTracker; +import org.neo4j.storageengine.api.cursor.StoreCursors; +import org.neo4j.token.TokenHolders; +import org.neo4j.token.api.TokenNotFoundException; + +public class InMemoryCountsStoreImpl implements CountsStorage, CountsAccessor { + + private final GraphStore graphStore; + private final TokenHolders tokenHolders; + + public InMemoryCountsStoreImpl( + GraphStore graphStore, + TokenHolders tokenHolders + ) { + + this.graphStore = graphStore; + this.tokenHolders = tokenHolders; + } + + @Override + public void start( + CursorContext cursorContext, StoreCursors storeCursors, MemoryTracker memoryTracker + ) { + + } + + @Override + public void checkpoint(FileFlushEvent fileFlushEvent, CursorContext cursorContext) { + + } + + @Override + public long nodeCount(int labelId, CursorContext cursorContext) { + if (labelId == -1) { + return graphStore.nodeCount(); + } + + String nodeLabel; + try { + nodeLabel = tokenHolders.labelTokens().getTokenById(labelId).name(); + } catch (TokenNotFoundException e) { + throw new RuntimeException(e); + } + return graphStore.nodes().nodeCount(NodeLabel.of(nodeLabel)); + } + + @Override + public long relationshipCount(int startLabelId, int typeId, int endLabelId, CursorContext cursorContext) { + // TODO: this is quite wrong + return graphStore.relationshipCount(); + } + + @Override + public boolean consistencyCheck( + ReporterFactory reporterFactory, + CursorContextFactory contextFactory, + int numThreads, + ProgressMonitorFactory progressMonitorFactory + ) { + return true; + } + + @Override + public CountsAccessor.Updater apply(long txId, boolean isLast, CursorContext cursorContext) { + throw new UnsupportedOperationException("Updates are not supported"); + } + + @Override + public void close() { + + } + + @Override + public void accept(CountsVisitor visitor, CursorContext cursorContext) { + tokenHolders.labelTokens().getAllTokens().forEach(labelToken -> { + visitor.visitNodeCount(labelToken.id(), nodeCount(labelToken.id(), cursorContext)); + }); + + visitor.visitRelationshipCount(-1, -1, -1, graphStore.relationshipCount()); + } +} diff --git a/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryMetaDataProviderImpl.java b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryMetaDataProviderImpl.java new file mode 100644 index 0000000000..77f9b72465 --- /dev/null +++ b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryMetaDataProviderImpl.java @@ -0,0 +1,201 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.internal.recordstorage.InMemoryLogVersionRepository59; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.context.TransactionIdSnapshot; +import org.neo4j.storageengine.api.ClosedTransactionMetadata; +import org.neo4j.storageengine.api.ExternalStoreId; +import org.neo4j.storageengine.api.MetadataProvider; +import org.neo4j.storageengine.api.StoreId; +import org.neo4j.storageengine.api.TransactionId; + +import java.io.IOException; +import java.util.Optional; +import java.util.UUID; + +public class InMemoryMetaDataProviderImpl implements MetadataProvider { + + private final ExternalStoreId externalStoreId; + private final InMemoryLogVersionRepository59 logVersionRepository; + private final InMemoryTransactionIdStoreImpl transactionIdStore; + + InMemoryMetaDataProviderImpl() { + this.logVersionRepository = new InMemoryLogVersionRepository59(); + this.externalStoreId = new ExternalStoreId(UUID.randomUUID()); + this.transactionIdStore = new InMemoryTransactionIdStoreImpl(); + } + + @Override + public ExternalStoreId getExternalStoreId() { + return this.externalStoreId; + } + + @Override + public ClosedTransactionMetadata getLastClosedTransaction() { + return this.transactionIdStore.getLastClosedTransaction(); + } + + @Override + public void setCurrentLogVersion(long version) { + logVersionRepository.setCurrentLogVersion(version); + } + + @Override + public long incrementAndGetVersion() { + return logVersionRepository.incrementAndGetVersion(); + } + + @Override + public void setCheckpointLogVersion(long version) { + logVersionRepository.setCheckpointLogVersion(version); + } + + @Override + public long incrementAndGetCheckpointLogVersion() { + return logVersionRepository.incrementAndGetCheckpointLogVersion(); + } + + @Override + public void transactionCommitted(long transactionId, int checksum, long commitTimestamp, long consensusIndex) { + transactionIdStore.transactionCommitted(transactionId, checksum, commitTimestamp, consensusIndex); + } + + @Override + public void setLastCommittedAndClosedTransactionId( + long transactionId, + int checksum, + long commitTimestamp, + long consensusIndex, + long byteOffset, + long logVersion + ) { + transactionIdStore.setLastCommittedAndClosedTransactionId( + transactionId, + checksum, + commitTimestamp, + consensusIndex, + byteOffset, + logVersion + ); + } + + @Override + public void transactionClosed( + long transactionId, + long logVersion, + long byteOffset, + int checksum, + long commitTimestamp, + long consensusIndex + ) { + this.transactionIdStore.transactionClosed( + transactionId, + logVersion, + byteOffset, + checksum, + commitTimestamp, + consensusIndex + ); + } + + @Override + public void resetLastClosedTransaction( + long transactionId, + long logVersion, + long byteOffset, + int checksum, + long commitTimestamp, + long consensusIndex + ) { + this.transactionIdStore.resetLastClosedTransaction( + transactionId, + logVersion, + byteOffset, + checksum, + commitTimestamp, + consensusIndex + ); + } + + @Override + public TransactionIdSnapshot getClosedTransactionSnapshot() { + return new TransactionIdSnapshot(this.getLastClosedTransactionId()); + } + + @Override + public void regenerateMetadata(StoreId storeId, UUID externalStoreUUID, CursorContext cursorContext) { + } + + @Override + public StoreId getStoreId() { + return StoreId.UNKNOWN; + } + + @Override + public void close() throws IOException { + } + + @Override + public long getCurrentLogVersion() { + return this.logVersionRepository.getCurrentLogVersion(); + } + + @Override + public long getCheckpointLogVersion() { + return this.logVersionRepository.getCheckpointLogVersion(); + } + + @Override + public long nextCommittingTransactionId() { + return this.transactionIdStore.nextCommittingTransactionId(); + } + + @Override + public long committingTransactionId() { + return this.transactionIdStore.committingTransactionId(); + } + + @Override + public long getLastCommittedTransactionId() { + return this.transactionIdStore.getLastCommittedTransactionId(); + } + + @Override + public TransactionId getLastCommittedTransaction() { + return this.transactionIdStore.getLastCommittedTransaction(); + } + + @Override + public long getLastClosedTransactionId() { + return this.transactionIdStore.getLastClosedTransactionId(); + } + + @Override + public Optional getDatabaseIdUuid(CursorContext cursorTracer) { + throw new IllegalStateException("Not supported"); + } + + @Override + public void setDatabaseIdUuid(UUID uuid, CursorContext cursorContext) { + throw new IllegalStateException("Not supported"); + } +} diff --git a/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryNodeCursor.java b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryNodeCursor.java new file mode 100644 index 0000000000..15d394b158 --- /dev/null +++ b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryNodeCursor.java @@ -0,0 +1,82 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.gds.api.GraphStore; +import org.neo4j.gds.compat.AbstractInMemoryNodeCursor; +import org.neo4j.storageengine.api.AllNodeScan; +import org.neo4j.storageengine.api.Degrees; +import org.neo4j.storageengine.api.LongReference; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.storageengine.api.RelationshipSelection; +import org.neo4j.storageengine.api.StoragePropertyCursor; +import org.neo4j.storageengine.api.StorageRelationshipTraversalCursor; +import org.neo4j.token.TokenHolders; + +public class InMemoryNodeCursor extends AbstractInMemoryNodeCursor { + + public InMemoryNodeCursor(GraphStore graphStore, TokenHolders tokenHolders) { + super(graphStore, tokenHolders); + } + + @Override + public boolean hasLabel() { + return hasAtLeastOneLabelForCurrentNode(); + } + + @Override + public Reference propertiesReference() { + return LongReference.longReference(getId()); + } + + @Override + public void properties(StoragePropertyCursor propertyCursor, PropertySelection selection) { + propertyCursor.initNodeProperties(propertiesReference(), selection); + } + + @Override + public void properties(StoragePropertyCursor propertyCursor) { + properties(propertyCursor, PropertySelection.ALL_PROPERTIES); + } + + @Override + public boolean supportsFastRelationshipsTo() { + return false; + } + + @Override + public void relationshipsTo( + StorageRelationshipTraversalCursor storageRelationshipTraversalCursor, + RelationshipSelection relationshipSelection, + long neighbourNodeReference + ) { + throw new UnsupportedOperationException(); + } + + @Override + public void degrees(RelationshipSelection selection, Degrees.Mutator mutator) { + } + + @Override + public boolean scanBatch(AllNodeScan allNodeScan, long sizeHint) { + return super.scanBatch(allNodeScan, (int) sizeHint); + } +} diff --git a/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryNodePropertyCursor.java b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryNodePropertyCursor.java new file mode 100644 index 0000000000..43b8fd4837 --- /dev/null +++ b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryNodePropertyCursor.java @@ -0,0 +1,45 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.gds.compat.AbstractInMemoryNodePropertyCursor; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.storageengine.api.LongReference; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.token.TokenHolders; + +public class InMemoryNodePropertyCursor extends AbstractInMemoryNodePropertyCursor { + + public InMemoryNodePropertyCursor(CypherGraphStore graphStore, TokenHolders tokenHolders) { + super(graphStore, tokenHolders); + } + + @Override + public void initNodeProperties(Reference reference, PropertySelection selection, long ownerReference) { + reset(); + setId(((LongReference) reference).id); + setPropertySelection(new InMemoryPropertySelectionImpl(selection)); + } + + @Override + public void initRelationshipProperties(Reference reference, PropertySelection selection, long ownerReference) { + } +} diff --git a/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryPropertyCursor.java b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryPropertyCursor.java new file mode 100644 index 0000000000..2b6499a1f9 --- /dev/null +++ b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryPropertyCursor.java @@ -0,0 +1,71 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.gds.compat.AbstractInMemoryPropertyCursor; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.storageengine.api.StorageNodeCursor; +import org.neo4j.storageengine.api.StorageRelationshipCursor; +import org.neo4j.token.TokenHolders; + +public class InMemoryPropertyCursor extends AbstractInMemoryPropertyCursor { + + public InMemoryPropertyCursor(CypherGraphStore graphStore, TokenHolders tokenHolders) { + super(graphStore, tokenHolders); + } + + @Override + public void initNodeProperties(Reference reference, PropertySelection selection, long ownerReference) { + if (this.delegate == null || !(this.delegate instanceof InMemoryNodePropertyCursor)) { + this.delegate = new InMemoryNodePropertyCursor(graphStore, tokenHolders); + } + + ((InMemoryNodePropertyCursor) delegate).initNodeProperties(reference, selection); + } + + @Override + public void initNodeProperties(StorageNodeCursor nodeCursor, PropertySelection selection) { + if (this.delegate == null || !(this.delegate instanceof InMemoryNodePropertyCursor)) { + this.delegate = new InMemoryNodePropertyCursor(graphStore, tokenHolders); + } + + ((InMemoryNodePropertyCursor) delegate).initNodeProperties(nodeCursor, selection); + } + + @Override + public void initRelationshipProperties(StorageRelationshipCursor relationshipCursor, PropertySelection selection) { + if (this.delegate == null || !(this.delegate instanceof InMemoryRelationshipPropertyCursor)) { + this.delegate = new InMemoryRelationshipPropertyCursor(graphStore, tokenHolders); + } + + ((InMemoryRelationshipPropertyCursor) delegate).initRelationshipProperties(relationshipCursor, selection); + } + + @Override + public void initRelationshipProperties(Reference reference, PropertySelection selection, long ownerReference) { + if (this.delegate == null || !(this.delegate instanceof InMemoryRelationshipPropertyCursor)) { + this.delegate = new InMemoryRelationshipPropertyCursor(graphStore, tokenHolders); + } + + ((InMemoryRelationshipPropertyCursor) delegate).initRelationshipProperties(reference, selection); + } +} diff --git a/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryPropertySelectionImpl.java b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryPropertySelectionImpl.java new file mode 100644 index 0000000000..5724e2829d --- /dev/null +++ b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryPropertySelectionImpl.java @@ -0,0 +1,55 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.gds.compat.InMemoryPropertySelection; +import org.neo4j.storageengine.api.PropertySelection; + +public class InMemoryPropertySelectionImpl implements InMemoryPropertySelection { + + private final PropertySelection propertySelection; + + public InMemoryPropertySelectionImpl(PropertySelection propertySelection) {this.propertySelection = propertySelection;} + + @Override + public boolean isLimited() { + return propertySelection.isLimited(); + } + + @Override + public int numberOfKeys() { + return propertySelection.numberOfKeys(); + } + + @Override + public int key(int index) { + return propertySelection.key(index); + } + + @Override + public boolean test(int key) { + return propertySelection.test(key); + } + + @Override + public boolean isKeysOnly() { + return propertySelection.isKeysOnly(); + } +} diff --git a/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryRelationshipPropertyCursor.java b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryRelationshipPropertyCursor.java new file mode 100644 index 0000000000..806f0b2565 --- /dev/null +++ b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryRelationshipPropertyCursor.java @@ -0,0 +1,60 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.gds.compat.AbstractInMemoryRelationshipPropertyCursor; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.gds.storageengine.InMemoryRelationshipCursor; +import org.neo4j.storageengine.api.LongReference; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.storageengine.api.StorageRelationshipCursor; +import org.neo4j.token.TokenHolders; + +public class InMemoryRelationshipPropertyCursor extends AbstractInMemoryRelationshipPropertyCursor { + + InMemoryRelationshipPropertyCursor(CypherGraphStore graphStore, TokenHolders tokenHolders) { + super(graphStore, tokenHolders); + } + + @Override + public void initNodeProperties( + Reference reference, PropertySelection propertySelection, long ownerReference + ) { + + } + + @Override + public void initRelationshipProperties( + Reference reference, PropertySelection propertySelection, long ownerReference + ) { + var relationshipId = ((LongReference) reference).id; + var relationshipCursor = new InMemoryRelationshipScanCursor(graphStore, tokenHolders); + relationshipCursor.single(relationshipId); + relationshipCursor.next(); + relationshipCursor.properties(this, new InMemoryPropertySelectionImpl(propertySelection)); + } + + @Override + public void initRelationshipProperties(StorageRelationshipCursor relationshipCursor, PropertySelection selection) { + var inMemoryRelationshipCursor = (InMemoryRelationshipCursor) relationshipCursor; + inMemoryRelationshipCursor.properties(this, selection); + } +} diff --git a/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryRelationshipScanCursor.java b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryRelationshipScanCursor.java new file mode 100644 index 0000000000..6f0c089ee8 --- /dev/null +++ b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryRelationshipScanCursor.java @@ -0,0 +1,61 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.internal.recordstorage.AbstractInMemoryRelationshipScanCursor; +import org.neo4j.storageengine.api.AllRelationshipsScan; +import org.neo4j.storageengine.api.LongReference; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.storageengine.api.StoragePropertyCursor; +import org.neo4j.token.TokenHolders; + +public class InMemoryRelationshipScanCursor extends AbstractInMemoryRelationshipScanCursor { + + public InMemoryRelationshipScanCursor( + CypherGraphStore graphStore, + TokenHolders tokenHolders + ) { + super(graphStore, tokenHolders); + } + + @Override + public void single(long reference, long sourceNodeReference, int type, long targetNodeReference) { + single(reference); + } + + @Override + public Reference propertiesReference() { + return LongReference.longReference(getId()); + } + + @Override + public void properties( + StoragePropertyCursor storagePropertyCursor, PropertySelection propertySelection + ) { + properties(storagePropertyCursor, new InMemoryPropertySelectionImpl(propertySelection)); + } + + @Override + public boolean scanBatch(AllRelationshipsScan allRelationshipsScan, long sizeHint) { + return super.scanBatch(allRelationshipsScan, (int) sizeHint); + } +} diff --git a/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryRelationshipTraversalCursor.java b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryRelationshipTraversalCursor.java new file mode 100644 index 0000000000..6cd7ebf031 --- /dev/null +++ b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryRelationshipTraversalCursor.java @@ -0,0 +1,47 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.gds.compat.AbstractInMemoryRelationshipTraversalCursor; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.storageengine.api.LongReference; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.storageengine.api.StoragePropertyCursor; +import org.neo4j.token.TokenHolders; + +public class InMemoryRelationshipTraversalCursor extends AbstractInMemoryRelationshipTraversalCursor { + + public InMemoryRelationshipTraversalCursor(CypherGraphStore graphStore, TokenHolders tokenHolders) { + super(graphStore, tokenHolders); + } + + @Override + public Reference propertiesReference() { + return LongReference.longReference(getId()); + } + + @Override + public void properties( + StoragePropertyCursor propertyCursor, PropertySelection selection + ) { + properties(propertyCursor, new InMemoryPropertySelectionImpl(selection)); + } +} diff --git a/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryStorageEngineFactory.java b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryStorageEngineFactory.java new file mode 100644 index 0000000000..95bd036fb0 --- /dev/null +++ b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryStorageEngineFactory.java @@ -0,0 +1,554 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.eclipse.collections.api.factory.Sets; +import org.eclipse.collections.api.set.ImmutableSet; +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.configuration.Config; +import org.neo4j.consistency.checking.ConsistencyFlags; +import org.neo4j.consistency.report.ConsistencySummaryStatistics; +import org.neo4j.dbms.database.readonly.DatabaseReadOnlyChecker; +import org.neo4j.function.ThrowingSupplier; +import org.neo4j.gds.annotation.SuppressForbidden; +import org.neo4j.gds.compat.Neo4jVersion; +import org.neo4j.gds.compat.StorageEngineFactoryIdProvider; +import org.neo4j.gds.compat.StorageEngineProxyApi; +import org.neo4j.index.internal.gbptree.RecoveryCleanupWorkCollector; +import org.neo4j.internal.batchimport.AdditionalInitialIds; +import org.neo4j.internal.batchimport.BatchImporter; +import org.neo4j.internal.batchimport.Configuration; +import org.neo4j.internal.batchimport.IncrementalBatchImporter; +import org.neo4j.internal.batchimport.IndexImporterFactory; +import org.neo4j.internal.batchimport.Monitor; +import org.neo4j.internal.batchimport.ReadBehaviour; +import org.neo4j.internal.batchimport.input.Collector; +import org.neo4j.internal.batchimport.input.Input; +import org.neo4j.internal.batchimport.input.LenientStoreInput; +import org.neo4j.internal.id.IdGeneratorFactory; +import org.neo4j.internal.id.ScanOnOpenReadOnlyIdGeneratorFactory; +import org.neo4j.internal.recordstorage.InMemoryStorageCommandReaderFactory59; +import org.neo4j.internal.recordstorage.StoreTokens; +import org.neo4j.internal.schema.IndexConfigCompleter; +import org.neo4j.internal.schema.SchemaRule; +import org.neo4j.internal.schema.SchemaState; +import org.neo4j.io.fs.FileSystemAbstraction; +import org.neo4j.io.layout.DatabaseLayout; +import org.neo4j.io.layout.Neo4jLayout; +import org.neo4j.io.layout.recordstorage.RecordDatabaseLayout; +import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.context.CursorContextFactory; +import org.neo4j.io.pagecache.tracing.PageCacheTracer; +import org.neo4j.kernel.KernelVersionRepository; +import org.neo4j.kernel.api.index.IndexProvidersAccess; +import org.neo4j.kernel.impl.api.index.IndexProviderMap; +import org.neo4j.kernel.impl.locking.LockManager; +import org.neo4j.kernel.impl.store.MetaDataStore; +import org.neo4j.kernel.impl.store.NeoStores; +import org.neo4j.kernel.impl.store.StoreFactory; +import org.neo4j.kernel.impl.store.StoreType; +import org.neo4j.kernel.impl.store.cursor.CachedStoreCursors; +import org.neo4j.kernel.impl.transaction.log.LogTailLogVersionsMetadata; +import org.neo4j.kernel.impl.transaction.log.LogTailMetadata; +import org.neo4j.lock.LockService; +import org.neo4j.logging.InternalLog; +import org.neo4j.logging.InternalLogProvider; +import org.neo4j.logging.NullLogProvider; +import org.neo4j.logging.internal.LogService; +import org.neo4j.memory.MemoryTracker; +import org.neo4j.monitoring.DatabaseHealth; +import org.neo4j.scheduler.JobScheduler; +import org.neo4j.storageengine.api.CommandReaderFactory; +import org.neo4j.storageengine.api.ConstraintRuleAccessor; +import org.neo4j.storageengine.api.LogFilesInitializer; +import org.neo4j.storageengine.api.MetadataProvider; +import org.neo4j.storageengine.api.SchemaRule44; +import org.neo4j.storageengine.api.StorageEngine; +import org.neo4j.storageengine.api.StorageEngineFactory; +import org.neo4j.storageengine.api.StorageFilesState; +import org.neo4j.storageengine.api.StoreId; +import org.neo4j.storageengine.api.StoreVersion; +import org.neo4j.storageengine.api.StoreVersionCheck; +import org.neo4j.storageengine.api.StoreVersionIdentifier; +import org.neo4j.storageengine.migration.StoreMigrationParticipant; +import org.neo4j.time.SystemNanoClock; +import org.neo4j.token.DelegatingTokenHolder; +import org.neo4j.token.ReadOnlyTokenCreator; +import org.neo4j.token.TokenHolders; +import org.neo4j.token.api.NamedToken; +import org.neo4j.token.api.TokenHolder; +import org.neo4j.token.api.TokensLoader; + +import java.io.IOException; +import java.io.OutputStream; +import java.io.PrintStream; +import java.io.UncheckedIOException; +import java.nio.file.OpenOption; +import java.nio.file.Path; +import java.time.Clock; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.UUID; +import java.util.function.Function; + +@ServiceProvider +public class InMemoryStorageEngineFactory implements StorageEngineFactory { + + static final String IN_MEMORY_STORAGE_ENGINE_NAME = "in-memory-59"; + + public InMemoryStorageEngineFactory() { + StorageEngineProxyApi.requireNeo4jVersion(Neo4jVersion.V_5_9, StorageEngineFactory.class); + } + + @Override + public byte id() { + return StorageEngineFactoryIdProvider.ID; + } + + @Override + public boolean storageExists(FileSystemAbstraction fileSystem, DatabaseLayout databaseLayout) { + return false; + } + + @Override + public StorageEngine instantiate( + FileSystemAbstraction fs, + Clock clock, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + TokenHolders tokenHolders, + SchemaState schemaState, + ConstraintRuleAccessor constraintSemantics, + IndexConfigCompleter indexConfigCompleter, + LockService lockService, + IdGeneratorFactory idGeneratorFactory, + DatabaseHealth databaseHealth, + InternalLogProvider internalLogProvider, + InternalLogProvider userLogProvider, + RecoveryCleanupWorkCollector recoveryCleanupWorkCollector, + LogTailMetadata logTailMetadata, + KernelVersionRepository kernelVersionRepository, + MemoryTracker memoryTracker, + CursorContextFactory contextFactory, + PageCacheTracer pageCacheTracer + ) { + StoreFactory factory = new StoreFactory( + databaseLayout, + config, + idGeneratorFactory, + pageCache, + pageCacheTracer, + fs, + internalLogProvider, + contextFactory, + false, + logTailMetadata + ); + + factory.openNeoStores(StoreType.LABEL_TOKEN).close(); + + return new InMemoryStorageEngineImpl( + databaseLayout, + tokenHolders + ); + } + + @Override + public Optional databaseIdUuid( + FileSystemAbstraction fs, DatabaseLayout databaseLayout, PageCache pageCache, CursorContext cursorContext + ) { + var fieldAccess = MetaDataStore.getFieldAccess( + pageCache, + RecordDatabaseLayout.convert(databaseLayout).metadataStore(), + databaseLayout.getDatabaseName(), + cursorContext + ); + + try { + return fieldAccess.readDatabaseUUID(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + @Override + public List migrationParticipants( + FileSystemAbstraction fileSystemAbstraction, + Config config, + PageCache pageCache, + JobScheduler jobScheduler, + LogService logService, + MemoryTracker memoryTracker, + PageCacheTracer pageCacheTracer, + CursorContextFactory cursorContextFactory, + boolean b + ) { + return List.of(); + } + + @Override + public DatabaseLayout databaseLayout( + Neo4jLayout neo4jLayout, String databaseName + ) { + return RecordDatabaseLayout.of(neo4jLayout, databaseName); + } + + @Override + public DatabaseLayout formatSpecificDatabaseLayout(DatabaseLayout plainLayout) { + return databaseLayout(plainLayout.getNeo4jLayout(), plainLayout.getDatabaseName()); + } + + @SuppressForbidden(reason = "This is the compat layer and we don't really need to go through the proxy") + @Override + public BatchImporter batchImporter( + DatabaseLayout databaseLayout, + FileSystemAbstraction fileSystemAbstraction, + PageCacheTracer pageCacheTracer, + Configuration configuration, + LogService logService, + PrintStream printStream, + boolean b, + AdditionalInitialIds additionalInitialIds, + Config config, + Monitor monitor, + JobScheduler jobScheduler, + Collector collector, + LogFilesInitializer logFilesInitializer, + IndexImporterFactory indexImporterFactory, + MemoryTracker memoryTracker, + CursorContextFactory cursorContextFactory + ) { + throw new UnsupportedOperationException("Batch Import into GDS is not supported"); + } + + @Override + public Input asBatchImporterInput( + DatabaseLayout databaseLayout, + FileSystemAbstraction fileSystemAbstraction, + PageCache pageCache, + PageCacheTracer pageCacheTracer, + Config config, + MemoryTracker memoryTracker, + ReadBehaviour readBehaviour, + boolean b, + CursorContextFactory cursorContextFactory, + LogTailMetadata logTailMetadata + ) { + NeoStores neoStores = (new StoreFactory( + databaseLayout, + config, + new ScanOnOpenReadOnlyIdGeneratorFactory(), + pageCache, + pageCacheTracer, + fileSystemAbstraction, + NullLogProvider.getInstance(), + cursorContextFactory, + false, + logTailMetadata + )).openAllNeoStores(); + return new LenientStoreInput( + neoStores, + readBehaviour.decorateTokenHolders(this.loadReadOnlyTokens(neoStores, true, cursorContextFactory)), + true, + cursorContextFactory, + readBehaviour + ); + } + + @Override + public long optimalAvailableConsistencyCheckerMemory( + FileSystemAbstraction fileSystemAbstraction, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache + ) { + return 0; + } + + @Override + public String name() { + return IN_MEMORY_STORAGE_ENGINE_NAME; + } + + @Override + public Set supportedFormats(boolean includeFormatsUnderDevelopment) { + return Set.of(IN_MEMORY_STORAGE_ENGINE_NAME); + } + + @Override + public boolean supportedFormat(String format, boolean includeFormatsUnderDevelopment) { + return format.equals(IN_MEMORY_STORAGE_ENGINE_NAME); + } + + @Override + public MetadataProvider transactionMetaDataStore( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + DatabaseReadOnlyChecker readOnlyChecker, + CursorContextFactory contextFactory, + LogTailLogVersionsMetadata logTailMetadata, + PageCacheTracer pageCacheTracer + ) { + return new InMemoryMetaDataProviderImpl(); + } + + @Override + public StoreVersionCheck versionCheck( + FileSystemAbstraction fileSystemAbstraction, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + LogService logService, + CursorContextFactory cursorContextFactory + ) { + return new InMemoryVersionCheck(); + } + + @Override + public List loadSchemaRules( + FileSystemAbstraction fileSystemAbstraction, + PageCache pageCache, + PageCacheTracer pageCacheTracer, + Config config, + DatabaseLayout databaseLayout, + boolean b, + Function function, + CursorContextFactory cursorContextFactory + ) { + return List.of(); + } + + @Override + public List load44SchemaRules( + FileSystemAbstraction fs, + PageCache pageCache, + PageCacheTracer pageCacheTracer, + Config config, + DatabaseLayout databaseLayout, + CursorContextFactory contextFactory, + LogTailLogVersionsMetadata logTailMetadata + ) { + return List.of(); + } + + @Override + public TokenHolders loadReadOnlyTokens( + FileSystemAbstraction fileSystemAbstraction, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + PageCacheTracer pageCacheTracer, + boolean lenient, + CursorContextFactory cursorContextFactory + ) { + StoreFactory factory = new StoreFactory( + databaseLayout, + config, + new ScanOnOpenReadOnlyIdGeneratorFactory(), + pageCache, + pageCacheTracer, + fileSystemAbstraction, + NullLogProvider.getInstance(), + cursorContextFactory, + false, + LogTailMetadata.EMPTY_LOG_TAIL + ); + try ( NeoStores stores = factory.openNeoStores( + StoreType.PROPERTY_KEY_TOKEN, StoreType.PROPERTY_KEY_TOKEN_NAME, + StoreType.LABEL_TOKEN, StoreType.LABEL_TOKEN_NAME, + StoreType.RELATIONSHIP_TYPE_TOKEN, StoreType.RELATIONSHIP_TYPE_TOKEN_NAME ) ) + { + return loadReadOnlyTokens(stores, lenient, cursorContextFactory); + } + } + + private TokenHolders loadReadOnlyTokens( + NeoStores stores, + boolean lenient, + CursorContextFactory cursorContextFactory + ) + { + try ( var cursorContext = cursorContextFactory.create("loadReadOnlyTokens"); + var storeCursors = new CachedStoreCursors( stores, cursorContext ) ) + { + stores.start( cursorContext ); + TokensLoader loader = lenient ? StoreTokens.allReadableTokens( stores ) : StoreTokens.allTokens( stores ); + TokenHolder propertyKeys = new DelegatingTokenHolder( ReadOnlyTokenCreator.READ_ONLY, TokenHolder.TYPE_PROPERTY_KEY ); + TokenHolder labels = new DelegatingTokenHolder( ReadOnlyTokenCreator.READ_ONLY, TokenHolder.TYPE_LABEL ); + TokenHolder relationshipTypes = new DelegatingTokenHolder( ReadOnlyTokenCreator.READ_ONLY, TokenHolder.TYPE_RELATIONSHIP_TYPE ); + + propertyKeys.setInitialTokens( lenient ? unique( loader.getPropertyKeyTokens( storeCursors ) ) : loader.getPropertyKeyTokens( storeCursors ) ); + labels.setInitialTokens( lenient ? unique( loader.getLabelTokens( storeCursors ) ) : loader.getLabelTokens( storeCursors ) ); + relationshipTypes.setInitialTokens( + lenient ? unique( loader.getRelationshipTypeTokens( storeCursors ) ) : loader.getRelationshipTypeTokens( storeCursors ) ); + return new TokenHolders( propertyKeys, labels, relationshipTypes ); + } + catch ( IOException e ) + { + throw new UncheckedIOException( e ); + } + } + + private static List unique( List tokens ) + { + if ( !tokens.isEmpty() ) + { + Set names = new HashSet<>( tokens.size() ); + int i = 0; + while ( i < tokens.size() ) + { + if ( names.add( tokens.get( i ).name() ) ) + { + i++; + } + else + { + // Remove the token at the given index, by replacing it with the last token in the list. + // This changes the order of elements, but can be done in constant time instead of linear time. + int lastIndex = tokens.size() - 1; + NamedToken endToken = tokens.remove( lastIndex ); + if ( i < lastIndex ) + { + tokens.set( i, endToken ); + } + } + } + } + return tokens; + } + + @Override + public CommandReaderFactory commandReaderFactory() { + return InMemoryStorageCommandReaderFactory59.INSTANCE; + } + + @Override + public void consistencyCheck( + FileSystemAbstraction fileSystem, + DatabaseLayout layout, + Config config, + PageCache pageCache, + IndexProviderMap indexProviders, + InternalLog log, + ConsistencySummaryStatistics summary, + int numberOfThreads, + long maxOffHeapCachingMemory, + OutputStream progressOutput, + boolean verbose, + ConsistencyFlags flags, + CursorContextFactory contextFactory, + PageCacheTracer pageCacheTracer, + LogTailMetadata logTailMetadata + ) { + // we can do no-op, since our "database" is _always_ consistent + } + + @Override + public ImmutableSet getStoreOpenOptions( + FileSystemAbstraction fs, + PageCache pageCache, + DatabaseLayout layout, + CursorContextFactory contextFactory + ) { + // Not sure about this, empty set is returned when the store files are in `little-endian` format + // See: `org.neo4j.kernel.impl.store.format.PageCacheOptionsSelector.select` + return Sets.immutable.empty(); + } + + @Override + public StoreId retrieveStoreId( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) throws IOException { + return StoreId.retrieveFromStore(fs, databaseLayout, pageCache, cursorContext); + } + + + @Override + public Optional versionInformation(StoreVersionIdentifier storeVersionIdentifier) { + return Optional.of(new InMemoryStoreVersion()); + } + + @Override + public void resetMetadata( + FileSystemAbstraction fileSystemAbstraction, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + CursorContextFactory cursorContextFactory, + PageCacheTracer pageCacheTracer, + StoreId storeId, + UUID externalStoreId + ) { + throw new UnsupportedOperationException(); + } + + @Override + public IncrementalBatchImporter incrementalBatchImporter( + DatabaseLayout databaseLayout, + FileSystemAbstraction fileSystem, + PageCacheTracer pageCacheTracer, + Configuration config, + LogService logService, + PrintStream progressOutput, + boolean verboseProgressOutput, + AdditionalInitialIds additionalInitialIds, + ThrowingSupplier logTailMetadataSupplier, + Config dbConfig, + Monitor monitor, + JobScheduler jobScheduler, + Collector badCollector, + LogFilesInitializer logFilesInitializer, + IndexImporterFactory indexImporterFactory, + MemoryTracker memoryTracker, + CursorContextFactory contextFactory, + IndexProvidersAccess indexProvidersAccess + ) { + throw new UnsupportedOperationException(); + } + + @Override + public LockManager createLockManager(Config config, SystemNanoClock clock) { + return LockManager.NO_LOCKS_LOCK_MANAGER; + } + + @Override + public List listStorageFiles( + FileSystemAbstraction fileSystem, DatabaseLayout databaseLayout + ) { + return Collections.emptyList(); + } + + @Override + public StorageFilesState checkStoreFileState( + FileSystemAbstraction fs, DatabaseLayout databaseLayout, PageCache pageCache + ) { + return StorageFilesState.recoveredState(); + } +} diff --git a/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryStorageEngineImpl.java b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryStorageEngineImpl.java new file mode 100644 index 0000000000..e52651080d --- /dev/null +++ b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryStorageEngineImpl.java @@ -0,0 +1,341 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.configuration.Config; +import org.neo4j.counts.CountsAccessor; +import org.neo4j.exceptions.KernelException; +import org.neo4j.gds.compat.TokenManager; +import org.neo4j.gds.config.GraphProjectConfig; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.gds.core.loading.GraphStoreCatalog; +import org.neo4j.gds.storageengine.InMemoryDatabaseCreationCatalog; +import org.neo4j.gds.storageengine.InMemoryTransactionStateVisitor; +import org.neo4j.internal.diagnostics.DiagnosticsLogger; +import org.neo4j.internal.recordstorage.InMemoryStorageReader59; +import org.neo4j.internal.schema.StorageEngineIndexingBehaviour; +import org.neo4j.io.layout.DatabaseLayout; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.tracing.DatabaseFlushEvent; +import org.neo4j.kernel.KernelVersion; +import org.neo4j.kernel.impl.store.stats.StoreEntityCounters; +import org.neo4j.kernel.lifecycle.Lifecycle; +import org.neo4j.kernel.lifecycle.LifecycleAdapter; +import org.neo4j.lock.LockGroup; +import org.neo4j.lock.LockService; +import org.neo4j.lock.LockTracer; +import org.neo4j.lock.ResourceLocker; +import org.neo4j.logging.InternalLog; +import org.neo4j.memory.MemoryTracker; +import org.neo4j.storageengine.api.CommandBatchToApply; +import org.neo4j.storageengine.api.CommandCreationContext; +import org.neo4j.storageengine.api.CommandStream; +import org.neo4j.storageengine.api.IndexUpdateListener; +import org.neo4j.storageengine.api.MetadataProvider; +import org.neo4j.storageengine.api.StorageCommand; +import org.neo4j.storageengine.api.StorageEngine; +import org.neo4j.storageengine.api.StorageEngineFactory; +import org.neo4j.storageengine.api.StorageLocks; +import org.neo4j.storageengine.api.StorageReader; +import org.neo4j.storageengine.api.StoreFileMetadata; +import org.neo4j.storageengine.api.StoreId; +import org.neo4j.storageengine.api.TransactionApplicationMode; +import org.neo4j.storageengine.api.cursor.StoreCursors; +import org.neo4j.storageengine.api.enrichment.Enrichment; +import org.neo4j.storageengine.api.enrichment.EnrichmentCommand; +import org.neo4j.storageengine.api.txstate.ReadableTransactionState; +import org.neo4j.storageengine.api.txstate.TxStateVisitor; +import org.neo4j.storageengine.api.txstate.validation.TransactionValidatorFactory; +import org.neo4j.time.SystemNanoClock; +import org.neo4j.token.TokenHolders; + +import java.util.Collection; +import java.util.List; +import java.util.stream.Collectors; + +import static org.neo4j.gds.utils.StringFormatting.formatWithLocale; + +public final class InMemoryStorageEngineImpl implements StorageEngine { + + public static final byte ID = 42; + private final MetadataProvider metadataProvider; + private final CypherGraphStore graphStore; + private final DatabaseLayout databaseLayout; + private final InMemoryTransactionStateVisitor txStateVisitor; + + private final CommandCreationContext commandCreationContext; + + private final TokenManager tokenManager; + private final InMemoryCountsStoreImpl countsStore; + + private static final StorageEngineIndexingBehaviour INDEXING_BEHAVIOUR = new StorageEngineIndexingBehaviour() { + @Override + public boolean useNodeIdsInRelationshipTokenIndex() { + return false; + } + + @Override + public boolean requireCoordinationLocks() { + return false; + } + + @Override + public int nodesPerPage() { + return 0; + } + + @Override + public int relationshipsPerPage() { + return 0; + } + }; + + InMemoryStorageEngineImpl( + DatabaseLayout databaseLayout, + TokenHolders tokenHolders + ) { + this.databaseLayout = databaseLayout; + this.graphStore = getGraphStoreFromCatalog(databaseLayout.getDatabaseName()); + this.txStateVisitor = new InMemoryTransactionStateVisitor(graphStore, tokenHolders); + this.commandCreationContext = new InMemoryCommandCreationContextImpl(); + this.tokenManager = new TokenManager( + tokenHolders, + InMemoryStorageEngineImpl.this.txStateVisitor, + InMemoryStorageEngineImpl.this.graphStore, + commandCreationContext + ); + InMemoryStorageEngineImpl.this.graphStore.initialize(tokenHolders); + this.countsStore = new InMemoryCountsStoreImpl(graphStore, tokenHolders); + this.metadataProvider = new InMemoryMetaDataProviderImpl(); + } + + private static CypherGraphStore getGraphStoreFromCatalog(String databaseName) { + var graphName = InMemoryDatabaseCreationCatalog.getRegisteredDbCreationGraphName(databaseName); + return (CypherGraphStore) GraphStoreCatalog.getAllGraphStores() + .filter(graphStoreWithUserNameAndConfig -> graphStoreWithUserNameAndConfig + .config() + .graphName() + .equals(graphName)) + .findFirst() + .orElseThrow(() -> new IllegalArgumentException(formatWithLocale( + "No graph with name `%s` was found in GraphStoreCatalog. Available graph names are %s", + graphName, + GraphStoreCatalog.getAllGraphStores() + .map(GraphStoreCatalog.GraphStoreWithUserNameAndConfig::config) + .map(GraphProjectConfig::graphName) + .collect(Collectors.toList()) + ))) + .graphStore(); + } + + @Override + public StoreEntityCounters storeEntityCounters() { + return new StoreEntityCounters() { + @Override + public long nodes() { + return graphStore.nodeCount(); + } + + @Override + public long relationships() { + return graphStore.relationshipCount(); + } + + @Override + public long properties() { + return graphStore.nodePropertyKeys().size() + graphStore.relationshipPropertyKeys().size(); + } + + @Override + public long relationshipTypes() { + return graphStore.relationshipTypes().size(); + } + + @Override + public long allNodesCountStore(CursorContext cursorContext) { + return graphStore.nodeCount(); + } + + @Override + public long allRelationshipsCountStore(CursorContext cursorContext) { + return graphStore.relationshipCount(); + } + }; + } + + @Override + public void preAllocateStoreFilesForCommands( + CommandBatchToApply commandBatchToApply, + TransactionApplicationMode transactionApplicationMode + ) { + } + + @Override + public StoreCursors createStorageCursors(CursorContext initialContext) { + return StoreCursors.NULL; + } + + @Override + public StorageLocks createStorageLocks(ResourceLocker locker) { + return new InMemoryStorageLocksImpl(locker); + } + + @Override + public List createCommands( + ReadableTransactionState state, + StorageReader storageReader, + CommandCreationContext creationContext, + LockTracer lockTracer, + TxStateVisitor.Decorator additionalTxStateVisitor, + CursorContext cursorContext, + StoreCursors storeCursors, + MemoryTracker memoryTracker + ) throws KernelException { + state.accept(txStateVisitor); + return List.of(); + } + + @Override + public void dumpDiagnostics(InternalLog internalLog, DiagnosticsLogger diagnosticsLogger) { + } + + @Override + public List createUpgradeCommands( + KernelVersion versionToUpgradeFrom, + KernelVersion versionToUpgradeTo + ) { + return List.of(); + } + + @Override + public EnrichmentCommand createEnrichmentCommand(KernelVersion kernelVersion, Enrichment enrichment) { + throw new UnsupportedOperationException(); + } + + @Override + public StoreId retrieveStoreId() { + return metadataProvider.getStoreId(); + } + + @Override + public StorageEngineIndexingBehaviour indexingBehaviour() { + return INDEXING_BEHAVIOUR; + } + + @Override + public StorageReader newReader() { + return new InMemoryStorageReader59(graphStore, tokenManager.tokenHolders(), countsStore); + } + + @Override + public void addIndexUpdateListener(IndexUpdateListener listener) { + + } + + @Override + public void apply(CommandBatchToApply batch, TransactionApplicationMode mode) { + } + + @Override + public void init() { + } + + @Override + public void start() { + + } + + @Override + public void stop() { + shutdown(); + } + + @Override + public void shutdown() { + InMemoryDatabaseCreationCatalog.removeDatabaseEntry(databaseLayout.getDatabaseName()); + } + + @Override + public void listStorageFiles( + Collection atomic, Collection replayable + ) { + + } + + @Override + public Lifecycle schemaAndTokensLifecycle() { + return new LifecycleAdapter() { + @Override + public void init() { + + } + }; + } + + @Override + public CountsAccessor countsAccessor() { + return countsStore; + } + + @Override + public MetadataProvider metadataProvider() { + return metadataProvider; + } + + @Override + public String name() { + return "gds in-memory storage engine"; + } + + @Override + public byte id() { + return ID; + } + + @Override + public CommandCreationContext newCommandCreationContext() { + return commandCreationContext; + } + + @Override + public TransactionValidatorFactory createTransactionValidatorFactory(StorageEngineFactory storageEngineFactory, Config config, SystemNanoClock clock) { + return TransactionValidatorFactory.EMPTY_VALIDATOR_FACTORY; + } + + @Override + public void lockRecoveryCommands( + CommandStream commands, LockService lockService, LockGroup lockGroup, TransactionApplicationMode mode + ) { + + } + + @Override + public void rollback(ReadableTransactionState txState, CursorContext cursorContext) { + // rollback is not supported but it is also called when we fail for something else + // that we do not support, such as removing node properties + // TODO: do we want to inspect the txState to infer if rollback was called explicitly or not? + } + + @Override + public void checkpoint(DatabaseFlushEvent flushEvent, CursorContext cursorContext) { + // checkpoint is not supported but it is also called when we fail for something else + // that we do not support, such as removing node properties + } +} diff --git a/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryStorageLocksImpl.java b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryStorageLocksImpl.java new file mode 100644 index 0000000000..0b49cdb34a --- /dev/null +++ b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryStorageLocksImpl.java @@ -0,0 +1,86 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.lock.LockTracer; +import org.neo4j.lock.ResourceLocker; +import org.neo4j.storageengine.api.StorageLocks; +import org.neo4j.storageengine.api.txstate.ReadableTransactionState; + +public class InMemoryStorageLocksImpl implements StorageLocks { + + InMemoryStorageLocksImpl(ResourceLocker locker) {} + + @Override + public void acquireExclusiveNodeLock(LockTracer lockTracer, long... ids) {} + + @Override + public void releaseExclusiveNodeLock(long... ids) {} + + @Override + public void acquireSharedNodeLock(LockTracer lockTracer, long... ids) {} + + @Override + public void releaseSharedNodeLock(long... ids) {} + + @Override + public void acquireExclusiveRelationshipLock(LockTracer lockTracer, long... ids) {} + + @Override + public void releaseExclusiveRelationshipLock(long... ids) {} + + @Override + public void acquireSharedRelationshipLock(LockTracer lockTracer, long... ids) {} + + @Override + public void releaseSharedRelationshipLock(long... ids) {} + + @Override + public void acquireRelationshipCreationLock( + LockTracer lockTracer, + long sourceNode, + long targetNode, + boolean sourceNodeAddedInTx, + boolean targetNodeAddedInTx + ) { + } + + @Override + public void acquireRelationshipDeletionLock( + LockTracer lockTracer, + long sourceNode, + long targetNode, + long relationship, + boolean relationshipAddedInTx, + boolean sourceNodeAddedInTx, + boolean targetNodeAddedInTx + ) { + } + + @Override + public void acquireNodeDeletionLock( + ReadableTransactionState readableTransactionState, + LockTracer lockTracer, + long node + ) {} + + @Override + public void acquireNodeLabelChangeLock(LockTracer lockTracer, long node, int labelId) {} +} diff --git a/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryStoreVersion.java b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryStoreVersion.java new file mode 100644 index 0000000000..fcd75bd343 --- /dev/null +++ b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryStoreVersion.java @@ -0,0 +1,68 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.storageengine.api.StoreVersion; +import org.neo4j.storageengine.api.format.Capability; +import org.neo4j.storageengine.api.format.CapabilityType; + +import java.util.Optional; + +public class InMemoryStoreVersion implements StoreVersion { + + public static final String STORE_VERSION = "gds-experimental"; + + @Override + public String getStoreVersionUserString() { + return "Unknown"; + } + + @Override + public Optional successorStoreVersion() { + return Optional.empty(); + } + + @Override + public String formatName() { + return getClass().getSimpleName(); + } + + @Override + public boolean onlyForMigration() { + return false; + } + + @Override + public boolean hasCapability(Capability capability) { + return false; + } + + @Override + public boolean hasCompatibleCapabilities( + StoreVersion otherVersion, CapabilityType type + ) { + return false; + } + + @Override + public String introductionNeo4jVersion() { + return "foo"; + } +} diff --git a/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryTransactionIdStoreImpl.java b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryTransactionIdStoreImpl.java new file mode 100644 index 0000000000..2e220e4d3b --- /dev/null +++ b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryTransactionIdStoreImpl.java @@ -0,0 +1,117 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.internal.recordstorage.AbstractTransactionIdStore; +import org.neo4j.io.pagecache.context.TransactionIdSnapshot; +import org.neo4j.kernel.impl.transaction.log.LogPosition; +import org.neo4j.storageengine.api.ClosedTransactionMetadata; +import org.neo4j.storageengine.api.TransactionId; +import org.neo4j.storageengine.api.TransactionIdStore; + +public class InMemoryTransactionIdStoreImpl extends AbstractTransactionIdStore { + + @Override + protected void initLastCommittedAndClosedTransactionId( + long previouslyCommittedTxId, + int checksum, + long previouslyCommittedTxCommitTimestamp, + long previouslyCommittedTxLogByteOffset, + long previouslyCommittedTxLogVersion + ) { + this.setLastCommittedAndClosedTransactionId( + previouslyCommittedTxId, + checksum, + previouslyCommittedTxCommitTimestamp, + TransactionIdStore.UNKNOWN_CONSENSUS_INDEX, + previouslyCommittedTxLogByteOffset, + previouslyCommittedTxLogVersion + ); + } + + @Override + public ClosedTransactionMetadata getLastClosedTransaction() { + long[] metaData = this.closedTransactionId.get(); + return new ClosedTransactionMetadata( + metaData[0], + new LogPosition(metaData[1], metaData[2]), + (int) metaData[3], + metaData[4], + metaData[5] + ); + } + + @Override + public TransactionIdSnapshot getClosedTransactionSnapshot() { + return new TransactionIdSnapshot(this.getLastClosedTransactionId()); + } + + @Override + protected TransactionId transactionId(long transactionId, int checksum, long commitTimestamp) { + return new TransactionId(transactionId, checksum, commitTimestamp, TransactionIdStore.UNKNOWN_CONSENSUS_INDEX); + } + + @Override + public void transactionCommitted(long transactionId, int checksum, long commitTimestamp, long consensusIndex) { + + } + + @Override + public void setLastCommittedAndClosedTransactionId( + long transactionId, + int checksum, + long commitTimestamp, + long consensusIndex, + long byteOffset, + long logVersion + ) { + + } + + @Override + public void transactionClosed( + long transactionId, + long logVersion, + long byteOffset, + int checksum, + long commitTimestamp, + long consensusIndex + ) { + this.closedTransactionId.offer( + transactionId, + new long[]{logVersion, byteOffset, checksum, commitTimestamp, consensusIndex} + ); + } + + @Override + public void resetLastClosedTransaction( + long transactionId, + long logVersion, + long byteOffset, + int checksum, + long commitTimestamp, + long consensusIndex + ) { + this.closedTransactionId.set( + transactionId, + new long[]{logVersion, byteOffset, checksum, commitTimestamp, consensusIndex} + ); + } +} diff --git a/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryVersionCheck.java b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryVersionCheck.java new file mode 100644 index 0000000000..fb196c18fb --- /dev/null +++ b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryVersionCheck.java @@ -0,0 +1,61 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.kernel.impl.store.format.FormatFamily; +import org.neo4j.storageengine.api.StoreVersionCheck; +import org.neo4j.storageengine.api.StoreVersionIdentifier; + +import static org.neo4j.gds.compat._59.InMemoryStoreVersion.STORE_VERSION; + +public class InMemoryVersionCheck implements StoreVersionCheck { + + private static final StoreVersionIdentifier STORE_IDENTIFIER = new StoreVersionIdentifier( + STORE_VERSION, + FormatFamily.STANDARD.name(), + 0, + 0 + ); + + @Override + public boolean isCurrentStoreVersionFullySupported(CursorContext cursorContext) { + return true; + } + + @Override + public MigrationCheckResult getAndCheckMigrationTargetVersion(String formatFamily, CursorContext cursorContext) { + return new StoreVersionCheck.MigrationCheckResult(MigrationOutcome.NO_OP, STORE_IDENTIFIER, null, null); + } + + @Override + public UpgradeCheckResult getAndCheckUpgradeTargetVersion(CursorContext cursorContext) { + return new StoreVersionCheck.UpgradeCheckResult(UpgradeOutcome.NO_OP, STORE_IDENTIFIER, null, null); + } + + @Override + public String getIntroductionVersionFromVersion(StoreVersionIdentifier storeVersionIdentifier) { + return STORE_VERSION; + } + + public StoreVersionIdentifier findLatestVersion(String s) { + return STORE_IDENTIFIER; + } +} diff --git a/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/StorageEngineProxyFactoryImpl.java b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/StorageEngineProxyFactoryImpl.java new file mode 100644 index 0000000000..81132c0c9d --- /dev/null +++ b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/StorageEngineProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jVersion; +import org.neo4j.gds.compat.StorageEngineProxyApi; +import org.neo4j.gds.compat.StorageEngineProxyFactory; + +@ServiceProvider +public class StorageEngineProxyFactoryImpl implements StorageEngineProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return version == Neo4jVersion.V_5_9; + } + + @Override + public StorageEngineProxyApi load() { + return new StorageEngineProxyImpl(); + } + + @Override + public String description() { + return "Storage Engine 5.9"; + } +} diff --git a/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/StorageEngineProxyImpl.java b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/StorageEngineProxyImpl.java new file mode 100644 index 0000000000..8cda8e608e --- /dev/null +++ b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/StorageEngineProxyImpl.java @@ -0,0 +1,156 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.common.Edition; +import org.neo4j.configuration.Config; +import org.neo4j.configuration.GraphDatabaseInternalSettings; +import org.neo4j.counts.CountsAccessor; +import org.neo4j.dbms.api.DatabaseManagementService; +import org.neo4j.gds.compat.AbstractInMemoryNodeCursor; +import org.neo4j.gds.compat.AbstractInMemoryNodePropertyCursor; +import org.neo4j.gds.compat.AbstractInMemoryRelationshipPropertyCursor; +import org.neo4j.gds.compat.AbstractInMemoryRelationshipTraversalCursor; +import org.neo4j.gds.compat.GdsDatabaseManagementServiceBuilder; +import org.neo4j.gds.compat.GraphDatabaseApiProxy; +import org.neo4j.gds.compat.StorageEngineProxyApi; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.graphdb.Direction; +import org.neo4j.graphdb.GraphDatabaseService; +import org.neo4j.internal.recordstorage.AbstractInMemoryRelationshipScanCursor; +import org.neo4j.internal.recordstorage.InMemoryStorageReader59; +import org.neo4j.io.layout.DatabaseLayout; +import org.neo4j.storageengine.api.CommandCreationContext; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.RelationshipSelection; +import org.neo4j.storageengine.api.StorageEngine; +import org.neo4j.storageengine.api.StorageEntityCursor; +import org.neo4j.storageengine.api.StoragePropertyCursor; +import org.neo4j.storageengine.api.StorageReader; +import org.neo4j.storageengine.api.StorageRelationshipTraversalCursor; +import org.neo4j.token.TokenHolders; + +import static org.neo4j.configuration.GraphDatabaseSettings.db_format; + +public class StorageEngineProxyImpl implements StorageEngineProxyApi { + + @Override + public CommandCreationContext inMemoryCommandCreationContext() { + return new InMemoryCommandCreationContextImpl(); + } + + @Override + public void initRelationshipTraversalCursorForRelType( + StorageRelationshipTraversalCursor cursor, + long sourceNodeId, + int relTypeToken + ) { + var relationshipSelection = RelationshipSelection.selection( + relTypeToken, + Direction.OUTGOING + ); + cursor.init(sourceNodeId, -1, relationshipSelection); + } + + @Override + public StorageReader inMemoryStorageReader( + CypherGraphStore graphStore, TokenHolders tokenHolders, CountsAccessor counts + ) { + return new InMemoryStorageReader59(graphStore, tokenHolders, counts); + } + + @Override + public StorageEngine createInMemoryStorageEngine(DatabaseLayout databaseLayout, TokenHolders tokenHolders) { + return new InMemoryStorageEngineImpl(databaseLayout, tokenHolders); + } + + @Override + public void createInMemoryDatabase( + DatabaseManagementService dbms, + String dbName, + Config config + ) { + config.set(db_format, InMemoryStorageEngineFactory.IN_MEMORY_STORAGE_ENGINE_NAME); + dbms.createDatabase(dbName, config); + } + + @Override + public GraphDatabaseService startAndGetInMemoryDatabase(DatabaseManagementService dbms, String dbName) { + dbms.startDatabase(dbName); + return dbms.database(dbName); + } + + @Override + public GdsDatabaseManagementServiceBuilder setSkipDefaultIndexesOnCreationSetting(GdsDatabaseManagementServiceBuilder dbmsBuilder) { + return dbmsBuilder.setConfig(GraphDatabaseInternalSettings.skip_default_indexes_on_creation, true); + } + + @Override + public AbstractInMemoryNodeCursor inMemoryNodeCursor(CypherGraphStore graphStore, TokenHolders tokenHolders) { + return new InMemoryNodeCursor(graphStore, tokenHolders); + } + + @Override + public AbstractInMemoryNodePropertyCursor inMemoryNodePropertyCursor( + CypherGraphStore graphStore, + TokenHolders tokenHolders + ) { + return new InMemoryNodePropertyCursor(graphStore, tokenHolders); + } + + @Override + public AbstractInMemoryRelationshipTraversalCursor inMemoryRelationshipTraversalCursor( + CypherGraphStore graphStore, TokenHolders tokenHolders + ) { + return new InMemoryRelationshipTraversalCursor(graphStore, tokenHolders); + } + + @Override + public AbstractInMemoryRelationshipScanCursor inMemoryRelationshipScanCursor( + CypherGraphStore graphStore, TokenHolders tokenHolders + ) { + return new InMemoryRelationshipScanCursor(graphStore, tokenHolders); + } + + @Override + public AbstractInMemoryRelationshipPropertyCursor inMemoryRelationshipPropertyCursor( + CypherGraphStore graphStore, TokenHolders tokenHolders + ) { + return new InMemoryRelationshipPropertyCursor(graphStore, tokenHolders); + } + + @Override + public void properties( + StorageEntityCursor storageCursor, StoragePropertyCursor propertyCursor, int[] propertySelection + ) { + PropertySelection selection; + if (propertySelection.length == 0) { + selection = PropertySelection.ALL_PROPERTIES; + } else { + selection = PropertySelection.selection(propertySelection); + } + storageCursor.properties(propertyCursor, selection); + } + + @Override + public Edition dbmsEdition(GraphDatabaseService databaseService) { + return GraphDatabaseApiProxy.dbmsInfo(databaseService).edition; + } +} diff --git a/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository59.java b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository59.java new file mode 100644 index 0000000000..793eae1cab --- /dev/null +++ b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository59.java @@ -0,0 +1,71 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.internal.recordstorage; + +import org.neo4j.storageengine.api.LogVersionRepository; + +import java.util.concurrent.atomic.AtomicLong; + +public class InMemoryLogVersionRepository59 implements LogVersionRepository { + + private final AtomicLong logVersion; + private final AtomicLong checkpointLogVersion; + + public InMemoryLogVersionRepository59() { + this(0, 0); + } + + private InMemoryLogVersionRepository59(long initialLogVersion, long initialCheckpointLogVersion) { + this.logVersion = new AtomicLong(); + this.checkpointLogVersion = new AtomicLong(); + this.logVersion.set(initialLogVersion); + this.checkpointLogVersion.set(initialCheckpointLogVersion); + } + + @Override + public void setCurrentLogVersion(long version) { + this.logVersion.set(version); + } + + @Override + public long incrementAndGetVersion() { + return this.logVersion.incrementAndGet(); + } + + @Override + public void setCheckpointLogVersion(long version) { + this.checkpointLogVersion.set(version); + } + + @Override + public long incrementAndGetCheckpointLogVersion() { + return this.checkpointLogVersion.incrementAndGet(); + } + + @Override + public long getCurrentLogVersion() { + return this.logVersion.get(); + } + + @Override + public long getCheckpointLogVersion() { + return this.checkpointLogVersion.get(); + } +} diff --git a/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory59.java b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory59.java new file mode 100644 index 0000000000..4aac7460e2 --- /dev/null +++ b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory59.java @@ -0,0 +1,43 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.internal.recordstorage; + +import org.neo4j.kernel.KernelVersion; +import org.neo4j.storageengine.api.CommandReader; +import org.neo4j.storageengine.api.CommandReaderFactory; + +public class InMemoryStorageCommandReaderFactory59 implements CommandReaderFactory { + + public static final CommandReaderFactory INSTANCE = new InMemoryStorageCommandReaderFactory59(); + + @Override + public CommandReader get(KernelVersion kernelVersion) { + switch (kernelVersion) { + case V4_2: + return LogCommandSerializationV4_2.INSTANCE; + case V4_3_D4: + return LogCommandSerializationV4_3_D3.INSTANCE; + case V5_0: + return LogCommandSerializationV5_0.INSTANCE; + default: + throw new IllegalArgumentException("Unsupported kernel version " + kernelVersion); + } + } +} diff --git a/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageReader59.java b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageReader59.java new file mode 100644 index 0000000000..ae8687d07a --- /dev/null +++ b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageReader59.java @@ -0,0 +1,332 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.internal.recordstorage; + +import org.eclipse.collections.api.set.primitive.IntSet; +import org.eclipse.collections.impl.set.immutable.primitive.ImmutableIntSetFactoryImpl; +import org.neo4j.common.EntityType; +import org.neo4j.common.TokenNameLookup; +import org.neo4j.counts.CountsAccessor; +import org.neo4j.gds.compat._59.InMemoryNodeCursor; +import org.neo4j.gds.compat._59.InMemoryPropertyCursor; +import org.neo4j.gds.compat._59.InMemoryRelationshipScanCursor; +import org.neo4j.gds.compat._59.InMemoryRelationshipTraversalCursor; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.internal.schema.ConstraintDescriptor; +import org.neo4j.internal.schema.IndexDescriptor; +import org.neo4j.internal.schema.IndexType; +import org.neo4j.internal.schema.SchemaDescriptor; +import org.neo4j.internal.schema.constraints.IndexBackedConstraintDescriptor; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.memory.MemoryTracker; +import org.neo4j.storageengine.api.AllNodeScan; +import org.neo4j.storageengine.api.AllRelationshipsScan; +import org.neo4j.storageengine.api.StorageNodeCursor; +import org.neo4j.storageengine.api.StoragePropertyCursor; +import org.neo4j.storageengine.api.StorageReader; +import org.neo4j.storageengine.api.StorageRelationshipScanCursor; +import org.neo4j.storageengine.api.StorageRelationshipTraversalCursor; +import org.neo4j.storageengine.api.StorageSchemaReader; +import org.neo4j.storageengine.api.cursor.StoreCursors; +import org.neo4j.token.TokenHolders; + +import java.util.Collection; +import java.util.Collections; +import java.util.Iterator; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Function; + +public class InMemoryStorageReader59 implements StorageReader { + + protected final CypherGraphStore graphStore; + protected final TokenHolders tokenHolders; + protected final CountsAccessor counts; + private final Map, Object> dependantState; + private boolean closed; + + public InMemoryStorageReader59( + CypherGraphStore graphStore, + TokenHolders tokenHolders, + CountsAccessor counts + ) { + this.graphStore = graphStore; + + this.tokenHolders = tokenHolders; + this.counts = counts; + this.dependantState = new ConcurrentHashMap<>(); + } + + @Override + public Collection uniquenessConstraintsGetRelated( + long[] changedLabels, + long[] unchangedLabels, + int[] propertyKeyIds, + boolean propertyKeyListIsComplete, + EntityType entityType + ) { + return Collections.emptyList(); + } + + @Override + public long relationshipsGetCount(CursorContext cursorTracer) { + return graphStore.relationshipCount(); + } + + @Override + public boolean nodeExists(long id, StoreCursors storeCursors) { + var originalId = graphStore.nodes().toOriginalNodeId(id); + return graphStore.nodes().containsOriginalId(originalId); + } + + @Override + public boolean relationshipExists(long id, StoreCursors storeCursors) { + return true; + } + + @Override + public StorageNodeCursor allocateNodeCursor( + CursorContext cursorContext, StoreCursors storeCursors + ) { + return new InMemoryNodeCursor(graphStore, tokenHolders); + } + + @Override + public StoragePropertyCursor allocatePropertyCursor( + CursorContext cursorContext, StoreCursors storeCursors, MemoryTracker memoryTracker + ) { + return new InMemoryPropertyCursor(graphStore, tokenHolders); + } + + @Override + public StorageRelationshipTraversalCursor allocateRelationshipTraversalCursor( + CursorContext cursorContext, StoreCursors storeCursors + ) { + return new InMemoryRelationshipTraversalCursor(graphStore, tokenHolders); + } + + @Override + public StorageRelationshipScanCursor allocateRelationshipScanCursor( + CursorContext cursorContext, StoreCursors storeCursors + ) { + return new InMemoryRelationshipScanCursor(graphStore, tokenHolders); + } + + @Override + public IndexDescriptor indexGetForSchemaAndType( + SchemaDescriptor descriptor, IndexType type + ) { + return null; + } + + @Override + public AllRelationshipsScan allRelationshipScan() { + return new AbstractInMemoryAllRelationshipScan() { + @Override + boolean scanRange(AbstractInMemoryRelationshipScanCursor cursor, long start, long stopInclusive) { + return cursor.scanRange(start, stopInclusive); + } + + @Override + public boolean scanBatch(long sizeHint, AbstractInMemoryRelationshipScanCursor cursor) { + return super.scanBatch(sizeHint, cursor); + } + }; + } + + @Override + public Iterator indexGetForSchema(SchemaDescriptor descriptor) { + return Collections.emptyIterator(); + } + + @Override + public Iterator indexesGetForLabel(int labelId) { + return Collections.emptyIterator(); + } + + @Override + public Iterator indexesGetForRelationshipType(int relationshipType) { + return Collections.emptyIterator(); + } + + @Override + public IndexDescriptor indexGetForName(String name) { + return null; + } + + @Override + public ConstraintDescriptor constraintGetForName(String name) { + return null; + } + + @Override + public boolean indexExists(IndexDescriptor index) { + return false; + } + + @Override + public Iterator indexesGetAll() { + return Collections.emptyIterator(); + } + + @Override + public Collection valueIndexesGetRelated( + long[] tokens, int propertyKeyId, EntityType entityType + ) { + return valueIndexesGetRelated(tokens, new int[]{propertyKeyId}, entityType); + } + + @Override + public Collection valueIndexesGetRelated( + long[] tokens, int[] propertyKeyIds, EntityType entityType + ) { + return Collections.emptyList(); + } + + @Override + public Collection uniquenessConstraintsGetRelated( + long[] labels, + int propertyKeyId, + EntityType entityType + ) { + return Collections.emptyList(); + } + + @Override + public Collection uniquenessConstraintsGetRelated( + long[] tokens, + int[] propertyKeyIds, + EntityType entityType + ) { + return Collections.emptyList(); + } + + @Override + public boolean hasRelatedSchema(long[] labels, int propertyKey, EntityType entityType) { + return false; + } + + @Override + public boolean hasRelatedSchema(int label, EntityType entityType) { + return false; + } + + @Override + public Iterator constraintsGetForSchema(SchemaDescriptor descriptor) { + return Collections.emptyIterator(); + } + + @Override + public boolean constraintExists(ConstraintDescriptor descriptor) { + return false; + } + + @Override + public Iterator constraintsGetForLabel(int labelId) { + return Collections.emptyIterator(); + } + + @Override + public Iterator constraintsGetForRelationshipType(int typeId) { + return Collections.emptyIterator(); + } + + @Override + public Iterator constraintsGetAll() { + return Collections.emptyIterator(); + } + + @Override + public IntSet constraintsGetPropertyTokensForLogicalKey(int token, EntityType entityType) { + return ImmutableIntSetFactoryImpl.INSTANCE.empty(); + } + + @Override + public Long indexGetOwningUniquenessConstraintId(IndexDescriptor index) { + return null; + } + + @Override + public long countsForNode(int labelId, CursorContext cursorContext) { + return counts.nodeCount(labelId, cursorContext); + } + + @Override + public long countsForRelationship(int startLabelId, int typeId, int endLabelId, CursorContext cursorContext) { + return counts.relationshipCount(startLabelId, typeId, endLabelId, cursorContext); + } + + @Override + public long nodesGetCount(CursorContext cursorContext) { + return graphStore.nodeCount(); + } + + @Override + public int labelCount() { + return graphStore.nodes().availableNodeLabels().size(); + } + + @Override + public int propertyKeyCount() { + int nodePropertyCount = graphStore + .schema() + .nodeSchema() + .allProperties() + .size(); + int relPropertyCount = graphStore + .schema() + .relationshipSchema() + .allProperties() + .size(); + return nodePropertyCount + relPropertyCount; + } + + @Override + public int relationshipTypeCount() { + return graphStore.schema().relationshipSchema().availableTypes().size(); + } + + @Override + public T getOrCreateSchemaDependantState(Class type, Function factory) { + return type.cast(dependantState.computeIfAbsent(type, key -> factory.apply(this))); + } + + @Override + public AllNodeScan allNodeScan() { + return new InMemoryNodeScan(); + } + + @Override + public void close() { + assert !closed; + closed = true; + } + + @Override + public StorageSchemaReader schemaSnapshot() { + return this; + } + + @Override + public TokenNameLookup tokenNameLookup() { + return tokenHolders; + } + +} diff --git a/gradle/dependencies.gradle b/gradle/dependencies.gradle index 4edcf871f6..f77b099992 100644 --- a/gradle/dependencies.gradle +++ b/gradle/dependencies.gradle @@ -9,6 +9,7 @@ ext { '5.6': properties.getOrDefault('neo4jVersion56', '5.6.0'), '5.7': properties.getOrDefault('neo4jVersion57', '5.7.0'), '5.8': properties.getOrDefault('neo4jVersion58', '5.8.0'), + '5.9': properties.getOrDefault('neo4jVersion59', '5.9.0'), ] neo4jDefault = neos.'4.4' From 587dbe0405c6d45b6ed234901143dfe1b363af4c Mon Sep 17 00:00:00 2001 From: yuval Date: Fri, 9 Jun 2023 12:51:49 +0200 Subject: [PATCH 053/273] Handle the version enum --- .../src/main/java/org/neo4j/gds/compat/Neo4jVersion.java | 5 +++++ .../src/test/java/org/neo4j/gds/SysInfoProcTest.java | 8 ++++++++ 2 files changed, 13 insertions(+) diff --git a/neo4j-adapter/src/main/java/org/neo4j/gds/compat/Neo4jVersion.java b/neo4j-adapter/src/main/java/org/neo4j/gds/compat/Neo4jVersion.java index 9beb328163..9e2b7a7612 100644 --- a/neo4j-adapter/src/main/java/org/neo4j/gds/compat/Neo4jVersion.java +++ b/neo4j-adapter/src/main/java/org/neo4j/gds/compat/Neo4jVersion.java @@ -36,6 +36,7 @@ public enum Neo4jVersion { V_5_6, V_5_7, V_5_8, + V_5_9, V_RC; @Override @@ -59,6 +60,8 @@ public String toString() { return "5.7"; case V_5_8: return "5.8"; + case V_5_9: + return "5.9"; case V_RC: return "rc"; default: @@ -148,6 +151,8 @@ static Neo4jVersion parse(String version) { } else if (minorVersion == 8) { return Neo4jVersion.V_5_8; } else if (minorVersion == 9) { + return Neo4jVersion.V_5_9; + } else if (minorVersion == 10) { return Neo4jVersion.V_RC; } } diff --git a/proc/sysinfo/src/test/java/org/neo4j/gds/SysInfoProcTest.java b/proc/sysinfo/src/test/java/org/neo4j/gds/SysInfoProcTest.java index 45f37d74c3..820d333fbc 100644 --- a/proc/sysinfo/src/test/java/org/neo4j/gds/SysInfoProcTest.java +++ b/proc/sysinfo/src/test/java/org/neo4j/gds/SysInfoProcTest.java @@ -216,6 +216,14 @@ void testSysInfoProc() throws IOException { "Neo4j 5.8" ); break; + case V_5_9: + expectedCompatibilities = Set.of( + "Neo4j Settings 5.9 (placeholder)", + "Neo4j Settings 5.9", + "Neo4j 5.9 (placeholder)", + "Neo4j 5.9" + ); + break; case V_RC: expectedCompatibilities = Set.of( "Neo4j Settings RC", From 5125424f709065fdae150f38a3abff496eb4011c Mon Sep 17 00:00:00 2001 From: Mats Rydberg Date: Fri, 9 Jun 2023 13:58:10 +0200 Subject: [PATCH 054/273] Fix tests to work with Neo4j 5.9 compat --- .../src/test/java/org/neo4j/gds/compat/Neo4jVersionTest.java | 3 ++- .../sysinfo/src/test/java/org/neo4j/gds/SysInfoProcTest.java | 5 +++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/neo4j-adapter/src/test/java/org/neo4j/gds/compat/Neo4jVersionTest.java b/neo4j-adapter/src/test/java/org/neo4j/gds/compat/Neo4jVersionTest.java index 999179b681..87a1b115a2 100644 --- a/neo4j-adapter/src/test/java/org/neo4j/gds/compat/Neo4jVersionTest.java +++ b/neo4j-adapter/src/test/java/org/neo4j/gds/compat/Neo4jVersionTest.java @@ -49,7 +49,8 @@ class Neo4jVersionTest { "5.6.0, V_5_6", "5.7.0, V_5_7", "5.8.0, V_5_8", - "5.9.0, V_RC", + "5.9.0, V_5_9", + "5.10.0, V_RC", }) void testParse(String input, Neo4jVersion expected) { assertEquals(expected.name(), Neo4jVersion.parse(input).name()); diff --git a/proc/sysinfo/src/test/java/org/neo4j/gds/SysInfoProcTest.java b/proc/sysinfo/src/test/java/org/neo4j/gds/SysInfoProcTest.java index 820d333fbc..04e90466d3 100644 --- a/proc/sysinfo/src/test/java/org/neo4j/gds/SysInfoProcTest.java +++ b/proc/sysinfo/src/test/java/org/neo4j/gds/SysInfoProcTest.java @@ -92,6 +92,11 @@ class SysInfoProcTest extends BaseProcTest { "Neo4j Settings 5.8", "Neo4j Settings 5.8 (placeholder)", + "Neo4j 5.9", + "Neo4j 5.9 (placeholder)", + "Neo4j Settings 5.9", + "Neo4j Settings 5.9 (placeholder)", + "Neo4j DEV", "Neo4j DEV (placeholder)", "Neo4j Settings DEV", From 58203b6c62c52a1621567b366b4e9663861b4c4e Mon Sep 17 00:00:00 2001 From: Veselin Nikolov Date: Fri, 9 Jun 2023 13:04:11 +0100 Subject: [PATCH 055/273] Fix license header --- .../_59/StorageEngineProxyFactoryImpl.java | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/compatibility/5.9/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_59/StorageEngineProxyFactoryImpl.java b/compatibility/5.9/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_59/StorageEngineProxyFactoryImpl.java index c08f9a2716..e43773a19d 100644 --- a/compatibility/5.9/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_59/StorageEngineProxyFactoryImpl.java +++ b/compatibility/5.9/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_59/StorageEngineProxyFactoryImpl.java @@ -1,8 +1,21 @@ /* * Copyright (c) "Neo4j" * Neo4j Sweden AB [http://neo4j.com] - * This file contains proprietary code that is only available via a commercial license from Neo4j. - * For more information, see https://neo4j.com/contact-us/ + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package org.neo4j.gds.compat._59; From a48ba6ceba1dba7509776cd8ad94069b0b1a0845 Mon Sep 17 00:00:00 2001 From: Mats Rydberg Date: Fri, 9 Jun 2023 14:17:08 +0200 Subject: [PATCH 056/273] Promote ScaleProperties out of beta --- .../ROOT/pages/algorithms/auxiliary.adoc | 4 +++- .../pages/algorithms/scale-properties.adoc | 3 --- .../algorithm-references.adoc | 19 +++++++++---------- 3 files changed, 12 insertions(+), 14 deletions(-) diff --git a/doc/modules/ROOT/pages/algorithms/auxiliary.adoc b/doc/modules/ROOT/pages/algorithms/auxiliary.adoc index 94dd223e9b..b1fc27debe 100644 --- a/doc/modules/ROOT/pages/algorithms/auxiliary.adoc +++ b/doc/modules/ROOT/pages/algorithms/auxiliary.adoc @@ -6,9 +6,11 @@ Auxiliary procedures are extra tools that can be useful in your workflow. + The Neo4j GDS library includes the following auxiliary procedures, grouped by quality tier: +* Production-ready +** xref:algorithms/scale-properties.adoc[Scale Properties] + * Beta ** xref:beta-algorithms/collapse-path.adoc[Collapse Path] -** xref:algorithms/scale-properties.adoc[Scale Properties] * Alpha ** xref:alpha-algorithms/one-hot-encoding.adoc#algorithms-one-hot-encoding-sample[One Hot Encoding] diff --git a/doc/modules/ROOT/pages/algorithms/scale-properties.adoc b/doc/modules/ROOT/pages/algorithms/scale-properties.adoc index 9ddb0babde..e22ab0310d 100644 --- a/doc/modules/ROOT/pages/algorithms/scale-properties.adoc +++ b/doc/modules/ROOT/pages/algorithms/scale-properties.adoc @@ -1,5 +1,4 @@ [[algorithms-scale-properties]] -[.beta] = Scale Properties :description: This section describes the Scale Properties algorithm in the Neo4j Graph Data Science library. :page-aliases: alpha-algorithms/scale-properties @@ -8,8 +7,6 @@ :entity: node :result: scaled properties -include::partial$/operations-reference/beta-note.adoc[] - [[algorithms-scale-properties-intro]] == Introduction diff --git a/doc/modules/ROOT/pages/operations-reference/algorithm-references.adoc b/doc/modules/ROOT/pages/operations-reference/algorithm-references.adoc index a7833b9992..9ba4d322bb 100644 --- a/doc/modules/ROOT/pages/operations-reference/algorithm-references.adoc +++ b/doc/modules/ROOT/pages/operations-reference/algorithm-references.adoc @@ -189,6 +189,15 @@ | `gds.randomWalk.stats.estimate` | `gds.randomWalk.stream` | `gds.randomWalk.stream.estimate` +.8+<.^|xref:algorithms/scale-properties.adoc[Scale Properties] +| `gds.scaleProperties.mutate` +| `gds.scaleProperties.mutate.estimate` +| `gds.scaleProperties.stream` +| `gds.scaleProperties.stream.estimate` +| `gds.scaleProperties.stats` +| `gds.scaleProperties.stats.estimate` +| `gds.scaleProperties.write` +| `gds.scaleProperties.write.estimate` |=== [[beta-tier]] @@ -248,16 +257,6 @@ | `gds.beta.influenceMaximization.celf.stream.estimate` | `gds.beta.influenceMaximization.celf.write` | `gds.beta.influenceMaximization.celf.write.estimate` -.8+<.^|xref:algorithms/scale-properties.adoc[Scale Properties] -| `gds.scaleProperties.mutate` -| `gds.scaleProperties.mutate.estimate` -| `gds.scaleProperties.stream` -| `gds.scaleProperties.stream.estimate` -| `gds.scaleProperties.stats` -| `gds.scaleProperties.stats.estimate` -| `gds.scaleProperties.write` -| `gds.scaleProperties.write.estimate` - |=== [[alpha-tier]] From 403412452d4af2eeb492bb55504c6875a056b3e4 Mon Sep 17 00:00:00 2001 From: Mats Rydberg Date: Fri, 9 Jun 2023 15:03:44 +0200 Subject: [PATCH 057/273] Use correct license header --- .../gds/compat/_59/Neo4jProxyFactoryImpl.java | 17 +++++++++++++++-- .../gds/compat/_59/SettingProxyFactoryImpl.java | 17 +++++++++++++++-- 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/compatibility/5.9/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_59/Neo4jProxyFactoryImpl.java b/compatibility/5.9/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_59/Neo4jProxyFactoryImpl.java index 26324fc643..c972ffc94a 100644 --- a/compatibility/5.9/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_59/Neo4jProxyFactoryImpl.java +++ b/compatibility/5.9/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_59/Neo4jProxyFactoryImpl.java @@ -1,8 +1,21 @@ /* * Copyright (c) "Neo4j" * Neo4j Sweden AB [http://neo4j.com] - * This file contains proprietary code that is only available via a commercial license from Neo4j. - * For more information, see https://neo4j.com/contact-us/ + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package org.neo4j.gds.compat._59; diff --git a/compatibility/5.9/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_59/SettingProxyFactoryImpl.java b/compatibility/5.9/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_59/SettingProxyFactoryImpl.java index 791ce29673..1a05f4b02f 100644 --- a/compatibility/5.9/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_59/SettingProxyFactoryImpl.java +++ b/compatibility/5.9/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_59/SettingProxyFactoryImpl.java @@ -1,8 +1,21 @@ /* * Copyright (c) "Neo4j" * Neo4j Sweden AB [http://neo4j.com] - * This file contains proprietary code that is only available via a commercial license from Neo4j. - * For more information, see https://neo4j.com/contact-us/ + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package org.neo4j.gds.compat._59; From 5e020abaa133790fa86b943825a3611d13968526 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Fri, 9 Jun 2023 16:41:54 +0200 Subject: [PATCH 058/273] Avoid double mapping in KSpanningTree --- .../main/java/org/neo4j/gds/kspanningtree/KSpanningTree.java | 2 +- .../gds/paths/kspanningtree/KSpanningTreeWriteProcTest.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/algo/src/main/java/org/neo4j/gds/kspanningtree/KSpanningTree.java b/algo/src/main/java/org/neo4j/gds/kspanningtree/KSpanningTree.java index dcdbd4f465..13560970dd 100644 --- a/algo/src/main/java/org/neo4j/gds/kspanningtree/KSpanningTree.java +++ b/algo/src/main/java/org/neo4j/gds/kspanningtree/KSpanningTree.java @@ -58,7 +58,7 @@ public KSpanningTree( super(progressTracker); this.graph = graph; this.minMax = minMax; - this.startNodeId = (int) graph.toMappedNodeId(startNodeId); + this.startNodeId = startNodeId; this.k = k; } diff --git a/proc/path-finding/src/test/java/org/neo4j/gds/paths/kspanningtree/KSpanningTreeWriteProcTest.java b/proc/path-finding/src/test/java/org/neo4j/gds/paths/kspanningtree/KSpanningTreeWriteProcTest.java index 9196c15190..0327691896 100644 --- a/proc/path-finding/src/test/java/org/neo4j/gds/paths/kspanningtree/KSpanningTreeWriteProcTest.java +++ b/proc/path-finding/src/test/java/org/neo4j/gds/paths/kspanningtree/KSpanningTreeWriteProcTest.java @@ -38,7 +38,7 @@ class KSpanningTreeWriteProcTest extends BaseProcTest { private static final String GRAPH_NAME = "graph"; - @Neo4jGraph + @Neo4jGraph(offsetIds = true) private static final String DB_CYPHER = "CREATE (a:Node {name:'a'})\n" + "CREATE (b:Node {name:'b'})\n" + From c0068f012186d31ec4bd984baeed5e2682c8af6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Mon, 12 Jun 2023 09:33:22 +0200 Subject: [PATCH 059/273] Remove caring about id offsets on algorithm level The algorithm expects to get mapped ids --- .../gds/kspanningtree/KSpanningTreeTest.java | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/algo/src/test/java/org/neo4j/gds/kspanningtree/KSpanningTreeTest.java b/algo/src/test/java/org/neo4j/gds/kspanningtree/KSpanningTreeTest.java index d41b8d5874..301f6c95cc 100644 --- a/algo/src/test/java/org/neo4j/gds/kspanningtree/KSpanningTreeTest.java +++ b/algo/src/test/java/org/neo4j/gds/kspanningtree/KSpanningTreeTest.java @@ -76,14 +76,6 @@ class KSpanningTreeTest { @Inject private IdFunction idFunction; - private static final int OFFSET = 5; - - @GdlGraph(idOffset = OFFSET, orientation = Orientation.UNDIRECTED, graphNamePrefix = "offset") - private static final String DB_CYPHER_WITH_OFFSET = DB_CYPHER; - - @Inject - private Graph offsetGraph; - private int a, b, c, d, x; @BeforeEach @@ -118,15 +110,6 @@ void testMinimumKSpanningTree() { assertThat(spanningTree.head(b)).isNotEqualTo(spanningTree.head(x)); } - @Test - void testNeoIdsWithOffset() { - var spanningTree = new KSpanningTree(graph, Prim.MIN_OPERATOR, 0, 2, ProgressTracker.NULL_TRACKER).compute(); - var otherSpanningTree = new KSpanningTree(offsetGraph, Prim.MIN_OPERATOR, OFFSET, 2, ProgressTracker.NULL_TRACKER).compute(); - - assertThat(spanningTree.parentArray().toArray()) - .containsExactly(otherSpanningTree.parentArray().toArray()); - } - @Test void shouldProduceSingleConnectedTree() { var factory = GdlFactory.of("CREATE" + From c874c9259a6c305e76a493ef7ed59aaff47dc910 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Mon, 12 Jun 2023 14:24:56 +0200 Subject: [PATCH 060/273] Fix ConductanceStream result it does not return node ids but community ids. --- .../org/neo4j/gds/conductance/ConductanceStreamSpec.java | 5 ++--- .../org/neo4j/gds/conductance/ConductanceStreamProcTest.java | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/proc/community/src/main/java/org/neo4j/gds/conductance/ConductanceStreamSpec.java b/proc/community/src/main/java/org/neo4j/gds/conductance/ConductanceStreamSpec.java index 2b3c0e9557..5f73ef899a 100644 --- a/proc/community/src/main/java/org/neo4j/gds/conductance/ConductanceStreamSpec.java +++ b/proc/community/src/main/java/org/neo4j/gds/conductance/ConductanceStreamSpec.java @@ -57,12 +57,11 @@ public ComputationResultConsumer computationResult.result() .map(result -> { - var graph = computationResult.graph(); var condunctances = result.communityConductances(); return LongStream .range(0, condunctances.capacity()) - .filter(c -> !Double.isNaN(condunctances.get(c))) - .mapToObj(c -> new StreamResult(graph.toOriginalNodeId(c), condunctances.get(c))); + .filter(community -> !Double.isNaN(condunctances.get(community))) + .mapToObj(community -> new StreamResult(community, condunctances.get(community))); }).orElseGet(Stream::empty) ); } diff --git a/proc/community/src/test/java/org/neo4j/gds/conductance/ConductanceStreamProcTest.java b/proc/community/src/test/java/org/neo4j/gds/conductance/ConductanceStreamProcTest.java index f4e30d26db..0d5c6e7b8f 100644 --- a/proc/community/src/test/java/org/neo4j/gds/conductance/ConductanceStreamProcTest.java +++ b/proc/community/src/test/java/org/neo4j/gds/conductance/ConductanceStreamProcTest.java @@ -38,7 +38,7 @@ class ConductanceStreamProcTest extends BaseProcTest { static final String GRAPH_NAME = "myGraph"; - @Neo4jGraph + @Neo4jGraph(offsetIds = true) @Language("Cypher") private static final String DB_CYPHER = "CREATE" + From bb548cbdec35cf78f7191f84d14cdfdbae6e33ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Mon, 12 Jun 2023 15:26:53 +0200 Subject: [PATCH 061/273] Fix triangle count on a node-filtered graph --- .../IntersectingTriangleCountFilteredGraphTest.java | 2 +- .../neo4j/gds/core/huge/CompositeAdjacencyList.java | 3 ++- .../gds/core/huge/NodeFilteredAdjacencyCursor.java | 10 +++++----- .../core/huge/NodeFilteredAdjacencyCursorTest.java | 13 ++++++++++++- 4 files changed, 20 insertions(+), 8 deletions(-) diff --git a/algo/src/test/java/org/neo4j/gds/triangle/IntersectingTriangleCountFilteredGraphTest.java b/algo/src/test/java/org/neo4j/gds/triangle/IntersectingTriangleCountFilteredGraphTest.java index 98501e94fb..9df195441e 100644 --- a/algo/src/test/java/org/neo4j/gds/triangle/IntersectingTriangleCountFilteredGraphTest.java +++ b/algo/src/test/java/org/neo4j/gds/triangle/IntersectingTriangleCountFilteredGraphTest.java @@ -40,7 +40,7 @@ class IntersectingTriangleCountFilteredGraphTest extends BaseTest { - @Neo4jGraph + @Neo4jGraph(offsetIds = true) static final String DB_CYPHER = "CREATE " + " (a1:A)" + ", (a2:A)" + diff --git a/core/src/main/java/org/neo4j/gds/core/huge/CompositeAdjacencyList.java b/core/src/main/java/org/neo4j/gds/core/huge/CompositeAdjacencyList.java index cd8e3eadea..82caedfba4 100644 --- a/core/src/main/java/org/neo4j/gds/core/huge/CompositeAdjacencyList.java +++ b/core/src/main/java/org/neo4j/gds/core/huge/CompositeAdjacencyList.java @@ -22,6 +22,7 @@ import org.jetbrains.annotations.Nullable; import org.neo4j.gds.api.AdjacencyCursor; import org.neo4j.gds.api.AdjacencyList; +import org.neo4j.gds.api.FilteredIdMap; import org.neo4j.gds.api.IdMap; import org.neo4j.gds.api.ImmutableMemoryInfo; @@ -65,7 +66,7 @@ static CompositeAdjacencyList withFilteredIdMap(List adjacencyLis assert filteredIdMap instanceof NodeFilteredGraph; var adjacencyCursorWrapperFactory = (AdjacencyCursorWrapperFactory) cursor -> new NodeFilteredAdjacencyCursor( cursor, - filteredIdMap + ((FilteredIdMap) filteredIdMap) ); var compositeAdjacencyCursorFactory = (CompositeAdjacencyCursorFactory) cursors -> { List wrappedCursors = cursors diff --git a/core/src/main/java/org/neo4j/gds/core/huge/NodeFilteredAdjacencyCursor.java b/core/src/main/java/org/neo4j/gds/core/huge/NodeFilteredAdjacencyCursor.java index eb0f9613b5..591141e255 100644 --- a/core/src/main/java/org/neo4j/gds/core/huge/NodeFilteredAdjacencyCursor.java +++ b/core/src/main/java/org/neo4j/gds/core/huge/NodeFilteredAdjacencyCursor.java @@ -22,18 +22,18 @@ import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import org.neo4j.gds.api.AdjacencyCursor; -import org.neo4j.gds.api.IdMap; +import org.neo4j.gds.api.FilteredIdMap; import static org.neo4j.gds.utils.StringFormatting.formatWithLocale; public class NodeFilteredAdjacencyCursor implements AdjacencyCursor { private final AdjacencyCursor innerCursor; - private final IdMap idMap; + private final FilteredIdMap idMap; private long nextLongValue; - NodeFilteredAdjacencyCursor(AdjacencyCursor innerCursor, IdMap idMap) { + NodeFilteredAdjacencyCursor(AdjacencyCursor innerCursor, FilteredIdMap idMap) { this.innerCursor = innerCursor; this.idMap = idMap; @@ -57,11 +57,11 @@ public int size() { public boolean hasNextVLong() { if (innerCursor.hasNextVLong()) { var innerNextLong = innerCursor.peekVLong(); - if (!idMap.containsOriginalId(innerNextLong)) { + if (!idMap.containsRootNodeId(innerNextLong)) { innerCursor.nextVLong(); return hasNextVLong(); } - this.nextLongValue = idMap.toMappedNodeId(innerNextLong); + this.nextLongValue = idMap.toFilteredNodeId(innerNextLong); return true; } return false; diff --git a/core/src/test/java/org/neo4j/gds/core/huge/NodeFilteredAdjacencyCursorTest.java b/core/src/test/java/org/neo4j/gds/core/huge/NodeFilteredAdjacencyCursorTest.java index e6aa2010a0..d0cf4b8a62 100644 --- a/core/src/test/java/org/neo4j/gds/core/huge/NodeFilteredAdjacencyCursorTest.java +++ b/core/src/test/java/org/neo4j/gds/core/huge/NodeFilteredAdjacencyCursorTest.java @@ -22,6 +22,7 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.neo4j.gds.api.AdjacencyCursor; +import org.neo4j.gds.api.FilteredIdMap; import java.util.ArrayList; import java.util.List; @@ -78,7 +79,7 @@ void shouldAdvanceWithFilter() { assertThat(adjacencyCursor.advance(9L)).isEqualTo(AdjacencyCursor.NOT_FOUND); } - static class FilteredDirectIdMap extends DirectIdMap { + static class FilteredDirectIdMap extends DirectIdMap implements FilteredIdMap { private final LongPredicate nodeFilter; @@ -91,5 +92,15 @@ static class FilteredDirectIdMap extends DirectIdMap { public boolean containsOriginalId(long originalNodeId) { return nodeFilter.test(originalNodeId); } + + @Override + public long toFilteredNodeId(long rootNodeId) { + return rootNodeId; + } + + @Override + public boolean containsRootNodeId(long rootNodeId) { + return nodeFilter.test(rootNodeId); + } } } From 95a29fe925cfb7520fa56d3c76cab00669491dc2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Mon, 12 Jun 2023 16:06:10 +0200 Subject: [PATCH 062/273] Remove casting for filtered case Co-authored-by: Paul Horn --- .../neo4j/gds/core/huge/CompositeAdjacencyList.java | 10 +++------- .../main/java/org/neo4j/gds/core/huge/UnionGraph.java | 8 ++++---- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/core/src/main/java/org/neo4j/gds/core/huge/CompositeAdjacencyList.java b/core/src/main/java/org/neo4j/gds/core/huge/CompositeAdjacencyList.java index 82caedfba4..c359606ac9 100644 --- a/core/src/main/java/org/neo4j/gds/core/huge/CompositeAdjacencyList.java +++ b/core/src/main/java/org/neo4j/gds/core/huge/CompositeAdjacencyList.java @@ -23,7 +23,6 @@ import org.neo4j.gds.api.AdjacencyCursor; import org.neo4j.gds.api.AdjacencyList; import org.neo4j.gds.api.FilteredIdMap; -import org.neo4j.gds.api.IdMap; import org.neo4j.gds.api.ImmutableMemoryInfo; import java.util.ArrayList; @@ -62,12 +61,9 @@ static CompositeAdjacencyList of(List adjacencyLists) { ); } - static CompositeAdjacencyList withFilteredIdMap(List adjacencyLists, IdMap filteredIdMap) { - assert filteredIdMap instanceof NodeFilteredGraph; - var adjacencyCursorWrapperFactory = (AdjacencyCursorWrapperFactory) cursor -> new NodeFilteredAdjacencyCursor( - cursor, - ((FilteredIdMap) filteredIdMap) - ); + static CompositeAdjacencyList withFilteredIdMap(List adjacencyLists, FilteredIdMap filteredIdMap) { + var adjacencyCursorWrapperFactory = (AdjacencyCursorWrapperFactory) cursor -> new NodeFilteredAdjacencyCursor(cursor, filteredIdMap); + var compositeAdjacencyCursorFactory = (CompositeAdjacencyCursorFactory) cursors -> { List wrappedCursors = cursors .stream() diff --git a/core/src/main/java/org/neo4j/gds/core/huge/UnionGraph.java b/core/src/main/java/org/neo4j/gds/core/huge/UnionGraph.java index 9855f19eb3..c0d914dc43 100644 --- a/core/src/main/java/org/neo4j/gds/core/huge/UnionGraph.java +++ b/core/src/main/java/org/neo4j/gds/core/huge/UnionGraph.java @@ -333,10 +333,10 @@ public CompositeAdjacencyList relationshipTopology() { .flatMap(Collection::stream) .map(Topology::adjacencyList) .collect(Collectors.toList()); - if (isNodeFilteredGraph()) { - return CompositeAdjacencyList.withFilteredIdMap(adjacencies, first); - } - return CompositeAdjacencyList.of(adjacencies); + + return first.asNodeFilteredGraph() + .map(graph -> CompositeAdjacencyList.withFilteredIdMap(adjacencies, graph)) + .orElseGet(() -> CompositeAdjacencyList.of(adjacencies)); } @Override From 7b501ac9cb544be9920afcb8b42c915d928f748a Mon Sep 17 00:00:00 2001 From: Brian Shi Date: Mon, 12 Jun 2023 16:53:07 +0100 Subject: [PATCH 063/273] Fix mutateResult recordHistogramValue --- .../pipeline/predict/MutateResult.java | 4 +- .../pipeline/predict/MutateResultTest.java | 38 +++++++++++++++++++ 2 files changed, 41 insertions(+), 1 deletion(-) create mode 100644 proc/machine-learning/src/test/java/org/neo4j/gds/ml/linkmodels/pipeline/predict/MutateResultTest.java diff --git a/proc/machine-learning/src/main/java/org/neo4j/gds/ml/linkmodels/pipeline/predict/MutateResult.java b/proc/machine-learning/src/main/java/org/neo4j/gds/ml/linkmodels/pipeline/predict/MutateResult.java index cf6e20e4c1..e1552d0269 100644 --- a/proc/machine-learning/src/main/java/org/neo4j/gds/ml/linkmodels/pipeline/predict/MutateResult.java +++ b/proc/machine-learning/src/main/java/org/neo4j/gds/ml/linkmodels/pipeline/predict/MutateResult.java @@ -90,7 +90,9 @@ void recordHistogramValue(double value) { return; } - histogram.recordValue(value); + //HISTOGRAM_PRECISION_DEFAULT hence numberOfSignificantValueDigits is 1E-5, so it can't separate 0 and 1E-5 + //Therefore we can floor at 1E-6 and smaller probabilities between 0 and 1E-6 is unnecessary. + if (value >= 1E-6) histogram.recordValue(value); else histogram.recordValue(1E-6); } Builder withSamplingStats(Map samplingStats) { diff --git a/proc/machine-learning/src/test/java/org/neo4j/gds/ml/linkmodels/pipeline/predict/MutateResultTest.java b/proc/machine-learning/src/test/java/org/neo4j/gds/ml/linkmodels/pipeline/predict/MutateResultTest.java new file mode 100644 index 0000000000..3af65170be --- /dev/null +++ b/proc/machine-learning/src/test/java/org/neo4j/gds/ml/linkmodels/pipeline/predict/MutateResultTest.java @@ -0,0 +1,38 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.ml.linkmodels.pipeline.predict; + +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThatNoException; + +class MutateResultTest { + + @Test + void shouldRecordResults() { + var resultWithHistogramBuilder = new MutateResult.Builder().withHistogram(); + assertThatNoException().isThrownBy(() -> { + resultWithHistogramBuilder.recordHistogramValue(0.9); + resultWithHistogramBuilder.recordHistogramValue(5E-10); + resultWithHistogramBuilder.recordHistogramValue(5E-20); + }); + } + +} \ No newline at end of file From b53543b0b7b5e396c7b787b894464c4da71dfdf2 Mon Sep 17 00:00:00 2001 From: Brian Shi Date: Tue, 13 Jun 2023 09:49:27 +0100 Subject: [PATCH 064/273] Fix checkstyle 24 --- .../gds/ml/linkmodels/pipeline/predict/MutateResultTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/proc/machine-learning/src/test/java/org/neo4j/gds/ml/linkmodels/pipeline/predict/MutateResultTest.java b/proc/machine-learning/src/test/java/org/neo4j/gds/ml/linkmodels/pipeline/predict/MutateResultTest.java index 3af65170be..f18d8f57a5 100644 --- a/proc/machine-learning/src/test/java/org/neo4j/gds/ml/linkmodels/pipeline/predict/MutateResultTest.java +++ b/proc/machine-learning/src/test/java/org/neo4j/gds/ml/linkmodels/pipeline/predict/MutateResultTest.java @@ -35,4 +35,4 @@ void shouldRecordResults() { }); } -} \ No newline at end of file +} From 758988e39f97101cc477eee25c4f4fd9ebf2940a Mon Sep 17 00:00:00 2001 From: Veselin Nikolov Date: Wed, 14 Jun 2023 06:49:05 +0100 Subject: [PATCH 065/273] Bump the Aura version after GDS 2.4.0+23 --- gradle/version.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle/version.gradle b/gradle/version.gradle index 611f7d7a2c..e2ae11ff6f 100644 --- a/gradle/version.gradle +++ b/gradle/version.gradle @@ -1,6 +1,6 @@ ext { gdsBaseVersion = '2.4.0' - gdsAuraVersion = '23' + gdsAuraVersion = '24' gdsVersion = gdsBaseVersion + (rootProject.hasProperty('aurads') ? "+${gdsAuraVersion}" : "") } From b728953f42dde5ffe744f02c0f6e79b11d58bb5b Mon Sep 17 00:00:00 2001 From: Martin Junghanns Date: Mon, 5 Jun 2023 12:56:20 +0200 Subject: [PATCH 066/273] Allow selection of idmap type in IdMapBehaviour impls --- .../org/neo4j/gds/core/IdMapBehavior.java | 15 +++++ .../neo4j/gds/core/OpenGdsIdMapBehavior.java | 6 ++ .../gds/core/loading/ArrayIdMapBuilder.java | 3 + .../gds/core/OpenGdsIdMapBehaviorTest.java | 56 +++++++++++++++++++ 4 files changed, 80 insertions(+) create mode 100644 core/src/test/java/org/neo4j/gds/core/OpenGdsIdMapBehaviorTest.java diff --git a/core/src/main/java/org/neo4j/gds/core/IdMapBehavior.java b/core/src/main/java/org/neo4j/gds/core/IdMapBehavior.java index 6540d712a7..0d3c6cff28 100644 --- a/core/src/main/java/org/neo4j/gds/core/IdMapBehavior.java +++ b/core/src/main/java/org/neo4j/gds/core/IdMapBehavior.java @@ -31,5 +31,20 @@ IdMapBuilder create( Optional nodeCount ); + /** + * Attempts to create an IdMapBuilder identified by the given id. + *

+ * If the id is not recognized, we fall back to the default behavior + * using {@link #create(int, Optional, Optional)}. + * + * @param id the id of the IdMapBuilder to create + */ + IdMapBuilder create( + byte id, + int concurrency, + Optional maxOriginalId, + Optional nodeCount + ); + MemoryEstimation memoryEstimation(); } diff --git a/core/src/main/java/org/neo4j/gds/core/OpenGdsIdMapBehavior.java b/core/src/main/java/org/neo4j/gds/core/OpenGdsIdMapBehavior.java index 9968f0c2f5..535ab0805f 100644 --- a/core/src/main/java/org/neo4j/gds/core/OpenGdsIdMapBehavior.java +++ b/core/src/main/java/org/neo4j/gds/core/OpenGdsIdMapBehavior.java @@ -40,6 +40,12 @@ public IdMapBuilder create( .orElseGet(GrowingArrayIdMapBuilder::of); } + @Override + public IdMapBuilder create(byte id, int concurrency, Optional maxOriginalId, Optional nodeCount) { + // We can only create array id maps in open GDS; no need to check the id. + return create(concurrency, maxOriginalId, nodeCount); + } + @Override public MemoryEstimation memoryEstimation() { return ArrayIdMap.memoryEstimation(); diff --git a/core/src/main/java/org/neo4j/gds/core/loading/ArrayIdMapBuilder.java b/core/src/main/java/org/neo4j/gds/core/loading/ArrayIdMapBuilder.java index 6b1e526a40..4edfabe868 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/ArrayIdMapBuilder.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/ArrayIdMapBuilder.java @@ -28,6 +28,9 @@ public final class ArrayIdMapBuilder implements IdMapBuilder { + public static final String NAME = "array"; + public static final byte ID = 0; + private final HugeLongArray array; private final long capacity; private final AtomicLong allocationIndex; diff --git a/core/src/test/java/org/neo4j/gds/core/OpenGdsIdMapBehaviorTest.java b/core/src/test/java/org/neo4j/gds/core/OpenGdsIdMapBehaviorTest.java new file mode 100644 index 0000000000..3fac7a4765 --- /dev/null +++ b/core/src/test/java/org/neo4j/gds/core/OpenGdsIdMapBehaviorTest.java @@ -0,0 +1,56 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.core; + +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.neo4j.gds.core.loading.ArrayIdMapBuilder; +import org.neo4j.gds.core.loading.GrowingArrayIdMapBuilder; + +import java.util.Optional; +import java.util.stream.Stream; + +import static org.assertj.core.api.Assertions.assertThat; + +class OpenGdsIdMapBehaviorTest { + + static Stream idMapBuildersById() { + return Stream.of( + Arguments.of(ArrayIdMapBuilder.ID, Optional.empty(), Optional.empty(), GrowingArrayIdMapBuilder.class), + Arguments.of(ArrayIdMapBuilder.ID, Optional.empty(), Optional.of(42L), ArrayIdMapBuilder.class), + Arguments.of((byte) 42, Optional.empty(), Optional.empty(), GrowingArrayIdMapBuilder.class), + Arguments.of((byte) 42, Optional.empty(), Optional.of(42L), ArrayIdMapBuilder.class) + ); + } + + @ParameterizedTest + @MethodSource("idMapBuildersById") + void shouldCreateIdMapBuilderById( + byte id, + Optional maxOriginalId, + Optional nodeCount, + Class idMapBuilderClazz + ) { + var idMapBuilder = new OpenGdsIdMapBehavior().create(id, 1, maxOriginalId, nodeCount); + assertThat(idMapBuilder).isInstanceOf(idMapBuilderClazz); + } + +} \ No newline at end of file From bdd69705c2084a93f8a8a839634a4a5f1fead430 Mon Sep 17 00:00:00 2001 From: Martin Junghanns Date: Mon, 5 Jun 2023 13:35:38 +0200 Subject: [PATCH 067/273] Allow setting idmap builder type in nodes builder --- .../gds/core/loading/construction/GraphFactory.java | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/core/src/main/java/org/neo4j/gds/core/loading/construction/GraphFactory.java b/core/src/main/java/org/neo4j/gds/core/loading/construction/GraphFactory.java index 467d9c86a6..7a512a0a00 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/construction/GraphFactory.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/construction/GraphFactory.java @@ -85,7 +85,8 @@ static NodesBuilder nodesBuilder( Optional hasProperties, Optional deduplicateIds, Optional concurrency, - Optional propertyState + Optional propertyState, + Optional idMapBuilderType ) { boolean labelInformation = nodeSchema .map(schema -> !(schema.availableLabels().isEmpty() && schema.containsOnlyAllNodesLabel())) @@ -97,11 +98,9 @@ static NodesBuilder nodesBuilder( ? Optional.of(maxOriginalId) : Optional.empty(); - var idMapBuilder = idMapBehavior.create( - threadCount, - maybeMaxOriginalId, - nodeCount - ); + var idMapBuilder = idMapBuilderType.isPresent() + ? idMapBehavior.create(idMapBuilderType.get(), threadCount, maybeMaxOriginalId, nodeCount) + : idMapBehavior.create(threadCount, maybeMaxOriginalId, nodeCount); boolean deduplicate = deduplicateIds.orElse(true); From 77429c231ae40bce83725b06efe52e9f68ccff4e Mon Sep 17 00:00:00 2001 From: Martin Junghanns Date: Tue, 6 Jun 2023 10:18:09 +0200 Subject: [PATCH 068/273] Remove NAME field --- .../main/java/org/neo4j/gds/core/loading/ArrayIdMapBuilder.java | 1 - .../test/java/org/neo4j/gds/core/OpenGdsIdMapBehaviorTest.java | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/core/src/main/java/org/neo4j/gds/core/loading/ArrayIdMapBuilder.java b/core/src/main/java/org/neo4j/gds/core/loading/ArrayIdMapBuilder.java index 4edfabe868..913d856a38 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/ArrayIdMapBuilder.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/ArrayIdMapBuilder.java @@ -28,7 +28,6 @@ public final class ArrayIdMapBuilder implements IdMapBuilder { - public static final String NAME = "array"; public static final byte ID = 0; private final HugeLongArray array; diff --git a/core/src/test/java/org/neo4j/gds/core/OpenGdsIdMapBehaviorTest.java b/core/src/test/java/org/neo4j/gds/core/OpenGdsIdMapBehaviorTest.java index 3fac7a4765..592898566c 100644 --- a/core/src/test/java/org/neo4j/gds/core/OpenGdsIdMapBehaviorTest.java +++ b/core/src/test/java/org/neo4j/gds/core/OpenGdsIdMapBehaviorTest.java @@ -53,4 +53,4 @@ void shouldCreateIdMapBuilderById( assertThat(idMapBuilder).isInstanceOf(idMapBuilderClazz); } -} \ No newline at end of file +} From 07f3a1db1394a9b5563221561fbd9b2b1e5404af Mon Sep 17 00:00:00 2001 From: Martin Junghanns Date: Mon, 5 Jun 2023 15:53:05 +0200 Subject: [PATCH 069/273] Add HighLimitIdMapBuilder --- .../core/loading/HighLimitIdMapBuilder.java | 103 ++++++++++++++++++ .../loading/HighLimitIdMapBuilderTest.java | 31 ++++++ .../gds/core/idmap/IdMapBuilderTest.java | 21 ++-- 3 files changed, 145 insertions(+), 10 deletions(-) create mode 100644 core/src/main/java/org/neo4j/gds/core/loading/HighLimitIdMapBuilder.java create mode 100644 core/src/test/java/org/neo4j/gds/core/loading/HighLimitIdMapBuilderTest.java diff --git a/core/src/main/java/org/neo4j/gds/core/loading/HighLimitIdMapBuilder.java b/core/src/main/java/org/neo4j/gds/core/loading/HighLimitIdMapBuilder.java new file mode 100644 index 0000000000..d2c99ef358 --- /dev/null +++ b/core/src/main/java/org/neo4j/gds/core/loading/HighLimitIdMapBuilder.java @@ -0,0 +1,103 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.core.loading; + +import org.neo4j.gds.api.IdMap; +import org.neo4j.gds.core.utils.paged.ShardedLongLongMap; +import org.neo4j.gds.utils.CloseableThreadLocal; + +public final class HighLimitIdMapBuilder implements IdMapBuilder { + + public static final String NAME = "highlimit"; + public static final byte ID = 3; + + private final ShardedLongLongMap.Builder intermediateIdMapBuilder; + private final IdMapBuilder internalIdMapBuilder; + + private final CloseableThreadLocal adders; + + public static HighLimitIdMapBuilder of(int concurrency, IdMapBuilder internalIdMapBuilder) { + return new HighLimitIdMapBuilder(concurrency, internalIdMapBuilder); + } + + private HighLimitIdMapBuilder(int concurrency, IdMapBuilder internalIdMapBuilder) { + // TODO: we might want to use the batched builder + this.intermediateIdMapBuilder = ShardedLongLongMap.builder(concurrency); + this.internalIdMapBuilder = internalIdMapBuilder; + this.adders = CloseableThreadLocal.withInitial(this::newBulkAdder); + } + + @Override + public IdMapAllocator allocate(int batchLength) { + var internalAllocator = this.internalIdMapBuilder.allocate(batchLength); + var allocator = this.adders.get(); + allocator.reset(batchLength, internalAllocator); + return allocator; + } + + @Override + public IdMap build(LabelInformation.Builder labelInformationBuilder, long highestNodeId, int concurrency) { + var intermediateIdMap = this.intermediateIdMapBuilder.build(); + var internalIdMap = this.internalIdMapBuilder.build(labelInformationBuilder, highestNodeId, concurrency); + + return new HighLimitIdMap(intermediateIdMap, internalIdMap); + } + + private BulkAdder newBulkAdder() { + return new BulkAdder(intermediateIdMapBuilder); + } + + private static final class BulkAdder implements IdMapAllocator { + + private final ShardedLongLongMap.Builder intermediateIdMapBuilder; + private IdMapAllocator internalAllocator; + private int batchLength = 0; + + private BulkAdder(ShardedLongLongMap.Builder intermediateIdMapBuilder) { + this.intermediateIdMapBuilder = intermediateIdMapBuilder; + } + + private void reset(int batchLength, IdMapAllocator internalAllocator) { + this.batchLength = batchLength; + this.internalAllocator = internalAllocator; + } + + @Override + public int allocatedSize() { + return this.batchLength; + } + + @Override + public void insert(long[] nodeIds) { + int length = this.batchLength; + var intermediateIdMapBuilder = this.intermediateIdMapBuilder; + + // Replace the original nodeIds with the intermediate ones + // in the input buffer as this one is reused by the caller + // to insert node labels and property values. + for (int i = 0; i < length; i++) { + nodeIds[i] = intermediateIdMapBuilder.addNode(nodeIds[i]); + } + + // Use intermediate ids for the internal id map builder. + internalAllocator.insert(nodeIds); + } + } +} diff --git a/core/src/test/java/org/neo4j/gds/core/loading/HighLimitIdMapBuilderTest.java b/core/src/test/java/org/neo4j/gds/core/loading/HighLimitIdMapBuilderTest.java new file mode 100644 index 0000000000..a1072f1a49 --- /dev/null +++ b/core/src/test/java/org/neo4j/gds/core/loading/HighLimitIdMapBuilderTest.java @@ -0,0 +1,31 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.core.loading; + +import org.neo4j.gds.core.idmap.IdMapBuilderTest; + +public class HighLimitIdMapBuilderTest extends IdMapBuilderTest { + + @Override + protected IdMapBuilder builder(long capacity, int concurrency) { + return HighLimitIdMapBuilder.of(concurrency, ArrayIdMapBuilder.of(capacity)); + + } +} diff --git a/test-utils/src/main/java/org/neo4j/gds/core/idmap/IdMapBuilderTest.java b/test-utils/src/main/java/org/neo4j/gds/core/idmap/IdMapBuilderTest.java index 412a52a9d3..a7b5742127 100644 --- a/test-utils/src/main/java/org/neo4j/gds/core/idmap/IdMapBuilderTest.java +++ b/test-utils/src/main/java/org/neo4j/gds/core/idmap/IdMapBuilderTest.java @@ -41,7 +41,7 @@ import java.util.Optional; import java.util.Random; import java.util.SplittableRandom; -import java.util.function.LongFunction; +import java.util.function.IntFunction; import java.util.stream.Collectors; import java.util.stream.LongStream; @@ -111,7 +111,7 @@ void testContains( @ForAll long seed ) { var originalIds = generateShuffledIds(idOffset, nodeCount, seed); - var idMapAndHighestId = buildFrom(originalIds, concurrency); + var idMapAndHighestId = buildFrom(originalIds.clone(), concurrency); var idMap = idMapAndHighestId.idMap(); var highestOriginalId = idMapAndHighestId.highestOriginalId(); @@ -152,7 +152,7 @@ void testToMappedNodeId( @ForAll long seed ) { var originalIds = generateShuffledIds(idOffset, nodeCount, seed); - var idMap = buildIdMapFrom(originalIds, concurrency); + var idMap = buildIdMapFrom(originalIds.clone(), concurrency); var mappedNodeIds = new long[originalIds.length]; var actualNodeCount = idMap.nodeCount(); @@ -202,7 +202,7 @@ void testToOriginalNodeId( @ForAll long seed ) { var originalIds = generateShuffledIds(idOffset, nodeCount, seed); - var idMap = buildIdMapFrom(originalIds, concurrency); + var idMap = buildIdMapFrom(originalIds.clone(), concurrency); var actualOriginalIds = new long[originalIds.length]; Arrays.fill(actualOriginalIds, -1); @@ -271,12 +271,12 @@ void testLabels( var expectedLabels = new HashMap>(); var noLabel = List.of(NodeLabel.ALL_NODES); - var idMap = buildFromWithLabels(originalIds, concurrency, originalId -> { + var idMap = buildFromWithLabels(originalIds.clone(), concurrency, index -> { var labelCount = rng.nextInt(allLabels.length); var labels = labelCount > 0 ? Arrays.stream(allLabels).limit(labelCount).collect(Collectors.toList()) : noLabel; - expectedLabels.put(originalId, labels); + expectedLabels.put(originalIds[index], labels); return labels; }).idMap(); @@ -308,7 +308,7 @@ private IdMapAndHighestId buildFrom(long[] originalIds, int concurrency) { private IdMapAndHighestId buildFromWithLabels( long[] originalIds, int concurrency, - LongFunction> labelFn + IntFunction> labelFn ) { return buildFromWithLabels(originalIds, concurrency, Optional.ofNullable(labelFn)); } @@ -316,7 +316,7 @@ private IdMapAndHighestId buildFromWithLabels( private IdMapAndHighestId buildFromWithLabels( long[] originalIds, int concurrency, - Optional>> labelFn + Optional>> labelFn ) { // number of ids we want to insert at once int batchLength = originalIds.length; @@ -330,8 +330,9 @@ private IdMapAndHighestId buildFromWithLabels( var labelInformationBuilder = labelFn.map(lFn -> { var multiLabelBuilder = LabelInformationBuilders.multiLabelWithCapacity(originalIds.length); - for (long originalNodeId : originalIds) { - lFn.apply(originalNodeId).forEach(label -> multiLabelBuilder.addNodeIdToLabel(label, originalNodeId)); + for (int i = 0; i < originalIds.length; i++) { + long originalId = originalIds[i]; + lFn.apply(i).forEach(label -> multiLabelBuilder.addNodeIdToLabel(label, originalId)); } return multiLabelBuilder; }).orElseGet(LabelInformationBuilders::allNodes); From b472e260f67c274c845b770e24996073a91ed871 Mon Sep 17 00:00:00 2001 From: Martin Junghanns Date: Tue, 6 Jun 2023 11:14:18 +0200 Subject: [PATCH 070/273] Use rootIdMap for id resolving in NodeProperty builder We use the root id map to access the innermost id mapping layer. For all id maps except HighLimit, this is the same instance. For the HighLimit id map, the root id map is the mapping from intermediate to mapped ids. Since we use the intermediate ids to write properties, we need to use that mapping to correctly resolve the values. It is somewhat hacky, I admit ... --- .../nodeproperties/DoubleArrayNodePropertiesBuilder.java | 5 +++-- .../loading/nodeproperties/DoubleNodePropertiesBuilder.java | 6 +++--- .../nodeproperties/FloatArrayNodePropertiesBuilder.java | 6 +++--- .../loading/nodeproperties/InnerNodePropertiesBuilder.java | 3 ++- .../nodeproperties/LongArrayNodePropertiesBuilder.java | 6 +++--- .../loading/nodeproperties/LongNodePropertiesBuilder.java | 5 +++-- 6 files changed, 17 insertions(+), 14 deletions(-) diff --git a/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/DoubleArrayNodePropertiesBuilder.java b/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/DoubleArrayNodePropertiesBuilder.java index 1bb7ac3fbb..7941b9f8ef 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/DoubleArrayNodePropertiesBuilder.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/DoubleArrayNodePropertiesBuilder.java @@ -60,7 +60,7 @@ public void setValue(long neoNodeId, Value value) { } @Override - public DoubleArrayNodePropertyValues build(long size, PartialIdMap idMap, long highestOriginalId) { + public DoubleArrayNodePropertyValues build(long size, IdMap idMap, long highestOriginalId) { var propertiesByNeoIds = builder.build(); var propertiesByMappedIdsBuilder = HugeSparseDoubleArrayArray.builder( @@ -71,6 +71,7 @@ public DoubleArrayNodePropertyValues build(long size, PartialIdMap idMap, long h var tasks = IntStream.range(0, concurrency).mapToObj(threadId -> (Runnable) () -> { var batch = drainingIterator.drainingBatch(); + var rootIdMap = idMap.rootIdMap(); while (drainingIterator.next(batch)) { var page = batch.page; @@ -79,7 +80,7 @@ public DoubleArrayNodePropertyValues build(long size, PartialIdMap idMap, long h for (int pageIndex = 0; pageIndex < end; pageIndex++) { var neoId = offset + pageIndex; - var mappedId = idMap.toMappedNodeId(neoId); + var mappedId = rootIdMap.toMappedNodeId(neoId); if (mappedId == IdMap.NOT_FOUND) { continue; } diff --git a/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/DoubleNodePropertiesBuilder.java b/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/DoubleNodePropertiesBuilder.java index e5b45a1e48..badec887ce 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/DoubleNodePropertiesBuilder.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/DoubleNodePropertiesBuilder.java @@ -21,7 +21,6 @@ import org.neo4j.gds.api.DefaultValue; import org.neo4j.gds.api.IdMap; -import org.neo4j.gds.api.PartialIdMap; import org.neo4j.gds.api.properties.nodes.DoubleNodePropertyValues; import org.neo4j.gds.collections.HugeSparseDoubleArray; import org.neo4j.gds.core.concurrency.ParallelUtil; @@ -84,7 +83,7 @@ public void setValue(long neoNodeId, Value value) { } @Override - public DoubleNodePropertyValues build(long size, PartialIdMap idMap, long highestOriginalId) { + public DoubleNodePropertyValues build(long size, IdMap idMap, long highestOriginalId) { var propertiesByNeoIds = builder.build(); var propertiesByMappedIdsBuilder = HugeSparseDoubleArray.builder( @@ -95,6 +94,7 @@ public DoubleNodePropertyValues build(long size, PartialIdMap idMap, long highes var tasks = IntStream.range(0, concurrency).mapToObj(threadId -> (Runnable) () -> { var batch = drainingIterator.drainingBatch(); + var rootIdMap = idMap.rootIdMap(); while (drainingIterator.next(batch)) { var page = batch.page; @@ -103,7 +103,7 @@ public DoubleNodePropertyValues build(long size, PartialIdMap idMap, long highes for (int pageIndex = 0; pageIndex < end; pageIndex++) { var neoId = offset + pageIndex; - var mappedId = idMap.toMappedNodeId(neoId); + var mappedId = rootIdMap.toMappedNodeId(neoId); if (mappedId == IdMap.NOT_FOUND) { continue; } diff --git a/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/FloatArrayNodePropertiesBuilder.java b/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/FloatArrayNodePropertiesBuilder.java index 86633552e9..a20ccd3cad 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/FloatArrayNodePropertiesBuilder.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/FloatArrayNodePropertiesBuilder.java @@ -21,7 +21,6 @@ import org.neo4j.gds.api.DefaultValue; import org.neo4j.gds.api.IdMap; -import org.neo4j.gds.api.PartialIdMap; import org.neo4j.gds.api.properties.nodes.FloatArrayNodePropertyValues; import org.neo4j.gds.collections.HugeSparseFloatArrayArray; import org.neo4j.gds.core.concurrency.ParallelUtil; @@ -58,7 +57,7 @@ public void setValue(long neoNodeId, Value value) { } @Override - public FloatArrayNodePropertyValues build(long size, PartialIdMap idMap, long highestOriginalId) { + public FloatArrayNodePropertyValues build(long size, IdMap idMap, long highestOriginalId) { var propertiesByNeoIds = builder.build(); var propertiesByMappedIdsBuilder = HugeSparseFloatArrayArray.builder( @@ -69,6 +68,7 @@ public FloatArrayNodePropertyValues build(long size, PartialIdMap idMap, long hi var tasks = IntStream.range(0, concurrency).mapToObj(threadId -> (Runnable) () -> { var batch = drainingIterator.drainingBatch(); + var rootIdMap = idMap.rootIdMap(); while (drainingIterator.next(batch)) { var page = batch.page; @@ -77,7 +77,7 @@ public FloatArrayNodePropertyValues build(long size, PartialIdMap idMap, long hi for (int pageIndex = 0; pageIndex < end; pageIndex++) { var neoId = offset + pageIndex; - var mappedId = idMap.toMappedNodeId(neoId); + var mappedId = rootIdMap.toMappedNodeId(neoId); if (mappedId == IdMap.NOT_FOUND) { continue; } diff --git a/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/InnerNodePropertiesBuilder.java b/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/InnerNodePropertiesBuilder.java index 4e61d9bd7f..2d07560769 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/InnerNodePropertiesBuilder.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/InnerNodePropertiesBuilder.java @@ -19,6 +19,7 @@ */ package org.neo4j.gds.core.loading.nodeproperties; +import org.neo4j.gds.api.IdMap; import org.neo4j.gds.api.PartialIdMap; import org.neo4j.gds.api.properties.nodes.NodePropertyValues; import org.neo4j.values.storable.Value; @@ -31,5 +32,5 @@ public interface InnerNodePropertiesBuilder { * Builds the underlying node properties and performs a remapping * to the internal id space using the given id map. */ - NodePropertyValues build(long size, PartialIdMap idMap, long highestOriginalId); + NodePropertyValues build(long size, IdMap idMap, long highestOriginalId); } diff --git a/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/LongArrayNodePropertiesBuilder.java b/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/LongArrayNodePropertiesBuilder.java index 6c3616b641..f46cb29215 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/LongArrayNodePropertiesBuilder.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/LongArrayNodePropertiesBuilder.java @@ -21,7 +21,6 @@ import org.neo4j.gds.api.DefaultValue; import org.neo4j.gds.api.IdMap; -import org.neo4j.gds.api.PartialIdMap; import org.neo4j.gds.api.properties.nodes.LongArrayNodePropertyValues; import org.neo4j.gds.collections.HugeSparseLongArrayArray; import org.neo4j.gds.core.concurrency.ParallelUtil; @@ -62,7 +61,7 @@ public void setValue(long nodeId, long[] value) { } @Override - public LongArrayNodePropertyValues build(long size, PartialIdMap idMap, long highestOriginalId) { + public LongArrayNodePropertyValues build(long size, IdMap idMap, long highestOriginalId) { var propertiesByNeoIds = builder.build(); var propertiesByMappedIdsBuilder = HugeSparseLongArrayArray.builder( @@ -73,6 +72,7 @@ public LongArrayNodePropertyValues build(long size, PartialIdMap idMap, long hig var tasks = IntStream.range(0, concurrency).mapToObj(threadId -> (Runnable) () -> { var batch = drainingIterator.drainingBatch(); + var rootIdMap = idMap.rootIdMap(); while (drainingIterator.next(batch)) { var page = batch.page; @@ -81,7 +81,7 @@ public LongArrayNodePropertyValues build(long size, PartialIdMap idMap, long hig for (int pageIndex = 0; pageIndex < end; pageIndex++) { var neoId = offset + pageIndex; - var mappedId = idMap.toMappedNodeId(neoId); + var mappedId = rootIdMap.toMappedNodeId(neoId); if (mappedId == IdMap.NOT_FOUND) { continue; } diff --git a/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/LongNodePropertiesBuilder.java b/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/LongNodePropertiesBuilder.java index e743d718d1..8fda1adb38 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/LongNodePropertiesBuilder.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/LongNodePropertiesBuilder.java @@ -92,7 +92,7 @@ public void setValue(long neoNodeId, Value value) { } @Override - public NodePropertyValues build(long size, PartialIdMap idMap, long highestOriginalId) { + public NodePropertyValues build(long size, IdMap idMap, long highestOriginalId) { var propertiesByNeoIds = builder.build(); var propertiesByMappedIdsBuilder = HugeSparseLongArray.builder( @@ -103,6 +103,7 @@ public NodePropertyValues build(long size, PartialIdMap idMap, long highestOrigi var tasks = IntStream.range(0, concurrency).mapToObj(threadId -> (Runnable) () -> { var batch = drainingIterator.drainingBatch(); + var rootIdMap = idMap.rootIdMap(); while (drainingIterator.next(batch)) { var page = batch.page; @@ -111,7 +112,7 @@ public NodePropertyValues build(long size, PartialIdMap idMap, long highestOrigi for (int pageIndex = 0; pageIndex < end; pageIndex++) { var neoId = offset + pageIndex; - var mappedId = idMap.toMappedNodeId(neoId); + var mappedId = rootIdMap.toMappedNodeId(neoId); if (mappedId == IdMap.NOT_FOUND) { continue; } From 74bac8e5159881d33a860a06f65462a50d59f399 Mon Sep 17 00:00:00 2001 From: Martin Junghanns Date: Tue, 6 Jun 2023 11:16:47 +0200 Subject: [PATCH 071/273] Remove HighLimitIdMapBuilder#NAME --- .../java/org/neo4j/gds/core/loading/HighLimitIdMapBuilder.java | 1 - 1 file changed, 1 deletion(-) diff --git a/core/src/main/java/org/neo4j/gds/core/loading/HighLimitIdMapBuilder.java b/core/src/main/java/org/neo4j/gds/core/loading/HighLimitIdMapBuilder.java index d2c99ef358..c8df1be4a3 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/HighLimitIdMapBuilder.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/HighLimitIdMapBuilder.java @@ -25,7 +25,6 @@ public final class HighLimitIdMapBuilder implements IdMapBuilder { - public static final String NAME = "highlimit"; public static final byte ID = 3; private final ShardedLongLongMap.Builder intermediateIdMapBuilder; From d0c7833e1a8d5932d6e1a4fd4a477e349d916176 Mon Sep 17 00:00:00 2001 From: Martin Junghanns Date: Tue, 6 Jun 2023 11:51:40 +0200 Subject: [PATCH 072/273] Use BatchedBuilder in HighLimitIdMapBuilder --- .../core/loading/HighLimitIdMapBuilder.java | 33 +++++--------- .../core/utils/paged/ShardedLongLongMap.java | 43 +++++++++++++++---- 2 files changed, 45 insertions(+), 31 deletions(-) diff --git a/core/src/main/java/org/neo4j/gds/core/loading/HighLimitIdMapBuilder.java b/core/src/main/java/org/neo4j/gds/core/loading/HighLimitIdMapBuilder.java index c8df1be4a3..d4f4ee28a0 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/HighLimitIdMapBuilder.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/HighLimitIdMapBuilder.java @@ -27,7 +27,7 @@ public final class HighLimitIdMapBuilder implements IdMapBuilder { public static final byte ID = 3; - private final ShardedLongLongMap.Builder intermediateIdMapBuilder; + private final ShardedLongLongMap.BatchedBuilder intermediateIdMapBuilder; private final IdMapBuilder internalIdMapBuilder; private final CloseableThreadLocal adders; @@ -37,8 +37,7 @@ public static HighLimitIdMapBuilder of(int concurrency, IdMapBuilder internalIdM } private HighLimitIdMapBuilder(int concurrency, IdMapBuilder internalIdMapBuilder) { - // TODO: we might want to use the batched builder - this.intermediateIdMapBuilder = ShardedLongLongMap.builder(concurrency); + this.intermediateIdMapBuilder = ShardedLongLongMap.batchedBuilder(concurrency, true); this.internalIdMapBuilder = internalIdMapBuilder; this.adders = CloseableThreadLocal.withInitial(this::newBulkAdder); } @@ -46,8 +45,9 @@ private HighLimitIdMapBuilder(int concurrency, IdMapBuilder internalIdMapBuilder @Override public IdMapAllocator allocate(int batchLength) { var internalAllocator = this.internalIdMapBuilder.allocate(batchLength); + var intermediateAllocator = this.intermediateIdMapBuilder.prepareBatch(batchLength); var allocator = this.adders.get(); - allocator.reset(batchLength, internalAllocator); + allocator.reset(batchLength, internalAllocator, intermediateAllocator); return allocator; } @@ -60,22 +60,19 @@ public IdMap build(LabelInformation.Builder labelInformationBuilder, long highes } private BulkAdder newBulkAdder() { - return new BulkAdder(intermediateIdMapBuilder); + return new BulkAdder(); } private static final class BulkAdder implements IdMapAllocator { - private final ShardedLongLongMap.Builder intermediateIdMapBuilder; + private IdMapAllocator intermediateAllocator; private IdMapAllocator internalAllocator; private int batchLength = 0; - private BulkAdder(ShardedLongLongMap.Builder intermediateIdMapBuilder) { - this.intermediateIdMapBuilder = intermediateIdMapBuilder; - } - - private void reset(int batchLength, IdMapAllocator internalAllocator) { + private void reset(int batchLength, IdMapAllocator internalAllocator, IdMapAllocator intermediateAllocator) { this.batchLength = batchLength; this.internalAllocator = internalAllocator; + this.intermediateAllocator = intermediateAllocator; } @Override @@ -85,18 +82,8 @@ public int allocatedSize() { @Override public void insert(long[] nodeIds) { - int length = this.batchLength; - var intermediateIdMapBuilder = this.intermediateIdMapBuilder; - - // Replace the original nodeIds with the intermediate ones - // in the input buffer as this one is reused by the caller - // to insert node labels and property values. - for (int i = 0; i < length; i++) { - nodeIds[i] = intermediateIdMapBuilder.addNode(nodeIds[i]); - } - - // Use intermediate ids for the internal id map builder. - internalAllocator.insert(nodeIds); + this.intermediateAllocator.insert(nodeIds); + this.internalAllocator.insert(nodeIds); } } } diff --git a/core/src/main/java/org/neo4j/gds/core/utils/paged/ShardedLongLongMap.java b/core/src/main/java/org/neo4j/gds/core/utils/paged/ShardedLongLongMap.java index 62c0f10ec2..dc4b1db032 100644 --- a/core/src/main/java/org/neo4j/gds/core/utils/paged/ShardedLongLongMap.java +++ b/core/src/main/java/org/neo4j/gds/core/utils/paged/ShardedLongLongMap.java @@ -48,7 +48,11 @@ public static Builder builder(int concurrency) { } public static BatchedBuilder batchedBuilder(int concurrency) { - return new BatchedBuilder(concurrency); + return batchedBuilder(concurrency, false); + } + + public static BatchedBuilder batchedBuilder(int concurrency, boolean overrideIds) { + return new BatchedBuilder(concurrency, overrideIds); } private ShardedLongLongMap( @@ -273,7 +277,7 @@ public static final class BatchedBuilder { private final int shardShift; private final int shardMask; - BatchedBuilder(int concurrency) { + BatchedBuilder(int concurrency, boolean overrideIds) { this.nodeCount = new AtomicLong(); int numberOfShards = numberOfShards(concurrency); this.shardShift = Long.SIZE - Integer.numberOfTrailingZeros(numberOfShards); @@ -281,11 +285,16 @@ public static final class BatchedBuilder { this.shards = IntStream.range(0, numberOfShards) .mapToObj(__ -> new Shard()) .toArray(Shard[]::new); - this.batches = CloseableThreadLocal.withInitial(() -> new Batch( - this.shards, - this.shardShift, - this.shardMask - )); + this.batches = CloseableThreadLocal.withInitial(() -> { + if (overrideIds) { + return new OverridingBatch(this.shards, this.shardShift, this.shardMask); + } + return new Batch( + this.shards, + this.shardShift, + this.shardMask + ); + }); } public Batch prepareBatch(int nodeCount) { @@ -316,7 +325,7 @@ public ShardedLongLongMap build(long maxOriginalId) { ); } - public static final class Batch implements IdMapAllocator { + public static class Batch implements IdMapAllocator { private final Shard[] shards; private final int shardShift; @@ -359,6 +368,24 @@ void initBatch(long startId, int length) { } } + /** + * An allocator/batch that overrides the incoming node ids with the generated mapped ids. + */ + private static final class OverridingBatch extends Batch { + + private OverridingBatch(Shard[] shards, int shardShift, int shardMask) { + super(shards, shardShift, shardMask); + } + + @Override + public void insert(long[] nodeIds) { + int length = allocatedSize(); + for (int i = 0; i < length; i++) { + nodeIds[i] = addNode(nodeIds[i]); + } + } + } + private static final class Shard extends MapShard { void addNode(long nodeId, long mappedId) { From e33ba24c50080edf3d6edc5dfc7a0f85a7aadcf5 Mon Sep 17 00:00:00 2001 From: Martin Junghanns Date: Tue, 6 Jun 2023 14:52:50 +0200 Subject: [PATCH 073/273] Resolve rootIdMap only when necessary and comment inline --- .../nodeproperties/DoubleArrayNodePropertiesBuilder.java | 5 ++--- .../nodeproperties/DoubleNodePropertiesBuilder.java | 6 +++--- .../nodeproperties/FloatArrayNodePropertiesBuilder.java | 6 +++--- .../nodeproperties/InnerNodePropertiesBuilder.java | 3 +-- .../nodeproperties/LongArrayNodePropertiesBuilder.java | 6 +++--- .../nodeproperties/LongNodePropertiesBuilder.java | 5 ++--- .../nodeproperties/NodePropertiesFromStoreBuilder.java | 9 ++++++++- 7 files changed, 22 insertions(+), 18 deletions(-) diff --git a/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/DoubleArrayNodePropertiesBuilder.java b/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/DoubleArrayNodePropertiesBuilder.java index 7941b9f8ef..1bb7ac3fbb 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/DoubleArrayNodePropertiesBuilder.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/DoubleArrayNodePropertiesBuilder.java @@ -60,7 +60,7 @@ public void setValue(long neoNodeId, Value value) { } @Override - public DoubleArrayNodePropertyValues build(long size, IdMap idMap, long highestOriginalId) { + public DoubleArrayNodePropertyValues build(long size, PartialIdMap idMap, long highestOriginalId) { var propertiesByNeoIds = builder.build(); var propertiesByMappedIdsBuilder = HugeSparseDoubleArrayArray.builder( @@ -71,7 +71,6 @@ public DoubleArrayNodePropertyValues build(long size, IdMap idMap, long highestO var tasks = IntStream.range(0, concurrency).mapToObj(threadId -> (Runnable) () -> { var batch = drainingIterator.drainingBatch(); - var rootIdMap = idMap.rootIdMap(); while (drainingIterator.next(batch)) { var page = batch.page; @@ -80,7 +79,7 @@ public DoubleArrayNodePropertyValues build(long size, IdMap idMap, long highestO for (int pageIndex = 0; pageIndex < end; pageIndex++) { var neoId = offset + pageIndex; - var mappedId = rootIdMap.toMappedNodeId(neoId); + var mappedId = idMap.toMappedNodeId(neoId); if (mappedId == IdMap.NOT_FOUND) { continue; } diff --git a/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/DoubleNodePropertiesBuilder.java b/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/DoubleNodePropertiesBuilder.java index badec887ce..e5b45a1e48 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/DoubleNodePropertiesBuilder.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/DoubleNodePropertiesBuilder.java @@ -21,6 +21,7 @@ import org.neo4j.gds.api.DefaultValue; import org.neo4j.gds.api.IdMap; +import org.neo4j.gds.api.PartialIdMap; import org.neo4j.gds.api.properties.nodes.DoubleNodePropertyValues; import org.neo4j.gds.collections.HugeSparseDoubleArray; import org.neo4j.gds.core.concurrency.ParallelUtil; @@ -83,7 +84,7 @@ public void setValue(long neoNodeId, Value value) { } @Override - public DoubleNodePropertyValues build(long size, IdMap idMap, long highestOriginalId) { + public DoubleNodePropertyValues build(long size, PartialIdMap idMap, long highestOriginalId) { var propertiesByNeoIds = builder.build(); var propertiesByMappedIdsBuilder = HugeSparseDoubleArray.builder( @@ -94,7 +95,6 @@ public DoubleNodePropertyValues build(long size, IdMap idMap, long highestOrigin var tasks = IntStream.range(0, concurrency).mapToObj(threadId -> (Runnable) () -> { var batch = drainingIterator.drainingBatch(); - var rootIdMap = idMap.rootIdMap(); while (drainingIterator.next(batch)) { var page = batch.page; @@ -103,7 +103,7 @@ public DoubleNodePropertyValues build(long size, IdMap idMap, long highestOrigin for (int pageIndex = 0; pageIndex < end; pageIndex++) { var neoId = offset + pageIndex; - var mappedId = rootIdMap.toMappedNodeId(neoId); + var mappedId = idMap.toMappedNodeId(neoId); if (mappedId == IdMap.NOT_FOUND) { continue; } diff --git a/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/FloatArrayNodePropertiesBuilder.java b/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/FloatArrayNodePropertiesBuilder.java index a20ccd3cad..86633552e9 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/FloatArrayNodePropertiesBuilder.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/FloatArrayNodePropertiesBuilder.java @@ -21,6 +21,7 @@ import org.neo4j.gds.api.DefaultValue; import org.neo4j.gds.api.IdMap; +import org.neo4j.gds.api.PartialIdMap; import org.neo4j.gds.api.properties.nodes.FloatArrayNodePropertyValues; import org.neo4j.gds.collections.HugeSparseFloatArrayArray; import org.neo4j.gds.core.concurrency.ParallelUtil; @@ -57,7 +58,7 @@ public void setValue(long neoNodeId, Value value) { } @Override - public FloatArrayNodePropertyValues build(long size, IdMap idMap, long highestOriginalId) { + public FloatArrayNodePropertyValues build(long size, PartialIdMap idMap, long highestOriginalId) { var propertiesByNeoIds = builder.build(); var propertiesByMappedIdsBuilder = HugeSparseFloatArrayArray.builder( @@ -68,7 +69,6 @@ public FloatArrayNodePropertyValues build(long size, IdMap idMap, long highestOr var tasks = IntStream.range(0, concurrency).mapToObj(threadId -> (Runnable) () -> { var batch = drainingIterator.drainingBatch(); - var rootIdMap = idMap.rootIdMap(); while (drainingIterator.next(batch)) { var page = batch.page; @@ -77,7 +77,7 @@ public FloatArrayNodePropertyValues build(long size, IdMap idMap, long highestOr for (int pageIndex = 0; pageIndex < end; pageIndex++) { var neoId = offset + pageIndex; - var mappedId = rootIdMap.toMappedNodeId(neoId); + var mappedId = idMap.toMappedNodeId(neoId); if (mappedId == IdMap.NOT_FOUND) { continue; } diff --git a/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/InnerNodePropertiesBuilder.java b/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/InnerNodePropertiesBuilder.java index 2d07560769..4e61d9bd7f 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/InnerNodePropertiesBuilder.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/InnerNodePropertiesBuilder.java @@ -19,7 +19,6 @@ */ package org.neo4j.gds.core.loading.nodeproperties; -import org.neo4j.gds.api.IdMap; import org.neo4j.gds.api.PartialIdMap; import org.neo4j.gds.api.properties.nodes.NodePropertyValues; import org.neo4j.values.storable.Value; @@ -32,5 +31,5 @@ public interface InnerNodePropertiesBuilder { * Builds the underlying node properties and performs a remapping * to the internal id space using the given id map. */ - NodePropertyValues build(long size, IdMap idMap, long highestOriginalId); + NodePropertyValues build(long size, PartialIdMap idMap, long highestOriginalId); } diff --git a/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/LongArrayNodePropertiesBuilder.java b/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/LongArrayNodePropertiesBuilder.java index f46cb29215..6c3616b641 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/LongArrayNodePropertiesBuilder.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/LongArrayNodePropertiesBuilder.java @@ -21,6 +21,7 @@ import org.neo4j.gds.api.DefaultValue; import org.neo4j.gds.api.IdMap; +import org.neo4j.gds.api.PartialIdMap; import org.neo4j.gds.api.properties.nodes.LongArrayNodePropertyValues; import org.neo4j.gds.collections.HugeSparseLongArrayArray; import org.neo4j.gds.core.concurrency.ParallelUtil; @@ -61,7 +62,7 @@ public void setValue(long nodeId, long[] value) { } @Override - public LongArrayNodePropertyValues build(long size, IdMap idMap, long highestOriginalId) { + public LongArrayNodePropertyValues build(long size, PartialIdMap idMap, long highestOriginalId) { var propertiesByNeoIds = builder.build(); var propertiesByMappedIdsBuilder = HugeSparseLongArrayArray.builder( @@ -72,7 +73,6 @@ public LongArrayNodePropertyValues build(long size, IdMap idMap, long highestOri var tasks = IntStream.range(0, concurrency).mapToObj(threadId -> (Runnable) () -> { var batch = drainingIterator.drainingBatch(); - var rootIdMap = idMap.rootIdMap(); while (drainingIterator.next(batch)) { var page = batch.page; @@ -81,7 +81,7 @@ public LongArrayNodePropertyValues build(long size, IdMap idMap, long highestOri for (int pageIndex = 0; pageIndex < end; pageIndex++) { var neoId = offset + pageIndex; - var mappedId = rootIdMap.toMappedNodeId(neoId); + var mappedId = idMap.toMappedNodeId(neoId); if (mappedId == IdMap.NOT_FOUND) { continue; } diff --git a/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/LongNodePropertiesBuilder.java b/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/LongNodePropertiesBuilder.java index 8fda1adb38..e743d718d1 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/LongNodePropertiesBuilder.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/LongNodePropertiesBuilder.java @@ -92,7 +92,7 @@ public void setValue(long neoNodeId, Value value) { } @Override - public NodePropertyValues build(long size, IdMap idMap, long highestOriginalId) { + public NodePropertyValues build(long size, PartialIdMap idMap, long highestOriginalId) { var propertiesByNeoIds = builder.build(); var propertiesByMappedIdsBuilder = HugeSparseLongArray.builder( @@ -103,7 +103,6 @@ public NodePropertyValues build(long size, IdMap idMap, long highestOriginalId) var tasks = IntStream.range(0, concurrency).mapToObj(threadId -> (Runnable) () -> { var batch = drainingIterator.drainingBatch(); - var rootIdMap = idMap.rootIdMap(); while (drainingIterator.next(batch)) { var page = batch.page; @@ -112,7 +111,7 @@ public NodePropertyValues build(long size, IdMap idMap, long highestOriginalId) for (int pageIndex = 0; pageIndex < end; pageIndex++) { var neoId = offset + pageIndex; - var mappedId = rootIdMap.toMappedNodeId(neoId); + var mappedId = idMap.toMappedNodeId(neoId); if (mappedId == IdMap.NOT_FOUND) { continue; } diff --git a/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/NodePropertiesFromStoreBuilder.java b/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/NodePropertiesFromStoreBuilder.java index df891258ee..e6f4e819e4 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/NodePropertiesFromStoreBuilder.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/NodePropertiesFromStoreBuilder.java @@ -24,6 +24,7 @@ import org.neo4j.gds.api.nodeproperties.ValueType; import org.neo4j.gds.api.properties.nodes.NodePropertyValues; import org.neo4j.gds.collections.HugeSparseCollections; +import org.neo4j.gds.core.loading.HighLimitIdMap; import org.neo4j.gds.core.loading.ValueConverter; import org.neo4j.gds.core.utils.mem.MemoryEstimation; import org.neo4j.gds.core.utils.mem.MemoryEstimations; @@ -90,7 +91,13 @@ public NodePropertyValues build(IdMap idMap) { } } - return innerBuilder.get().build(idMap.nodeCount(), idMap, idMap.highestOriginalId()); + // For HighLimitIdMap, we need to use the rootIdMap to resolve intermediate + // node ids correctly. The rootIdMap in that case is the mapping between + // intermediate and mapped node ids. The imported property values are associated + // with the intermediate node ids. + var actualIdMap = (idMap instanceof HighLimitIdMap) ? idMap.rootIdMap() : idMap; + + return innerBuilder.get().build(idMap.nodeCount(), actualIdMap, idMap.highestOriginalId()); } // This is synchronized as we want to prevent the creation of multiple InnerNodePropertiesBuilders of which only once survives. From a2d46b3749892bbffa7f1d768e426fe9b2db11c7 Mon Sep 17 00:00:00 2001 From: Martin Junghanns Date: Tue, 6 Jun 2023 15:00:18 +0200 Subject: [PATCH 074/273] Improve internal naming and docs in HL IdMapBuilder MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Max Kießling --- .../core/loading/HighLimitIdMapBuilder.java | 29 ++++++++++--------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/core/src/main/java/org/neo4j/gds/core/loading/HighLimitIdMapBuilder.java b/core/src/main/java/org/neo4j/gds/core/loading/HighLimitIdMapBuilder.java index d4f4ee28a0..a6587c40b0 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/HighLimitIdMapBuilder.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/HighLimitIdMapBuilder.java @@ -27,34 +27,37 @@ public final class HighLimitIdMapBuilder implements IdMapBuilder { public static final byte ID = 3; - private final ShardedLongLongMap.BatchedBuilder intermediateIdMapBuilder; - private final IdMapBuilder internalIdMapBuilder; + private final ShardedLongLongMap.BatchedBuilder originalToIntermediateMapping; + private final IdMapBuilder intermediateToInternalMapping; - private final CloseableThreadLocal adders; + private final CloseableThreadLocal bulkAdders; public static HighLimitIdMapBuilder of(int concurrency, IdMapBuilder internalIdMapBuilder) { return new HighLimitIdMapBuilder(concurrency, internalIdMapBuilder); } private HighLimitIdMapBuilder(int concurrency, IdMapBuilder internalIdMapBuilder) { - this.intermediateIdMapBuilder = ShardedLongLongMap.batchedBuilder(concurrency, true); - this.internalIdMapBuilder = internalIdMapBuilder; - this.adders = CloseableThreadLocal.withInitial(this::newBulkAdder); + // We use a builder that overrides the node ids in the input batch with the + // generated intermediate node ids. This is necessary for downstream label + // and property processing. + this.originalToIntermediateMapping = ShardedLongLongMap.batchedBuilder(concurrency, true); + this.intermediateToInternalMapping = internalIdMapBuilder; + this.bulkAdders = CloseableThreadLocal.withInitial(this::newBulkAdder); } @Override public IdMapAllocator allocate(int batchLength) { - var internalAllocator = this.internalIdMapBuilder.allocate(batchLength); - var intermediateAllocator = this.intermediateIdMapBuilder.prepareBatch(batchLength); - var allocator = this.adders.get(); - allocator.reset(batchLength, internalAllocator, intermediateAllocator); - return allocator; + var batch = this.originalToIntermediateMapping.prepareBatch(batchLength); + var internalAllocator = this.intermediateToInternalMapping.allocate(batchLength); + var bulkAdder = this.bulkAdders.get(); + bulkAdder.reset(batchLength, internalAllocator, batch); + return bulkAdder; } @Override public IdMap build(LabelInformation.Builder labelInformationBuilder, long highestNodeId, int concurrency) { - var intermediateIdMap = this.intermediateIdMapBuilder.build(); - var internalIdMap = this.internalIdMapBuilder.build(labelInformationBuilder, highestNodeId, concurrency); + var intermediateIdMap = this.originalToIntermediateMapping.build(); + var internalIdMap = this.intermediateToInternalMapping.build(labelInformationBuilder, highestNodeId, concurrency); return new HighLimitIdMap(intermediateIdMap, internalIdMap); } From 2617a1c82e02c0a0e4d9a50bf2af03284a44d97f Mon Sep 17 00:00:00 2001 From: Martin Junghanns Date: Tue, 6 Jun 2023 16:47:14 +0200 Subject: [PATCH 075/273] Export idMapBuilderType --- core/src/main/java/org/neo4j/gds/api/GraphStore.java | 2 ++ .../java/org/neo4j/gds/api/GraphStoreAdapter.java | 5 +++++ .../org/neo4j/gds/core/loading/CSRGraphStore.java | 5 +++++ .../java/org/neo4j/gds/core/io/MetaDataStore.java | 1 + .../java/org/neo4j/gds/core/io/file/GraphInfo.java | 1 + .../gds/core/io/file/csv/CsvGraphInfoVisitor.java | 3 +++ .../core/io/file/csv/CsvGraphInfoVisitorTest.java | 12 +++++++++++- .../io/file/csv/GraphStoreToCsvExporterTest.java | 2 ++ 8 files changed, 30 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/org/neo4j/gds/api/GraphStore.java b/core/src/main/java/org/neo4j/gds/api/GraphStore.java index 750b415371..1aacc2ba72 100644 --- a/core/src/main/java/org/neo4j/gds/api/GraphStore.java +++ b/core/src/main/java/org/neo4j/gds/api/GraphStore.java @@ -47,6 +47,8 @@ public interface GraphStore { DatabaseId databaseId(); + byte idMapBuilderType(); + GraphSchema schema(); ZonedDateTime modificationTime(); diff --git a/core/src/main/java/org/neo4j/gds/api/GraphStoreAdapter.java b/core/src/main/java/org/neo4j/gds/api/GraphStoreAdapter.java index 8c805a99a4..abbfbd6ae0 100644 --- a/core/src/main/java/org/neo4j/gds/api/GraphStoreAdapter.java +++ b/core/src/main/java/org/neo4j/gds/api/GraphStoreAdapter.java @@ -54,6 +54,11 @@ public DatabaseId databaseId() { return graphStore.databaseId(); } + @Override + public byte idMapBuilderType() { + return graphStore.idMapBuilderType(); + } + @Override public Capabilities capabilities() { return graphStore.capabilities(); diff --git a/core/src/main/java/org/neo4j/gds/core/loading/CSRGraphStore.java b/core/src/main/java/org/neo4j/gds/core/loading/CSRGraphStore.java index 1b2065e07a..4f4a9f3903 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/CSRGraphStore.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/CSRGraphStore.java @@ -149,6 +149,11 @@ public DatabaseId databaseId() { return databaseId; } + @Override + public byte idMapBuilderType() { + return 0; + } + @Override public GraphSchema schema() { return schema; diff --git a/io/core/src/main/java/org/neo4j/gds/core/io/MetaDataStore.java b/io/core/src/main/java/org/neo4j/gds/core/io/MetaDataStore.java index 1d96439bf3..913ebb6a61 100644 --- a/io/core/src/main/java/org/neo4j/gds/core/io/MetaDataStore.java +++ b/io/core/src/main/java/org/neo4j/gds/core/io/MetaDataStore.java @@ -45,6 +45,7 @@ static MetaDataStore of(GraphStore graphStore) { )); GraphInfo graphInfo = ImmutableGraphInfo.of( graphStore.databaseId(), + graphStore.idMapBuilderType(), graphStore.nodeCount(), graphStore.nodes().highestOriginalId(), relTypeCounts, diff --git a/io/core/src/main/java/org/neo4j/gds/core/io/file/GraphInfo.java b/io/core/src/main/java/org/neo4j/gds/core/io/file/GraphInfo.java index d2b7cc2361..25a0928755 100644 --- a/io/core/src/main/java/org/neo4j/gds/core/io/file/GraphInfo.java +++ b/io/core/src/main/java/org/neo4j/gds/core/io/file/GraphInfo.java @@ -29,6 +29,7 @@ @ValueClass public interface GraphInfo { DatabaseId databaseId(); + byte idMapBuilderType(); long nodeCount(); long maxOriginalId(); Map relationshipTypeCounts(); diff --git a/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/CsvGraphInfoVisitor.java b/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/CsvGraphInfoVisitor.java index 24fb3f72e4..7cef017346 100644 --- a/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/CsvGraphInfoVisitor.java +++ b/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/CsvGraphInfoVisitor.java @@ -33,6 +33,7 @@ public class CsvGraphInfoVisitor implements SingleRowVisitor { public static final String GRAPH_INFO_FILE_NAME = "graph_info.csv"; public static final String DATABASE_NAME_COLUMN_NAME = "databaseName"; + public static final String ID_MAP_BUILDER_TYPE_COLUMN_NAME = "idMapBuilderType"; public static final String NODE_COUNT_COLUMN_NAME = "nodeCount"; public static final String MAX_ORIGINAL_ID_COLUMN_NAME = "maxOriginalId"; public static final String REL_TYPE_COUNTS_COLUMN_NAME = "relTypeCounts"; @@ -61,6 +62,7 @@ public void export(GraphInfo graphInfo) { this.csvWriter.writeRow( graphInfo.databaseId().databaseName(), + Byte.toString(graphInfo.idMapBuilderType()), Long.toString(graphInfo.nodeCount()), Long.toString(graphInfo.maxOriginalId()), CsvMapUtil.relationshipCountsToString(graphInfo.relationshipTypeCounts()), @@ -80,6 +82,7 @@ public void close() { private void writeHeader() { this.csvWriter.writeRow( DATABASE_NAME_COLUMN_NAME, + ID_MAP_BUILDER_TYPE_COLUMN_NAME, NODE_COUNT_COLUMN_NAME, MAX_ORIGINAL_ID_COLUMN_NAME, REL_TYPE_COUNTS_COLUMN_NAME, diff --git a/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/CsvGraphInfoVisitorTest.java b/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/CsvGraphInfoVisitorTest.java index 26b326ab28..f6dc6e231c 100644 --- a/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/CsvGraphInfoVisitorTest.java +++ b/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/CsvGraphInfoVisitorTest.java @@ -39,7 +39,15 @@ void shouldExportGraphInfo() { CsvGraphInfoVisitor graphInfoVisitor = new CsvGraphInfoVisitor(tempDir); var relationshipTypeCounts = Map.of(RelationshipType.of("REL1"), 42L, RelationshipType.of("REL2"), 1337L); var inverseIndexedRelTypes = List.of(RelationshipType.of("REL1"),RelationshipType.of("REL2")); - graphInfoVisitor.export(ImmutableGraphInfo.of(databaseId, 1337L, 19L, relationshipTypeCounts, inverseIndexedRelTypes)); + var graphInfo = ImmutableGraphInfo.builder() + .databaseId(databaseId) + .idMapBuilderType((byte) 42) + .nodeCount(1337L) + .maxOriginalId(19L) + .relationshipTypeCounts(relationshipTypeCounts) + .inverseIndexedRelationshipTypes(inverseIndexedRelTypes) + .build(); + graphInfoVisitor.export(graphInfo); graphInfoVisitor.close(); assertCsvFiles(List.of(GRAPH_INFO_FILE_NAME)); @@ -49,6 +57,7 @@ void shouldExportGraphInfo() { defaultHeaderColumns(), List.of( databaseId.databaseName(), + Byte.toString((byte) 42), Long.toString(1337L), Long.toString(19L), CsvMapUtil.relationshipCountsToString(relationshipTypeCounts), @@ -62,6 +71,7 @@ void shouldExportGraphInfo() { protected List defaultHeaderColumns() { return List.of( CsvGraphInfoVisitor.DATABASE_NAME_COLUMN_NAME, + CsvGraphInfoVisitor.ID_MAP_BUILDER_TYPE_COLUMN_NAME, CsvGraphInfoVisitor.NODE_COUNT_COLUMN_NAME, CsvGraphInfoVisitor.MAX_ORIGINAL_ID_COLUMN_NAME, CsvGraphInfoVisitor.REL_TYPE_COUNTS_COLUMN_NAME, diff --git a/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/GraphStoreToCsvExporterTest.java b/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/GraphStoreToCsvExporterTest.java index 84a8e93d64..644424d9e1 100644 --- a/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/GraphStoreToCsvExporterTest.java +++ b/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/GraphStoreToCsvExporterTest.java @@ -462,6 +462,7 @@ public long valueCount() { List.of( List.of( CsvGraphInfoVisitor.DATABASE_NAME_COLUMN_NAME, + CsvGraphInfoVisitor.ID_MAP_BUILDER_TYPE_COLUMN_NAME, CsvGraphInfoVisitor.NODE_COUNT_COLUMN_NAME, CsvGraphInfoVisitor.MAX_ORIGINAL_ID_COLUMN_NAME, CsvGraphInfoVisitor.REL_TYPE_COUNTS_COLUMN_NAME, @@ -469,6 +470,7 @@ public long valueCount() { ), List.of( graphStore.databaseId().databaseName(), + Byte.toString(graphStore.idMapBuilderType()), Long.toString(graphStore.nodeCount()), Long.toString(graphStore.nodes().highestOriginalId()), CsvMapUtil.relationshipCountsToString(Map.of(RelationshipType.of("REL2"), 6L, RelationshipType.of("REL1"), 6L)), From 62392108bc5cc34e120600c532c475ae03f38ffc Mon Sep 17 00:00:00 2001 From: Martin Junghanns Date: Tue, 6 Jun 2023 17:48:44 +0200 Subject: [PATCH 076/273] Expose type id on IdMap --- .../java/org/neo4j/gds/api/GraphAdapter.java | 5 +++++ .../main/java/org/neo4j/gds/api/GraphStore.java | 2 -- .../org/neo4j/gds/api/GraphStoreAdapter.java | 5 ----- core/src/main/java/org/neo4j/gds/api/IdMap.java | 10 ++++++++++ .../java/org/neo4j/gds/api/IdMapAdapter.java | 5 +++++ .../java/org/neo4j/gds/core/huge/HugeGraph.java | 5 +++++ .../java/org/neo4j/gds/core/huge/UnionGraph.java | 5 +++++ .../org/neo4j/gds/core/loading/ArrayIdMap.java | 5 +++++ .../neo4j/gds/core/loading/CSRGraphStore.java | 5 ----- .../gds/core/loading/FilteredLabeledIdMap.java | 5 +++++ .../neo4j/gds/core/loading/HighLimitIdMap.java | 5 +++++ .../gds/core/loading/ScanningNodesImporter.java | 7 ++++++- .../org/neo4j/gds/core/io/MetaDataStore.java | 16 +++++++++++++++- .../io/file/csv/GraphStoreToCsvExporterTest.java | 2 +- .../org/neo4j/gds/core/huge/DirectIdMap.java | 5 +++++ .../org/neo4j/gds/core/loading/TestIdMap.java | 5 +++++ 16 files changed, 77 insertions(+), 15 deletions(-) diff --git a/core/src/main/java/org/neo4j/gds/api/GraphAdapter.java b/core/src/main/java/org/neo4j/gds/api/GraphAdapter.java index 894cbf7164..57f844a70e 100644 --- a/core/src/main/java/org/neo4j/gds/api/GraphAdapter.java +++ b/core/src/main/java/org/neo4j/gds/api/GraphAdapter.java @@ -47,6 +47,11 @@ public Graph graph() { return graph; } + @Override + public byte typeId() { + return graph.typeId(); + } + @Override public long relationshipCount() { return graph.relationshipCount(); diff --git a/core/src/main/java/org/neo4j/gds/api/GraphStore.java b/core/src/main/java/org/neo4j/gds/api/GraphStore.java index 1aacc2ba72..750b415371 100644 --- a/core/src/main/java/org/neo4j/gds/api/GraphStore.java +++ b/core/src/main/java/org/neo4j/gds/api/GraphStore.java @@ -47,8 +47,6 @@ public interface GraphStore { DatabaseId databaseId(); - byte idMapBuilderType(); - GraphSchema schema(); ZonedDateTime modificationTime(); diff --git a/core/src/main/java/org/neo4j/gds/api/GraphStoreAdapter.java b/core/src/main/java/org/neo4j/gds/api/GraphStoreAdapter.java index abbfbd6ae0..8c805a99a4 100644 --- a/core/src/main/java/org/neo4j/gds/api/GraphStoreAdapter.java +++ b/core/src/main/java/org/neo4j/gds/api/GraphStoreAdapter.java @@ -54,11 +54,6 @@ public DatabaseId databaseId() { return graphStore.databaseId(); } - @Override - public byte idMapBuilderType() { - return graphStore.idMapBuilderType(); - } - @Override public Capabilities capabilities() { return graphStore.capabilities(); diff --git a/core/src/main/java/org/neo4j/gds/api/IdMap.java b/core/src/main/java/org/neo4j/gds/api/IdMap.java index 7ca084730e..bf0b2bf579 100644 --- a/core/src/main/java/org/neo4j/gds/api/IdMap.java +++ b/core/src/main/java/org/neo4j/gds/api/IdMap.java @@ -43,6 +43,16 @@ public interface IdMap extends PartialIdMap, NodeIterator, BatchNodeIterable { */ long NOT_FOUND = -1; + /** + * Used for IdMap implementations that do not require a type definition. + */ + byte NO_TYPE = -1; + + /** + * A unique identifier for this type of IdMap. + */ + byte typeId(); + /** * Map original nodeId to mapped nodeId * diff --git a/core/src/main/java/org/neo4j/gds/api/IdMapAdapter.java b/core/src/main/java/org/neo4j/gds/api/IdMapAdapter.java index 116d932fda..6f95135e44 100644 --- a/core/src/main/java/org/neo4j/gds/api/IdMapAdapter.java +++ b/core/src/main/java/org/neo4j/gds/api/IdMapAdapter.java @@ -38,6 +38,11 @@ public IdMapAdapter(IdMap idMap) { this.idMap = idMap; } + @Override + public byte typeId() { + return idMap.typeId(); + } + @Override public Collection batchIterables( long batchSize diff --git a/core/src/main/java/org/neo4j/gds/core/huge/HugeGraph.java b/core/src/main/java/org/neo4j/gds/core/huge/HugeGraph.java index 31333dd98c..707712753c 100644 --- a/core/src/main/java/org/neo4j/gds/core/huge/HugeGraph.java +++ b/core/src/main/java/org/neo4j/gds/core/huge/HugeGraph.java @@ -393,6 +393,11 @@ public long toMappedNodeId(long originalNodeId) { return idMap.toMappedNodeId(originalNodeId); } + @Override + public byte typeId() { + return idMap.typeId(); + } + @Override public long toOriginalNodeId(long mappedNodeId) { return idMap.toOriginalNodeId(mappedNodeId); diff --git a/core/src/main/java/org/neo4j/gds/core/huge/UnionGraph.java b/core/src/main/java/org/neo4j/gds/core/huge/UnionGraph.java index c0d914dc43..aed180ca4b 100644 --- a/core/src/main/java/org/neo4j/gds/core/huge/UnionGraph.java +++ b/core/src/main/java/org/neo4j/gds/core/huge/UnionGraph.java @@ -150,6 +150,11 @@ public long toMappedNodeId(long originalNodeId) { return first.toMappedNodeId(originalNodeId); } + @Override + public byte typeId() { + return first.typeId(); + } + @Override public long toOriginalNodeId(long mappedNodeId) { return first.toOriginalNodeId(mappedNodeId); diff --git a/core/src/main/java/org/neo4j/gds/core/loading/ArrayIdMap.java b/core/src/main/java/org/neo4j/gds/core/loading/ArrayIdMap.java index cb7011084a..8ccd7e67e2 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/ArrayIdMap.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/ArrayIdMap.java @@ -89,6 +89,11 @@ public long toMappedNodeId(long originalNodeId) { return originalToInternalIds.get(originalNodeId); } + @Override + public byte typeId() { + return ArrayIdMapBuilder.ID; + } + @Override public long toOriginalNodeId(long mappedNodeId) { return internalToOriginalIds.get(mappedNodeId); diff --git a/core/src/main/java/org/neo4j/gds/core/loading/CSRGraphStore.java b/core/src/main/java/org/neo4j/gds/core/loading/CSRGraphStore.java index 4f4a9f3903..1b2065e07a 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/CSRGraphStore.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/CSRGraphStore.java @@ -149,11 +149,6 @@ public DatabaseId databaseId() { return databaseId; } - @Override - public byte idMapBuilderType() { - return 0; - } - @Override public GraphSchema schema() { return schema; diff --git a/core/src/main/java/org/neo4j/gds/core/loading/FilteredLabeledIdMap.java b/core/src/main/java/org/neo4j/gds/core/loading/FilteredLabeledIdMap.java index 137fa50044..b079a7e138 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/FilteredLabeledIdMap.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/FilteredLabeledIdMap.java @@ -38,6 +38,11 @@ public FilteredLabeledIdMap(IdMap originalToRootIdMap, LabeledIdMap rootToFilter this.rootToFilteredIdMap = rootToFilteredIdMap; } + @Override + public byte typeId() { + return originalToRootIdMap.typeId(); + } + @Override public OptionalLong rootNodeCount() { return originalToRootIdMap.rootNodeCount(); diff --git a/core/src/main/java/org/neo4j/gds/core/loading/HighLimitIdMap.java b/core/src/main/java/org/neo4j/gds/core/loading/HighLimitIdMap.java index a05b4e0019..b4353d03d6 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/HighLimitIdMap.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/HighLimitIdMap.java @@ -37,6 +37,11 @@ public HighLimitIdMap(ShardedLongLongMap intermediateIdMap, IdMap internalIdMap) this.highToLowIdSpace = intermediateIdMap; } + @Override + public byte typeId() { + return HighLimitIdMapBuilder.ID; + } + @Override public long toOriginalNodeId(long mappedNodeId) { return highToLowIdSpace.toOriginalNodeId(super.toOriginalNodeId(mappedNodeId)); diff --git a/core/src/main/java/org/neo4j/gds/core/loading/ScanningNodesImporter.java b/core/src/main/java/org/neo4j/gds/core/loading/ScanningNodesImporter.java index 93d3a3863d..5bc86c2f77 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/ScanningNodesImporter.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/ScanningNodesImporter.java @@ -199,7 +199,12 @@ public Nodes build() { importPropertiesFromIndex(idMap, nodeProperties); } - return Nodes.of(idMap, this.propertyMappingsByLabel, nodeProperties, PropertyState.PERSISTENT); + return Nodes.of( + idMap, + this.propertyMappingsByLabel, + nodeProperties, + PropertyState.PERSISTENT + ); } private void importPropertiesFromIndex( diff --git a/io/core/src/main/java/org/neo4j/gds/core/io/MetaDataStore.java b/io/core/src/main/java/org/neo4j/gds/core/io/MetaDataStore.java index 913ebb6a61..5a4af6e8f2 100644 --- a/io/core/src/main/java/org/neo4j/gds/core/io/MetaDataStore.java +++ b/io/core/src/main/java/org/neo4j/gds/core/io/MetaDataStore.java @@ -21,12 +21,14 @@ import org.neo4j.gds.annotation.ValueClass; import org.neo4j.gds.api.GraphStore; +import org.neo4j.gds.api.IdMap; import org.neo4j.gds.api.schema.NodeSchema; import org.neo4j.gds.api.schema.PropertySchema; import org.neo4j.gds.api.schema.RelationshipSchema; import org.neo4j.gds.core.io.file.GraphInfo; import org.neo4j.gds.core.io.file.ImmutableGraphInfo; +import java.util.Locale; import java.util.Map; import java.util.function.Function; import java.util.stream.Collectors; @@ -43,9 +45,21 @@ static MetaDataStore of(GraphStore graphStore) { Function.identity(), graphStore::relationshipCount )); + + var idMapTypeId = graphStore.nodes().typeId(); + + if (idMapTypeId == IdMap.NO_TYPE) { + throw new IllegalArgumentException(String.format( + Locale.US, + "Cannot write graph store with untyped id map. Got instance of `%s`", + graphStore.nodes() + .getClass() + .getSimpleName())); + } + GraphInfo graphInfo = ImmutableGraphInfo.of( graphStore.databaseId(), - graphStore.idMapBuilderType(), + idMapTypeId, graphStore.nodeCount(), graphStore.nodes().highestOriginalId(), relTypeCounts, diff --git a/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/GraphStoreToCsvExporterTest.java b/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/GraphStoreToCsvExporterTest.java index 644424d9e1..d10c597c06 100644 --- a/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/GraphStoreToCsvExporterTest.java +++ b/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/GraphStoreToCsvExporterTest.java @@ -470,7 +470,7 @@ public long valueCount() { ), List.of( graphStore.databaseId().databaseName(), - Byte.toString(graphStore.idMapBuilderType()), + Byte.toString(graphStore.nodes().typeId()), Long.toString(graphStore.nodeCount()), Long.toString(graphStore.nodes().highestOriginalId()), CsvMapUtil.relationshipCountsToString(Map.of(RelationshipType.of("REL2"), 6L, RelationshipType.of("REL1"), 6L)), diff --git a/test-utils/src/main/java/org/neo4j/gds/core/huge/DirectIdMap.java b/test-utils/src/main/java/org/neo4j/gds/core/huge/DirectIdMap.java index 6846829f57..1dfc688312 100644 --- a/test-utils/src/main/java/org/neo4j/gds/core/huge/DirectIdMap.java +++ b/test-utils/src/main/java/org/neo4j/gds/core/huge/DirectIdMap.java @@ -39,6 +39,11 @@ public DirectIdMap(long nodeCount) { this.nodeCount = nodeCount; } + @Override + public byte typeId() { + return NO_TYPE; + } + @Override public long toMappedNodeId(long originalNodeId) { return originalNodeId; diff --git a/test-utils/src/main/java/org/neo4j/gds/core/loading/TestIdMap.java b/test-utils/src/main/java/org/neo4j/gds/core/loading/TestIdMap.java index d62278f279..b35cee684c 100644 --- a/test-utils/src/main/java/org/neo4j/gds/core/loading/TestIdMap.java +++ b/test-utils/src/main/java/org/neo4j/gds/core/loading/TestIdMap.java @@ -50,6 +50,11 @@ private TestIdMap( this.highestOriginalId = highestOriginalId; } + @Override + public byte typeId() { + return NO_TYPE; + } + @Override public long toOriginalNodeId(long mappedNodeId) { return this.reverseMap.getOrDefault(mappedNodeId, NOT_FOUND); From 4454e102592f777fa6a5df4146309192ee17019b Mon Sep 17 00:00:00 2001 From: Martin Junghanns Date: Wed, 7 Jun 2023 10:45:42 +0200 Subject: [PATCH 077/273] Add idMapBuilderType to GraphInfo serialization --- .../org/neo4j/gds/core/io/file/csv/GraphInfoLoader.java | 7 ++++++- .../neo4j/gds/core/io/file/csv/GraphInfoLoaderTest.java | 9 +++++++-- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/GraphInfoLoader.java b/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/GraphInfoLoader.java index 94554de100..ad59002f56 100644 --- a/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/GraphInfoLoader.java +++ b/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/GraphInfoLoader.java @@ -31,6 +31,7 @@ import com.fasterxml.jackson.dataformat.csv.CsvSchema; import org.neo4j.gds.RelationshipType; import org.neo4j.gds.api.DatabaseId; +import org.neo4j.gds.api.IdMap; import org.neo4j.gds.core.io.file.GraphInfo; import org.neo4j.gds.core.io.file.ImmutableGraphInfo; @@ -42,6 +43,7 @@ import java.util.Arrays; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.UUID; import java.util.stream.Collectors; @@ -65,12 +67,12 @@ public GraphInfo load() { var databaseId = DatabaseId.from(line.databaseName); return ImmutableGraphInfo.builder() .databaseId(databaseId) + .idMapBuilderType(line.idMapBuilderType) .nodeCount(line.nodeCount) .maxOriginalId(line.maxOriginalId) .relationshipTypeCounts(line.relTypeCounts) .inverseIndexedRelationshipTypes(line.inverseIndexedRelTypes) .build(); - } catch (IOException e) { throw new UncheckedIOException(e); } @@ -85,6 +87,9 @@ public static class GraphInfoLine { @JsonProperty String databaseName; + @JsonProperty + byte idMapBuilderType = IdMap.NO_TYPE; + @JsonProperty long nodeCount; diff --git a/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/GraphInfoLoaderTest.java b/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/GraphInfoLoaderTest.java index 1d95bb5b64..8d93b4758f 100644 --- a/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/GraphInfoLoaderTest.java +++ b/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/GraphInfoLoaderTest.java @@ -25,6 +25,7 @@ import org.junit.jupiter.api.io.TempDir; import org.neo4j.gds.RelationshipType; import org.neo4j.gds.api.DatabaseId; +import org.neo4j.gds.api.IdMap; import java.io.IOException; import java.nio.file.Path; @@ -43,8 +44,8 @@ void shouldLoadGraphInfo(@TempDir Path exportDir) throws IOException { var databaseId = DatabaseId.from("my-database"); var graphInfoFile = exportDir.resolve(GRAPH_INFO_FILE_NAME).toFile(); var lines = List.of( - String.join(", ", "databaseName", "nodeCount", "maxOriginalId", "relTypeCounts", "inverseIndexedRelTypes"), - String.join(", ", "my-database", "19", "1337", "REL;42", "REL;REL1") + String.join(", ", "databaseName", "nodeCount", "maxOriginalId", "relTypeCounts", "inverseIndexedRelTypes","idMapBuilderType"), + String.join(", ", "my-database", "19", "1337", "REL;42", "REL;REL1", "1") ); FileUtils.writeLines(graphInfoFile, lines); @@ -55,6 +56,8 @@ void shouldLoadGraphInfo(@TempDir Path exportDir) throws IOException { assertThat(graphInfo.databaseId()).isEqualTo(databaseId); assertThat(graphInfo.databaseId().databaseName()).isEqualTo("my-database"); + assertThat(graphInfo.idMapBuilderType()).isEqualTo((byte) 1); + assertThat(graphInfo.nodeCount()).isEqualTo(19L); assertThat(graphInfo.maxOriginalId()).isEqualTo(1337L); @@ -85,6 +88,8 @@ void shouldLoadGraphInfoWithDatabaseId(@TempDir Path exportDir) throws IOExcepti assertThat(graphInfo.databaseId()).isEqualTo(databaseId); assertThat(graphInfo.databaseId().databaseName()).isEqualTo("my-database"); + assertThat(graphInfo.idMapBuilderType()).isEqualTo(IdMap.NO_TYPE); + assertThat(graphInfo.nodeCount()).isEqualTo(19L); assertThat(graphInfo.maxOriginalId()).isEqualTo(1337L); From b4ed8902e6b6be73db7f42d3362753c75e5d4a87 Mon Sep 17 00:00:00 2001 From: Martin Junghanns Date: Wed, 7 Jun 2023 10:46:11 +0200 Subject: [PATCH 078/273] Use NO_TYPE as fallback in NodesBuilder --- .../gds/core/loading/construction/GraphFactory.java | 9 ++++++--- .../neo4j/gds/core/io/file/FileToGraphStoreImporter.java | 1 + 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/org/neo4j/gds/core/loading/construction/GraphFactory.java b/core/src/main/java/org/neo4j/gds/core/loading/construction/GraphFactory.java index 7a512a0a00..7871f561fc 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/construction/GraphFactory.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/construction/GraphFactory.java @@ -98,9 +98,12 @@ static NodesBuilder nodesBuilder( ? Optional.of(maxOriginalId) : Optional.empty(); - var idMapBuilder = idMapBuilderType.isPresent() - ? idMapBehavior.create(idMapBuilderType.get(), threadCount, maybeMaxOriginalId, nodeCount) - : idMapBehavior.create(threadCount, maybeMaxOriginalId, nodeCount); + var idMapBuilder = idMapBehavior.create( + idMapBuilderType.orElse(IdMap.NO_TYPE), + threadCount, + maybeMaxOriginalId, + nodeCount + ); boolean deduplicate = deduplicateIds.orElse(true); diff --git a/io/core/src/main/java/org/neo4j/gds/core/io/file/FileToGraphStoreImporter.java b/io/core/src/main/java/org/neo4j/gds/core/io/file/FileToGraphStoreImporter.java index 1c12cc3996..f40955fbca 100644 --- a/io/core/src/main/java/org/neo4j/gds/core/io/file/FileToGraphStoreImporter.java +++ b/io/core/src/main/java/org/neo4j/gds/core/io/file/FileToGraphStoreImporter.java @@ -156,6 +156,7 @@ private Nodes importNodes(FileInput fileInput) { .concurrency(concurrency) .nodeCount(fileInput.graphInfo().nodeCount()) .deduplicateIds(false) + .idMapBuilderType(fileInput.graphInfo().idMapBuilderType()) .build(); nodeVisitorBuilder.withNodeSchema(nodeSchema); nodeVisitorBuilder.withNodesBuilder(nodesBuilder); From 6c57c7c251d39fe7c12cc0036330ef13e38f34c5 Mon Sep 17 00:00:00 2001 From: Martin Junghanns Date: Wed, 7 Jun 2023 11:08:59 +0200 Subject: [PATCH 079/273] Support adding id map builder type in GdlFactory --- .../src/main/java/org/neo4j/gds/gdl/GdlFactory.java | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/test-utils/src/main/java/org/neo4j/gds/gdl/GdlFactory.java b/test-utils/src/main/java/org/neo4j/gds/gdl/GdlFactory.java index c90fb85e49..06f3063cb3 100644 --- a/test-utils/src/main/java/org/neo4j/gds/gdl/GdlFactory.java +++ b/test-utils/src/main/java/org/neo4j/gds/gdl/GdlFactory.java @@ -78,6 +78,7 @@ public final class GdlFactory extends CSRGraphStoreFactory graphName, Optional graphProjectConfig, Optional nodeIdFunction, - Optional graphCapabilities + Optional graphCapabilities, + Optional idMapBuilderType ) { var config = graphProjectConfig.orElseGet(() -> ImmutableGraphProjectFromGdlConfig.builder() .username(userName.orElse(Username.EMPTY_USERNAME.username())) @@ -123,7 +125,8 @@ static GdlFactory gdlFactory( config, graphDimensions, databaseId.orElse(GdlSupportPerMethodExtension.DATABASE_ID), - capabilities + capabilities, + idMapBuilderType.orElse(IdMap.NO_TYPE) ); } @@ -132,7 +135,8 @@ private GdlFactory( GraphProjectFromGdlConfig graphProjectConfig, GraphDimensions graphDimensions, DatabaseId databaseId, - Capabilities capabilities + Capabilities capabilities, + byte idMapBuilderType ) { super( graphProjectConfig, @@ -142,6 +146,7 @@ private GdlFactory( ); this.gdlHandler = gdlHandler; this.databaseId = databaseId; + this.idMapBuilderType = idMapBuilderType; } public long nodeId(String variable) { @@ -185,6 +190,7 @@ private Nodes loadNodes() { .hasProperties(true) .concurrency(1) .propertyState(graphProjectConfig.propertyState()) + .idMapBuilderType(this.idMapBuilderType) .build(); gdlHandler.getVertices().forEach(vertex -> { From 2cc5a80f608cfde7b9cacde9c501438f7e8ef8b8 Mon Sep 17 00:00:00 2001 From: Martin Junghanns Date: Wed, 7 Jun 2023 11:17:03 +0200 Subject: [PATCH 080/273] Add HighLimitIdMap as additional option for OpenGdsIdMapBehaviour --- .../org/neo4j/gds/core/OpenGdsIdMapBehavior.java | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/org/neo4j/gds/core/OpenGdsIdMapBehavior.java b/core/src/main/java/org/neo4j/gds/core/OpenGdsIdMapBehavior.java index 535ab0805f..ca971234ae 100644 --- a/core/src/main/java/org/neo4j/gds/core/OpenGdsIdMapBehavior.java +++ b/core/src/main/java/org/neo4j/gds/core/OpenGdsIdMapBehavior.java @@ -22,6 +22,7 @@ import org.neo4j.gds.core.loading.ArrayIdMap; import org.neo4j.gds.core.loading.ArrayIdMapBuilder; import org.neo4j.gds.core.loading.GrowingArrayIdMapBuilder; +import org.neo4j.gds.core.loading.HighLimitIdMapBuilder; import org.neo4j.gds.core.loading.IdMapBuilder; import org.neo4j.gds.core.utils.mem.MemoryEstimation; @@ -42,8 +43,16 @@ public IdMapBuilder create( @Override public IdMapBuilder create(byte id, int concurrency, Optional maxOriginalId, Optional nodeCount) { - // We can only create array id maps in open GDS; no need to check the id. - return create(concurrency, maxOriginalId, nodeCount); + switch (id) { + case ArrayIdMapBuilder.ID: { + return create(concurrency, maxOriginalId, nodeCount); + } + case HighLimitIdMapBuilder.ID: { + return HighLimitIdMapBuilder.of(concurrency, create(concurrency, maxOriginalId, nodeCount)); + } + default: + return create(concurrency, maxOriginalId, nodeCount); + } } @Override From 6be7e31098a1f08cabff23174ffacd4732e6ff6a Mon Sep 17 00:00:00 2001 From: Martin Junghanns Date: Wed, 7 Jun 2023 11:17:37 +0200 Subject: [PATCH 081/273] Add test for CSV roundtrip with explicit idmap type --- ...svToGraphStoreImporterIntegrationTest.java | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/CsvToGraphStoreImporterIntegrationTest.java b/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/CsvToGraphStoreImporterIntegrationTest.java index d1b080ac1c..d6e8d8f047 100644 --- a/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/CsvToGraphStoreImporterIntegrationTest.java +++ b/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/CsvToGraphStoreImporterIntegrationTest.java @@ -19,6 +19,7 @@ */ package org.neo4j.gds.core.io.file.csv; +import org.immutables.value.Value; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; import org.junit.jupiter.params.ParameterizedTest; @@ -31,7 +32,11 @@ import org.neo4j.gds.compat.Neo4jProxy; import org.neo4j.gds.core.io.file.GraphStoreToFileExporterConfig; import org.neo4j.gds.core.io.file.GraphStoreToFileExporterConfigImpl; +import org.neo4j.gds.core.loading.ArrayIdMap; +import org.neo4j.gds.core.loading.ArrayIdMapBuilder; +import org.neo4j.gds.core.loading.CSRGraphStore; import org.neo4j.gds.core.loading.Capabilities.WriteMode; +import org.neo4j.gds.core.loading.HighLimitIdMapBuilder; import org.neo4j.gds.core.loading.ImmutableStaticCapabilities; import org.neo4j.gds.core.utils.progress.EmptyTaskRegistryFactory; import org.neo4j.gds.extension.GdlExtension; @@ -154,6 +159,23 @@ void shouldImportCapabilities(WriteMode writeMode) { .isEqualTo(ImmutableStaticCapabilities.of(writeMode)); } + @ParameterizedTest + @ValueSource(bytes = {ArrayIdMapBuilder.ID, HighLimitIdMapBuilder.ID}) + void shouldConsiderIdMapBuilderType(byte idMapBuilderType) { + var graphStore = GdlFactory.builder() + .gdlGraph("()-[]->()") + .idMapBuilderType(idMapBuilderType) + .build() + .build(); + + GraphStoreToCsvExporter.create(graphStore, exportConfig(1), graphLocation).run(); + + var importer = new CsvToGraphStoreImporter(1, graphLocation, Neo4jProxy.testLog(), EmptyTaskRegistryFactory.INSTANCE); + var userGraphStore = importer.run(); + + assertThat(userGraphStore.graphStore().nodes().typeId()).isEqualTo(idMapBuilderType); + } + private GraphStoreToFileExporterConfig exportConfig(int concurrency) { return GraphStoreToFileExporterConfigImpl.builder() .exportName("my-export") From ae6bbfabcbdf35095c1bef88625eb3e00dc10602 Mon Sep 17 00:00:00 2001 From: Martin Junghanns Date: Wed, 7 Jun 2023 12:51:58 +0200 Subject: [PATCH 082/273] Fix checkstyle --- .../java/org/neo4j/gds/core/io/file/csv/GraphInfoLoader.java | 1 - .../io/file/csv/CsvToGraphStoreImporterIntegrationTest.java | 3 --- 2 files changed, 4 deletions(-) diff --git a/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/GraphInfoLoader.java b/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/GraphInfoLoader.java index ad59002f56..6899ee1a84 100644 --- a/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/GraphInfoLoader.java +++ b/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/GraphInfoLoader.java @@ -43,7 +43,6 @@ import java.util.Arrays; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.UUID; import java.util.stream.Collectors; diff --git a/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/CsvToGraphStoreImporterIntegrationTest.java b/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/CsvToGraphStoreImporterIntegrationTest.java index d6e8d8f047..e7460478f6 100644 --- a/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/CsvToGraphStoreImporterIntegrationTest.java +++ b/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/CsvToGraphStoreImporterIntegrationTest.java @@ -19,7 +19,6 @@ */ package org.neo4j.gds.core.io.file.csv; -import org.immutables.value.Value; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; import org.junit.jupiter.params.ParameterizedTest; @@ -32,9 +31,7 @@ import org.neo4j.gds.compat.Neo4jProxy; import org.neo4j.gds.core.io.file.GraphStoreToFileExporterConfig; import org.neo4j.gds.core.io.file.GraphStoreToFileExporterConfigImpl; -import org.neo4j.gds.core.loading.ArrayIdMap; import org.neo4j.gds.core.loading.ArrayIdMapBuilder; -import org.neo4j.gds.core.loading.CSRGraphStore; import org.neo4j.gds.core.loading.Capabilities.WriteMode; import org.neo4j.gds.core.loading.HighLimitIdMapBuilder; import org.neo4j.gds.core.loading.ImmutableStaticCapabilities; From f2fd2109b4732be3534e71ef2bd987316ba594c7 Mon Sep 17 00:00:00 2001 From: Martin Junghanns Date: Wed, 7 Jun 2023 13:25:55 +0200 Subject: [PATCH 083/273] Replace byte with String identifier --- core/src/main/java/org/neo4j/gds/api/GraphAdapter.java | 2 +- core/src/main/java/org/neo4j/gds/api/IdMap.java | 4 ++-- core/src/main/java/org/neo4j/gds/api/IdMapAdapter.java | 2 +- core/src/main/java/org/neo4j/gds/core/IdMapBehavior.java | 2 +- .../main/java/org/neo4j/gds/core/OpenGdsIdMapBehavior.java | 5 +++-- core/src/main/java/org/neo4j/gds/core/huge/HugeGraph.java | 2 +- core/src/main/java/org/neo4j/gds/core/huge/UnionGraph.java | 2 +- .../main/java/org/neo4j/gds/core/loading/ArrayIdMap.java | 2 +- .../java/org/neo4j/gds/core/loading/ArrayIdMapBuilder.java | 2 +- .../org/neo4j/gds/core/loading/FilteredLabeledIdMap.java | 2 +- .../java/org/neo4j/gds/core/loading/HighLimitIdMap.java | 2 +- .../org/neo4j/gds/core/loading/HighLimitIdMapBuilder.java | 2 +- .../neo4j/gds/core/loading/construction/GraphFactory.java | 2 +- .../java/org/neo4j/gds/core/OpenGdsIdMapBehaviorTest.java | 7 ++++--- .../main/java/org/neo4j/gds/core/io/file/GraphInfo.java | 2 +- .../neo4j/gds/core/io/file/csv/CsvGraphInfoVisitor.java | 2 +- .../org/neo4j/gds/core/io/file/csv/GraphInfoLoader.java | 2 +- .../gds/core/io/file/csv/CsvGraphInfoVisitorTest.java | 5 +++-- .../file/csv/CsvToGraphStoreImporterIntegrationTest.java | 4 ++-- .../gds/core/io/file/csv/GraphStoreToCsvExporterTest.java | 2 +- .../src/main/java/org/neo4j/gds/core/huge/DirectIdMap.java | 2 +- .../main/java/org/neo4j/gds/core/loading/TestIdMap.java | 2 +- test-utils/src/main/java/org/neo4j/gds/gdl/GdlFactory.java | 6 +++--- 23 files changed, 34 insertions(+), 31 deletions(-) diff --git a/core/src/main/java/org/neo4j/gds/api/GraphAdapter.java b/core/src/main/java/org/neo4j/gds/api/GraphAdapter.java index 57f844a70e..9518003235 100644 --- a/core/src/main/java/org/neo4j/gds/api/GraphAdapter.java +++ b/core/src/main/java/org/neo4j/gds/api/GraphAdapter.java @@ -48,7 +48,7 @@ public Graph graph() { } @Override - public byte typeId() { + public String typeId() { return graph.typeId(); } diff --git a/core/src/main/java/org/neo4j/gds/api/IdMap.java b/core/src/main/java/org/neo4j/gds/api/IdMap.java index bf0b2bf579..467bf4694f 100644 --- a/core/src/main/java/org/neo4j/gds/api/IdMap.java +++ b/core/src/main/java/org/neo4j/gds/api/IdMap.java @@ -46,12 +46,12 @@ public interface IdMap extends PartialIdMap, NodeIterator, BatchNodeIterable { /** * Used for IdMap implementations that do not require a type definition. */ - byte NO_TYPE = -1; + String NO_TYPE = "unsupported"; /** * A unique identifier for this type of IdMap. */ - byte typeId(); + String typeId(); /** * Map original nodeId to mapped nodeId diff --git a/core/src/main/java/org/neo4j/gds/api/IdMapAdapter.java b/core/src/main/java/org/neo4j/gds/api/IdMapAdapter.java index 6f95135e44..dd43ab27ab 100644 --- a/core/src/main/java/org/neo4j/gds/api/IdMapAdapter.java +++ b/core/src/main/java/org/neo4j/gds/api/IdMapAdapter.java @@ -39,7 +39,7 @@ public IdMapAdapter(IdMap idMap) { } @Override - public byte typeId() { + public String typeId() { return idMap.typeId(); } diff --git a/core/src/main/java/org/neo4j/gds/core/IdMapBehavior.java b/core/src/main/java/org/neo4j/gds/core/IdMapBehavior.java index 0d3c6cff28..a02da6eb0a 100644 --- a/core/src/main/java/org/neo4j/gds/core/IdMapBehavior.java +++ b/core/src/main/java/org/neo4j/gds/core/IdMapBehavior.java @@ -40,7 +40,7 @@ IdMapBuilder create( * @param id the id of the IdMapBuilder to create */ IdMapBuilder create( - byte id, + String id, int concurrency, Optional maxOriginalId, Optional nodeCount diff --git a/core/src/main/java/org/neo4j/gds/core/OpenGdsIdMapBehavior.java b/core/src/main/java/org/neo4j/gds/core/OpenGdsIdMapBehavior.java index ca971234ae..c125a26ab2 100644 --- a/core/src/main/java/org/neo4j/gds/core/OpenGdsIdMapBehavior.java +++ b/core/src/main/java/org/neo4j/gds/core/OpenGdsIdMapBehavior.java @@ -26,6 +26,7 @@ import org.neo4j.gds.core.loading.IdMapBuilder; import org.neo4j.gds.core.utils.mem.MemoryEstimation; +import java.util.Locale; import java.util.Optional; public class OpenGdsIdMapBehavior implements IdMapBehavior { @@ -42,8 +43,8 @@ public IdMapBuilder create( } @Override - public IdMapBuilder create(byte id, int concurrency, Optional maxOriginalId, Optional nodeCount) { - switch (id) { + public IdMapBuilder create(String id, int concurrency, Optional maxOriginalId, Optional nodeCount) { + switch (id.toLowerCase(Locale.US)) { case ArrayIdMapBuilder.ID: { return create(concurrency, maxOriginalId, nodeCount); } diff --git a/core/src/main/java/org/neo4j/gds/core/huge/HugeGraph.java b/core/src/main/java/org/neo4j/gds/core/huge/HugeGraph.java index 707712753c..e158334f68 100644 --- a/core/src/main/java/org/neo4j/gds/core/huge/HugeGraph.java +++ b/core/src/main/java/org/neo4j/gds/core/huge/HugeGraph.java @@ -394,7 +394,7 @@ public long toMappedNodeId(long originalNodeId) { } @Override - public byte typeId() { + public String typeId() { return idMap.typeId(); } diff --git a/core/src/main/java/org/neo4j/gds/core/huge/UnionGraph.java b/core/src/main/java/org/neo4j/gds/core/huge/UnionGraph.java index aed180ca4b..49f02c6d11 100644 --- a/core/src/main/java/org/neo4j/gds/core/huge/UnionGraph.java +++ b/core/src/main/java/org/neo4j/gds/core/huge/UnionGraph.java @@ -151,7 +151,7 @@ public long toMappedNodeId(long originalNodeId) { } @Override - public byte typeId() { + public String typeId() { return first.typeId(); } diff --git a/core/src/main/java/org/neo4j/gds/core/loading/ArrayIdMap.java b/core/src/main/java/org/neo4j/gds/core/loading/ArrayIdMap.java index 8ccd7e67e2..c8d597bae5 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/ArrayIdMap.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/ArrayIdMap.java @@ -90,7 +90,7 @@ public long toMappedNodeId(long originalNodeId) { } @Override - public byte typeId() { + public String typeId() { return ArrayIdMapBuilder.ID; } diff --git a/core/src/main/java/org/neo4j/gds/core/loading/ArrayIdMapBuilder.java b/core/src/main/java/org/neo4j/gds/core/loading/ArrayIdMapBuilder.java index 913d856a38..1e68da0b90 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/ArrayIdMapBuilder.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/ArrayIdMapBuilder.java @@ -28,7 +28,7 @@ public final class ArrayIdMapBuilder implements IdMapBuilder { - public static final byte ID = 0; + public static final String ID = "array"; private final HugeLongArray array; private final long capacity; diff --git a/core/src/main/java/org/neo4j/gds/core/loading/FilteredLabeledIdMap.java b/core/src/main/java/org/neo4j/gds/core/loading/FilteredLabeledIdMap.java index b079a7e138..f061c6aa2b 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/FilteredLabeledIdMap.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/FilteredLabeledIdMap.java @@ -39,7 +39,7 @@ public FilteredLabeledIdMap(IdMap originalToRootIdMap, LabeledIdMap rootToFilter } @Override - public byte typeId() { + public String typeId() { return originalToRootIdMap.typeId(); } diff --git a/core/src/main/java/org/neo4j/gds/core/loading/HighLimitIdMap.java b/core/src/main/java/org/neo4j/gds/core/loading/HighLimitIdMap.java index b4353d03d6..b64de68a55 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/HighLimitIdMap.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/HighLimitIdMap.java @@ -38,7 +38,7 @@ public HighLimitIdMap(ShardedLongLongMap intermediateIdMap, IdMap internalIdMap) } @Override - public byte typeId() { + public String typeId() { return HighLimitIdMapBuilder.ID; } diff --git a/core/src/main/java/org/neo4j/gds/core/loading/HighLimitIdMapBuilder.java b/core/src/main/java/org/neo4j/gds/core/loading/HighLimitIdMapBuilder.java index a6587c40b0..fe3d69c622 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/HighLimitIdMapBuilder.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/HighLimitIdMapBuilder.java @@ -25,7 +25,7 @@ public final class HighLimitIdMapBuilder implements IdMapBuilder { - public static final byte ID = 3; + public static final String ID = "highlimit"; private final ShardedLongLongMap.BatchedBuilder originalToIntermediateMapping; private final IdMapBuilder intermediateToInternalMapping; diff --git a/core/src/main/java/org/neo4j/gds/core/loading/construction/GraphFactory.java b/core/src/main/java/org/neo4j/gds/core/loading/construction/GraphFactory.java index 7871f561fc..c53803b9f3 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/construction/GraphFactory.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/construction/GraphFactory.java @@ -86,7 +86,7 @@ static NodesBuilder nodesBuilder( Optional deduplicateIds, Optional concurrency, Optional propertyState, - Optional idMapBuilderType + Optional idMapBuilderType ) { boolean labelInformation = nodeSchema .map(schema -> !(schema.availableLabels().isEmpty() && schema.containsOnlyAllNodesLabel())) diff --git a/core/src/test/java/org/neo4j/gds/core/OpenGdsIdMapBehaviorTest.java b/core/src/test/java/org/neo4j/gds/core/OpenGdsIdMapBehaviorTest.java index 592898566c..bc41af57a7 100644 --- a/core/src/test/java/org/neo4j/gds/core/OpenGdsIdMapBehaviorTest.java +++ b/core/src/test/java/org/neo4j/gds/core/OpenGdsIdMapBehaviorTest.java @@ -22,6 +22,7 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; +import org.neo4j.gds.api.IdMap; import org.neo4j.gds.core.loading.ArrayIdMapBuilder; import org.neo4j.gds.core.loading.GrowingArrayIdMapBuilder; @@ -36,15 +37,15 @@ static Stream idMapBuildersById() { return Stream.of( Arguments.of(ArrayIdMapBuilder.ID, Optional.empty(), Optional.empty(), GrowingArrayIdMapBuilder.class), Arguments.of(ArrayIdMapBuilder.ID, Optional.empty(), Optional.of(42L), ArrayIdMapBuilder.class), - Arguments.of((byte) 42, Optional.empty(), Optional.empty(), GrowingArrayIdMapBuilder.class), - Arguments.of((byte) 42, Optional.empty(), Optional.of(42L), ArrayIdMapBuilder.class) + Arguments.of(IdMap.NO_TYPE, Optional.empty(), Optional.empty(), GrowingArrayIdMapBuilder.class), + Arguments.of(IdMap.NO_TYPE, Optional.empty(), Optional.of(42L), ArrayIdMapBuilder.class) ); } @ParameterizedTest @MethodSource("idMapBuildersById") void shouldCreateIdMapBuilderById( - byte id, + String id, Optional maxOriginalId, Optional nodeCount, Class idMapBuilderClazz diff --git a/io/core/src/main/java/org/neo4j/gds/core/io/file/GraphInfo.java b/io/core/src/main/java/org/neo4j/gds/core/io/file/GraphInfo.java index 25a0928755..96c9ea45d0 100644 --- a/io/core/src/main/java/org/neo4j/gds/core/io/file/GraphInfo.java +++ b/io/core/src/main/java/org/neo4j/gds/core/io/file/GraphInfo.java @@ -29,7 +29,7 @@ @ValueClass public interface GraphInfo { DatabaseId databaseId(); - byte idMapBuilderType(); + String idMapBuilderType(); long nodeCount(); long maxOriginalId(); Map relationshipTypeCounts(); diff --git a/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/CsvGraphInfoVisitor.java b/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/CsvGraphInfoVisitor.java index 7cef017346..799134da38 100644 --- a/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/CsvGraphInfoVisitor.java +++ b/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/CsvGraphInfoVisitor.java @@ -62,7 +62,7 @@ public void export(GraphInfo graphInfo) { this.csvWriter.writeRow( graphInfo.databaseId().databaseName(), - Byte.toString(graphInfo.idMapBuilderType()), + graphInfo.idMapBuilderType(), Long.toString(graphInfo.nodeCount()), Long.toString(graphInfo.maxOriginalId()), CsvMapUtil.relationshipCountsToString(graphInfo.relationshipTypeCounts()), diff --git a/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/GraphInfoLoader.java b/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/GraphInfoLoader.java index 6899ee1a84..2191cd9836 100644 --- a/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/GraphInfoLoader.java +++ b/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/GraphInfoLoader.java @@ -87,7 +87,7 @@ public static class GraphInfoLine { String databaseName; @JsonProperty - byte idMapBuilderType = IdMap.NO_TYPE; + String idMapBuilderType = IdMap.NO_TYPE; @JsonProperty long nodeCount; diff --git a/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/CsvGraphInfoVisitorTest.java b/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/CsvGraphInfoVisitorTest.java index f6dc6e231c..e4001d7d2f 100644 --- a/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/CsvGraphInfoVisitorTest.java +++ b/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/CsvGraphInfoVisitorTest.java @@ -36,12 +36,13 @@ class CsvGraphInfoVisitorTest extends CsvVisitorTest { @Test void shouldExportGraphInfo() { DatabaseId databaseId = DatabaseId.random(); + String idMapBuilderType = "custom"; CsvGraphInfoVisitor graphInfoVisitor = new CsvGraphInfoVisitor(tempDir); var relationshipTypeCounts = Map.of(RelationshipType.of("REL1"), 42L, RelationshipType.of("REL2"), 1337L); var inverseIndexedRelTypes = List.of(RelationshipType.of("REL1"),RelationshipType.of("REL2")); var graphInfo = ImmutableGraphInfo.builder() .databaseId(databaseId) - .idMapBuilderType((byte) 42) + .idMapBuilderType(idMapBuilderType) .nodeCount(1337L) .maxOriginalId(19L) .relationshipTypeCounts(relationshipTypeCounts) @@ -57,7 +58,7 @@ void shouldExportGraphInfo() { defaultHeaderColumns(), List.of( databaseId.databaseName(), - Byte.toString((byte) 42), + idMapBuilderType, Long.toString(1337L), Long.toString(19L), CsvMapUtil.relationshipCountsToString(relationshipTypeCounts), diff --git a/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/CsvToGraphStoreImporterIntegrationTest.java b/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/CsvToGraphStoreImporterIntegrationTest.java index e7460478f6..aa8784ddad 100644 --- a/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/CsvToGraphStoreImporterIntegrationTest.java +++ b/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/CsvToGraphStoreImporterIntegrationTest.java @@ -157,8 +157,8 @@ void shouldImportCapabilities(WriteMode writeMode) { } @ParameterizedTest - @ValueSource(bytes = {ArrayIdMapBuilder.ID, HighLimitIdMapBuilder.ID}) - void shouldConsiderIdMapBuilderType(byte idMapBuilderType) { + @ValueSource(strings = {ArrayIdMapBuilder.ID, HighLimitIdMapBuilder.ID}) + void shouldConsiderIdMapBuilderType(String idMapBuilderType) { var graphStore = GdlFactory.builder() .gdlGraph("()-[]->()") .idMapBuilderType(idMapBuilderType) diff --git a/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/GraphStoreToCsvExporterTest.java b/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/GraphStoreToCsvExporterTest.java index d10c597c06..2c82139428 100644 --- a/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/GraphStoreToCsvExporterTest.java +++ b/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/GraphStoreToCsvExporterTest.java @@ -470,7 +470,7 @@ public long valueCount() { ), List.of( graphStore.databaseId().databaseName(), - Byte.toString(graphStore.nodes().typeId()), + graphStore.nodes().typeId(), Long.toString(graphStore.nodeCount()), Long.toString(graphStore.nodes().highestOriginalId()), CsvMapUtil.relationshipCountsToString(Map.of(RelationshipType.of("REL2"), 6L, RelationshipType.of("REL1"), 6L)), diff --git a/test-utils/src/main/java/org/neo4j/gds/core/huge/DirectIdMap.java b/test-utils/src/main/java/org/neo4j/gds/core/huge/DirectIdMap.java index 1dfc688312..173dc7ffaa 100644 --- a/test-utils/src/main/java/org/neo4j/gds/core/huge/DirectIdMap.java +++ b/test-utils/src/main/java/org/neo4j/gds/core/huge/DirectIdMap.java @@ -40,7 +40,7 @@ public DirectIdMap(long nodeCount) { } @Override - public byte typeId() { + public String typeId() { return NO_TYPE; } diff --git a/test-utils/src/main/java/org/neo4j/gds/core/loading/TestIdMap.java b/test-utils/src/main/java/org/neo4j/gds/core/loading/TestIdMap.java index b35cee684c..17da4853bd 100644 --- a/test-utils/src/main/java/org/neo4j/gds/core/loading/TestIdMap.java +++ b/test-utils/src/main/java/org/neo4j/gds/core/loading/TestIdMap.java @@ -51,7 +51,7 @@ private TestIdMap( } @Override - public byte typeId() { + public String typeId() { return NO_TYPE; } diff --git a/test-utils/src/main/java/org/neo4j/gds/gdl/GdlFactory.java b/test-utils/src/main/java/org/neo4j/gds/gdl/GdlFactory.java index 06f3063cb3..1c68a21ef7 100644 --- a/test-utils/src/main/java/org/neo4j/gds/gdl/GdlFactory.java +++ b/test-utils/src/main/java/org/neo4j/gds/gdl/GdlFactory.java @@ -78,7 +78,7 @@ public final class GdlFactory extends CSRGraphStoreFactory graphProjectConfig, Optional nodeIdFunction, Optional graphCapabilities, - Optional idMapBuilderType + Optional idMapBuilderType ) { var config = graphProjectConfig.orElseGet(() -> ImmutableGraphProjectFromGdlConfig.builder() .username(userName.orElse(Username.EMPTY_USERNAME.username())) @@ -136,7 +136,7 @@ private GdlFactory( GraphDimensions graphDimensions, DatabaseId databaseId, Capabilities capabilities, - byte idMapBuilderType + String idMapBuilderType ) { super( graphProjectConfig, From 1ba8ab9491384ba637209173cad42daa894f96cc Mon Sep 17 00:00:00 2001 From: Martin Junghanns Date: Wed, 7 Jun 2023 14:23:50 +0200 Subject: [PATCH 084/273] Add nested type id to high limit type id --- .../gds/core/loading/HighLimitIdMap.java | 13 ++++++++- .../gds/core/loading/HighLimitIdMapTest.java | 27 +++++++++++++++++++ 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/org/neo4j/gds/core/loading/HighLimitIdMap.java b/core/src/main/java/org/neo4j/gds/core/loading/HighLimitIdMap.java index b64de68a55..9e1aefcad3 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/HighLimitIdMap.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/HighLimitIdMap.java @@ -39,7 +39,7 @@ public HighLimitIdMap(ShardedLongLongMap intermediateIdMap, IdMap internalIdMap) @Override public String typeId() { - return HighLimitIdMapBuilder.ID; + return HighLimitIdMapBuilder.ID + "-" + super.typeId(); } @Override @@ -76,6 +76,17 @@ public Optional withFilteredLabels(Collection nodeLabe .map(filteredIdMap -> new FilteredHighLimitIdMap(this.highToLowIdSpace, filteredIdMap)); } + public static boolean isHighLimitIdMap(String typeId) { + return typeId.startsWith(HighLimitIdMapBuilder.ID); + } + + public static Optional innerTypeId(String typeId) { + var separatorIndex = typeId.indexOf("-"); + return isHighLimitIdMap(typeId) && separatorIndex > 0 && separatorIndex < typeId.length() - 1 + ? Optional.of(typeId.substring(separatorIndex + 1)) + : Optional.empty(); + } + static final class FilteredHighLimitIdMap extends HighLimitIdMap implements FilteredIdMap { private final FilteredIdMap filteredIdMap; diff --git a/core/src/test/java/org/neo4j/gds/core/loading/HighLimitIdMapTest.java b/core/src/test/java/org/neo4j/gds/core/loading/HighLimitIdMapTest.java index e630127315..3a3302fc74 100644 --- a/core/src/test/java/org/neo4j/gds/core/loading/HighLimitIdMapTest.java +++ b/core/src/test/java/org/neo4j/gds/core/loading/HighLimitIdMapTest.java @@ -29,6 +29,8 @@ import org.neo4j.gds.core.utils.paged.ShardedLongLongMap; import java.util.List; +import java.util.Optional; +import java.util.stream.LongStream; import static org.assertj.core.api.Assertions.assertThat; import static org.neo4j.gds.api.IdMap.NOT_FOUND; @@ -68,6 +70,31 @@ void shouldHandleUnmappedIds() { assertThat(highLimitIdMap.containsOriginalId(1337)).isFalse(); } + @Test + void shouldReturnCorrectTypeIds() { + long[] nodes = LongStream.range(0, 42).toArray(); + var builder = HighLimitIdMapBuilder.of(1, ArrayIdMapBuilder.of(nodes.length)); + builder.allocate(nodes.length).insert(nodes); + var idMap = builder.build(LabelInformationBuilders.allNodes(), nodes.length - 1, 1); + + assertThat(idMap.typeId()).contains(HighLimitIdMapBuilder.ID).contains(ArrayIdMapBuilder.ID); + assertThat(HighLimitIdMap.innerTypeId(idMap.typeId()).get()).isEqualTo(ArrayIdMapBuilder.ID); + } + + @Test + void shouldReturnCorrectInnerTypeId() { + assertThat(HighLimitIdMap.innerTypeId(HighLimitIdMapBuilder.ID)).isEqualTo(Optional.empty()); + assertThat(HighLimitIdMap.innerTypeId(HighLimitIdMapBuilder.ID + "-foobar")).isEqualTo(Optional.of("foobar")); + assertThat(HighLimitIdMap.innerTypeId("foobar")).isEqualTo(Optional.empty()); + assertThat(HighLimitIdMap.innerTypeId(HighLimitIdMapBuilder.ID + "-")).isEqualTo(Optional.empty()); + } + + @Test + void shouldIdentifyHighLimitIdMaps() { + assertThat(HighLimitIdMap.isHighLimitIdMap(HighLimitIdMapBuilder.ID)).isTrue(); + assertThat(HighLimitIdMap.isHighLimitIdMap("foobar")).isFalse(); + } + @Nested class FilteredHighLimitIdMap { From fa67260a02cfe6e2e8e35bb4d09ccb8ff47911f9 Mon Sep 17 00:00:00 2001 From: Martin Junghanns Date: Wed, 7 Jun 2023 14:24:15 +0200 Subject: [PATCH 085/273] Identify inner type in IdMap behaviours --- .../neo4j/gds/core/OpenGdsIdMapBehavior.java | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/core/src/main/java/org/neo4j/gds/core/OpenGdsIdMapBehavior.java b/core/src/main/java/org/neo4j/gds/core/OpenGdsIdMapBehavior.java index c125a26ab2..ea64308f92 100644 --- a/core/src/main/java/org/neo4j/gds/core/OpenGdsIdMapBehavior.java +++ b/core/src/main/java/org/neo4j/gds/core/OpenGdsIdMapBehavior.java @@ -22,6 +22,7 @@ import org.neo4j.gds.core.loading.ArrayIdMap; import org.neo4j.gds.core.loading.ArrayIdMapBuilder; import org.neo4j.gds.core.loading.GrowingArrayIdMapBuilder; +import org.neo4j.gds.core.loading.HighLimitIdMap; import org.neo4j.gds.core.loading.HighLimitIdMapBuilder; import org.neo4j.gds.core.loading.IdMapBuilder; import org.neo4j.gds.core.utils.mem.MemoryEstimation; @@ -44,16 +45,17 @@ public IdMapBuilder create( @Override public IdMapBuilder create(String id, int concurrency, Optional maxOriginalId, Optional nodeCount) { - switch (id.toLowerCase(Locale.US)) { - case ArrayIdMapBuilder.ID: { - return create(concurrency, maxOriginalId, nodeCount); - } - case HighLimitIdMapBuilder.ID: { - return HighLimitIdMapBuilder.of(concurrency, create(concurrency, maxOriginalId, nodeCount)); - } - default: - return create(concurrency, maxOriginalId, nodeCount); + var idLowerCase = id.toLowerCase(Locale.US); + if (idLowerCase.equals(ArrayIdMapBuilder.ID)) { + return create(concurrency, maxOriginalId, nodeCount); } + if (HighLimitIdMap.isHighLimitIdMap(idLowerCase)) { + var innerBuilder = HighLimitIdMap.innerTypeId(idLowerCase) + .map(innerId -> create(innerId, concurrency, maxOriginalId, nodeCount)) + .orElseGet(() -> create(concurrency, maxOriginalId, nodeCount)); + return HighLimitIdMapBuilder.of(concurrency, innerBuilder); + } + return create(concurrency, maxOriginalId, nodeCount); } @Override From 114d769b4ee2058783364cd016e02915ff3a4f9c Mon Sep 17 00:00:00 2001 From: Martin Junghanns Date: Wed, 7 Jun 2023 14:43:19 +0200 Subject: [PATCH 086/273] Reject high limit as nested type --- .../java/org/neo4j/gds/core/loading/HighLimitIdMap.java | 9 ++++++--- .../org/neo4j/gds/core/loading/HighLimitIdMapTest.java | 1 + 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/org/neo4j/gds/core/loading/HighLimitIdMap.java b/core/src/main/java/org/neo4j/gds/core/loading/HighLimitIdMap.java index 9e1aefcad3..184ea5c486 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/HighLimitIdMap.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/HighLimitIdMap.java @@ -19,6 +19,7 @@ */ package org.neo4j.gds.core.loading; +import org.jetbrains.annotations.TestOnly; import org.neo4j.gds.NodeLabel; import org.neo4j.gds.api.FilteredIdMap; import org.neo4j.gds.api.IdMap; @@ -82,9 +83,11 @@ public static boolean isHighLimitIdMap(String typeId) { public static Optional innerTypeId(String typeId) { var separatorIndex = typeId.indexOf("-"); - return isHighLimitIdMap(typeId) && separatorIndex > 0 && separatorIndex < typeId.length() - 1 - ? Optional.of(typeId.substring(separatorIndex + 1)) - : Optional.empty(); + if (isHighLimitIdMap(typeId) && separatorIndex > 0 && separatorIndex < typeId.length() - 1) { + var substring = typeId.substring(separatorIndex + 1); + return substring.equals(HighLimitIdMapBuilder.ID) ? Optional.empty() : Optional.of(substring); + } + return Optional.empty(); } static final class FilteredHighLimitIdMap extends HighLimitIdMap implements FilteredIdMap { diff --git a/core/src/test/java/org/neo4j/gds/core/loading/HighLimitIdMapTest.java b/core/src/test/java/org/neo4j/gds/core/loading/HighLimitIdMapTest.java index 3a3302fc74..1ca29bc402 100644 --- a/core/src/test/java/org/neo4j/gds/core/loading/HighLimitIdMapTest.java +++ b/core/src/test/java/org/neo4j/gds/core/loading/HighLimitIdMapTest.java @@ -85,6 +85,7 @@ void shouldReturnCorrectTypeIds() { void shouldReturnCorrectInnerTypeId() { assertThat(HighLimitIdMap.innerTypeId(HighLimitIdMapBuilder.ID)).isEqualTo(Optional.empty()); assertThat(HighLimitIdMap.innerTypeId(HighLimitIdMapBuilder.ID + "-foobar")).isEqualTo(Optional.of("foobar")); + assertThat(HighLimitIdMap.innerTypeId(HighLimitIdMapBuilder.ID + "-" + HighLimitIdMapBuilder.ID)).isEqualTo(Optional.empty()); assertThat(HighLimitIdMap.innerTypeId("foobar")).isEqualTo(Optional.empty()); assertThat(HighLimitIdMap.innerTypeId(HighLimitIdMapBuilder.ID + "-")).isEqualTo(Optional.empty()); } From 52f489275c2620c3ffc9e8e7387fcb5519b5ccba Mon Sep 17 00:00:00 2001 From: Martin Junghanns Date: Wed, 7 Jun 2023 14:54:59 +0200 Subject: [PATCH 087/273] Fix tests --- .../main/java/org/neo4j/gds/core/loading/HighLimitIdMap.java | 3 +-- .../neo4j/gds/core/loading/HighLimitIdMapBuilderTest.java | 1 - .../src/main/java/org/neo4j/gds/core/io/MetaDataStore.java | 2 +- .../io/file/csv/CsvToGraphStoreImporterIntegrationTest.java | 2 +- .../org/neo4j/gds/core/io/file/csv/GraphInfoLoaderTest.java | 5 +++-- 5 files changed, 6 insertions(+), 7 deletions(-) diff --git a/core/src/main/java/org/neo4j/gds/core/loading/HighLimitIdMap.java b/core/src/main/java/org/neo4j/gds/core/loading/HighLimitIdMap.java index 184ea5c486..7cecb160cc 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/HighLimitIdMap.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/HighLimitIdMap.java @@ -19,7 +19,6 @@ */ package org.neo4j.gds.core.loading; -import org.jetbrains.annotations.TestOnly; import org.neo4j.gds.NodeLabel; import org.neo4j.gds.api.FilteredIdMap; import org.neo4j.gds.api.IdMap; @@ -82,7 +81,7 @@ public static boolean isHighLimitIdMap(String typeId) { } public static Optional innerTypeId(String typeId) { - var separatorIndex = typeId.indexOf("-"); + var separatorIndex = typeId.indexOf('-'); if (isHighLimitIdMap(typeId) && separatorIndex > 0 && separatorIndex < typeId.length() - 1) { var substring = typeId.substring(separatorIndex + 1); return substring.equals(HighLimitIdMapBuilder.ID) ? Optional.empty() : Optional.of(substring); diff --git a/core/src/test/java/org/neo4j/gds/core/loading/HighLimitIdMapBuilderTest.java b/core/src/test/java/org/neo4j/gds/core/loading/HighLimitIdMapBuilderTest.java index a1072f1a49..3b8a8f2fee 100644 --- a/core/src/test/java/org/neo4j/gds/core/loading/HighLimitIdMapBuilderTest.java +++ b/core/src/test/java/org/neo4j/gds/core/loading/HighLimitIdMapBuilderTest.java @@ -26,6 +26,5 @@ public class HighLimitIdMapBuilderTest extends IdMapBuilderTest { @Override protected IdMapBuilder builder(long capacity, int concurrency) { return HighLimitIdMapBuilder.of(concurrency, ArrayIdMapBuilder.of(capacity)); - } } diff --git a/io/core/src/main/java/org/neo4j/gds/core/io/MetaDataStore.java b/io/core/src/main/java/org/neo4j/gds/core/io/MetaDataStore.java index 5a4af6e8f2..13646fa18a 100644 --- a/io/core/src/main/java/org/neo4j/gds/core/io/MetaDataStore.java +++ b/io/core/src/main/java/org/neo4j/gds/core/io/MetaDataStore.java @@ -48,7 +48,7 @@ static MetaDataStore of(GraphStore graphStore) { var idMapTypeId = graphStore.nodes().typeId(); - if (idMapTypeId == IdMap.NO_TYPE) { + if (idMapTypeId.equals(IdMap.NO_TYPE)) { throw new IllegalArgumentException(String.format( Locale.US, "Cannot write graph store with untyped id map. Got instance of `%s`", diff --git a/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/CsvToGraphStoreImporterIntegrationTest.java b/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/CsvToGraphStoreImporterIntegrationTest.java index aa8784ddad..4c56e49c8c 100644 --- a/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/CsvToGraphStoreImporterIntegrationTest.java +++ b/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/CsvToGraphStoreImporterIntegrationTest.java @@ -170,7 +170,7 @@ void shouldConsiderIdMapBuilderType(String idMapBuilderType) { var importer = new CsvToGraphStoreImporter(1, graphLocation, Neo4jProxy.testLog(), EmptyTaskRegistryFactory.INSTANCE); var userGraphStore = importer.run(); - assertThat(userGraphStore.graphStore().nodes().typeId()).isEqualTo(idMapBuilderType); + assertThat(userGraphStore.graphStore().nodes().typeId()).startsWith(idMapBuilderType); } private GraphStoreToFileExporterConfig exportConfig(int concurrency) { diff --git a/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/GraphInfoLoaderTest.java b/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/GraphInfoLoaderTest.java index 8d93b4758f..c889cae05d 100644 --- a/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/GraphInfoLoaderTest.java +++ b/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/GraphInfoLoaderTest.java @@ -26,6 +26,7 @@ import org.neo4j.gds.RelationshipType; import org.neo4j.gds.api.DatabaseId; import org.neo4j.gds.api.IdMap; +import org.neo4j.gds.core.loading.ArrayIdMapBuilder; import java.io.IOException; import java.nio.file.Path; @@ -45,7 +46,7 @@ void shouldLoadGraphInfo(@TempDir Path exportDir) throws IOException { var graphInfoFile = exportDir.resolve(GRAPH_INFO_FILE_NAME).toFile(); var lines = List.of( String.join(", ", "databaseName", "nodeCount", "maxOriginalId", "relTypeCounts", "inverseIndexedRelTypes","idMapBuilderType"), - String.join(", ", "my-database", "19", "1337", "REL;42", "REL;REL1", "1") + String.join(", ", "my-database", "19", "1337", "REL;42", "REL;REL1", ArrayIdMapBuilder.ID) ); FileUtils.writeLines(graphInfoFile, lines); @@ -56,7 +57,7 @@ void shouldLoadGraphInfo(@TempDir Path exportDir) throws IOException { assertThat(graphInfo.databaseId()).isEqualTo(databaseId); assertThat(graphInfo.databaseId().databaseName()).isEqualTo("my-database"); - assertThat(graphInfo.idMapBuilderType()).isEqualTo((byte) 1); + assertThat(graphInfo.idMapBuilderType()).isEqualTo(ArrayIdMapBuilder.ID); assertThat(graphInfo.nodeCount()).isEqualTo(19L); assertThat(graphInfo.maxOriginalId()).isEqualTo(1337L); From 90471d6310b893311643fca3697f36e041409045 Mon Sep 17 00:00:00 2001 From: Martin Junghanns Date: Thu, 8 Jun 2023 15:14:07 +0200 Subject: [PATCH 088/273] Remove ShardedIdMap --- .../org/neo4j/gds/utils/GdsFeatureToggles.java | 1 - .../production-deployment/feature-toggles.adoc | 14 -------------- .../src/main/java/org/neo4j/gds/SysInfoProc.java | 1 - .../test/java/org/neo4j/gds/SysInfoProcTest.java | 1 - 4 files changed, 17 deletions(-) diff --git a/core-utils/src/main/java/org/neo4j/gds/utils/GdsFeatureToggles.java b/core-utils/src/main/java/org/neo4j/gds/utils/GdsFeatureToggles.java index a9017ae1de..3bc4ac60f3 100644 --- a/core-utils/src/main/java/org/neo4j/gds/utils/GdsFeatureToggles.java +++ b/core-utils/src/main/java/org/neo4j/gds/utils/GdsFeatureToggles.java @@ -32,7 +32,6 @@ public enum GdsFeatureToggles { USE_PARALLEL_PROPERTY_VALUE_INDEX(false), USE_PARTITIONED_SCAN(false), USE_BIT_ID_MAP(true), - USE_SHARDED_ID_MAP(false), USE_UNCOMPRESSED_ADJACENCY_LIST(false), USE_PACKED_ADJACENCY_LIST(false), USE_REORDERED_ADJACENCY_LIST(false), diff --git a/doc/modules/ROOT/pages/production-deployment/feature-toggles.adoc b/doc/modules/ROOT/pages/production-deployment/feature-toggles.adoc index f2b1fac5e6..96dd7fc79f 100644 --- a/doc/modules/ROOT/pages/production-deployment/feature-toggles.adoc +++ b/doc/modules/ROOT/pages/production-deployment/feature-toggles.adoc @@ -20,20 +20,6 @@ To switch to the more memory intensive implementation used in GDS Community Edit CALL gds.features.useBitIdMap(false) ---- -[.enterprise-edition] -[[sharded-id-map-feature-toggle]] -== ShardedIdMap Feature Toggle - -The xref:production-deployment/feature-toggles.adoc#bit-id-map-feature-toggle[BitIdMap] is optimized for a low memory footprint when used together with a Neo4j database. -However, its memory footprint is not optimal if the range of possible original node ids exceeds the node count significantly. -In this situation the `ShardedIdMap` can be used to significantly reduce the required memory of the graph projection. -To enable the sharded id map the following procedure call can be used: - -[source, cypher, role=noplay] ----- -CALL gds.features.useShardedIdMap(true) ----- - [[packed-adjacency-list-feature-toggle]] == Packed Adjacency List Toggle diff --git a/proc/sysinfo/src/main/java/org/neo4j/gds/SysInfoProc.java b/proc/sysinfo/src/main/java/org/neo4j/gds/SysInfoProc.java index 096db14b00..67e57de64a 100644 --- a/proc/sysinfo/src/main/java/org/neo4j/gds/SysInfoProc.java +++ b/proc/sysinfo/src/main/java/org/neo4j/gds/SysInfoProc.java @@ -167,7 +167,6 @@ private static void features(Stream.Builder builder) { )) .add(value("featurePartitionedScan", GdsFeatureToggles.USE_PARTITIONED_SCAN.isEnabled())) .add(value("featureBitIdMap", GdsFeatureToggles.USE_BIT_ID_MAP.isEnabled())) - .add(value("featureShardedIdMap", GdsFeatureToggles.USE_SHARDED_ID_MAP.isEnabled())) .add(value( "featureUncompressedAdjacencyList", GdsFeatureToggles.USE_UNCOMPRESSED_ADJACENCY_LIST.isEnabled() diff --git a/proc/sysinfo/src/test/java/org/neo4j/gds/SysInfoProcTest.java b/proc/sysinfo/src/test/java/org/neo4j/gds/SysInfoProcTest.java index 04e90466d3..30d1430862 100644 --- a/proc/sysinfo/src/test/java/org/neo4j/gds/SysInfoProcTest.java +++ b/proc/sysinfo/src/test/java/org/neo4j/gds/SysInfoProcTest.java @@ -311,7 +311,6 @@ void testSysInfoProc() throws IOException { .containsEntry("featureParallelPropertyValueIndex", GdsFeatureToggles.USE_PARALLEL_PROPERTY_VALUE_INDEX.isEnabled()) .containsEntry("featurePartitionedScan", GdsFeatureToggles.USE_PARTITIONED_SCAN.isEnabled()) .containsEntry("featureBitIdMap", GdsFeatureToggles.USE_BIT_ID_MAP.isEnabled()) - .containsEntry("featureShardedIdMap", GdsFeatureToggles.USE_SHARDED_ID_MAP.isEnabled()) .containsEntry("featureUncompressedAdjacencyList", GdsFeatureToggles.USE_UNCOMPRESSED_ADJACENCY_LIST.isEnabled()) .containsEntry("featurePackedAdjacencyList", GdsFeatureToggles.USE_PACKED_ADJACENCY_LIST.isEnabled()) .containsEntry("featureReorderedAdjacencyList", GdsFeatureToggles.USE_REORDERED_ADJACENCY_LIST.isEnabled()) From be26f1d2cf17d859364b6d2ed6840088bee9faa5 Mon Sep 17 00:00:00 2001 From: Martin Junghanns Date: Thu, 8 Jun 2023 16:25:05 +0200 Subject: [PATCH 089/273] Remove obsolete NOTE in arrow project docs --- doc/modules/ROOT/pages/graph-project-apache-arrow.adoc | 7 ------- 1 file changed, 7 deletions(-) diff --git a/doc/modules/ROOT/pages/graph-project-apache-arrow.adoc b/doc/modules/ROOT/pages/graph-project-apache-arrow.adoc index 92005eff4b..024e8876e4 100644 --- a/doc/modules/ROOT/pages/graph-project-apache-arrow.adoc +++ b/doc/modules/ROOT/pages/graph-project-apache-arrow.adoc @@ -147,13 +147,6 @@ The server acknowledges the action by returning a JSON document including the na } ---- -[NOTE] -==== -Node identifiers are represented by long values in the range of 0 to 2^63. -If the input node id space is sparse and contains very large node id values, one might observe a high memory footprint for the projected graph. -In these situations, the memory footprint of the graph could be reduced by switching to another xref:production-deployment/feature-toggles.adoc#sharded-id-map-feature-toggle[id map implementation]. -==== - [[arrow-send-relationships]] == Sending relationship records via PUT as a Flight stream From 937747378ef4c325c77f4f6eae8cfec264d94ff1 Mon Sep 17 00:00:00 2001 From: Martin Junghanns Date: Thu, 8 Jun 2023 15:36:10 +0200 Subject: [PATCH 090/273] Enable CsvToGraphStoreImporterIntegrationEETest and adapt GdlFactory --- .../src/main/java/org/neo4j/gds/gdl/GdlFactory.java | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/test-utils/src/main/java/org/neo4j/gds/gdl/GdlFactory.java b/test-utils/src/main/java/org/neo4j/gds/gdl/GdlFactory.java index 1c68a21ef7..a2977ed1ff 100644 --- a/test-utils/src/main/java/org/neo4j/gds/gdl/GdlFactory.java +++ b/test-utils/src/main/java/org/neo4j/gds/gdl/GdlFactory.java @@ -95,6 +95,7 @@ static GdlFactory gdlFactory( Optional userName, Optional graphName, Optional graphProjectConfig, + Optional highestNodeId, Optional nodeIdFunction, Optional graphCapabilities, Optional idMapBuilderType @@ -115,7 +116,7 @@ static GdlFactory gdlFactory( .setDefaultEdgeLabel(RelationshipType.ALL_RELATIONSHIPS.name) .buildFromString(config.gdlGraph()); - var graphDimensions = GraphDimensionsGdlReader.of(gdlHandler); + var graphDimensions = GraphDimensionsGdlReader.of(gdlHandler, highestNodeId); // NOTE: We don't really have a database, but GDL is for testing to work as if we had a database var capabilities = graphCapabilities.orElseGet(() -> ImmutableStaticCapabilities.of(WriteMode.LOCAL)); @@ -188,6 +189,7 @@ private Nodes loadNodes() { .maxOriginalId(dimensions.highestPossibleNodeCount() - 1) .hasLabelInformation(true) .hasProperties(true) + .deduplicateIds(false) .concurrency(1) .propertyState(graphProjectConfig.propertyState()) .idMapBuilderType(this.idMapBuilderType) @@ -381,14 +383,14 @@ protected ProgressTracker progressTracker() { private static final class GraphDimensionsGdlReader { - static GraphDimensions of(GDLHandler gdlHandler) { + static GraphDimensions of(GDLHandler gdlHandler, Optional highestNodeId) { var nodeCount = gdlHandler.getVertices().size(); - var highestId = gdlHandler + var highestId = highestNodeId.orElseGet(() -> gdlHandler .getVertices() .stream() .map(Vertex::getId) .max(Long::compareTo) - .orElse((long) nodeCount); + .orElse((long) nodeCount)); var relCount = gdlHandler.getEdges().size(); var nodePropertyDimensions = new HashMap>(); From 2128c83049d0b340ac4a239f855678bda759435f Mon Sep 17 00:00:00 2001 From: Martin Junghanns Date: Mon, 12 Jun 2023 13:16:34 +0200 Subject: [PATCH 091/273] Remove capacity check in NodesBatchBuffer#add The check had been previously introduced to avoid the situation where nodes are added to the database while running a native projection leading to a potential AIOOB on the pre-sized data structures. This check is still in the #offer() method which is the one that is being called on from the node scanners. The #add() method is called from the NodesBuilder (construction) code path where this constraint is not necessary. Removing this check allows us to work nicely with high limit id map where the external highest id is not the one we use for sizing the internal data structures. --- .../gds/core/loading/NodesBatchBuffer.java | 4 -- .../core/loading/NodesBatchBufferTest.java | 40 ------------------- 2 files changed, 44 deletions(-) diff --git a/core/src/main/java/org/neo4j/gds/core/loading/NodesBatchBuffer.java b/core/src/main/java/org/neo4j/gds/core/loading/NodesBatchBuffer.java index da86e7fbc7..9f027630bb 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/NodesBatchBuffer.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/NodesBatchBuffer.java @@ -120,10 +120,6 @@ public boolean offer(final NodeReference record) { } public void add(long nodeId, PropertyReference propertyReference, long[] labels) { - if (nodeId >= this.highestPossibleNodeCount) { - return; - } - int len = length++; buffer[len] = nodeId; if (properties != null) { diff --git a/core/src/test/java/org/neo4j/gds/core/loading/NodesBatchBufferTest.java b/core/src/test/java/org/neo4j/gds/core/loading/NodesBatchBufferTest.java index 84d764b70f..8360521d23 100644 --- a/core/src/test/java/org/neo4j/gds/core/loading/NodesBatchBufferTest.java +++ b/core/src/test/java/org/neo4j/gds/core/loading/NodesBatchBufferTest.java @@ -29,46 +29,6 @@ class NodesBatchBufferTest { - @Test - void shouldIgnoreNodesThatAreOutOfBoundsOnAdd() { - var nodesBatchBuffer = new NodesBatchBufferBuilder() - .capacity(3) - .highestPossibleNodeCount(43) - .build(); - - // within range - nodesBatchBuffer.add(21, Neo4jProxy.noPropertyReference(), new long[0]); - // end of range - nodesBatchBuffer.add(42, Neo4jProxy.noPropertyReference(), new long[0]); - // out of range - nodesBatchBuffer.add(84, Neo4jProxy.noPropertyReference(), new long[0]); - - assertThat(nodesBatchBuffer) - .returns(2, RecordsBatchBuffer::length) - .returns(new long[]{21, 42, 0}, RecordsBatchBuffer::batch); - } - - @Test - void shouldIgnoreNodesThatAreOutOfBoundsOnAddWithLabelInformation() { - var nodesBatchBuffer = new NodesBatchBufferBuilder() - .capacity(3) - .highestPossibleNodeCount(43) - .hasLabelInformation(true) - .nodeLabelIds(LongHashSet.from(0)) - .build(); - - // within range - nodesBatchBuffer.add(21, Neo4jProxy.noPropertyReference(), new long[]{0}); - // end of range - nodesBatchBuffer.add(42, Neo4jProxy.noPropertyReference(), new long[]{0}); - // out of range - nodesBatchBuffer.add(84, Neo4jProxy.noPropertyReference(), new long[]{0}); - - assertThat(nodesBatchBuffer) - .returns(2, RecordsBatchBuffer::length) - .returns(new long[]{21, 42, 0}, RecordsBatchBuffer::batch); - } - @Test void shouldIgnoreNodesThatAreOutOfBoundsOnOffer() { var nodesBatchBuffer = new NodesBatchBufferBuilder() From 02d305af82f05a0c812e2c7f4bcddbc0ba77b838 Mon Sep 17 00:00:00 2001 From: Martin Junghanns Date: Mon, 12 Jun 2023 13:41:44 +0200 Subject: [PATCH 092/273] Forward highest intermediate id to nested idmap builders --- .../org/neo4j/gds/core/OpenGdsIdMapBehavior.java | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/org/neo4j/gds/core/OpenGdsIdMapBehavior.java b/core/src/main/java/org/neo4j/gds/core/OpenGdsIdMapBehavior.java index ea64308f92..88f73b80a4 100644 --- a/core/src/main/java/org/neo4j/gds/core/OpenGdsIdMapBehavior.java +++ b/core/src/main/java/org/neo4j/gds/core/OpenGdsIdMapBehavior.java @@ -50,11 +50,16 @@ public IdMapBuilder create(String id, int concurrency, Optional maxOrigina return create(concurrency, maxOriginalId, nodeCount); } if (HighLimitIdMap.isHighLimitIdMap(idLowerCase)) { + // We do not pass in the highest original id to the nested id map builder + // since initializing a HighLimitIdMap is typically a situation where the + // external ids may exceed the storage capabilities of the nested id map. + // Instead, we _know_ that the highest original id for the nested id map + // will be nodeCount - 1 as this is what the HighLimitIdMap guarantees. + var maxIntermediateId = nodeCount.map(nc -> nc - 1); var innerBuilder = HighLimitIdMap.innerTypeId(idLowerCase) - .map(innerId -> create(innerId, concurrency, maxOriginalId, nodeCount)) - .orElseGet(() -> create(concurrency, maxOriginalId, nodeCount)); - return HighLimitIdMapBuilder.of(concurrency, innerBuilder); - } + .map(innerId -> create(innerId, concurrency, maxIntermediateId, nodeCount)) + .orElseGet(() -> create(concurrency, maxIntermediateId, nodeCount)); + return HighLimitIdMapBuilder.of(concurrency, innerBuilder); } return create(concurrency, maxOriginalId, nodeCount); } From 99aee388be19e5c216dcce13836d5c6ca70562f0 Mon Sep 17 00:00:00 2001 From: Martin Junghanns Date: Mon, 12 Jun 2023 15:20:52 +0200 Subject: [PATCH 093/273] Pass highest intermediate id to nested idmap builder in HL --- .../java/org/neo4j/gds/core/loading/HighLimitIdMapBuilder.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/org/neo4j/gds/core/loading/HighLimitIdMapBuilder.java b/core/src/main/java/org/neo4j/gds/core/loading/HighLimitIdMapBuilder.java index fe3d69c622..0d101f6d2e 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/HighLimitIdMapBuilder.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/HighLimitIdMapBuilder.java @@ -57,7 +57,8 @@ public IdMapAllocator allocate(int batchLength) { @Override public IdMap build(LabelInformation.Builder labelInformationBuilder, long highestNodeId, int concurrency) { var intermediateIdMap = this.originalToIntermediateMapping.build(); - var internalIdMap = this.intermediateToInternalMapping.build(labelInformationBuilder, highestNodeId, concurrency); + var highestIntermediateId = intermediateIdMap.size() - 1; + var internalIdMap = this.intermediateToInternalMapping.build(labelInformationBuilder, highestIntermediateId, concurrency); return new HighLimitIdMap(intermediateIdMap, internalIdMap); } From 3a84c89e78850a3eb294ae2622da0c5b52e1b3c8 Mon Sep 17 00:00:00 2001 From: Martin Junghanns Date: Mon, 12 Jun 2023 15:21:21 +0200 Subject: [PATCH 094/273] Validate input constraints when HighLimit is requested When HighLimit is used, we require the node count to properly size the label bit sets. Alternatively, we could just let them grow which is a bit more runtime overhead. When HighLimit is used, we cannot use deduplication as this requires an additional HABS which might fail on creation due to size limitations. --- .../loading/construction/GraphFactory.java | 63 +++++++++++++------ .../loading/construction/NodesBuilder.java | 3 +- 2 files changed, 47 insertions(+), 19 deletions(-) diff --git a/core/src/main/java/org/neo4j/gds/core/loading/construction/GraphFactory.java b/core/src/main/java/org/neo4j/gds/core/loading/construction/GraphFactory.java index c53803b9f3..e24f3947e8 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/construction/GraphFactory.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/construction/GraphFactory.java @@ -43,6 +43,8 @@ import org.neo4j.gds.core.concurrency.Pools; import org.neo4j.gds.core.huge.HugeGraph; import org.neo4j.gds.core.huge.HugeGraphBuilder; +import org.neo4j.gds.core.loading.HighLimitIdMap; +import org.neo4j.gds.core.loading.HighLimitIdMapBuilder; import org.neo4j.gds.core.loading.IdMapBuilder; import org.neo4j.gds.core.loading.ImmutableImportMetaData; import org.neo4j.gds.core.loading.ImportSizing; @@ -98,36 +100,60 @@ static NodesBuilder nodesBuilder( ? Optional.of(maxOriginalId) : Optional.empty(); + var idMapType = idMapBuilderType.orElse(IdMap.NO_TYPE); var idMapBuilder = idMapBehavior.create( - idMapBuilderType.orElse(IdMap.NO_TYPE), + idMapType, threadCount, maybeMaxOriginalId, nodeCount ); boolean deduplicate = deduplicateIds.orElse(true); + long maxIntermediateId = maxOriginalId; + + if (HighLimitIdMap.isHighLimitIdMap(idMapType)) { + // If the requested id map is high limit, we need to make sure that + // internal data structures are sized accordingly. Using the highest + // original id will potentially fail due to size limitations. + if (nodeCount.isPresent()) { + maxIntermediateId = nodeCount.get() - 1; + } else { + throw new IllegalArgumentException("Cannot use high limit id map without node count."); + } + if (deduplicate) { + // We internally use HABS for deduplication, which is being initialized + // with max original id. This is fine for all id maps except high limit, + // where original ids can exceed the supported HABS range. + throw new IllegalArgumentException("Cannot use high limit id map with deduplication."); + } + } - return nodeSchema.map(schema -> fromSchema( - maxOriginalId, - idMapBuilder, - threadCount, - schema, - labelInformation, - deduplicate - )).orElseGet(() -> new NodesBuilder( - maxOriginalId, - threadCount, - NodesBuilderContext.lazy(threadCount), - idMapBuilder, - labelInformation, - hasProperties.orElse(false), - deduplicate, - __ -> propertyState.orElse(PropertyState.PERSISTENT) - )); + return nodeSchema.isPresent() + ? fromSchema( + maxOriginalId, + maxIntermediateId, + idMapBuilder, + threadCount, + nodeSchema.get(), + labelInformation, + deduplicate + ) + : new NodesBuilder( + maxOriginalId, + maxIntermediateId, + threadCount, + NodesBuilderContext.lazy(threadCount), + idMapBuilder, + labelInformation, + hasProperties.orElse(false), + deduplicate, + __ -> propertyState.orElse(PropertyState.PERSISTENT) + ); } private static NodesBuilder fromSchema( long maxOriginalId, + long maxIntermediateId, IdMapBuilder idMapBuilder, int concurrency, NodeSchema nodeSchema, @@ -136,6 +162,7 @@ private static NodesBuilder fromSchema( ) { return new NodesBuilder( maxOriginalId, + maxIntermediateId, concurrency, NodesBuilderContext.fixed(nodeSchema, concurrency), idMapBuilder, diff --git a/core/src/main/java/org/neo4j/gds/core/loading/construction/NodesBuilder.java b/core/src/main/java/org/neo4j/gds/core/loading/construction/NodesBuilder.java index 9dad02bc63..1988bfe093 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/construction/NodesBuilder.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/construction/NodesBuilder.java @@ -78,6 +78,7 @@ public final class NodesBuilder { NodesBuilder( long maxOriginalId, + long maxIntermediateId, int concurrency, NodesBuilderContext nodesBuilderContext, IdMapBuilder idMapBuilder, @@ -93,7 +94,7 @@ public final class NodesBuilder { this.propertyStates = propertyStates; this.labelInformationBuilder = !hasLabelInformation ? LabelInformationBuilders.allNodes() - : LabelInformationBuilders.multiLabelWithCapacity(maxOriginalId + 1); + : LabelInformationBuilders.multiLabelWithCapacity(maxIntermediateId + 1); this.importedNodes = new LongAdder(); this.nodeImporter = new NodeImporterBuilder() From a0d39994ac6dce79c76fe51a7ccc07d1530b6e84 Mon Sep 17 00:00:00 2001 From: Martin Junghanns Date: Mon, 12 Jun 2023 15:26:50 +0200 Subject: [PATCH 095/273] Remove superfluous highestNodeId on GdlFactory --- .../src/main/java/org/neo4j/gds/gdl/GdlFactory.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test-utils/src/main/java/org/neo4j/gds/gdl/GdlFactory.java b/test-utils/src/main/java/org/neo4j/gds/gdl/GdlFactory.java index a2977ed1ff..430de23226 100644 --- a/test-utils/src/main/java/org/neo4j/gds/gdl/GdlFactory.java +++ b/test-utils/src/main/java/org/neo4j/gds/gdl/GdlFactory.java @@ -95,7 +95,6 @@ static GdlFactory gdlFactory( Optional userName, Optional graphName, Optional graphProjectConfig, - Optional highestNodeId, Optional nodeIdFunction, Optional graphCapabilities, Optional idMapBuilderType @@ -116,7 +115,7 @@ static GdlFactory gdlFactory( .setDefaultEdgeLabel(RelationshipType.ALL_RELATIONSHIPS.name) .buildFromString(config.gdlGraph()); - var graphDimensions = GraphDimensionsGdlReader.of(gdlHandler, highestNodeId); + var graphDimensions = GraphDimensionsGdlReader.of(gdlHandler); // NOTE: We don't really have a database, but GDL is for testing to work as if we had a database var capabilities = graphCapabilities.orElseGet(() -> ImmutableStaticCapabilities.of(WriteMode.LOCAL)); @@ -187,6 +186,7 @@ public CSRGraphStore build() { private Nodes loadNodes() { var nodesBuilder = GraphFactory.initNodesBuilder() .maxOriginalId(dimensions.highestPossibleNodeCount() - 1) + .nodeCount(gdlHandler.getVertices().size()) .hasLabelInformation(true) .hasProperties(true) .deduplicateIds(false) @@ -383,14 +383,14 @@ protected ProgressTracker progressTracker() { private static final class GraphDimensionsGdlReader { - static GraphDimensions of(GDLHandler gdlHandler, Optional highestNodeId) { + static GraphDimensions of(GDLHandler gdlHandler) { var nodeCount = gdlHandler.getVertices().size(); - var highestId = highestNodeId.orElseGet(() -> gdlHandler + var highestId = gdlHandler .getVertices() .stream() .map(Vertex::getId) .max(Long::compareTo) - .orElse((long) nodeCount)); + .orElse((long) nodeCount - 1); var relCount = gdlHandler.getEdges().size(); var nodePropertyDimensions = new HashMap>(); From c5b040d536d119270663cc9eae36d4634967bba5 Mon Sep 17 00:00:00 2001 From: Martin Junghanns Date: Mon, 12 Jun 2023 16:14:05 +0200 Subject: [PATCH 096/273] Fix checkstyle issues --- .../org/neo4j/gds/core/loading/construction/GraphFactory.java | 1 - 1 file changed, 1 deletion(-) diff --git a/core/src/main/java/org/neo4j/gds/core/loading/construction/GraphFactory.java b/core/src/main/java/org/neo4j/gds/core/loading/construction/GraphFactory.java index e24f3947e8..35bf2d06aa 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/construction/GraphFactory.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/construction/GraphFactory.java @@ -44,7 +44,6 @@ import org.neo4j.gds.core.huge.HugeGraph; import org.neo4j.gds.core.huge.HugeGraphBuilder; import org.neo4j.gds.core.loading.HighLimitIdMap; -import org.neo4j.gds.core.loading.HighLimitIdMapBuilder; import org.neo4j.gds.core.loading.IdMapBuilder; import org.neo4j.gds.core.loading.ImmutableImportMetaData; import org.neo4j.gds.core.loading.ImportSizing; From 423d7fe6530055121ec888ccb93715019d78d3f1 Mon Sep 17 00:00:00 2001 From: Veselin Nikolov Date: Wed, 14 Jun 2023 10:42:26 +0100 Subject: [PATCH 097/273] Add Neo4j 5.9 support --- README.adoc | 5 +++-- build.gradle | 2 ++ settings.gradle | 6 ++++++ 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/README.adoc b/README.adoc index fc50589906..4e0e4901c2 100644 --- a/README.adoc +++ b/README.adoc @@ -26,9 +26,10 @@ When installing GDS manually, please refer to the below compatibility matrix: .Compatibility matrix (italicized version is in development) |=== |GDS version | Neo4j version | Java Version -.9+<.^|_GDS 2.4.x_ +.10+<.^|_GDS 2.4.x_ +|Neo4j 5.9.0 +.9+.^|Java 17 |Neo4j 5.8.0 -.8+.^|Java 17 |Neo4j 5.7.0 |Neo4j 5.6.0 |Neo4j 5.3.0 diff --git a/build.gradle b/build.gradle index 6d2713de31..16f260a93b 100644 --- a/build.gradle +++ b/build.gradle @@ -35,6 +35,7 @@ ext { project(':neo4j-kernel-adapter-5.6'), project(':neo4j-kernel-adapter-5.7'), project(':neo4j-kernel-adapter-5.8'), + project(':neo4j-kernel-adapter-5.9'), ], 'storage-engine-adapter': [ project(':storage-engine-adapter-4.4'), @@ -46,6 +47,7 @@ ext { project(':storage-engine-adapter-5.6'), project(':storage-engine-adapter-5.7'), project(':storage-engine-adapter-5.8'), + project(':storage-engine-adapter-5.9'), ] ] } diff --git a/settings.gradle b/settings.gradle index 9379ed4f1c..a73bc99bc3 100644 --- a/settings.gradle +++ b/settings.gradle @@ -154,6 +154,9 @@ project(':neo4j-kernel-adapter-5.7').projectDir = file('compatibility/5.7/neo4j- include('neo4j-kernel-adapter-5.8') project(':neo4j-kernel-adapter-5.8').projectDir = file('compatibility/5.8/neo4j-kernel-adapter') +include('neo4j-kernel-adapter-5.9') +project(':neo4j-kernel-adapter-5.9').projectDir = file('compatibility/5.9/neo4j-kernel-adapter') + include('neo4j-kernel-adapter-api') project(':neo4j-kernel-adapter-api').projectDir = file('compatibility/api/neo4j-kernel-adapter') @@ -256,6 +259,9 @@ project(':storage-engine-adapter-5.7').projectDir = file('compatibility/5.7/stor include('storage-engine-adapter-5.8') project(':storage-engine-adapter-5.8').projectDir = file('compatibility/5.8/storage-engine-adapter') +include('storage-engine-adapter-5.9') +project(':storage-engine-adapter-5.9').projectDir = file('compatibility/5.9/storage-engine-adapter') + include('storage-engine-adapter-api') project(':storage-engine-adapter-api').projectDir = file('compatibility/api/storage-engine-adapter') From 87760c7b0ff07ebc5f4488d28f7411501eddafd0 Mon Sep 17 00:00:00 2001 From: Nicola Vitucci Date: Wed, 14 Jun 2023 12:41:07 +0200 Subject: [PATCH 098/273] Merge pull request #7736 from nvitucci/update-install-server Clarify procedure allowlist check --- doc/modules/ROOT/pages/installation/index.adoc | 2 +- doc/modules/ROOT/pages/installation/neo4j-server.adoc | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/doc/modules/ROOT/pages/installation/index.adoc b/doc/modules/ROOT/pages/installation/index.adoc index 04fe1483b4..d7f9ec926d 100644 --- a/doc/modules/ROOT/pages/installation/index.adoc +++ b/doc/modules/ROOT/pages/installation/index.adoc @@ -65,7 +65,7 @@ CALL gds.debug.sysInfo(); In order to make use of certain features of the GDS library, additional configuration may be necessary. For example, exporting graphs xref:graph-catalog-export-ops.adoc#catalog-graph-export-csv[to CSV files] requires the configuration parameter `gds.export.location` to be set to the folder in which exported graphs are to be stored. -You can find the list of all the configuration options xref:production-deployment/configuration-settings/[here]. +You can find the list of all the configuration options xref:production-deployment/configuration-settings.adoc[here]. Refer to the <<_installation_methods,installation methods>> for details on how to edit a Neo4j database configuration depending on the Neo4j deployment. == Reference diff --git a/doc/modules/ROOT/pages/installation/neo4j-server.adoc b/doc/modules/ROOT/pages/installation/neo4j-server.adoc index 65ea46ed68..47e81ff9af 100644 --- a/doc/modules/ROOT/pages/installation/neo4j-server.adoc +++ b/doc/modules/ROOT/pages/installation/neo4j-server.adoc @@ -15,11 +15,12 @@ dbms.security.procedures.unrestricted=gds.* This configuration entry is necessary because the GDS library accesses low-level components of Neo4j to maximise performance. + -4. Check if the procedure allowlist is enabled in the `$NEO4J_HOME/conf/neo4j.conf` file and add the GDS library if necessary: +4. Check if the procedure allowlist is enabled in the `$NEO4J_HOME/conf/neo4j.conf` file, namely if the `dbms.security.procedures.allowlist` option is _not_ commented out with a leading `#` sign. In this case, add the GDS library to the allowlist: + ---- dbms.security.procedures.allowlist=gds.* ---- +You can find more information on allow listing in the link:https://neo4j.com/docs/operations-manual/current/security/securing-extensions/#allow-listing[Operations Manual]. + 5. Restart the Neo4j Server. From e90c4577ed7b5411dad5e6582345d89edf84919b Mon Sep 17 00:00:00 2001 From: yuval Date: Wed, 14 Jun 2023 13:33:05 +0200 Subject: [PATCH 099/273] Updated supported versions --- .../ROOT/pages/installation/supported-neo4j-versions.adoc | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/modules/ROOT/pages/installation/supported-neo4j-versions.adoc b/doc/modules/ROOT/pages/installation/supported-neo4j-versions.adoc index 203c8963c7..c6403d6c75 100644 --- a/doc/modules/ROOT/pages/installation/supported-neo4j-versions.adoc +++ b/doc/modules/ROOT/pages/installation/supported-neo4j-versions.adoc @@ -10,6 +10,7 @@ If your version of GDS or Neo4j is not listed in the matrix, you should upgrade. [opts=header] |=== | Neo4j version | Neo4j Graph Data Science +| `5.9` | `2.4`, `2.3.9` or later | `5.8` | `2.4`, `2.3.6` or later | `5.7` | `2.4`, `2.3.3` or later | `5.6` | `2.4`, `2.3.2` or later From 34e502ec1ac42057d32751d60aa382414ddea9eb Mon Sep 17 00:00:00 2001 From: Nicola Vitucci Date: Thu, 15 Jun 2023 09:29:03 +0100 Subject: [PATCH 100/273] Fix issue with package.json --- doc/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/package.json b/doc/package.json index ac9c910e0e..5df0a3afb3 100644 --- a/doc/package.json +++ b/doc/package.json @@ -9,7 +9,7 @@ "start:publish": "nodemon -e adoc --exec \"npm run build:publish && npm run serve\"", "serve": "node server.js", "build": "antora preview.yml --stacktrace --log-format=pretty", - "build-verify": "antora --stacktrace --fetch preview.yml --log-format=json --log-level=info --log-file ./build/log/log.json", + "build-verify": "antora --stacktrace --fetch preview.yml --log-format=json --log-level=info --log-file ./build/log/log.json" }, "keywords": [ "antora", From 7e4418e016642ae993439db83635db1849d26cbf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Thu, 15 Jun 2023 14:52:20 +0200 Subject: [PATCH 101/273] Handle latest rc --- gradle/dependencies.gradle | 2 ++ .../src/main/java/org/neo4j/gds/compat/Neo4jVersion.java | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/gradle/dependencies.gradle b/gradle/dependencies.gradle index f77b099992..30da1f749a 100644 --- a/gradle/dependencies.gradle +++ b/gradle/dependencies.gradle @@ -28,6 +28,7 @@ ext { '5.7': '2.13.10', '5.8': '2.13.10', '5.9': '2.13.10', + '5.10': '2.13.10', ] log4js = [ @@ -41,6 +42,7 @@ ext { '5.7': '2.20.0', '5.8': '2.20.0', '5.9': '2.20.0', + '5.10': '2.20.0', ] ver = [ diff --git a/neo4j-adapter/src/main/java/org/neo4j/gds/compat/Neo4jVersion.java b/neo4j-adapter/src/main/java/org/neo4j/gds/compat/Neo4jVersion.java index 9e2b7a7612..080384ad0f 100644 --- a/neo4j-adapter/src/main/java/org/neo4j/gds/compat/Neo4jVersion.java +++ b/neo4j-adapter/src/main/java/org/neo4j/gds/compat/Neo4jVersion.java @@ -71,7 +71,7 @@ public String toString() { public MajorMinorVersion semanticVersion() { if (this == V_RC) { - return ImmutableMajorMinorVersion.of(5, 9); + return ImmutableMajorMinorVersion.of(5, 10); } String version = toString(); From 81c6b67d8a2c97bde192d98e33a1a3ff07d69230 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Thu, 15 Jun 2023 10:23:19 +0200 Subject: [PATCH 102/273] Propagate exceptions from CountingCypherRecordLoader --- .../core/loading/CountingCypherRecordLoader.java | 16 ++++++++++++++++ .../gds/core/loading/CypherRecordLoader.java | 2 +- .../core/loading/ResultCountingSubscriber.java | 10 +++++++++- 3 files changed, 26 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/org/neo4j/gds/core/loading/CountingCypherRecordLoader.java b/core/src/main/java/org/neo4j/gds/core/loading/CountingCypherRecordLoader.java index dd1b91827f..03faa07f48 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/CountingCypherRecordLoader.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/CountingCypherRecordLoader.java @@ -21,8 +21,10 @@ import org.neo4j.gds.api.GraphLoaderContext; import org.neo4j.gds.config.GraphProjectFromCypherConfig; +import org.neo4j.graphdb.security.AuthorizationViolationException; import org.neo4j.kernel.impl.coreapi.InternalTransaction; +import java.util.Locale; import java.util.Set; class CountingCypherRecordLoader extends CypherRecordLoader { @@ -45,6 +47,20 @@ class CountingCypherRecordLoader extends CypherRecordLoader { BatchLoadResult loadSingleBatch(InternalTransaction tx, int bufferSize) { var subscriber = new ResultCountingSubscriber(); CypherLoadingUtils.consume(runLoadingQuery(tx, subscriber)); + + if (subscriber.error().isPresent()) { + Throwable cause = subscriber.error().get(); + if (cause instanceof AuthorizationViolationException) { + throw new IllegalArgumentException(String.format( + Locale.US, + "Query must be read only. Query: [%s]", + this.loadQuery + )); + } + + throw new RuntimeException(cause); + } + return new BatchLoadResult(subscriber.rows(), -1L); } diff --git a/core/src/main/java/org/neo4j/gds/core/loading/CypherRecordLoader.java b/core/src/main/java/org/neo4j/gds/core/loading/CypherRecordLoader.java index b52313c777..672b1e2ca0 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/CypherRecordLoader.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/CypherRecordLoader.java @@ -62,7 +62,7 @@ String toLowerCase() { final GraphLoaderContext loadingContext; private final long recordCount; - private final String loadQuery; + protected final String loadQuery; private final QueryExecutionEngine executionEngine; private final TransactionalContextFactory contextFactory; diff --git a/core/src/main/java/org/neo4j/gds/core/loading/ResultCountingSubscriber.java b/core/src/main/java/org/neo4j/gds/core/loading/ResultCountingSubscriber.java index 5df91e8a67..d44e9dabe9 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/ResultCountingSubscriber.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/ResultCountingSubscriber.java @@ -23,8 +23,16 @@ import org.neo4j.kernel.impl.query.QuerySubscriber; import org.neo4j.values.AnyValue; +import java.util.Optional; + class ResultCountingSubscriber implements QuerySubscriber { private long rows = 0; + private Optional error = Optional.empty(); + + Optional error() + { + return error; + } public long rows() { return rows; @@ -53,7 +61,7 @@ public void onRecordCompleted() { @Override public void onError(Throwable throwable) { - + this.error = Optional.of(throwable); } @Override From 297de6dae52dc578836bd82bbc3b75a9e4aff2cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Thu, 15 Jun 2023 10:24:14 +0200 Subject: [PATCH 103/273] Avoid trying to commit failed transactions This lead to TransactionTerminatedException on dev --- .../src/main/java/org/neo4j/gds/QueryRunner.java | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/test-utils/src/main/java/org/neo4j/gds/QueryRunner.java b/test-utils/src/main/java/org/neo4j/gds/QueryRunner.java index dfaa7c26ae..b80df0e009 100644 --- a/test-utils/src/main/java/org/neo4j/gds/QueryRunner.java +++ b/test-utils/src/main/java/org/neo4j/gds/QueryRunner.java @@ -35,9 +35,11 @@ import java.util.function.Function; import static java.util.Collections.emptyMap; +import static org.junit.jupiter.api.Assertions.fail; import static org.neo4j.gds.compat.GraphDatabaseApiProxy.applyInTransaction; import static org.neo4j.gds.compat.GraphDatabaseApiProxy.runInTransaction; import static org.neo4j.gds.compat.GraphDatabaseApiProxy.runQueryWithoutClosingTheResult; +import static org.neo4j.gds.utils.StringFormatting.formatWithLocale; import static org.neo4j.internal.kernel.api.security.AccessMode.Static.READ; public final class QueryRunner { @@ -203,6 +205,15 @@ public static void runQueryWithResultConsumer( }); } + public static void runFailingQuery(GraphDatabaseService db, String query, Map queryParameters, Consumer exceptionConsumer) { + try { + QueryRunner.runQueryWithResultConsumer(db, query, queryParameters, Result::resultAsString); + fail(formatWithLocale("Expected an exception to be thrown by query:\n%s", query)); + } catch (Throwable e) { + exceptionConsumer.accept(e); + } + } + private static KernelTransaction.Revertable withUsername(Transaction tx, String username, String databaseName) { InternalTransaction topLevelTransaction = (InternalTransaction) tx; AuthSubject subject = topLevelTransaction.securityContext().subject(); From 820039cf2db518e86a8aedef8b0e03a5de5c0e47 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Thu, 15 Jun 2023 14:40:36 +0200 Subject: [PATCH 104/273] Catch only exceptions and avoid isPresent + get on Optional Co-authored-by: Paul Horn --- .../core/loading/CountingCypherRecordLoader.java | 9 ++++----- .../gds/core/loading/ResultCountingSubscriber.java | 14 ++++++++++---- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/core/src/main/java/org/neo4j/gds/core/loading/CountingCypherRecordLoader.java b/core/src/main/java/org/neo4j/gds/core/loading/CountingCypherRecordLoader.java index 03faa07f48..3755610730 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/CountingCypherRecordLoader.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/CountingCypherRecordLoader.java @@ -48,9 +48,8 @@ BatchLoadResult loadSingleBatch(InternalTransaction tx, int bufferSize) { var subscriber = new ResultCountingSubscriber(); CypherLoadingUtils.consume(runLoadingQuery(tx, subscriber)); - if (subscriber.error().isPresent()) { - Throwable cause = subscriber.error().get(); - if (cause instanceof AuthorizationViolationException) { + subscriber.error().ifPresent(e -> { + if (e instanceof AuthorizationViolationException) { throw new IllegalArgumentException(String.format( Locale.US, "Query must be read only. Query: [%s]", @@ -58,8 +57,8 @@ BatchLoadResult loadSingleBatch(InternalTransaction tx, int bufferSize) { )); } - throw new RuntimeException(cause); - } + throw new RuntimeException(e); + }); return new BatchLoadResult(subscriber.rows(), -1L); } diff --git a/core/src/main/java/org/neo4j/gds/core/loading/ResultCountingSubscriber.java b/core/src/main/java/org/neo4j/gds/core/loading/ResultCountingSubscriber.java index d44e9dabe9..518fd5ef98 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/ResultCountingSubscriber.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/ResultCountingSubscriber.java @@ -20,6 +20,7 @@ package org.neo4j.gds.core.loading; import org.neo4j.graphdb.QueryStatistics; +import org.neo4j.kernel.impl.query.QueryExecutionKernelException; import org.neo4j.kernel.impl.query.QuerySubscriber; import org.neo4j.values.AnyValue; @@ -27,10 +28,9 @@ class ResultCountingSubscriber implements QuerySubscriber { private long rows = 0; - private Optional error = Optional.empty(); + private Optional error = Optional.empty(); - Optional error() - { + Optional error() { return error; } @@ -61,7 +61,13 @@ public void onRecordCompleted() { @Override public void onError(Throwable throwable) { - this.error = Optional.of(throwable); + if (throwable instanceof RuntimeException) { + this.error = Optional.of((RuntimeException) throwable); + } else if (throwable instanceof QueryExecutionKernelException) { + this.error = Optional.of(((QueryExecutionKernelException) throwable).asUserException()); + } else { + this.error = Optional.of(new RuntimeException(throwable)); + } } @Override From 3a105c1f1e63fc3e392924deee740fafcc743280 Mon Sep 17 00:00:00 2001 From: Nicola Vitucci Date: Thu, 15 Jun 2023 16:02:33 +0100 Subject: [PATCH 105/273] Fix PDF link --- doc/modules/ROOT/pages/algorithms/degree-centrality.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/modules/ROOT/pages/algorithms/degree-centrality.adoc b/doc/modules/ROOT/pages/algorithms/degree-centrality.adoc index 47a0436e3f..a4970752c2 100644 --- a/doc/modules/ROOT/pages/algorithms/degree-centrality.adoc +++ b/doc/modules/ROOT/pages/algorithms/degree-centrality.adoc @@ -27,7 +27,7 @@ It can be applied to heterogenous graphs, however the algorithm will not calcula For more information on this algorithm, see: -* http://leonidzhukov.net/hse/2014/socialnetworks/papers/freeman79-centrality.pdf[Linton C. Freeman: Centrality in Social Networks Conceptual Clarification, 1979.^] +* https://www.cin.ufpe.br/%7Erbcp/taia/Freeman1979-centrality.pdf[Linton C. Freeman: Centrality in Social Networks Conceptual Clarification, 1979.^] [[algorithms-degree-centrality-usecase]] From 6414581076525841d9dc939fce4dcbb97c8154b2 Mon Sep 17 00:00:00 2001 From: ioannispan Date: Thu, 15 Jun 2023 12:36:27 +0200 Subject: [PATCH 106/273] Fix bug with jumping scanning degree --- .../neo4j/gds/kcore/KCoreDecomposition.java | 7 ++++ .../gds/kcore/KCoreDecompositionTask.java | 6 ++-- .../gds/kcore/KCoreDecompositionTest.java | 33 +++++++++++++++++++ 3 files changed, 44 insertions(+), 2 deletions(-) diff --git a/algo/src/main/java/org/neo4j/gds/kcore/KCoreDecomposition.java b/algo/src/main/java/org/neo4j/gds/kcore/KCoreDecomposition.java index c3da74dcfb..cde9cf42b1 100644 --- a/algo/src/main/java/org/neo4j/gds/kcore/KCoreDecomposition.java +++ b/algo/src/main/java/org/neo4j/gds/kcore/KCoreDecomposition.java @@ -116,6 +116,7 @@ public KCoreDecompositionResult compute() { for (var task : tasks) { task.setScanningDegree(scanningDegree); + task.setPhase(KCoreDecompositionTask.KCoreDecompositionPhase.SCAN); } RunWithConcurrency.builder().tasks(tasks).concurrency(concurrency).run(); @@ -129,8 +130,14 @@ public KCoreDecompositionResult compute() { if (nextScanningDegree == scanningDegree) { degeneracy = scanningDegree; + + for (var task : tasks) { + task.setPhase(KCoreDecompositionTask.KCoreDecompositionPhase.ACT); + } + RunWithConcurrency.builder().tasks(tasks).concurrency(concurrency).run(); scanningDegree++; + } else { //this is a minor optimization not in paper: // if we do not do any updates this round, let's skip directly to the smallest active degree remaining diff --git a/algo/src/main/java/org/neo4j/gds/kcore/KCoreDecompositionTask.java b/algo/src/main/java/org/neo4j/gds/kcore/KCoreDecompositionTask.java index 3f5f9dacb2..854b08dfe9 100644 --- a/algo/src/main/java/org/neo4j/gds/kcore/KCoreDecompositionTask.java +++ b/algo/src/main/java/org/neo4j/gds/kcore/KCoreDecompositionTask.java @@ -80,13 +80,15 @@ void updateNodeProvider(NodeProvider nodeProvider) { public void run() { if (phase == KCoreDecompositionPhase.SCAN) { scan(); - phase = KCoreDecompositionPhase.ACT; } else { act(); - phase = KCoreDecompositionPhase.SCAN; } } + void setPhase(KCoreDecompositionPhase phase) { + this.phase = phase; + } + private void scan() { long upperBound = nodeProvider.size(); diff --git a/algo/src/test/java/org/neo4j/gds/kcore/KCoreDecompositionTest.java b/algo/src/test/java/org/neo4j/gds/kcore/KCoreDecompositionTest.java index decaeeac21..5f7d54f6c7 100644 --- a/algo/src/test/java/org/neo4j/gds/kcore/KCoreDecompositionTest.java +++ b/algo/src/test/java/org/neo4j/gds/kcore/KCoreDecompositionTest.java @@ -215,4 +215,37 @@ void shouldComputeCoreDecomposition() { } } + + @GdlExtension + @Nested + class K4Graph { + @GdlGraph(orientation = Orientation.UNDIRECTED) + private static final String DB_CYPHER = + "CREATE " + + " (a:node)," + + " (b:node)," + + " (c:node)," + + " (d:node)," + + + "(a)-[:R]->(b)," + + "(a)-[:R]->(c)," + + "(a)-[:R]->(d)," + + "(b)-[:R]->(c)," + + "(b)-[:R]->(d)," + + "(c)-[:R]->(d)"; + + + @Inject + TestGraph graph; + + @Test + void shouldAdvanceScanningDegreeCorrectly() { + var kcore = new KCoreDecomposition(graph, 1, ProgressTracker.NULL_TRACKER, 1).compute(); + assertThat(kcore.degeneracy()).isEqualTo(3); + var coreValues = kcore.coreValues(); + + assertThat(coreValues.toArray()).isEqualTo(new int[]{3, 3, 3, 3}); + + } + } } From 5b761e1bafe4fbe4af70a00181ffd90daaa12308 Mon Sep 17 00:00:00 2001 From: ioannispan Date: Thu, 15 Jun 2023 16:04:12 +0200 Subject: [PATCH 107/273] Handle negative node ids for source/target --- .../gds/paths/ShortestPathConfigTest.java | 28 +++++++++++++++++++ .../java/org/neo4j/gds/config/NodeConfig.java | 10 ++++++- .../ShortestPathYensStreamProcTest.java | 2 +- .../gds/paths/traverse/BfsStreamProcTest.java | 1 + 4 files changed, 39 insertions(+), 2 deletions(-) diff --git a/algo/src/test/java/org/neo4j/gds/paths/ShortestPathConfigTest.java b/algo/src/test/java/org/neo4j/gds/paths/ShortestPathConfigTest.java index 2354bcef27..7700e92605 100644 --- a/algo/src/test/java/org/neo4j/gds/paths/ShortestPathConfigTest.java +++ b/algo/src/test/java/org/neo4j/gds/paths/ShortestPathConfigTest.java @@ -22,6 +22,7 @@ import org.junit.jupiter.api.Test; import org.neo4j.gds.core.CypherMapWrapper; import org.neo4j.gds.paths.dijkstra.config.ShortestPathDijkstraStreamConfigImpl; +import org.neo4j.gds.paths.yens.config.ShortestPathYensStreamConfigImpl; import org.neo4j.kernel.impl.core.NodeEntity; import static org.assertj.core.api.Assertions.assertThat; @@ -55,6 +56,33 @@ void shouldAllowNodeIds() { assertThat(config.targetNode()).isEqualTo(1337L); } + @Test + void shouldNotAllowNegativeSourceNode() { + + var config = ShortestPathYensStreamConfigImpl.builder() + .k(1) + .sourceNode(-1337) + .targetNode(0); + + assertThatThrownBy(() -> config.build()) + .hasMessageContaining("Negative node ids are not supported for the field `sourceNode`"); + + } + + @Test + void shouldNotAllowNegativeTargetNode() { + + var config = ShortestPathYensStreamConfigImpl.builder() + .k(1) + .sourceNode(0) + .targetNode(-1337); + + assertThatThrownBy(() -> config.build()) + .hasMessageContaining("Negative node ids are not supported for the field `targetNode`"); + + } + + @Test void shouldThrowErrorOnUnsupportedType() { var cypherMapWrapper = CypherMapWrapper diff --git a/core/src/main/java/org/neo4j/gds/config/NodeConfig.java b/core/src/main/java/org/neo4j/gds/config/NodeConfig.java index 2d352d5acb..f462963cd4 100644 --- a/core/src/main/java/org/neo4j/gds/config/NodeConfig.java +++ b/core/src/main/java/org/neo4j/gds/config/NodeConfig.java @@ -29,7 +29,15 @@ static long parseNodeId(Object input, String field) { if (input instanceof Node) { return ((Node) input).getId(); } else if (input instanceof Number) { - return ((Number) input).longValue(); + var number = ((Number) input).longValue(); + if (number < 0) { + throw new IllegalArgumentException(formatWithLocale( + "Negative node ids are not supported for the field `%s`", + field, + input.getClass().getSimpleName() + )); + } + return number; } throw new IllegalArgumentException(formatWithLocale( diff --git a/proc/path-finding/src/test/java/org/neo4j/gds/paths/sourcetarget/ShortestPathYensStreamProcTest.java b/proc/path-finding/src/test/java/org/neo4j/gds/paths/sourcetarget/ShortestPathYensStreamProcTest.java index 91872423e2..b0dfeeeebe 100644 --- a/proc/path-finding/src/test/java/org/neo4j/gds/paths/sourcetarget/ShortestPathYensStreamProcTest.java +++ b/proc/path-finding/src/test/java/org/neo4j/gds/paths/sourcetarget/ShortestPathYensStreamProcTest.java @@ -200,7 +200,7 @@ void testStream() { assertCypherResult(query, expected); }); } - + @AfterEach void teardown() { GraphStoreCatalog.removeAllLoadedGraphs(); diff --git a/proc/path-finding/src/test/java/org/neo4j/gds/paths/traverse/BfsStreamProcTest.java b/proc/path-finding/src/test/java/org/neo4j/gds/paths/traverse/BfsStreamProcTest.java index 750c772d58..61df240f67 100644 --- a/proc/path-finding/src/test/java/org/neo4j/gds/paths/traverse/BfsStreamProcTest.java +++ b/proc/path-finding/src/test/java/org/neo4j/gds/paths/traverse/BfsStreamProcTest.java @@ -247,4 +247,5 @@ void failOnInvalidEndNode() { assertError(query, "Target nodes do not exist in the in-memory graph: ['42']"); } + } From 13e730b2ca37e328e7102cdd688044c09fd3be42 Mon Sep 17 00:00:00 2001 From: ioannispan Date: Thu, 15 Jun 2023 16:34:12 +0200 Subject: [PATCH 108/273] Handle negative node ids for sourceNodes/ targetNodes --- .../gds/pagerank/PageRankConfigTest.java | 75 ++++++++++++++++++ .../gds/paths/traverse/BfsConfigTest.java | 79 +++++++++++++++++++ .../gds/config/ConfigNodesValidations.java | 12 +++ .../neo4j/gds/config/SourceNodesConfig.java | 2 +- .../neo4j/gds/config/TargetNodesConfig.java | 2 +- .../gds/pagerank/ArticleRankProcTest.java | 22 ------ .../gds/pagerank/EigenvectorProcTest.java | 21 ----- .../randomwalk/RandomWalkStreamProcTest.java | 29 +------ .../gds/paths/traverse/BfsStreamProcTest.java | 15 +--- .../gds/paths/traverse/DfsStreamProcTest.java | 24 ------ 10 files changed, 170 insertions(+), 111 deletions(-) create mode 100644 algo/src/test/java/org/neo4j/gds/pagerank/PageRankConfigTest.java create mode 100644 algo/src/test/java/org/neo4j/gds/paths/traverse/BfsConfigTest.java diff --git a/algo/src/test/java/org/neo4j/gds/pagerank/PageRankConfigTest.java b/algo/src/test/java/org/neo4j/gds/pagerank/PageRankConfigTest.java new file mode 100644 index 0000000000..4aa6ff1aa7 --- /dev/null +++ b/algo/src/test/java/org/neo4j/gds/pagerank/PageRankConfigTest.java @@ -0,0 +1,75 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.pagerank; + +import org.junit.jupiter.api.Test; +import org.neo4j.gds.api.GraphStore; +import org.neo4j.gds.extension.GdlExtension; +import org.neo4j.gds.extension.GdlGraph; +import org.neo4j.gds.extension.Inject; + +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +@GdlExtension +public class PageRankConfigTest { + + @GdlGraph + private static final String DB_CYPHER = + "CREATE " + + " (a:node)," + + " (b:node)," + + "(a)-[:R]->(b)"; + + + @Inject + GraphStore graphStore; + + @Test + void shouldNotAllowNegativeSourceNodes() { + + var config = PageRankStreamConfigImpl.builder() + .sourceNodes(List.of(-1337)).build(); + + assertThatThrownBy(() -> config.graphStoreValidation( + graphStore, + config.nodeLabelIdentifiers(graphStore), + config.internalRelationshipTypes(graphStore) + )) + .hasMessageContaining("Negative node ids are not supported for the field `sourceNodes`"); + + } + + @Test + void shouldNotAllowNonExistantSourceNodes() { + + var config = PageRankStreamConfigImpl.builder() + .sourceNodes(List.of(421337)).build(); + + assertThatThrownBy(() -> config.graphStoreValidation( + graphStore, + config.nodeLabelIdentifiers(graphStore), + config.internalRelationshipTypes(graphStore) + )) + .hasMessageContaining("sourceNodes nodes do not exist in the in-memory graph: ['421337']"); + + } +} diff --git a/algo/src/test/java/org/neo4j/gds/paths/traverse/BfsConfigTest.java b/algo/src/test/java/org/neo4j/gds/paths/traverse/BfsConfigTest.java new file mode 100644 index 0000000000..5dfa110355 --- /dev/null +++ b/algo/src/test/java/org/neo4j/gds/paths/traverse/BfsConfigTest.java @@ -0,0 +1,79 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.paths.traverse; + +import org.junit.jupiter.api.Test; +import org.neo4j.gds.api.GraphStore; +import org.neo4j.gds.extension.GdlExtension; +import org.neo4j.gds.extension.GdlGraph; +import org.neo4j.gds.extension.IdFunction; +import org.neo4j.gds.extension.Inject; + +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +@GdlExtension +public class BfsConfigTest { + + @GdlGraph + private static final String DB_CYPHER = + "CREATE " + + " (a:node)," + + " (b:node)," + + "(a)-[:R]->(b)"; + + + @Inject + GraphStore graphStore; + + @Inject + IdFunction idFunction; + + @Test + void shouldNotAllowNegativeTargetNodes() { + var config = BfsStreamConfigImpl.builder() + .sourceNode(0L) + .targetNodes(List.of(idFunction.of("a"), -1337)).build(); + + assertThatThrownBy(() -> config.graphStoreValidation( + graphStore, + config.nodeLabelIdentifiers(graphStore), + config.internalRelationshipTypes(graphStore) + )) + .hasMessageContaining("Negative node ids are not supported for the field `targetNodes`"); + + } + + @Test + void failOnInvalidEndNodes() { + + var config = BfsStreamConfigImpl.builder() + .sourceNode(idFunction.of("a")) + .targetNodes(List.of(idFunction.of("b"), 421337)).build(); + + assertThatThrownBy(() -> config.graphStoreValidation( + graphStore, + config.nodeLabelIdentifiers(graphStore), + config.internalRelationshipTypes(graphStore) + )) + .hasMessageContaining("targetNodes nodes do not exist in the in-memory graph: ['421337']"); + } +} diff --git a/core/src/main/java/org/neo4j/gds/config/ConfigNodesValidations.java b/core/src/main/java/org/neo4j/gds/config/ConfigNodesValidations.java index 8174e228fa..f5d00deaec 100644 --- a/core/src/main/java/org/neo4j/gds/config/ConfigNodesValidations.java +++ b/core/src/main/java/org/neo4j/gds/config/ConfigNodesValidations.java @@ -40,6 +40,17 @@ static void validateNodes( String nodeDescription ) { if (!neoNodesToValidate.isEmpty()) { + + for (var neoNode : neoNodesToValidate) { + + if (neoNode < 0) { + throw new IllegalArgumentException(formatWithLocale( + "Negative node ids are not supported for the field `%s`", + nodeDescription + )); + } + } + var missingNodes = neoNodesToValidate .stream() .filter(targetNode -> labelFilteredGraphNotContainsNode( @@ -58,6 +69,7 @@ static void validateNodes( StringJoining.join(missingNodes) )); } + } } diff --git a/core/src/main/java/org/neo4j/gds/config/SourceNodesConfig.java b/core/src/main/java/org/neo4j/gds/config/SourceNodesConfig.java index 1aeada122f..389bb26482 100644 --- a/core/src/main/java/org/neo4j/gds/config/SourceNodesConfig.java +++ b/core/src/main/java/org/neo4j/gds/config/SourceNodesConfig.java @@ -45,7 +45,7 @@ default void validateSourceLabels( Collection selectedLabels, Collection selectedRelationshipTypes ) { - validateNodes(graphStore, sourceNodes(), selectedLabels, "Source"); + validateNodes(graphStore, sourceNodes(), selectedLabels, "sourceNodes"); } } diff --git a/core/src/main/java/org/neo4j/gds/config/TargetNodesConfig.java b/core/src/main/java/org/neo4j/gds/config/TargetNodesConfig.java index 34cd290af5..a8aabba796 100644 --- a/core/src/main/java/org/neo4j/gds/config/TargetNodesConfig.java +++ b/core/src/main/java/org/neo4j/gds/config/TargetNodesConfig.java @@ -51,6 +51,6 @@ default void validateTargetNodes( Collection selectedLabels, Collection selectedRelationshipTypes ) { - validateNodes(graphStore, targetNodes(), selectedLabels, "Target"); + validateNodes(graphStore, targetNodes(), selectedLabels, "targetNodes"); } } diff --git a/proc/centrality/src/test/java/org/neo4j/gds/pagerank/ArticleRankProcTest.java b/proc/centrality/src/test/java/org/neo4j/gds/pagerank/ArticleRankProcTest.java index 5f8b890b13..f332bcbd70 100644 --- a/proc/centrality/src/test/java/org/neo4j/gds/pagerank/ArticleRankProcTest.java +++ b/proc/centrality/src/test/java/org/neo4j/gds/pagerank/ArticleRankProcTest.java @@ -33,7 +33,6 @@ import org.neo4j.gds.scaling.Mean; import org.neo4j.gds.scaling.MinMax; import org.neo4j.gds.scaling.NoneScaler; -import org.neo4j.kernel.impl.core.NodeEntity; import java.util.List; import java.util.Map; @@ -163,27 +162,6 @@ void streamWithSourceNodes() { )); } - @Test - void failOnMissingSourceNodes() { - var sourceNodes = List.of( - new NodeEntity(null, 42), - new NodeEntity(null, 1337) - ); - - var query = GdsCypher.call(GRAPH_NAME) - .algo("articleRank") - .streamMode() - .addPlaceholder("sourceNodes", "sources") - .yields(); - - - assertThatThrownBy(() -> runQuery(query, Map.of("sources", sourceNodes))) - .hasRootCauseExactlyInstanceOf(IllegalArgumentException.class) - .hasMessageContaining("Source nodes do not exist in the in-memory graph") - .hasMessageContaining("['1337', '42']"); - } - - @Test void write() { String propertyKey = "pr"; diff --git a/proc/centrality/src/test/java/org/neo4j/gds/pagerank/EigenvectorProcTest.java b/proc/centrality/src/test/java/org/neo4j/gds/pagerank/EigenvectorProcTest.java index ff3b58ebaf..fa5df4c146 100644 --- a/proc/centrality/src/test/java/org/neo4j/gds/pagerank/EigenvectorProcTest.java +++ b/proc/centrality/src/test/java/org/neo4j/gds/pagerank/EigenvectorProcTest.java @@ -33,7 +33,6 @@ import org.neo4j.gds.scaling.Mean; import org.neo4j.gds.scaling.MinMax; import org.neo4j.gds.scaling.NoneScaler; -import org.neo4j.kernel.impl.core.NodeEntity; import java.util.List; import java.util.Map; @@ -165,26 +164,6 @@ void streamWithSourceNodes() { )); } - @Test - void failOnMissingSourceNodes() { - var sourceNodes = List.of( - new NodeEntity(null, 42), - new NodeEntity(null, 1337) - ); - - var query = GdsCypher.call(GRAPH_NAME) - .algo("eigenvector") - .streamMode() - .addPlaceholder("sourceNodes", "sources") - .yields(); - - - assertThatThrownBy(() -> runQuery(query, Map.of("sources", sourceNodes))) - .hasRootCauseExactlyInstanceOf(IllegalArgumentException.class) - .hasMessageContaining("Source nodes do not exist in the in-memory graph") - .hasMessageContaining("['1337', '42']"); - } - @Test void write() { String propertyKey = "pr"; diff --git a/proc/path-finding/src/test/java/org/neo4j/gds/paths/randomwalk/RandomWalkStreamProcTest.java b/proc/path-finding/src/test/java/org/neo4j/gds/paths/randomwalk/RandomWalkStreamProcTest.java index 07fe99d3c5..8bd1aece1a 100644 --- a/proc/path-finding/src/test/java/org/neo4j/gds/paths/randomwalk/RandomWalkStreamProcTest.java +++ b/proc/path-finding/src/test/java/org/neo4j/gds/paths/randomwalk/RandomWalkStreamProcTest.java @@ -191,34 +191,7 @@ void shouldStopWhenStreamIsNotLongerConsumed() { // we're done or fail the test assertThat(pool.getActiveCount()).isEqualTo(0); } - - @Test - void shouldThrowOnUnknownStartNode() { - String query = GdsCypher.call(DEFAULT_GRAPH_NAME) - .algo("gds", "randomWalk") - .streamMode() - .addParameter("walksPerNode", 3) - .addParameter("walkLength", 10) - .addParameter("sourceNodes", 42) - .yields(); - - assertError(query, "Source nodes do not exist in the in-memory graph: ['42']"); - } - - @Test - void shouldThrowOnUnselectedStartNode() { - String query = GdsCypher.call(DEFAULT_GRAPH_NAME) - .algo("gds", "randomWalk") - .streamMode() - .addParameter("walksPerNode", 3) - .addParameter("walkLength", 10) - .addParameter("sourceNodes", 3) - .addParameter("nodeLabels", List.of("Node1", "Node2")) - .yields(); - - assertError(query, "Source nodes do not exist in the in-memory graph for the labels ['Node1', 'Node2']: ['3']"); - } - + @Test void shouldRunMemoryEstimation() { String query = GdsCypher.call(DEFAULT_GRAPH_NAME) diff --git a/proc/path-finding/src/test/java/org/neo4j/gds/paths/traverse/BfsStreamProcTest.java b/proc/path-finding/src/test/java/org/neo4j/gds/paths/traverse/BfsStreamProcTest.java index 61df240f67..32fc1a5756 100644 --- a/proc/path-finding/src/test/java/org/neo4j/gds/paths/traverse/BfsStreamProcTest.java +++ b/proc/path-finding/src/test/java/org/neo4j/gds/paths/traverse/BfsStreamProcTest.java @@ -229,23 +229,10 @@ void failOnInvalidSourceNode() { String query = GdsCypher.call(DEFAULT_GRAPH_NAME) .algo("bfs") .streamMode() - .addParameter("sourceNode", 42) + .addParameter("sourceNode", 4242) .yields(); assertError(query, "Source node does not exist in the in-memory graph: `42`"); } - @Test - void failOnInvalidEndNode() { - loadCompleteGraph(DEFAULT_GRAPH_NAME); - String query = GdsCypher.call(DEFAULT_GRAPH_NAME) - .algo("bfs") - .streamMode() - .addParameter("sourceNode", 0) - .addParameter("targetNodes", List.of(0, 42, 1)) - .yields(); - - assertError(query, "Target nodes do not exist in the in-memory graph: ['42']"); - } - } diff --git a/proc/path-finding/src/test/java/org/neo4j/gds/paths/traverse/DfsStreamProcTest.java b/proc/path-finding/src/test/java/org/neo4j/gds/paths/traverse/DfsStreamProcTest.java index 2034081792..605d3d5e24 100644 --- a/proc/path-finding/src/test/java/org/neo4j/gds/paths/traverse/DfsStreamProcTest.java +++ b/proc/path-finding/src/test/java/org/neo4j/gds/paths/traverse/DfsStreamProcTest.java @@ -228,28 +228,4 @@ static Stream pathQueryBuilders() { ); } - @Test - void failOnInvalidSourceNode() { - loadCompleteGraph(DEFAULT_GRAPH_NAME); - String query = GdsCypher.call(DEFAULT_GRAPH_NAME) - .algo("dfs") - .streamMode() - .addParameter("sourceNode", 42) - .yields(); - - assertError(query, "Source node does not exist in the in-memory graph: `42`"); - } - - @Test - void failOnInvalidEndNode() { - loadCompleteGraph(DEFAULT_GRAPH_NAME); - String query = GdsCypher.call(DEFAULT_GRAPH_NAME) - .algo("dfs") - .streamMode() - .addParameter("sourceNode", 0) - .addParameter("targetNodes", Arrays.asList(0, 42, 1)) - .yields(); - - assertError(query, "Target nodes do not exist in the in-memory graph: ['42']"); - } } From b521ef96e72a497b5e11c3a1a4e209775a0360c2 Mon Sep 17 00:00:00 2001 From: ioannispan Date: Mon, 19 Jun 2023 10:04:55 +0200 Subject: [PATCH 109/273] Rename MutateProperty interface --- .../java/org/neo4j/gds/test/TestMutateConfig.java | 8 +++++--- .../config/ApproxMaxKCutMutateConfig.java | 4 ++-- .../BetweennessCentralityMutateConfig.java | 4 ++-- .../closeness/ClosenessCentralityMutateConfig.java | 4 ++-- .../gds/degree/DegreeCentralityMutateConfig.java | 4 ++-- .../gds/embeddings/fastrp/FastRPMutateConfig.java | 4 ++-- .../graphsage/algo/GraphSageMutateConfig.java | 4 ++-- .../gds/embeddings/hashgnn/HashGNNMutateConfig.java | 4 ++-- .../embeddings/node2vec/Node2VecMutateConfig.java | 4 ++-- .../InfluenceMaximizationMutateConfig.java | 4 ++-- .../neo4j/gds/k1coloring/K1ColoringMutateConfig.java | 4 ++-- .../gds/kcore/KCoreDecompositionMutateConfig.java | 4 ++-- .../org/neo4j/gds/kmeans/KmeansMutateConfig.java | 4 ++-- .../LabelPropagationMutateConfig.java | 4 ++-- .../org/neo4j/gds/leiden/LeidenMutateConfig.java | 4 ++-- .../org/neo4j/gds/louvain/LouvainMutateConfig.java | 4 ++-- .../ModularityOptimizationMutateConfig.java | 4 ++-- .../org/neo4j/gds/pagerank/PageRankMutateConfig.java | 4 ++-- .../scaleproperties/ScalePropertiesMutateConfig.java | 4 ++-- .../filteredknn/FilteredKnnMutateConfig.java | 5 +++-- .../neo4j/gds/similarity/knn/KnnMutateConfig.java | 4 ++-- .../nodesim/NodeSimilarityMutateConfig.java | 5 +++-- .../gds/spanningtree/SpanningTreeMutateConfig.java | 5 +++-- .../neo4j/gds/steiner/SteinerTreeMutateConfig.java | 5 +++-- .../LocalClusteringCoefficientMutateConfig.java | 5 +++-- .../gds/triangle/TriangleCountMutateConfig.java | 4 ++-- .../main/java/org/neo4j/gds/wcc/WccMutateConfig.java | 4 ++-- ...ertyConfig.java => MutateNodePropertyConfig.java} | 2 +- ...igTest.java => MutateNodePropertyConfigTest.java} | 12 ++++++++---- .../org/neo4j/gds/ml/pipeline/NodePropertyStep.java | 2 +- .../java/org/neo4j/gds/ml/pipeline/Pipeline.java | 2 +- .../org/neo4j/gds/ml/pipeline/TrainingPipeline.java | 2 +- .../pipeline/ExecutableNodePropertyStepTestUtil.java | 2 +- .../LinkPredictionTrainPipelineExecutorTest.java | 2 +- .../neo4j/gds/beta/pregel/PregelProcedureConfig.java | 4 ++-- .../java/org/neo4j/gds/harmonic/CentralityScore.java | 6 ++++-- .../main/java/org/neo4j/gds/GraphStoreUpdater.java | 6 +++--- .../neo4j/gds/MutateNodePropertyListFunction.java | 4 ++-- .../gds/MutatePropertyComputationResultConsumer.java | 4 ++-- .../LinkPredictionPredictPipelineMutateConfig.java | 6 ++++-- ...odeClassificationPredictPipelineMutateConfig.java | 6 ++++-- .../NodeRegressionPredictPipelineMutateConfig.java | 5 +++-- .../LinkPredictionPredictPipelineExecutorTest.java | 2 +- 43 files changed, 101 insertions(+), 83 deletions(-) rename core/src/main/java/org/neo4j/gds/config/{MutatePropertyConfig.java => MutateNodePropertyConfig.java} (97%) rename core/src/test/java/org/neo4j/gds/config/{MutatePropertyConfigTest.java => MutateNodePropertyConfigTest.java} (78%) diff --git a/algo-test/src/main/java/org/neo4j/gds/test/TestMutateConfig.java b/algo-test/src/main/java/org/neo4j/gds/test/TestMutateConfig.java index 31fd6d65d4..1e1484c66e 100644 --- a/algo-test/src/main/java/org/neo4j/gds/test/TestMutateConfig.java +++ b/algo-test/src/main/java/org/neo4j/gds/test/TestMutateConfig.java @@ -21,17 +21,19 @@ import org.neo4j.gds.annotation.Configuration; import org.neo4j.gds.annotation.ValueClass; -import org.neo4j.gds.config.MutatePropertyConfig; +import org.neo4j.gds.config.MutateNodePropertyConfig; import org.neo4j.gds.core.CypherMapWrapper; @ValueClass @Configuration @SuppressWarnings("immutables:subtype") -public interface TestMutateConfig extends TestConfig, MutatePropertyConfig { +public interface TestMutateConfig extends TestConfig, MutateNodePropertyConfig { default boolean throwOnEstimate() { return false; - }; + } + + ; static TestMutateConfig of(CypherMapWrapper config) { return new TestMutateConfigImpl(config); diff --git a/algo/src/main/java/org/neo4j/gds/approxmaxkcut/config/ApproxMaxKCutMutateConfig.java b/algo/src/main/java/org/neo4j/gds/approxmaxkcut/config/ApproxMaxKCutMutateConfig.java index 3c7856f472..2eb2f46852 100644 --- a/algo/src/main/java/org/neo4j/gds/approxmaxkcut/config/ApproxMaxKCutMutateConfig.java +++ b/algo/src/main/java/org/neo4j/gds/approxmaxkcut/config/ApproxMaxKCutMutateConfig.java @@ -21,13 +21,13 @@ import org.neo4j.gds.annotation.Configuration; import org.neo4j.gds.annotation.ValueClass; -import org.neo4j.gds.config.MutatePropertyConfig; +import org.neo4j.gds.config.MutateNodePropertyConfig; import org.neo4j.gds.core.CypherMapWrapper; @ValueClass @Configuration @SuppressWarnings("immutables:subtype") -public interface ApproxMaxKCutMutateConfig extends ApproxMaxKCutConfig, MutatePropertyConfig { +public interface ApproxMaxKCutMutateConfig extends ApproxMaxKCutConfig, MutateNodePropertyConfig { static ApproxMaxKCutMutateConfig of(CypherMapWrapper config) { return new ApproxMaxKCutMutateConfigImpl(config); diff --git a/algo/src/main/java/org/neo4j/gds/betweenness/BetweennessCentralityMutateConfig.java b/algo/src/main/java/org/neo4j/gds/betweenness/BetweennessCentralityMutateConfig.java index 981f361ad4..b482d3bedd 100644 --- a/algo/src/main/java/org/neo4j/gds/betweenness/BetweennessCentralityMutateConfig.java +++ b/algo/src/main/java/org/neo4j/gds/betweenness/BetweennessCentralityMutateConfig.java @@ -21,13 +21,13 @@ import org.neo4j.gds.annotation.Configuration; import org.neo4j.gds.annotation.ValueClass; -import org.neo4j.gds.config.MutatePropertyConfig; +import org.neo4j.gds.config.MutateNodePropertyConfig; import org.neo4j.gds.core.CypherMapWrapper; @ValueClass @Configuration @SuppressWarnings("immutables:subtype") -public interface BetweennessCentralityMutateConfig extends BetweennessCentralityBaseConfig, MutatePropertyConfig { +public interface BetweennessCentralityMutateConfig extends BetweennessCentralityBaseConfig, MutateNodePropertyConfig { static BetweennessCentralityMutateConfig of(CypherMapWrapper config) { return new BetweennessCentralityMutateConfigImpl(config); diff --git a/algo/src/main/java/org/neo4j/gds/closeness/ClosenessCentralityMutateConfig.java b/algo/src/main/java/org/neo4j/gds/closeness/ClosenessCentralityMutateConfig.java index b1ffb92afe..2e235b9644 100644 --- a/algo/src/main/java/org/neo4j/gds/closeness/ClosenessCentralityMutateConfig.java +++ b/algo/src/main/java/org/neo4j/gds/closeness/ClosenessCentralityMutateConfig.java @@ -21,11 +21,11 @@ import org.neo4j.gds.annotation.Configuration; -import org.neo4j.gds.config.MutatePropertyConfig; +import org.neo4j.gds.config.MutateNodePropertyConfig; import org.neo4j.gds.core.CypherMapWrapper; @Configuration -public interface ClosenessCentralityMutateConfig extends ClosenessCentralityConfig, MutatePropertyConfig { +public interface ClosenessCentralityMutateConfig extends ClosenessCentralityConfig, MutateNodePropertyConfig { static ClosenessCentralityMutateConfig of(CypherMapWrapper config) { return new ClosenessCentralityMutateConfigImpl(config); diff --git a/algo/src/main/java/org/neo4j/gds/degree/DegreeCentralityMutateConfig.java b/algo/src/main/java/org/neo4j/gds/degree/DegreeCentralityMutateConfig.java index c74bb0d029..1dc9ae7b10 100644 --- a/algo/src/main/java/org/neo4j/gds/degree/DegreeCentralityMutateConfig.java +++ b/algo/src/main/java/org/neo4j/gds/degree/DegreeCentralityMutateConfig.java @@ -21,13 +21,13 @@ import org.neo4j.gds.annotation.Configuration; import org.neo4j.gds.annotation.ValueClass; -import org.neo4j.gds.config.MutatePropertyConfig; +import org.neo4j.gds.config.MutateNodePropertyConfig; import org.neo4j.gds.core.CypherMapWrapper; @ValueClass @Configuration @SuppressWarnings("immutables:subtype") -public interface DegreeCentralityMutateConfig extends DegreeCentralityConfig, MutatePropertyConfig { +public interface DegreeCentralityMutateConfig extends DegreeCentralityConfig, MutateNodePropertyConfig { static DegreeCentralityMutateConfig of(CypherMapWrapper config) { return new DegreeCentralityMutateConfigImpl(config); diff --git a/algo/src/main/java/org/neo4j/gds/embeddings/fastrp/FastRPMutateConfig.java b/algo/src/main/java/org/neo4j/gds/embeddings/fastrp/FastRPMutateConfig.java index 96a22efff0..2cec705664 100644 --- a/algo/src/main/java/org/neo4j/gds/embeddings/fastrp/FastRPMutateConfig.java +++ b/algo/src/main/java/org/neo4j/gds/embeddings/fastrp/FastRPMutateConfig.java @@ -21,13 +21,13 @@ import org.neo4j.gds.annotation.Configuration; import org.neo4j.gds.annotation.ValueClass; -import org.neo4j.gds.config.MutatePropertyConfig; +import org.neo4j.gds.config.MutateNodePropertyConfig; import org.neo4j.gds.core.CypherMapWrapper; @ValueClass @Configuration @SuppressWarnings("immutables:subtype") -public interface FastRPMutateConfig extends FastRPBaseConfig, MutatePropertyConfig { +public interface FastRPMutateConfig extends FastRPBaseConfig, MutateNodePropertyConfig { static FastRPMutateConfig of(CypherMapWrapper userInput) { return new FastRPMutateConfigImpl(userInput); diff --git a/algo/src/main/java/org/neo4j/gds/embeddings/graphsage/algo/GraphSageMutateConfig.java b/algo/src/main/java/org/neo4j/gds/embeddings/graphsage/algo/GraphSageMutateConfig.java index 56c931feef..23e10f6f4f 100644 --- a/algo/src/main/java/org/neo4j/gds/embeddings/graphsage/algo/GraphSageMutateConfig.java +++ b/algo/src/main/java/org/neo4j/gds/embeddings/graphsage/algo/GraphSageMutateConfig.java @@ -21,13 +21,13 @@ import org.neo4j.gds.annotation.Configuration; import org.neo4j.gds.annotation.ValueClass; -import org.neo4j.gds.config.MutatePropertyConfig; +import org.neo4j.gds.config.MutateNodePropertyConfig; import org.neo4j.gds.core.CypherMapWrapper; @ValueClass @Configuration @SuppressWarnings("immutables:subtype") -public interface GraphSageMutateConfig extends GraphSageBaseConfig, MutatePropertyConfig { +public interface GraphSageMutateConfig extends GraphSageBaseConfig, MutateNodePropertyConfig { long serialVersionUID = 0x42L; diff --git a/algo/src/main/java/org/neo4j/gds/embeddings/hashgnn/HashGNNMutateConfig.java b/algo/src/main/java/org/neo4j/gds/embeddings/hashgnn/HashGNNMutateConfig.java index 3eb33296b0..425f687a1e 100644 --- a/algo/src/main/java/org/neo4j/gds/embeddings/hashgnn/HashGNNMutateConfig.java +++ b/algo/src/main/java/org/neo4j/gds/embeddings/hashgnn/HashGNNMutateConfig.java @@ -20,11 +20,11 @@ package org.neo4j.gds.embeddings.hashgnn; import org.neo4j.gds.annotation.Configuration; -import org.neo4j.gds.config.MutatePropertyConfig; +import org.neo4j.gds.config.MutateNodePropertyConfig; import org.neo4j.gds.core.CypherMapWrapper; @Configuration -public interface HashGNNMutateConfig extends HashGNNConfig, MutatePropertyConfig { +public interface HashGNNMutateConfig extends HashGNNConfig, MutateNodePropertyConfig { static HashGNNMutateConfig of(CypherMapWrapper config) { return new HashGNNMutateConfigImpl(config); diff --git a/algo/src/main/java/org/neo4j/gds/embeddings/node2vec/Node2VecMutateConfig.java b/algo/src/main/java/org/neo4j/gds/embeddings/node2vec/Node2VecMutateConfig.java index 6a75985f6d..712926bea9 100644 --- a/algo/src/main/java/org/neo4j/gds/embeddings/node2vec/Node2VecMutateConfig.java +++ b/algo/src/main/java/org/neo4j/gds/embeddings/node2vec/Node2VecMutateConfig.java @@ -21,13 +21,13 @@ import org.neo4j.gds.annotation.Configuration; import org.neo4j.gds.annotation.ValueClass; -import org.neo4j.gds.config.MutatePropertyConfig; +import org.neo4j.gds.config.MutateNodePropertyConfig; import org.neo4j.gds.core.CypherMapWrapper; @ValueClass @Configuration @SuppressWarnings("immutables:subtype") -public interface Node2VecMutateConfig extends Node2VecBaseConfig, MutatePropertyConfig { +public interface Node2VecMutateConfig extends Node2VecBaseConfig, MutateNodePropertyConfig { static Node2VecMutateConfig of(CypherMapWrapper userInput) { return new Node2VecMutateConfigImpl(userInput); diff --git a/algo/src/main/java/org/neo4j/gds/influenceMaximization/InfluenceMaximizationMutateConfig.java b/algo/src/main/java/org/neo4j/gds/influenceMaximization/InfluenceMaximizationMutateConfig.java index 97c901db9d..d8b1f85df1 100644 --- a/algo/src/main/java/org/neo4j/gds/influenceMaximization/InfluenceMaximizationMutateConfig.java +++ b/algo/src/main/java/org/neo4j/gds/influenceMaximization/InfluenceMaximizationMutateConfig.java @@ -20,11 +20,11 @@ package org.neo4j.gds.influenceMaximization; import org.neo4j.gds.annotation.Configuration; -import org.neo4j.gds.config.MutatePropertyConfig; +import org.neo4j.gds.config.MutateNodePropertyConfig; import org.neo4j.gds.core.CypherMapWrapper; @Configuration -public interface InfluenceMaximizationMutateConfig extends InfluenceMaximizationBaseConfig, MutatePropertyConfig { +public interface InfluenceMaximizationMutateConfig extends InfluenceMaximizationBaseConfig, MutateNodePropertyConfig { static InfluenceMaximizationMutateConfig of(CypherMapWrapper userInput) { return new InfluenceMaximizationMutateConfigImpl(userInput); diff --git a/algo/src/main/java/org/neo4j/gds/k1coloring/K1ColoringMutateConfig.java b/algo/src/main/java/org/neo4j/gds/k1coloring/K1ColoringMutateConfig.java index 519a7ca256..38330a1807 100644 --- a/algo/src/main/java/org/neo4j/gds/k1coloring/K1ColoringMutateConfig.java +++ b/algo/src/main/java/org/neo4j/gds/k1coloring/K1ColoringMutateConfig.java @@ -21,13 +21,13 @@ import org.neo4j.gds.annotation.Configuration; import org.neo4j.gds.annotation.ValueClass; -import org.neo4j.gds.config.MutatePropertyConfig; +import org.neo4j.gds.config.MutateNodePropertyConfig; import org.neo4j.gds.core.CypherMapWrapper; @ValueClass @Configuration @SuppressWarnings("immutables:subtype") -public interface K1ColoringMutateConfig extends K1ColoringConfig, MutatePropertyConfig { +public interface K1ColoringMutateConfig extends K1ColoringConfig, MutateNodePropertyConfig { static K1ColoringMutateConfig of(CypherMapWrapper userInput) { return new K1ColoringMutateConfigImpl(userInput); } diff --git a/algo/src/main/java/org/neo4j/gds/kcore/KCoreDecompositionMutateConfig.java b/algo/src/main/java/org/neo4j/gds/kcore/KCoreDecompositionMutateConfig.java index bcd8ea2aad..584973a1c9 100644 --- a/algo/src/main/java/org/neo4j/gds/kcore/KCoreDecompositionMutateConfig.java +++ b/algo/src/main/java/org/neo4j/gds/kcore/KCoreDecompositionMutateConfig.java @@ -20,11 +20,11 @@ package org.neo4j.gds.kcore; import org.neo4j.gds.annotation.Configuration; -import org.neo4j.gds.config.MutatePropertyConfig; +import org.neo4j.gds.config.MutateNodePropertyConfig; import org.neo4j.gds.core.CypherMapWrapper; @Configuration -public interface KCoreDecompositionMutateConfig extends KCoreDecompositionBaseConfig, MutatePropertyConfig { +public interface KCoreDecompositionMutateConfig extends KCoreDecompositionBaseConfig, MutateNodePropertyConfig { static KCoreDecompositionMutateConfig of(CypherMapWrapper userInput) { return new KCoreDecompositionMutateConfigImpl(userInput); diff --git a/algo/src/main/java/org/neo4j/gds/kmeans/KmeansMutateConfig.java b/algo/src/main/java/org/neo4j/gds/kmeans/KmeansMutateConfig.java index 04eb8a32a6..e77279582e 100644 --- a/algo/src/main/java/org/neo4j/gds/kmeans/KmeansMutateConfig.java +++ b/algo/src/main/java/org/neo4j/gds/kmeans/KmeansMutateConfig.java @@ -21,13 +21,13 @@ import org.neo4j.gds.annotation.Configuration; import org.neo4j.gds.annotation.ValueClass; -import org.neo4j.gds.config.MutatePropertyConfig; +import org.neo4j.gds.config.MutateNodePropertyConfig; import org.neo4j.gds.core.CypherMapWrapper; @ValueClass @Configuration @SuppressWarnings("immutables:subtype") -public interface KmeansMutateConfig extends KmeansBaseConfig, MutatePropertyConfig { +public interface KmeansMutateConfig extends KmeansBaseConfig, MutateNodePropertyConfig { static KmeansMutateConfig of(CypherMapWrapper userInput) { return new KmeansMutateConfigImpl(userInput); diff --git a/algo/src/main/java/org/neo4j/gds/labelpropagation/LabelPropagationMutateConfig.java b/algo/src/main/java/org/neo4j/gds/labelpropagation/LabelPropagationMutateConfig.java index a5a910e60c..5cc34e56d6 100644 --- a/algo/src/main/java/org/neo4j/gds/labelpropagation/LabelPropagationMutateConfig.java +++ b/algo/src/main/java/org/neo4j/gds/labelpropagation/LabelPropagationMutateConfig.java @@ -21,13 +21,13 @@ import org.neo4j.gds.annotation.Configuration; import org.neo4j.gds.annotation.ValueClass; -import org.neo4j.gds.config.MutatePropertyConfig; +import org.neo4j.gds.config.MutateNodePropertyConfig; import org.neo4j.gds.core.CypherMapWrapper; @ValueClass @Configuration @SuppressWarnings("immutables:subtype") -public interface LabelPropagationMutateConfig extends LabelPropagationBaseConfig, MutatePropertyConfig { +public interface LabelPropagationMutateConfig extends LabelPropagationBaseConfig, MutateNodePropertyConfig { static LabelPropagationMutateConfig of(CypherMapWrapper userInput) { return new LabelPropagationMutateConfigImpl(userInput); diff --git a/algo/src/main/java/org/neo4j/gds/leiden/LeidenMutateConfig.java b/algo/src/main/java/org/neo4j/gds/leiden/LeidenMutateConfig.java index 0f14102d0f..0ee812a534 100644 --- a/algo/src/main/java/org/neo4j/gds/leiden/LeidenMutateConfig.java +++ b/algo/src/main/java/org/neo4j/gds/leiden/LeidenMutateConfig.java @@ -20,11 +20,11 @@ package org.neo4j.gds.leiden; import org.neo4j.gds.annotation.Configuration; -import org.neo4j.gds.config.MutatePropertyConfig; +import org.neo4j.gds.config.MutateNodePropertyConfig; import org.neo4j.gds.core.CypherMapWrapper; @Configuration -public interface LeidenMutateConfig extends LeidenBaseConfig, MutatePropertyConfig { +public interface LeidenMutateConfig extends LeidenBaseConfig, MutateNodePropertyConfig { static LeidenMutateConfig of(CypherMapWrapper userInput) { return new LeidenMutateConfigImpl(userInput); diff --git a/algo/src/main/java/org/neo4j/gds/louvain/LouvainMutateConfig.java b/algo/src/main/java/org/neo4j/gds/louvain/LouvainMutateConfig.java index 3ee19b1790..0b8d575869 100644 --- a/algo/src/main/java/org/neo4j/gds/louvain/LouvainMutateConfig.java +++ b/algo/src/main/java/org/neo4j/gds/louvain/LouvainMutateConfig.java @@ -21,13 +21,13 @@ import org.neo4j.gds.annotation.Configuration; import org.neo4j.gds.annotation.ValueClass; -import org.neo4j.gds.config.MutatePropertyConfig; +import org.neo4j.gds.config.MutateNodePropertyConfig; import org.neo4j.gds.core.CypherMapWrapper; @ValueClass @Configuration @SuppressWarnings("immutables:subtype") -public interface LouvainMutateConfig extends LouvainBaseConfig, MutatePropertyConfig { +public interface LouvainMutateConfig extends LouvainBaseConfig, MutateNodePropertyConfig { static LouvainMutateConfig of(CypherMapWrapper userInput) { return new LouvainMutateConfigImpl(userInput); diff --git a/algo/src/main/java/org/neo4j/gds/modularityoptimization/ModularityOptimizationMutateConfig.java b/algo/src/main/java/org/neo4j/gds/modularityoptimization/ModularityOptimizationMutateConfig.java index 9e0ac155dc..e74d936d5e 100644 --- a/algo/src/main/java/org/neo4j/gds/modularityoptimization/ModularityOptimizationMutateConfig.java +++ b/algo/src/main/java/org/neo4j/gds/modularityoptimization/ModularityOptimizationMutateConfig.java @@ -21,13 +21,13 @@ import org.neo4j.gds.annotation.Configuration; import org.neo4j.gds.annotation.ValueClass; -import org.neo4j.gds.config.MutatePropertyConfig; +import org.neo4j.gds.config.MutateNodePropertyConfig; import org.neo4j.gds.core.CypherMapWrapper; @ValueClass @Configuration @SuppressWarnings("immutables:subtype") -public interface ModularityOptimizationMutateConfig extends ModularityOptimizationConfig, MutatePropertyConfig { +public interface ModularityOptimizationMutateConfig extends ModularityOptimizationConfig, MutateNodePropertyConfig { static ModularityOptimizationMutateConfig of(CypherMapWrapper userInput) { return new ModularityOptimizationMutateConfigImpl(userInput); diff --git a/algo/src/main/java/org/neo4j/gds/pagerank/PageRankMutateConfig.java b/algo/src/main/java/org/neo4j/gds/pagerank/PageRankMutateConfig.java index 4179283b83..8e4714eed9 100644 --- a/algo/src/main/java/org/neo4j/gds/pagerank/PageRankMutateConfig.java +++ b/algo/src/main/java/org/neo4j/gds/pagerank/PageRankMutateConfig.java @@ -21,13 +21,13 @@ import org.neo4j.gds.annotation.Configuration; import org.neo4j.gds.annotation.ValueClass; -import org.neo4j.gds.config.MutatePropertyConfig; +import org.neo4j.gds.config.MutateNodePropertyConfig; import org.neo4j.gds.core.CypherMapWrapper; @ValueClass @Configuration @SuppressWarnings("immutables:subtype") -public interface PageRankMutateConfig extends PageRankConfig, MutatePropertyConfig { +public interface PageRankMutateConfig extends PageRankConfig, MutateNodePropertyConfig { static PageRankMutateConfig of(CypherMapWrapper userInput) { return new PageRankMutateConfigImpl(userInput); diff --git a/algo/src/main/java/org/neo4j/gds/scaleproperties/ScalePropertiesMutateConfig.java b/algo/src/main/java/org/neo4j/gds/scaleproperties/ScalePropertiesMutateConfig.java index d71b9192c0..cb55d86186 100644 --- a/algo/src/main/java/org/neo4j/gds/scaleproperties/ScalePropertiesMutateConfig.java +++ b/algo/src/main/java/org/neo4j/gds/scaleproperties/ScalePropertiesMutateConfig.java @@ -20,12 +20,12 @@ package org.neo4j.gds.scaleproperties; import org.neo4j.gds.annotation.Configuration; -import org.neo4j.gds.config.MutatePropertyConfig; +import org.neo4j.gds.config.MutateNodePropertyConfig; import org.neo4j.gds.core.CypherMapWrapper; @Configuration @SuppressWarnings("immutables:subtype") -public interface ScalePropertiesMutateConfig extends ScalePropertiesBaseConfig, MutatePropertyConfig { +public interface ScalePropertiesMutateConfig extends ScalePropertiesBaseConfig, MutateNodePropertyConfig { static ScalePropertiesMutateConfig of(CypherMapWrapper userInput) { return new ScalePropertiesMutateConfigImpl(userInput); diff --git a/algo/src/main/java/org/neo4j/gds/similarity/filteredknn/FilteredKnnMutateConfig.java b/algo/src/main/java/org/neo4j/gds/similarity/filteredknn/FilteredKnnMutateConfig.java index 2ed1e183ea..ac235f3e75 100644 --- a/algo/src/main/java/org/neo4j/gds/similarity/filteredknn/FilteredKnnMutateConfig.java +++ b/algo/src/main/java/org/neo4j/gds/similarity/filteredknn/FilteredKnnMutateConfig.java @@ -21,14 +21,15 @@ import org.neo4j.gds.annotation.Configuration; import org.neo4j.gds.annotation.ValueClass; -import org.neo4j.gds.config.MutatePropertyConfig; +import org.neo4j.gds.config.MutateNodePropertyConfig; import org.neo4j.gds.config.MutateRelationshipConfig; import org.neo4j.gds.core.CypherMapWrapper; @ValueClass @Configuration @SuppressWarnings("immutables:subtype") -public interface FilteredKnnMutateConfig extends FilteredKnnBaseConfig, MutatePropertyConfig, MutateRelationshipConfig { +public interface FilteredKnnMutateConfig extends FilteredKnnBaseConfig, + MutateNodePropertyConfig, MutateRelationshipConfig { static FilteredKnnMutateConfig of(CypherMapWrapper config) { return new FilteredKnnMutateConfigImpl(config); diff --git a/algo/src/main/java/org/neo4j/gds/similarity/knn/KnnMutateConfig.java b/algo/src/main/java/org/neo4j/gds/similarity/knn/KnnMutateConfig.java index 8cf2746da6..2c87c0a99c 100644 --- a/algo/src/main/java/org/neo4j/gds/similarity/knn/KnnMutateConfig.java +++ b/algo/src/main/java/org/neo4j/gds/similarity/knn/KnnMutateConfig.java @@ -21,14 +21,14 @@ import org.neo4j.gds.annotation.Configuration; import org.neo4j.gds.annotation.ValueClass; -import org.neo4j.gds.config.MutatePropertyConfig; +import org.neo4j.gds.config.MutateNodePropertyConfig; import org.neo4j.gds.config.MutateRelationshipConfig; import org.neo4j.gds.core.CypherMapWrapper; @ValueClass @Configuration @SuppressWarnings("immutables:subtype") -public interface KnnMutateConfig extends KnnBaseConfig, MutatePropertyConfig, MutateRelationshipConfig { +public interface KnnMutateConfig extends KnnBaseConfig, MutateNodePropertyConfig, MutateRelationshipConfig { static KnnMutateConfig of(CypherMapWrapper config) { return new KnnMutateConfigImpl(config); diff --git a/algo/src/main/java/org/neo4j/gds/similarity/nodesim/NodeSimilarityMutateConfig.java b/algo/src/main/java/org/neo4j/gds/similarity/nodesim/NodeSimilarityMutateConfig.java index ab5ec264d8..e7a0afce43 100644 --- a/algo/src/main/java/org/neo4j/gds/similarity/nodesim/NodeSimilarityMutateConfig.java +++ b/algo/src/main/java/org/neo4j/gds/similarity/nodesim/NodeSimilarityMutateConfig.java @@ -21,14 +21,15 @@ import org.neo4j.gds.annotation.Configuration; import org.neo4j.gds.annotation.ValueClass; -import org.neo4j.gds.config.MutatePropertyConfig; +import org.neo4j.gds.config.MutateNodePropertyConfig; import org.neo4j.gds.config.MutateRelationshipConfig; import org.neo4j.gds.core.CypherMapWrapper; @ValueClass @Configuration @SuppressWarnings("immutables:subtype") -public interface NodeSimilarityMutateConfig extends NodeSimilarityBaseConfig, MutatePropertyConfig, MutateRelationshipConfig { +public interface NodeSimilarityMutateConfig extends NodeSimilarityBaseConfig, + MutateNodePropertyConfig, MutateRelationshipConfig { static NodeSimilarityMutateConfig of(CypherMapWrapper userInput) { NodeSimilarityMutateConfig config = new NodeSimilarityMutateConfigImpl(userInput); diff --git a/algo/src/main/java/org/neo4j/gds/spanningtree/SpanningTreeMutateConfig.java b/algo/src/main/java/org/neo4j/gds/spanningtree/SpanningTreeMutateConfig.java index 0477c38dcf..0a08f7c3cd 100644 --- a/algo/src/main/java/org/neo4j/gds/spanningtree/SpanningTreeMutateConfig.java +++ b/algo/src/main/java/org/neo4j/gds/spanningtree/SpanningTreeMutateConfig.java @@ -21,14 +21,15 @@ import org.neo4j.gds.annotation.Configuration; import org.neo4j.gds.annotation.ValueClass; -import org.neo4j.gds.config.MutatePropertyConfig; +import org.neo4j.gds.config.MutateNodePropertyConfig; import org.neo4j.gds.config.MutateRelationshipConfig; import org.neo4j.gds.core.CypherMapWrapper; @ValueClass @Configuration @SuppressWarnings("immutables:subtype") -public interface SpanningTreeMutateConfig extends SpanningTreeBaseConfig, MutatePropertyConfig, MutateRelationshipConfig { +public interface SpanningTreeMutateConfig extends SpanningTreeBaseConfig, + MutateNodePropertyConfig, MutateRelationshipConfig { static SpanningTreeMutateConfig of(CypherMapWrapper userInput) { return new SpanningTreeMutateConfigImpl(userInput); diff --git a/algo/src/main/java/org/neo4j/gds/steiner/SteinerTreeMutateConfig.java b/algo/src/main/java/org/neo4j/gds/steiner/SteinerTreeMutateConfig.java index 4a08ee5163..4a30034f7f 100644 --- a/algo/src/main/java/org/neo4j/gds/steiner/SteinerTreeMutateConfig.java +++ b/algo/src/main/java/org/neo4j/gds/steiner/SteinerTreeMutateConfig.java @@ -21,14 +21,15 @@ import org.neo4j.gds.annotation.Configuration; import org.neo4j.gds.annotation.ValueClass; -import org.neo4j.gds.config.MutatePropertyConfig; +import org.neo4j.gds.config.MutateNodePropertyConfig; import org.neo4j.gds.config.MutateRelationshipConfig; import org.neo4j.gds.core.CypherMapWrapper; @ValueClass @Configuration @SuppressWarnings("immutables:subtype") -public interface SteinerTreeMutateConfig extends SteinerTreeBaseConfig, MutateRelationshipConfig, MutatePropertyConfig { +public interface SteinerTreeMutateConfig extends SteinerTreeBaseConfig, MutateRelationshipConfig, + MutateNodePropertyConfig { static SteinerTreeMutateConfig of(CypherMapWrapper userInput) { return new SteinerTreeMutateConfigImpl(userInput); diff --git a/algo/src/main/java/org/neo4j/gds/triangle/LocalClusteringCoefficientMutateConfig.java b/algo/src/main/java/org/neo4j/gds/triangle/LocalClusteringCoefficientMutateConfig.java index 0ac7da9981..541b2637c8 100644 --- a/algo/src/main/java/org/neo4j/gds/triangle/LocalClusteringCoefficientMutateConfig.java +++ b/algo/src/main/java/org/neo4j/gds/triangle/LocalClusteringCoefficientMutateConfig.java @@ -21,13 +21,14 @@ import org.neo4j.gds.annotation.Configuration; import org.neo4j.gds.annotation.ValueClass; -import org.neo4j.gds.config.MutatePropertyConfig; +import org.neo4j.gds.config.MutateNodePropertyConfig; import org.neo4j.gds.core.CypherMapWrapper; @ValueClass @Configuration @SuppressWarnings("immutables:subtype") -public interface LocalClusteringCoefficientMutateConfig extends LocalClusteringCoefficientBaseConfig, MutatePropertyConfig { +public interface LocalClusteringCoefficientMutateConfig extends LocalClusteringCoefficientBaseConfig, + MutateNodePropertyConfig { static LocalClusteringCoefficientMutateConfig of(CypherMapWrapper userInput) { return new LocalClusteringCoefficientMutateConfigImpl(userInput); diff --git a/algo/src/main/java/org/neo4j/gds/triangle/TriangleCountMutateConfig.java b/algo/src/main/java/org/neo4j/gds/triangle/TriangleCountMutateConfig.java index b47c567ea7..b0371c4841 100644 --- a/algo/src/main/java/org/neo4j/gds/triangle/TriangleCountMutateConfig.java +++ b/algo/src/main/java/org/neo4j/gds/triangle/TriangleCountMutateConfig.java @@ -21,13 +21,13 @@ import org.neo4j.gds.annotation.Configuration; import org.neo4j.gds.annotation.ValueClass; -import org.neo4j.gds.config.MutatePropertyConfig; +import org.neo4j.gds.config.MutateNodePropertyConfig; import org.neo4j.gds.core.CypherMapWrapper; @ValueClass @Configuration @SuppressWarnings("immutables:subtype") -public interface TriangleCountMutateConfig extends TriangleCountBaseConfig, MutatePropertyConfig { +public interface TriangleCountMutateConfig extends TriangleCountBaseConfig, MutateNodePropertyConfig { static TriangleCountMutateConfig of(CypherMapWrapper userInput) { return new TriangleCountMutateConfigImpl(userInput); diff --git a/algo/src/main/java/org/neo4j/gds/wcc/WccMutateConfig.java b/algo/src/main/java/org/neo4j/gds/wcc/WccMutateConfig.java index ed7781ffd5..24be870b96 100644 --- a/algo/src/main/java/org/neo4j/gds/wcc/WccMutateConfig.java +++ b/algo/src/main/java/org/neo4j/gds/wcc/WccMutateConfig.java @@ -21,13 +21,13 @@ import org.neo4j.gds.annotation.Configuration; import org.neo4j.gds.annotation.ValueClass; -import org.neo4j.gds.config.MutatePropertyConfig; +import org.neo4j.gds.config.MutateNodePropertyConfig; import org.neo4j.gds.core.CypherMapWrapper; @ValueClass @Configuration @SuppressWarnings("immutables:subtype") -public interface WccMutateConfig extends WccBaseConfig, MutatePropertyConfig { +public interface WccMutateConfig extends WccBaseConfig, MutateNodePropertyConfig { static WccMutateConfig of(CypherMapWrapper userInput) { return new WccMutateConfigImpl(userInput); diff --git a/core/src/main/java/org/neo4j/gds/config/MutatePropertyConfig.java b/core/src/main/java/org/neo4j/gds/config/MutateNodePropertyConfig.java similarity index 97% rename from core/src/main/java/org/neo4j/gds/config/MutatePropertyConfig.java rename to core/src/main/java/org/neo4j/gds/config/MutateNodePropertyConfig.java index 3788a4e35c..3be55dfd4e 100644 --- a/core/src/main/java/org/neo4j/gds/config/MutatePropertyConfig.java +++ b/core/src/main/java/org/neo4j/gds/config/MutateNodePropertyConfig.java @@ -31,7 +31,7 @@ import static org.neo4j.gds.core.StringIdentifierValidations.validateNoWhiteCharacter; import static org.neo4j.gds.utils.StringFormatting.formatWithLocale; -public interface MutatePropertyConfig extends MutateConfig { +public interface MutateNodePropertyConfig extends MutateConfig { String MUTATE_PROPERTY_KEY = "mutateProperty"; diff --git a/core/src/test/java/org/neo4j/gds/config/MutatePropertyConfigTest.java b/core/src/test/java/org/neo4j/gds/config/MutateNodePropertyConfigTest.java similarity index 78% rename from core/src/test/java/org/neo4j/gds/config/MutatePropertyConfigTest.java rename to core/src/test/java/org/neo4j/gds/config/MutateNodePropertyConfigTest.java index eef6481ab7..fa43a25bfe 100644 --- a/core/src/test/java/org/neo4j/gds/config/MutatePropertyConfigTest.java +++ b/core/src/test/java/org/neo4j/gds/config/MutateNodePropertyConfigTest.java @@ -27,19 +27,23 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy; -class MutatePropertyConfigTest { +class MutateNodePropertyConfigTest { @Test void testMutateFailsOnExistingToken() { var graphStore = GdlFactory.of("(a {foo: 42})").build(); - MutatePropertyConfig config = TestMutatePropertyConfigImpl.builder().mutateProperty("foo").build(); + MutateNodePropertyConfig config = TestMutatePropertyConfigImpl.builder().mutateProperty("foo").build(); - assertThatThrownBy(() -> config.validateMutateProperty(graphStore, config.nodeLabelIdentifiers(graphStore), List.of())) + assertThatThrownBy(() -> config.validateMutateProperty( + graphStore, + config.nodeLabelIdentifiers(graphStore), + List.of() + )) .hasMessageContaining("Node property `foo` already exists in the in-memory graph."); } @Configuration - interface TestMutatePropertyConfig extends AlgoBaseConfig, MutatePropertyConfig { + interface TestMutatePropertyConfig extends AlgoBaseConfig, MutateNodePropertyConfig { } } diff --git a/pipeline/src/main/java/org/neo4j/gds/ml/pipeline/NodePropertyStep.java b/pipeline/src/main/java/org/neo4j/gds/ml/pipeline/NodePropertyStep.java index cf7e7c9344..1abd73bb46 100644 --- a/pipeline/src/main/java/org/neo4j/gds/ml/pipeline/NodePropertyStep.java +++ b/pipeline/src/main/java/org/neo4j/gds/ml/pipeline/NodePropertyStep.java @@ -43,7 +43,7 @@ import java.util.Objects; import java.util.stream.Collectors; -import static org.neo4j.gds.config.MutatePropertyConfig.MUTATE_PROPERTY_KEY; +import static org.neo4j.gds.config.MutateNodePropertyConfig.MUTATE_PROPERTY_KEY; import static org.neo4j.gds.ml.pipeline.NodePropertyStepContextConfig.CONTEXT_NODE_LABELS; import static org.neo4j.gds.ml.pipeline.NodePropertyStepContextConfig.CONTEXT_RELATIONSHIP_TYPES; diff --git a/pipeline/src/main/java/org/neo4j/gds/ml/pipeline/Pipeline.java b/pipeline/src/main/java/org/neo4j/gds/ml/pipeline/Pipeline.java index 9387a5f07e..4a79a398f3 100644 --- a/pipeline/src/main/java/org/neo4j/gds/ml/pipeline/Pipeline.java +++ b/pipeline/src/main/java/org/neo4j/gds/ml/pipeline/Pipeline.java @@ -29,7 +29,7 @@ import java.util.stream.Collectors; import java.util.stream.Stream; -import static org.neo4j.gds.config.MutatePropertyConfig.MUTATE_PROPERTY_KEY; +import static org.neo4j.gds.config.MutateNodePropertyConfig.MUTATE_PROPERTY_KEY; import static org.neo4j.gds.utils.StringFormatting.formatWithLocale; public interface Pipeline extends ToMapConvertible { diff --git a/pipeline/src/main/java/org/neo4j/gds/ml/pipeline/TrainingPipeline.java b/pipeline/src/main/java/org/neo4j/gds/ml/pipeline/TrainingPipeline.java index 078490a1c4..5f1e77fb4d 100644 --- a/pipeline/src/main/java/org/neo4j/gds/ml/pipeline/TrainingPipeline.java +++ b/pipeline/src/main/java/org/neo4j/gds/ml/pipeline/TrainingPipeline.java @@ -32,7 +32,7 @@ import java.util.Map; import java.util.stream.Collectors; -import static org.neo4j.gds.config.MutatePropertyConfig.MUTATE_PROPERTY_KEY; +import static org.neo4j.gds.config.MutateNodePropertyConfig.MUTATE_PROPERTY_KEY; import static org.neo4j.gds.utils.StringFormatting.formatWithLocale; public abstract class TrainingPipeline implements Pipeline { diff --git a/pipeline/src/test/java/org/neo4j/gds/ml/pipeline/ExecutableNodePropertyStepTestUtil.java b/pipeline/src/test/java/org/neo4j/gds/ml/pipeline/ExecutableNodePropertyStepTestUtil.java index 2e5f879407..258116ab46 100644 --- a/pipeline/src/test/java/org/neo4j/gds/ml/pipeline/ExecutableNodePropertyStepTestUtil.java +++ b/pipeline/src/test/java/org/neo4j/gds/ml/pipeline/ExecutableNodePropertyStepTestUtil.java @@ -38,7 +38,7 @@ import java.util.Optional; import java.util.UUID; -import static org.neo4j.gds.config.MutatePropertyConfig.MUTATE_PROPERTY_KEY; +import static org.neo4j.gds.config.MutateNodePropertyConfig.MUTATE_PROPERTY_KEY; public class ExecutableNodePropertyStepTestUtil { diff --git a/pipeline/src/test/java/org/neo4j/gds/ml/pipeline/linkPipeline/train/LinkPredictionTrainPipelineExecutorTest.java b/pipeline/src/test/java/org/neo4j/gds/ml/pipeline/linkPipeline/train/LinkPredictionTrainPipelineExecutorTest.java index 57a84c9f9f..2a6f2dc266 100644 --- a/pipeline/src/test/java/org/neo4j/gds/ml/pipeline/linkPipeline/train/LinkPredictionTrainPipelineExecutorTest.java +++ b/pipeline/src/test/java/org/neo4j/gds/ml/pipeline/linkPipeline/train/LinkPredictionTrainPipelineExecutorTest.java @@ -82,7 +82,7 @@ import static org.neo4j.gds.TestSupport.assertMemoryRange; import static org.neo4j.gds.assertj.Extractors.keepingFixedNumberOfDecimals; import static org.neo4j.gds.assertj.Extractors.removingThreadId; -import static org.neo4j.gds.config.MutatePropertyConfig.MUTATE_PROPERTY_KEY; +import static org.neo4j.gds.config.MutateNodePropertyConfig.MUTATE_PROPERTY_KEY; import static org.neo4j.gds.ml.pipeline.ExecutableNodePropertyStepTestUtil.TestNodePropertyStepWithFixedEstimation; import static org.neo4j.gds.ml.pipeline.PipelineExecutor.DatasetSplits.FEATURE_INPUT; import static org.neo4j.gds.ml.pipeline.PipelineExecutor.DatasetSplits.TEST; diff --git a/pregel/src/main/java/org/neo4j/gds/beta/pregel/PregelProcedureConfig.java b/pregel/src/main/java/org/neo4j/gds/beta/pregel/PregelProcedureConfig.java index 0f9d0fa81d..6ef12ba025 100644 --- a/pregel/src/main/java/org/neo4j/gds/beta/pregel/PregelProcedureConfig.java +++ b/pregel/src/main/java/org/neo4j/gds/beta/pregel/PregelProcedureConfig.java @@ -22,7 +22,7 @@ import org.immutables.value.Value; import org.neo4j.gds.annotation.Configuration; import org.neo4j.gds.annotation.ValueClass; -import org.neo4j.gds.config.MutatePropertyConfig; +import org.neo4j.gds.config.MutateNodePropertyConfig; import org.neo4j.gds.config.WritePropertyConfig; import org.neo4j.gds.core.CypherMapWrapper; @@ -32,7 +32,7 @@ public interface PregelProcedureConfig extends PregelConfig, WritePropertyConfig, - MutatePropertyConfig { + MutateNodePropertyConfig { @Value.Default default String writeProperty() { diff --git a/proc/centrality/src/main/java/org/neo4j/gds/harmonic/CentralityScore.java b/proc/centrality/src/main/java/org/neo4j/gds/harmonic/CentralityScore.java index de0846170f..bf145bdaaa 100644 --- a/proc/centrality/src/main/java/org/neo4j/gds/harmonic/CentralityScore.java +++ b/proc/centrality/src/main/java/org/neo4j/gds/harmonic/CentralityScore.java @@ -21,7 +21,7 @@ import org.jetbrains.annotations.Nullable; import org.neo4j.gds.api.ProcedureReturnColumns; -import org.neo4j.gds.config.MutatePropertyConfig; +import org.neo4j.gds.config.MutateNodePropertyConfig; import org.neo4j.gds.config.WritePropertyConfig; import org.neo4j.gds.result.AbstractCentralityResultBuilder; @@ -114,7 +114,9 @@ public Mutate buildResult() { preProcessingMillis, computeMillis, writeMillis, - config instanceof MutatePropertyConfig ? ((MutatePropertyConfig) config).mutateProperty() : "", + config instanceof MutateNodePropertyConfig + ? ((MutateNodePropertyConfig) config).mutateProperty() + : "", centralityHistogram ); } diff --git a/proc/common/src/main/java/org/neo4j/gds/GraphStoreUpdater.java b/proc/common/src/main/java/org/neo4j/gds/GraphStoreUpdater.java index c580962461..422151b5c0 100644 --- a/proc/common/src/main/java/org/neo4j/gds/GraphStoreUpdater.java +++ b/proc/common/src/main/java/org/neo4j/gds/GraphStoreUpdater.java @@ -20,7 +20,7 @@ package org.neo4j.gds; import org.neo4j.gds.api.GraphStore; -import org.neo4j.gds.config.MutatePropertyConfig; +import org.neo4j.gds.config.MutateNodePropertyConfig; import org.neo4j.gds.core.huge.FilteredNodePropertyValues; import org.neo4j.gds.core.write.ImmutableNodeProperty; import org.neo4j.gds.core.write.NodeProperty; @@ -39,14 +39,14 @@ public final class GraphStoreUpdater { private GraphStoreUpdater() {} - public static , ALGO_RESULT, CONFIG extends MutatePropertyConfig> void UpdateGraphStore( + public static , ALGO_RESULT, CONFIG extends MutateNodePropertyConfig> void UpdateGraphStore( AbstractResultBuilder resultBuilder, ComputationResult computationResult, ExecutionContext executionContext, final List nodePropertyList ) { var graph = computationResult.graph(); - MutatePropertyConfig mutatePropertyConfig = computationResult.config(); + MutateNodePropertyConfig mutatePropertyConfig = computationResult.config(); var maybeTranslatedProperties = graph .asNodeFilteredGraph() diff --git a/proc/common/src/main/java/org/neo4j/gds/MutateNodePropertyListFunction.java b/proc/common/src/main/java/org/neo4j/gds/MutateNodePropertyListFunction.java index 232caf2580..04a3e5f8a0 100644 --- a/proc/common/src/main/java/org/neo4j/gds/MutateNodePropertyListFunction.java +++ b/proc/common/src/main/java/org/neo4j/gds/MutateNodePropertyListFunction.java @@ -19,7 +19,7 @@ */ package org.neo4j.gds; -import org.neo4j.gds.config.MutatePropertyConfig; +import org.neo4j.gds.config.MutateNodePropertyConfig; /** * A generics adaptation interface? @@ -27,6 +27,6 @@ public interface MutateNodePropertyListFunction< ALGO extends Algorithm, ALGO_RESULT, - CONFIG extends MutatePropertyConfig + CONFIG extends MutateNodePropertyConfig > extends NodePropertyListFunction { } diff --git a/proc/common/src/main/java/org/neo4j/gds/MutatePropertyComputationResultConsumer.java b/proc/common/src/main/java/org/neo4j/gds/MutatePropertyComputationResultConsumer.java index c0c0d7c72b..34d0205365 100644 --- a/proc/common/src/main/java/org/neo4j/gds/MutatePropertyComputationResultConsumer.java +++ b/proc/common/src/main/java/org/neo4j/gds/MutatePropertyComputationResultConsumer.java @@ -19,12 +19,12 @@ */ package org.neo4j.gds; -import org.neo4j.gds.config.MutatePropertyConfig; +import org.neo4j.gds.config.MutateNodePropertyConfig; import org.neo4j.gds.executor.ComputationResult; import org.neo4j.gds.executor.ExecutionContext; import org.neo4j.gds.result.AbstractResultBuilder; -public class MutatePropertyComputationResultConsumer, ALGO_RESULT, CONFIG extends MutatePropertyConfig, RESULT> +public class MutatePropertyComputationResultConsumer, ALGO_RESULT, CONFIG extends MutateNodePropertyConfig, RESULT> extends MutateComputationResultConsumer { private final MutateNodePropertyListFunction nodePropertyListFunction; diff --git a/proc/machine-learning/src/main/java/org/neo4j/gds/ml/linkmodels/pipeline/predict/LinkPredictionPredictPipelineMutateConfig.java b/proc/machine-learning/src/main/java/org/neo4j/gds/ml/linkmodels/pipeline/predict/LinkPredictionPredictPipelineMutateConfig.java index 35e5170ac1..59d5e7ecbb 100644 --- a/proc/machine-learning/src/main/java/org/neo4j/gds/ml/linkmodels/pipeline/predict/LinkPredictionPredictPipelineMutateConfig.java +++ b/proc/machine-learning/src/main/java/org/neo4j/gds/ml/linkmodels/pipeline/predict/LinkPredictionPredictPipelineMutateConfig.java @@ -21,13 +21,15 @@ import org.immutables.value.Value; import org.neo4j.gds.annotation.Configuration; -import org.neo4j.gds.config.MutatePropertyConfig; +import org.neo4j.gds.config.MutateNodePropertyConfig; import org.neo4j.gds.config.MutateRelationshipConfig; import org.neo4j.gds.core.CypherMapWrapper; @Configuration @SuppressWarnings("immutables:subtype") -public interface LinkPredictionPredictPipelineMutateConfig extends LinkPredictionPredictPipelineBaseConfig, MutateRelationshipConfig, MutatePropertyConfig { +public interface LinkPredictionPredictPipelineMutateConfig extends LinkPredictionPredictPipelineBaseConfig, + MutateRelationshipConfig, + MutateNodePropertyConfig { @Value.Default @Override default String mutateProperty() { diff --git a/proc/machine-learning/src/main/java/org/neo4j/gds/ml/pipeline/node/classification/predict/NodeClassificationPredictPipelineMutateConfig.java b/proc/machine-learning/src/main/java/org/neo4j/gds/ml/pipeline/node/classification/predict/NodeClassificationPredictPipelineMutateConfig.java index 77ac8d862a..12c1c5c7d3 100644 --- a/proc/machine-learning/src/main/java/org/neo4j/gds/ml/pipeline/node/classification/predict/NodeClassificationPredictPipelineMutateConfig.java +++ b/proc/machine-learning/src/main/java/org/neo4j/gds/ml/pipeline/node/classification/predict/NodeClassificationPredictPipelineMutateConfig.java @@ -24,7 +24,7 @@ import org.neo4j.gds.RelationshipType; import org.neo4j.gds.annotation.Configuration; import org.neo4j.gds.api.GraphStore; -import org.neo4j.gds.config.MutatePropertyConfig; +import org.neo4j.gds.config.MutateNodePropertyConfig; import org.neo4j.gds.core.CypherMapWrapper; import java.util.Collection; @@ -33,7 +33,9 @@ @Configuration @SuppressWarnings("immutables:subtype") -public interface NodeClassificationPredictPipelineMutateConfig extends NodeClassificationPredictPipelineMutateOrWriteConfig, MutatePropertyConfig { +public interface NodeClassificationPredictPipelineMutateConfig extends + NodeClassificationPredictPipelineMutateOrWriteConfig, + MutateNodePropertyConfig { static NodeClassificationPredictPipelineMutateConfig of(String username, CypherMapWrapper config) { return new NodeClassificationPredictPipelineMutateConfigImpl(username, config); } diff --git a/proc/machine-learning/src/main/java/org/neo4j/gds/ml/pipeline/node/regression/predict/NodeRegressionPredictPipelineMutateConfig.java b/proc/machine-learning/src/main/java/org/neo4j/gds/ml/pipeline/node/regression/predict/NodeRegressionPredictPipelineMutateConfig.java index 4559907e89..525c89b28f 100644 --- a/proc/machine-learning/src/main/java/org/neo4j/gds/ml/pipeline/node/regression/predict/NodeRegressionPredictPipelineMutateConfig.java +++ b/proc/machine-learning/src/main/java/org/neo4j/gds/ml/pipeline/node/regression/predict/NodeRegressionPredictPipelineMutateConfig.java @@ -20,11 +20,12 @@ package org.neo4j.gds.ml.pipeline.node.regression.predict; import org.neo4j.gds.annotation.Configuration; -import org.neo4j.gds.config.MutatePropertyConfig; +import org.neo4j.gds.config.MutateNodePropertyConfig; import org.neo4j.gds.core.CypherMapWrapper; @Configuration -public interface NodeRegressionPredictPipelineMutateConfig extends NodeRegressionPredictPipelineBaseConfig, MutatePropertyConfig { +public interface NodeRegressionPredictPipelineMutateConfig extends NodeRegressionPredictPipelineBaseConfig, + MutateNodePropertyConfig { String graphName(); diff --git a/proc/machine-learning/src/test/java/org/neo4j/gds/ml/linkmodels/pipeline/predict/LinkPredictionPredictPipelineExecutorTest.java b/proc/machine-learning/src/test/java/org/neo4j/gds/ml/linkmodels/pipeline/predict/LinkPredictionPredictPipelineExecutorTest.java index 85226ce93f..de7097a49b 100644 --- a/proc/machine-learning/src/test/java/org/neo4j/gds/ml/linkmodels/pipeline/predict/LinkPredictionPredictPipelineExecutorTest.java +++ b/proc/machine-learning/src/test/java/org/neo4j/gds/ml/linkmodels/pipeline/predict/LinkPredictionPredictPipelineExecutorTest.java @@ -69,7 +69,7 @@ import static org.neo4j.gds.assertj.Extractors.removingThreadId; import static org.neo4j.gds.assertj.Extractors.replaceTimings; import static org.neo4j.gds.compat.TestLog.INFO; -import static org.neo4j.gds.config.MutatePropertyConfig.MUTATE_PROPERTY_KEY; +import static org.neo4j.gds.config.MutateNodePropertyConfig.MUTATE_PROPERTY_KEY; import static org.neo4j.gds.ml.pipeline.linkPipeline.LinkPredictionTrainingPipeline.MODEL_TYPE; @GdlExtension From 5673382b93ff08493b71bb0990c51774be3d72b7 Mon Sep 17 00:00:00 2001 From: ioannispan Date: Mon, 19 Jun 2023 11:45:57 +0200 Subject: [PATCH 110/273] Fix cherry-picking mistake --- .../java/org/neo4j/gds/paths/traverse/BfsStreamProcTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/proc/path-finding/src/test/java/org/neo4j/gds/paths/traverse/BfsStreamProcTest.java b/proc/path-finding/src/test/java/org/neo4j/gds/paths/traverse/BfsStreamProcTest.java index 32fc1a5756..37968f03d0 100644 --- a/proc/path-finding/src/test/java/org/neo4j/gds/paths/traverse/BfsStreamProcTest.java +++ b/proc/path-finding/src/test/java/org/neo4j/gds/paths/traverse/BfsStreamProcTest.java @@ -232,7 +232,7 @@ void failOnInvalidSourceNode() { .addParameter("sourceNode", 4242) .yields(); - assertError(query, "Source node does not exist in the in-memory graph: `42`"); + assertError(query, "Source node does not exist in the in-memory graph: `4242"); } } From 95e72ddb92138bccec59809d16c789a5e45ec439 Mon Sep 17 00:00:00 2001 From: ioannispan Date: Mon, 19 Jun 2023 10:36:49 +0200 Subject: [PATCH 111/273] Create a separate mutate relationship config that skips checking property existance MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Florentin Dörre --- .../org/neo4j/gds/test/TestMutateConfig.java | 4 +- .../filteredknn/FilteredKnnMutateConfig.java | 4 +- .../gds/similarity/knn/KnnMutateConfig.java | 4 +- .../nodesim/NodeSimilarityMutateConfig.java | 4 +- .../SpanningTreeMutateConfig.java | 4 +- .../gds/steiner/SteinerTreeMutateConfig.java | 7 +-- .../MutateRelationshipPropertyConfig.java | 40 +++++++++++++++ .../MutateRelationshipPropertyConfigTest.java | 49 +++++++++++++++++++ ...PredictionPredictPipelineMutateConfig.java | 4 +- 9 files changed, 104 insertions(+), 16 deletions(-) create mode 100644 core/src/main/java/org/neo4j/gds/config/MutateRelationshipPropertyConfig.java create mode 100644 core/src/test/java/org/neo4j/gds/config/MutateRelationshipPropertyConfigTest.java diff --git a/algo-test/src/main/java/org/neo4j/gds/test/TestMutateConfig.java b/algo-test/src/main/java/org/neo4j/gds/test/TestMutateConfig.java index 1e1484c66e..314e898be4 100644 --- a/algo-test/src/main/java/org/neo4j/gds/test/TestMutateConfig.java +++ b/algo-test/src/main/java/org/neo4j/gds/test/TestMutateConfig.java @@ -32,9 +32,7 @@ public interface TestMutateConfig extends TestConfig, MutateNodePropertyConfig { default boolean throwOnEstimate() { return false; } - - ; - + static TestMutateConfig of(CypherMapWrapper config) { return new TestMutateConfigImpl(config); } diff --git a/algo/src/main/java/org/neo4j/gds/similarity/filteredknn/FilteredKnnMutateConfig.java b/algo/src/main/java/org/neo4j/gds/similarity/filteredknn/FilteredKnnMutateConfig.java index ac235f3e75..d94c099243 100644 --- a/algo/src/main/java/org/neo4j/gds/similarity/filteredknn/FilteredKnnMutateConfig.java +++ b/algo/src/main/java/org/neo4j/gds/similarity/filteredknn/FilteredKnnMutateConfig.java @@ -21,15 +21,15 @@ import org.neo4j.gds.annotation.Configuration; import org.neo4j.gds.annotation.ValueClass; -import org.neo4j.gds.config.MutateNodePropertyConfig; import org.neo4j.gds.config.MutateRelationshipConfig; +import org.neo4j.gds.config.MutateRelationshipPropertyConfig; import org.neo4j.gds.core.CypherMapWrapper; @ValueClass @Configuration @SuppressWarnings("immutables:subtype") public interface FilteredKnnMutateConfig extends FilteredKnnBaseConfig, - MutateNodePropertyConfig, MutateRelationshipConfig { + MutateRelationshipPropertyConfig, MutateRelationshipConfig { static FilteredKnnMutateConfig of(CypherMapWrapper config) { return new FilteredKnnMutateConfigImpl(config); diff --git a/algo/src/main/java/org/neo4j/gds/similarity/knn/KnnMutateConfig.java b/algo/src/main/java/org/neo4j/gds/similarity/knn/KnnMutateConfig.java index 2c87c0a99c..39b7c9d277 100644 --- a/algo/src/main/java/org/neo4j/gds/similarity/knn/KnnMutateConfig.java +++ b/algo/src/main/java/org/neo4j/gds/similarity/knn/KnnMutateConfig.java @@ -21,14 +21,14 @@ import org.neo4j.gds.annotation.Configuration; import org.neo4j.gds.annotation.ValueClass; -import org.neo4j.gds.config.MutateNodePropertyConfig; import org.neo4j.gds.config.MutateRelationshipConfig; +import org.neo4j.gds.config.MutateRelationshipPropertyConfig; import org.neo4j.gds.core.CypherMapWrapper; @ValueClass @Configuration @SuppressWarnings("immutables:subtype") -public interface KnnMutateConfig extends KnnBaseConfig, MutateNodePropertyConfig, MutateRelationshipConfig { +public interface KnnMutateConfig extends KnnBaseConfig, MutateRelationshipPropertyConfig, MutateRelationshipConfig { static KnnMutateConfig of(CypherMapWrapper config) { return new KnnMutateConfigImpl(config); diff --git a/algo/src/main/java/org/neo4j/gds/similarity/nodesim/NodeSimilarityMutateConfig.java b/algo/src/main/java/org/neo4j/gds/similarity/nodesim/NodeSimilarityMutateConfig.java index e7a0afce43..02fb342231 100644 --- a/algo/src/main/java/org/neo4j/gds/similarity/nodesim/NodeSimilarityMutateConfig.java +++ b/algo/src/main/java/org/neo4j/gds/similarity/nodesim/NodeSimilarityMutateConfig.java @@ -21,15 +21,15 @@ import org.neo4j.gds.annotation.Configuration; import org.neo4j.gds.annotation.ValueClass; -import org.neo4j.gds.config.MutateNodePropertyConfig; import org.neo4j.gds.config.MutateRelationshipConfig; +import org.neo4j.gds.config.MutateRelationshipPropertyConfig; import org.neo4j.gds.core.CypherMapWrapper; @ValueClass @Configuration @SuppressWarnings("immutables:subtype") public interface NodeSimilarityMutateConfig extends NodeSimilarityBaseConfig, - MutateNodePropertyConfig, MutateRelationshipConfig { + MutateRelationshipPropertyConfig, MutateRelationshipConfig { static NodeSimilarityMutateConfig of(CypherMapWrapper userInput) { NodeSimilarityMutateConfig config = new NodeSimilarityMutateConfigImpl(userInput); diff --git a/algo/src/main/java/org/neo4j/gds/spanningtree/SpanningTreeMutateConfig.java b/algo/src/main/java/org/neo4j/gds/spanningtree/SpanningTreeMutateConfig.java index 0a08f7c3cd..c12d7b3dcd 100644 --- a/algo/src/main/java/org/neo4j/gds/spanningtree/SpanningTreeMutateConfig.java +++ b/algo/src/main/java/org/neo4j/gds/spanningtree/SpanningTreeMutateConfig.java @@ -21,15 +21,15 @@ import org.neo4j.gds.annotation.Configuration; import org.neo4j.gds.annotation.ValueClass; -import org.neo4j.gds.config.MutateNodePropertyConfig; import org.neo4j.gds.config.MutateRelationshipConfig; +import org.neo4j.gds.config.MutateRelationshipPropertyConfig; import org.neo4j.gds.core.CypherMapWrapper; @ValueClass @Configuration @SuppressWarnings("immutables:subtype") public interface SpanningTreeMutateConfig extends SpanningTreeBaseConfig, - MutateNodePropertyConfig, MutateRelationshipConfig { + MutateRelationshipPropertyConfig, MutateRelationshipConfig { static SpanningTreeMutateConfig of(CypherMapWrapper userInput) { return new SpanningTreeMutateConfigImpl(userInput); diff --git a/algo/src/main/java/org/neo4j/gds/steiner/SteinerTreeMutateConfig.java b/algo/src/main/java/org/neo4j/gds/steiner/SteinerTreeMutateConfig.java index 4a30034f7f..b85d996f34 100644 --- a/algo/src/main/java/org/neo4j/gds/steiner/SteinerTreeMutateConfig.java +++ b/algo/src/main/java/org/neo4j/gds/steiner/SteinerTreeMutateConfig.java @@ -21,15 +21,16 @@ import org.neo4j.gds.annotation.Configuration; import org.neo4j.gds.annotation.ValueClass; -import org.neo4j.gds.config.MutateNodePropertyConfig; import org.neo4j.gds.config.MutateRelationshipConfig; +import org.neo4j.gds.config.MutateRelationshipPropertyConfig; import org.neo4j.gds.core.CypherMapWrapper; @ValueClass @Configuration @SuppressWarnings("immutables:subtype") -public interface SteinerTreeMutateConfig extends SteinerTreeBaseConfig, MutateRelationshipConfig, - MutateNodePropertyConfig { +public interface SteinerTreeMutateConfig extends SteinerTreeBaseConfig, + MutateRelationshipConfig, + MutateRelationshipPropertyConfig { static SteinerTreeMutateConfig of(CypherMapWrapper userInput) { return new SteinerTreeMutateConfigImpl(userInput); diff --git a/core/src/main/java/org/neo4j/gds/config/MutateRelationshipPropertyConfig.java b/core/src/main/java/org/neo4j/gds/config/MutateRelationshipPropertyConfig.java new file mode 100644 index 0000000000..ec876ff497 --- /dev/null +++ b/core/src/main/java/org/neo4j/gds/config/MutateRelationshipPropertyConfig.java @@ -0,0 +1,40 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.config; + +import org.jetbrains.annotations.Nullable; +import org.neo4j.gds.annotation.Configuration; + +import static org.neo4j.gds.core.StringIdentifierValidations.emptyToNull; +import static org.neo4j.gds.core.StringIdentifierValidations.validateNoWhiteCharacter; + +public interface MutateRelationshipPropertyConfig extends MutateConfig { + + String MUTATE_PROPERTY_KEY = "mutateProperty"; + + @Configuration.ConvertWith(method = "validateProperty") + @Configuration.Key(MUTATE_PROPERTY_KEY) + String mutateProperty(); + + static @Nullable String validateProperty(String input) { + return validateNoWhiteCharacter(emptyToNull(input), "mutateProperty"); + } + +} diff --git a/core/src/test/java/org/neo4j/gds/config/MutateRelationshipPropertyConfigTest.java b/core/src/test/java/org/neo4j/gds/config/MutateRelationshipPropertyConfigTest.java new file mode 100644 index 0000000000..9afe4b2fa8 --- /dev/null +++ b/core/src/test/java/org/neo4j/gds/config/MutateRelationshipPropertyConfigTest.java @@ -0,0 +1,49 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.config; + +import org.junit.jupiter.api.Test; +import org.neo4j.gds.annotation.Configuration; +import org.neo4j.gds.gdl.GdlFactory; + +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThatNoException; + +class MutateRelationshipPropertyConfigTest { + + @Test + void testMutateFailsOnExistingToken() { + var graphStore = GdlFactory.of("(a {foo: 42})").build(); + TestMutateRelationshipPropertyConfig config = TestMutateRelationshipPropertyConfigImpl.builder().mutateProperty("foo").build(); + + assertThatNoException().isThrownBy(() -> config.graphStoreValidation( + graphStore, + config.nodeLabelIdentifiers(graphStore), + List.of() + )); + + } + + @Configuration + interface TestMutateRelationshipPropertyConfig extends AlgoBaseConfig, MutateRelationshipPropertyConfig { + + } +} diff --git a/proc/machine-learning/src/main/java/org/neo4j/gds/ml/linkmodels/pipeline/predict/LinkPredictionPredictPipelineMutateConfig.java b/proc/machine-learning/src/main/java/org/neo4j/gds/ml/linkmodels/pipeline/predict/LinkPredictionPredictPipelineMutateConfig.java index 59d5e7ecbb..f7705b922e 100644 --- a/proc/machine-learning/src/main/java/org/neo4j/gds/ml/linkmodels/pipeline/predict/LinkPredictionPredictPipelineMutateConfig.java +++ b/proc/machine-learning/src/main/java/org/neo4j/gds/ml/linkmodels/pipeline/predict/LinkPredictionPredictPipelineMutateConfig.java @@ -21,15 +21,15 @@ import org.immutables.value.Value; import org.neo4j.gds.annotation.Configuration; -import org.neo4j.gds.config.MutateNodePropertyConfig; import org.neo4j.gds.config.MutateRelationshipConfig; +import org.neo4j.gds.config.MutateRelationshipPropertyConfig; import org.neo4j.gds.core.CypherMapWrapper; @Configuration @SuppressWarnings("immutables:subtype") public interface LinkPredictionPredictPipelineMutateConfig extends LinkPredictionPredictPipelineBaseConfig, MutateRelationshipConfig, - MutateNodePropertyConfig { + MutateRelationshipPropertyConfig { @Value.Default @Override default String mutateProperty() { From ff865741dab789c3cdc4af559a217d53d7071c2c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Tue, 20 Jun 2023 15:14:36 +0200 Subject: [PATCH 112/273] Post 2.4.0 --- doc/modules/ROOT/pages/management-ops/utility-functions.adoc | 2 +- examples/pregel-bootstrap/build.gradle | 2 +- gradle/version.gradle | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/modules/ROOT/pages/management-ops/utility-functions.adoc b/doc/modules/ROOT/pages/management-ops/utility-functions.adoc index bfd53b8d28..99d1981c85 100644 --- a/doc/modules/ROOT/pages/management-ops/utility-functions.adoc +++ b/doc/modules/ROOT/pages/management-ops/utility-functions.adoc @@ -27,7 +27,7 @@ RETURN gds.version() AS version [opts="header"] |=== | version -| "2.4.0" +| "2.4.1" |=== -- diff --git a/examples/pregel-bootstrap/build.gradle b/examples/pregel-bootstrap/build.gradle index c7dc7cd32f..d0b7e9ef65 100644 --- a/examples/pregel-bootstrap/build.gradle +++ b/examples/pregel-bootstrap/build.gradle @@ -7,7 +7,7 @@ plugins { ext { // Make sure these are the same as your installation of GDS and Neo4j - gdsVersion = '2.4.0' + gdsVersion = '2.4.1' neo4jVersion = '5.8.0' // Necessary to generate value classes for Pregel configs diff --git a/gradle/version.gradle b/gradle/version.gradle index e2ae11ff6f..94198224a2 100644 --- a/gradle/version.gradle +++ b/gradle/version.gradle @@ -1,5 +1,5 @@ ext { - gdsBaseVersion = '2.4.0' + gdsBaseVersion = '2.4.1' gdsAuraVersion = '24' gdsVersion = gdsBaseVersion + (rootProject.hasProperty('aurads') ? "+${gdsAuraVersion}" : "") From fd9ed1e42f3202104940961a0dc068f2d4750ad0 Mon Sep 17 00:00:00 2001 From: Mats Rydberg Date: Mon, 19 Jun 2023 17:28:40 +0200 Subject: [PATCH 113/273] Use a long instead of an AtomicLong Because it's about 4x faster --- .../main/java/org/neo4j/gds/scaling/ScalarScaler.java | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/algo-common/src/main/java/org/neo4j/gds/scaling/ScalarScaler.java b/algo-common/src/main/java/org/neo4j/gds/scaling/ScalarScaler.java index 0c003b08c9..d042eb2d09 100644 --- a/algo-common/src/main/java/org/neo4j/gds/scaling/ScalarScaler.java +++ b/algo-common/src/main/java/org/neo4j/gds/scaling/ScalarScaler.java @@ -25,7 +25,6 @@ import java.util.List; import java.util.Map; -import java.util.concurrent.atomic.AtomicLong; public abstract class ScalarScaler implements Scaler { @@ -72,13 +71,13 @@ abstract static class AggregatesComputer implements Runnable { private final Partition partition; private final NodePropertyValues properties; private final ProgressTracker progressTracker; - private final AtomicLong nodeCountOmittingMissingProperties; + private long nodeCountOmittingMissingProperties; AggregatesComputer(Partition partition, NodePropertyValues property, ProgressTracker progressTracker) { this.partition = partition; this.properties = property; this.progressTracker = progressTracker; - this.nodeCountOmittingMissingProperties = new AtomicLong(); + this.nodeCountOmittingMissingProperties = 0L; } abstract void compute(double propertyValue); @@ -89,7 +88,7 @@ public void run() { for (long nodeId = partition.startNode(); nodeId < end; nodeId++) { var propertyValue = properties.doubleValue(nodeId); if (!Double.isNaN(propertyValue)) { - nodeCountOmittingMissingProperties.incrementAndGet(); + ++nodeCountOmittingMissingProperties; compute(propertyValue); } } @@ -97,7 +96,7 @@ public void run() { } long nodeCountOmittingMissingValues() { - return nodeCountOmittingMissingProperties.get(); + return nodeCountOmittingMissingProperties; } } } From 7d4b39622316087f50acd68087ef8fd6784430e2 Mon Sep 17 00:00:00 2001 From: Max Kiessling Date: Tue, 20 Jun 2023 22:35:04 +0200 Subject: [PATCH 114/273] Revert paging of ChunkedAdjacencyLists --- .../gds/api/compress/AdjacencyCompressor.java | 2 +- .../common/AdjacencyCompression.java | 56 ++-- .../common/ZigZagLongDecoding.java | 52 +--- .../compression/packed/PackedCompressor.java | 7 +- .../uncompressed/RawCompressor.java | 47 ++- .../varlong/DeltaVarLongCompressor.java | 10 +- .../core/loading/ChunkedAdjacencyLists.java | 282 ++++++------------ .../common/AdjacencyCompressionTest.java | 126 ++++---- .../loading/ChunkedAdjacencyListsTest.java | 209 ++++++------- .../gds/core/loading/CypherFactoryTest.java | 131 ++++---- .../gds/core/loading/NativeFactoryTest.java | 24 +- 11 files changed, 393 insertions(+), 553 deletions(-) diff --git a/core/src/main/java/org/neo4j/gds/api/compress/AdjacencyCompressor.java b/core/src/main/java/org/neo4j/gds/api/compress/AdjacencyCompressor.java index b39614be5d..103cc2a7e6 100644 --- a/core/src/main/java/org/neo4j/gds/api/compress/AdjacencyCompressor.java +++ b/core/src/main/java/org/neo4j/gds/api/compress/AdjacencyCompressor.java @@ -49,7 +49,7 @@ public interface AdjacencyCompressor extends AutoCloseable { * @param degree The number of targets stored in `targets`. * @return the degree of the compressed adjacency list */ - int compress(long nodeId, long[] targets, long[][][] properties, int degree); + int compress(long nodeId, long[] targets, long[][] properties, int degree); /** * Closing this compressor will release some internal data structures, making them eligible for garbage collection. diff --git a/core/src/main/java/org/neo4j/gds/core/compression/common/AdjacencyCompression.java b/core/src/main/java/org/neo4j/gds/core/compression/common/AdjacencyCompression.java index 8a11e8c50e..a6e6b8d0be 100644 --- a/core/src/main/java/org/neo4j/gds/core/compression/common/AdjacencyCompression.java +++ b/core/src/main/java/org/neo4j/gds/core/compression/common/AdjacencyCompression.java @@ -20,7 +20,6 @@ package org.neo4j.gds.core.compression.common; import com.carrotsearch.hppc.sorting.IndirectSort; -import org.apache.commons.lang3.mutable.MutableInt; import org.neo4j.gds.api.compress.AdjacencyCompressor; import org.neo4j.gds.api.compress.LongArrayBuffer; import org.neo4j.gds.core.Aggregation; @@ -39,12 +38,13 @@ public final class AdjacencyCompression { /** * Decompress the given {@code array} into the given {@code into}. - * After this, {@link org.neo4j.gds.api.compress.LongArrayBuffer#length} will reflect the number of decompressed values + * After this, {@link org.neo4j.gds.api.compress.LongArrayBuffer#length} will reflect the number of decompressed + * values * that are in the {@link org.neo4j.gds.api.compress.LongArrayBuffer#buffer}. */ public static void zigZagUncompressFrom( LongArrayBuffer into, - byte[][] targets, + byte[] targets, int compressedValues, int limit, AdjacencyCompressor.ValueMapper mapper @@ -56,7 +56,7 @@ public static void zigZagUncompressFrom( public static void zigZagUncompressFrom( long[] into, - byte[][] targets, + byte[] targets, int compressedValues, int limit, AdjacencyCompressor.ValueMapper mapper @@ -77,7 +77,7 @@ public static int applyDeltaEncoding(long[] data, int length, Aggregation aggreg // TODO: requires lots of additional memory ... inline indirect sort to make reuse of - to be created - buffers public static int applyDeltaEncoding( LongArrayBuffer data, - long[][][] weights, + long[][] weights, long[][] sortedWeights, Aggregation[] aggregations, boolean noAggregation @@ -96,7 +96,7 @@ public static int applyDeltaEncoding( public static int applyDeltaEncoding( long[] data, int length, - long[][][] unsortedWeights, + long[][] unsortedWeights, long[][] sortedWeights, Aggregation[] aggregations, boolean noAggregation @@ -233,21 +233,23 @@ public static void prefixSumDeltaEncodedValues(long[] values, int length) { * Applies delta encoding to the given {@code values}. * Weights are not encoded. * - * @param order Ordered indices into {@code values} and {@code weights} for consuming these in ascending value order. + * @param order Ordered indices into {@code values} and {@code weights} for consuming these in ascending + * value order. * @param values Relationships represented by target node ID. * @param outValues Sorted, delta-encoded and optionally aggregated relationships. * @param unsortedWeights Relationship properties by key, ordered by {@code order}. - * @param outWeights Sorted and optionally aggregated relationship properties. + * @param sortedWeights Sorted and optionally aggregated relationship properties. * @param length Number of relationships (degree of source node) to process. - * @param aggregations Aggregations to apply to parallel edges. One per relationship property key in {@code weights}. + * @param aggregations Aggregations to apply to parallel edges. One per relationship property key in + * {@code weights}. * @param noAggregation Is true iff all aggregations are NONE. */ private static int applyDelta( int[] order, long[] values, long[] outValues, - long[][][] unsortedWeights, - long[][] outWeights, + long[][] unsortedWeights, + long[][] sortedWeights, int length, Aggregation[] aggregations, boolean noAggregation @@ -256,41 +258,29 @@ private static int applyDelta( long value = values[firstSortIdx]; long delta; - int[] chunkLengths = new int[unsortedWeights[0].length]; - int totalChunkLength = 0; - for (int i = 0; i < unsortedWeights[0].length; i++) { - totalChunkLength += unsortedWeights[0][i].length; - chunkLengths[i] = totalChunkLength; - } - - var pageIndex = new MutableInt(); - var indexInPage = new MutableInt(); - findPosition(chunkLengths, firstSortIdx, pageIndex, indexInPage); - outValues[0] = values[firstSortIdx]; for (int i = 0; i < unsortedWeights.length; i++) { - outWeights[i][0] = unsortedWeights[i][pageIndex.intValue()][indexInPage.intValue()]; + sortedWeights[i][0] = unsortedWeights[i][firstSortIdx]; } int in = 1, out = 1; for (; in < length; ++in) { final int sortIdx = order[in]; - findPosition(chunkLengths, sortIdx, pageIndex, indexInPage); delta = values[sortIdx] - value; value = values[sortIdx]; if (delta > 0L || noAggregation) { for (int i = 0; i < unsortedWeights.length; i++) { - outWeights[i][out] = unsortedWeights[i][pageIndex.intValue()][indexInPage.intValue()]; + sortedWeights[i][out] = unsortedWeights[i][sortIdx]; } outValues[out++] = delta; } else { for (int i = 0; i < unsortedWeights.length; i++) { Aggregation aggregation = aggregations[i]; int existingIdx = out - 1; - long[] outWeight = outWeights[i]; + long[] outWeight = sortedWeights[i]; double existingWeight = Double.longBitsToDouble(outWeight[existingIdx]); - double newWeight = Double.longBitsToDouble(unsortedWeights[i][pageIndex.intValue()][indexInPage.intValue()]); + double newWeight = Double.longBitsToDouble(unsortedWeights[i][sortIdx]); newWeight = aggregation.merge(existingWeight, newWeight); outWeight[existingIdx] = Double.doubleToLongBits(newWeight); } @@ -299,18 +289,6 @@ private static int applyDelta( return out; } - public static void findPosition(int[] chunkLengths, int position, MutableInt pageIndex, MutableInt indexInPage) { - int chunkPosition = Arrays.binarySearch(chunkLengths, position); - if (chunkPosition >= 0) { - pageIndex.setValue(chunkPosition + 1); - indexInPage.setValue(0); - } else { - int index = -chunkPosition - 1; - pageIndex.setValue(index); - indexInPage.setValue(position - (index == 0 ? 0 : chunkLengths[index - 1])); - } - } - private AdjacencyCompression() { } } diff --git a/core/src/main/java/org/neo4j/gds/core/compression/common/ZigZagLongDecoding.java b/core/src/main/java/org/neo4j/gds/core/compression/common/ZigZagLongDecoding.java index d2dfa86d13..374c294b8f 100644 --- a/core/src/main/java/org/neo4j/gds/core/compression/common/ZigZagLongDecoding.java +++ b/core/src/main/java/org/neo4j/gds/core/compression/common/ZigZagLongDecoding.java @@ -32,16 +32,24 @@ public long map(long value) { } } - public static int zigZagUncompress(byte[] chunk, int numberOfBytes, long[] out) { + public static int zigZagUncompress(byte[] compressedData, int length, long[] uncompressedData) { + return zigZagUncompress(compressedData, length, uncompressedData, Identity.INSTANCE); + } + + static int zigZagUncompress( + byte[] compressedData, + int length, + long[] uncompressedData, + AdjacencyCompressor.ValueMapper mapper + ) { long input, startValue = 0L, value = 0L; int into = 0, shift = 0, offset = 0; - - while (offset < numberOfBytes) { - input = chunk[offset++]; + while (offset < length) { + input = compressedData[offset++]; value += (input & 127L) << shift; if ((input & 128L) == 128L) { startValue += ((value >>> 1L) ^ -(value & 1L)); - out[into++] = startValue; + uncompressedData[into++] = mapper.map(startValue); value = 0L; shift = 0; } else { @@ -51,40 +59,6 @@ public static int zigZagUncompress(byte[] chunk, int numberOfBytes, long[] out) return into; } - public static int zigZagUncompress(byte[][] chunks, int numberOfBytes, long[] out) { - return zigZagUncompress(chunks, numberOfBytes, out, Identity.INSTANCE); - } - - public static int zigZagUncompress( - byte[][] chunks, - int numberOfBytes, - long[] out, - AdjacencyCompressor.ValueMapper mapper - ) { - int currentChunk = 0; - long input, startValue = 0L, value = 0L; - int into = 0, shift = 0; - - while (numberOfBytes > 0) { - var chunk = chunks[currentChunk++]; - var bytesToConsumeForChunk = Math.min(numberOfBytes, chunk.length); - for (int offset = 0; offset < bytesToConsumeForChunk; offset++) { - input = chunk[offset]; - value += (input & 127L) << shift; - if ((input & 128L) == 128L) { - startValue += ((value >>> 1L) ^ -(value & 1L)); - out[into++] = mapper.map(startValue); - value = 0L; - shift = 0; - } else { - shift += 7; - } - } - numberOfBytes -= bytesToConsumeForChunk; - } - return into; - } - private ZigZagLongDecoding() { throw new UnsupportedOperationException("No instances"); } diff --git a/core/src/main/java/org/neo4j/gds/core/compression/packed/PackedCompressor.java b/core/src/main/java/org/neo4j/gds/core/compression/packed/PackedCompressor.java index cef5cda7bd..4f222e1137 100644 --- a/core/src/main/java/org/neo4j/gds/core/compression/packed/PackedCompressor.java +++ b/core/src/main/java/org/neo4j/gds/core/compression/packed/PackedCompressor.java @@ -50,7 +50,8 @@ public static AdjacencyCompressorFactory factory( boolean noAggregation, MemoryTracker memoryTracker ) { - AdjacencyListBuilder[] propertyBuilders = new AdjacencyListBuilder[propertyMappings.numberOfMappings()]; + AdjacencyListBuilder[] propertyBuilders = new AdjacencyListBuilder[propertyMappings + .numberOfMappings()]; Arrays.setAll(propertyBuilders, i -> adjacencyListBuilderFactory.newAdjacencyPropertiesBuilder(memoryTracker)); return new Factory( @@ -171,7 +172,7 @@ private PackedCompressor( } @Override - public int compress(long nodeId, long[] targets, long[][][] properties, int degree) { + public int compress(long nodeId, long[] targets, long[][] properties, int degree) { if (properties != null) { return packWithProperties( nodeId, @@ -191,7 +192,7 @@ public int compress(long nodeId, long[] targets, long[][][] properties, int degr private int packWithProperties( long nodeId, long[] targets, - long[][][] unsortedProperties, + long[][] unsortedProperties, int degree ) { long[][] sortedProperties = new long[unsortedProperties.length][degree]; diff --git a/core/src/main/java/org/neo4j/gds/core/compression/uncompressed/RawCompressor.java b/core/src/main/java/org/neo4j/gds/core/compression/uncompressed/RawCompressor.java index badfd097cb..a589f4dff5 100644 --- a/core/src/main/java/org/neo4j/gds/core/compression/uncompressed/RawCompressor.java +++ b/core/src/main/java/org/neo4j/gds/core/compression/uncompressed/RawCompressor.java @@ -20,7 +20,6 @@ package org.neo4j.gds.core.compression.uncompressed; import com.carrotsearch.hppc.sorting.IndirectSort; -import org.apache.commons.lang3.mutable.MutableInt; import org.jetbrains.annotations.Nullable; import org.neo4j.gds.PropertyMappings; import org.neo4j.gds.api.AdjacencyList; @@ -40,8 +39,6 @@ import java.util.Arrays; import java.util.function.LongSupplier; -import static org.neo4j.gds.core.compression.common.AdjacencyCompression.findPosition; - public final class RawCompressor implements AdjacencyCompressor { public static AdjacencyCompressorFactory factory( @@ -52,8 +49,10 @@ public static AdjacencyCompressorFactory factory( boolean noAggregation, MemoryTracker memoryTracker ) { - @SuppressWarnings("unchecked") - AdjacencyListBuilder[] propertyBuilders = new AdjacencyListBuilder[propertyMappings.numberOfMappings()]; + @SuppressWarnings( + "unchecked" + ) AdjacencyListBuilder[] propertyBuilders = new AdjacencyListBuilder[propertyMappings + .numberOfMappings()]; Arrays.setAll(propertyBuilders, i -> adjacencyListBuilderFactory.newAdjacencyPropertiesBuilder(memoryTracker)); return new Factory( @@ -158,7 +157,7 @@ private RawCompressor( } @Override - public int compress(long nodeId, long[] targets, long[][][] properties, int degree) { + public int compress(long nodeId, long[] targets, long[][] properties, int degree) { if (properties != null) { return withProperties( nodeId, @@ -168,7 +167,9 @@ public int compress(long nodeId, long[] targets, long[][][] properties, int degr ); } else { return withoutProperties( - nodeId, targets, degree + nodeId, + targets, + degree ); } } @@ -222,32 +223,21 @@ private int aggregate(long[] values, int length, Aggregation aggregation) { private int aggregateWithProperties( long[] values, int length, - long[][][] unsortedProperties, + long[][] unsortedProperties, long[][] sortedProperties, Aggregation[] aggregations ) { int[] order = IndirectSort.mergesort(0, length, new AscendingLongComparator(values)); + long[] outValues = new long[length]; + int firstSortIdx = order[0]; long value = values[firstSortIdx]; long delta; - - int[] chunkLengths = new int[unsortedProperties[0].length]; - int totalChunkLength = 0; - for (int i = 0; i < unsortedProperties[0].length; i++) { - totalChunkLength += unsortedProperties[0][i].length; - chunkLengths[i] = totalChunkLength; - } - - var pageIndex = new MutableInt(); - var indexInPage = new MutableInt(); - findPosition(chunkLengths, firstSortIdx, pageIndex, indexInPage); - - long[] outValues = new long[length]; outValues[0] = value; for (int i = 0; i < unsortedProperties.length; i++) { - sortedProperties[i][0] = unsortedProperties[i][pageIndex.intValue()][indexInPage.intValue()]; + sortedProperties[i][0] = unsortedProperties[i][firstSortIdx]; } int in = 1, out = 1; @@ -255,10 +245,9 @@ private int aggregateWithProperties( if (this.noAggregation) { for (; in < length; ++in) { int sortIdx = order[in]; - findPosition(chunkLengths, sortIdx, pageIndex, indexInPage); for (int i = 0; i < unsortedProperties.length; i++) { - sortedProperties[i][out] = unsortedProperties[i][pageIndex.intValue()][indexInPage.intValue()]; + sortedProperties[i][out] = unsortedProperties[i][sortIdx]; } outValues[out++] = values[sortIdx]; @@ -266,13 +255,12 @@ private int aggregateWithProperties( } else { for (; in < length; ++in) { int sortIdx = order[in]; - findPosition(chunkLengths, sortIdx, pageIndex, indexInPage); delta = values[sortIdx] - value; value = values[sortIdx]; if (delta > 0L) { for (int i = 0; i < unsortedProperties.length; i++) { - sortedProperties[i][out] = unsortedProperties[i][pageIndex.intValue()][indexInPage.intValue()]; + sortedProperties[i][out] = unsortedProperties[i][sortIdx]; } outValues[out++] = value; } else { @@ -281,7 +269,7 @@ private int aggregateWithProperties( int existingIdx = out - 1; long[] outProperty = sortedProperties[i]; double existingProperty = Double.longBitsToDouble(outProperty[existingIdx]); - double newProperty = Double.longBitsToDouble(unsortedProperties[i][pageIndex.intValue()][indexInPage.intValue()]); + double newProperty = Double.longBitsToDouble(unsortedProperties[i][sortIdx]); newProperty = aggregation.merge(existingProperty, newProperty); outProperty[existingIdx] = Double.doubleToLongBits(newProperty); } @@ -290,6 +278,9 @@ private int aggregateWithProperties( } System.arraycopy(outValues, 0, values, 0, out); + for (int i = 0; i < sortedProperties.length; i++) { + System.arraycopy(sortedProperties[i], 0, unsortedProperties[i], 0, out); + } return out; @@ -298,7 +289,7 @@ private int aggregateWithProperties( private int withProperties( long nodeId, long[] targets, - long[][][] unsortedProperties, + long[][] unsortedProperties, int degree ) { long[][] sortedProperties = new long[unsortedProperties.length][degree]; diff --git a/core/src/main/java/org/neo4j/gds/core/compression/varlong/DeltaVarLongCompressor.java b/core/src/main/java/org/neo4j/gds/core/compression/varlong/DeltaVarLongCompressor.java index 19547efe4c..310e3cb3d3 100644 --- a/core/src/main/java/org/neo4j/gds/core/compression/varlong/DeltaVarLongCompressor.java +++ b/core/src/main/java/org/neo4j/gds/core/compression/varlong/DeltaVarLongCompressor.java @@ -49,8 +49,10 @@ public static AdjacencyCompressorFactory factory( boolean noAggregation, MemoryTracker memoryTracker ) { - @SuppressWarnings("unchecked") - AdjacencyListBuilder[] propertyBuilders = new AdjacencyListBuilder[propertyMappings.numberOfMappings()]; + @SuppressWarnings( + "unchecked" + ) AdjacencyListBuilder[] propertyBuilders = new AdjacencyListBuilder[propertyMappings + .numberOfMappings()]; Arrays.setAll(propertyBuilders, i -> adjacencyListBuilderFactory.newAdjacencyPropertiesBuilder(memoryTracker)); return new Factory( @@ -157,7 +159,7 @@ private DeltaVarLongCompressor( } @Override - public int compress(long nodeId, long[] targets, long[][][] properties, int degree) { + public int compress(long nodeId, long[] targets, long[][] properties, int degree) { if (properties != null) { return applyVariableDeltaEncodingWithProperties( nodeId, @@ -210,7 +212,7 @@ private int applyVariableDeltaEncodingWithoutProperties(long nodeId, long[] targ private int applyVariableDeltaEncodingWithProperties( long nodeId, long[] targets, - long[][][] unsortedProperties, + long[][] unsortedProperties, int degree ) { // buffer contains uncompressed, unsorted target list diff --git a/core/src/main/java/org/neo4j/gds/core/loading/ChunkedAdjacencyLists.java b/core/src/main/java/org/neo4j/gds/core/loading/ChunkedAdjacencyLists.java index d9e56dcae1..eadb65bb24 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/ChunkedAdjacencyLists.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/ChunkedAdjacencyLists.java @@ -20,10 +20,10 @@ package org.neo4j.gds.core.loading; import org.neo4j.gds.collections.DrainingIterator; -import org.neo4j.gds.collections.HugeSparseByteArrayArrayList; +import org.neo4j.gds.collections.HugeSparseByteArrayList; import org.neo4j.gds.collections.HugeSparseCollections; import org.neo4j.gds.collections.HugeSparseIntList; -import org.neo4j.gds.collections.HugeSparseLongArrayArrayList; +import org.neo4j.gds.collections.HugeSparseLongArrayList; import org.neo4j.gds.collections.HugeSparseLongList; import org.neo4j.gds.core.utils.mem.MemoryEstimation; import org.neo4j.gds.core.utils.mem.MemoryEstimations; @@ -40,20 +40,19 @@ import static org.neo4j.gds.core.compression.common.VarLongEncoding.zigZag; import static org.neo4j.gds.core.loading.AdjacencyPreAggregation.IGNORE_VALUE; import static org.neo4j.gds.mem.BitUtil.ceilDiv; +import static org.neo4j.gds.utils.StringFormatting.formatWithLocale; public final class ChunkedAdjacencyLists { - private static final byte[][] EMPTY_BYTES_BYTES = new byte[0][]; - private static final long[][] EMPTY_PROPERTIES = new long[0][]; + private static final byte[] EMPTY_BYTES = new byte[0]; + private static final long[] EMPTY_PROPERTIES = new long[0]; - private final HugeSparseByteArrayArrayList targetLists; - private final HugeSparseLongArrayArrayList[] properties; + private final HugeSparseByteArrayList targetLists; + private final HugeSparseLongArrayList[] properties; private final HugeSparseIntList positions; private final HugeSparseLongList lastValues; private final HugeSparseIntList lengths; - private byte[] compressBuffer = new byte[1 << 20]; // 1MiB - public static MemoryEstimation memoryEstimation(long avgDegree, long nodeCount, int propertyCount) { // Best case scenario: // Difference between node identifiers in each adjacency list is 1. @@ -73,7 +72,10 @@ public static MemoryEstimation memoryEstimation(long avgDegree, long nodeCount, .fixed("positions", HugeSparseCollections.estimateInt(nodeCount, nodeCount)) .fixed("lengths", HugeSparseCollections.estimateInt(nodeCount, nodeCount)) .fixed("lastValues", HugeSparseCollections.estimateLong(nodeCount, nodeCount)) - .fixed("properties", HugeSparseCollections.estimateLongArray(nodeCount, nodeCount, (int) avgDegree).times(propertyCount)) + .fixed( + "properties", + HugeSparseCollections.estimateLongArray(nodeCount, nodeCount, (int) avgDegree).times(propertyCount) + ) .build(); } @@ -89,15 +91,15 @@ public static ChunkedAdjacencyLists of(int numberOfProperties, long initialCapac } private ChunkedAdjacencyLists(int numberOfProperties, long initialCapacity) { - this.targetLists = HugeSparseByteArrayArrayList.of(EMPTY_BYTES_BYTES, initialCapacity); + this.targetLists = HugeSparseByteArrayList.of(EMPTY_BYTES, initialCapacity); this.positions = HugeSparseIntList.of(0, initialCapacity); this.lastValues = HugeSparseLongList.of(0, initialCapacity); this.lengths = HugeSparseIntList.of(0, initialCapacity); if (numberOfProperties > 0) { - this.properties = new HugeSparseLongArrayArrayList[numberOfProperties]; - Arrays.setAll(this.properties, i -> HugeSparseLongArrayArrayList.of(EMPTY_PROPERTIES, initialCapacity)); + this.properties = new HugeSparseLongArrayList[numberOfProperties]; + Arrays.setAll(this.properties, i -> HugeSparseLongArrayList.of(EMPTY_PROPERTIES, initialCapacity)); } else { this.properties = null; } @@ -128,14 +130,11 @@ public void add(long index, long[] targets, int start, int end, int valuesToAdd) } var position = positions.get(index); - if (requiredBytes > this.compressBuffer.length) { - this.compressBuffer = new byte[BitUtil.nextHighestPowerOfTwo(requiredBytes)]; - } - encodeVLongs(targets, start, end, this.compressBuffer, 0); + var compressedTargets = ensureCompressedTargetsCapacity(index, position, requiredBytes); - copyTargetData(index, position, this.compressBuffer, requiredBytes, this.targetLists); + var newPosition = encodeVLongs(targets, start, end, compressedTargets, position); - positions.set(index, position + requiredBytes); + positions.set(index, newPosition); this.lastValues.set(index, currentLastValue); this.lengths.addTo(index, valuesToAdd); @@ -153,192 +152,81 @@ public void add(long index, long[] targets, int start, int end, int valuesToAdd) public void add(long index, long[] targets, long[][] allProperties, int start, int end, int targetsToAdd) { // write properties for (int i = 0; i < allProperties.length; i++) { - var length = lengths.get(index); - copyPropertiesData( - index, - length, - allProperties[i], - start, - end, - targets, - targetsToAdd, - this.properties[i] - ); + addProperties(index, targets, allProperties[i], start, end, i, targetsToAdd); } // write values add(index, targets, start, end, targetsToAdd); } - static final int[] NEXT_CHUNK_LENGTH = { - 32, - 32, - 64, - 64, - 128, - 128, - 256, - 256, - 512, - 512, - 1024, - 1024, - 2048, - 4096, - 8192, - 16384, - 32768, - 65536, - 131072, - 262144, - 524288, - 1048576, - 2097152, - 4194304, - }; - - // This method and the one after that ("copyPropertiesData") are the same except for the underlying primitive array type. - // Unfortunately, writing this method to be generic over the primitive array type requires us to use methods - // that are either reflection based or not optimizable by C2, and it shows on benchmarks. - // Any changes made here should also be done in "copyPropertiesData". - private static void copyTargetData( + private void addProperties( long index, - int targetPos, - byte[] buffer, - int bufferLength, - HugeSparseByteArrayArrayList dataLists + long[] targets, + long[] properties, + int start, + int end, + int propertyIndex, + int propertiesToAdd ) { - byte[][] compressedData = dataLists.get(index); + var length = lengths.get(index); - // last chunk, write pos = posInLastChunk - int posInLastChunk = targetPos; - for (int targetPage = 0; targetPage < compressedData.length - 1; targetPage++) { - byte[] compressedTarget = compressedData[targetPage]; - posInLastChunk -= compressedTarget.length; - } + var currentProperties = ensurePropertyCapacity(index, length, propertiesToAdd, propertyIndex); - // can we fit everything in the last chunk? - if (compressedData.length > 0) { - byte[] lastChunk = compressedData[compressedData.length - 1]; - if (lastChunk.length >= posInLastChunk + bufferLength) { - // fits in last chunk - System.arraycopy(buffer, 0, lastChunk, posInLastChunk, bufferLength); - return; + if (propertiesToAdd == end - start) { + System.arraycopy(properties, start, currentProperties, length, propertiesToAdd); + } else { + var writePos = length; + for (int i = 0; i < (end - start); i++) { + if (targets[start + i] != IGNORE_VALUE) { + currentProperties[writePos++] = properties[start + i]; + } } } + } - // how many bytes do we need to write into the last chunk - int fitsInLastChunk = compressedData.length == 0 - ? 0 - : compressedData[compressedData.length - 1].length - posInLastChunk; - - int newChunkLengthAtLeast = bufferLength - fitsInLastChunk; - - // the next chunk size grows slowly depending on the number of chunks we already have - int nextChunkLevel = Math.min(compressedData.length, NEXT_CHUNK_LENGTH.length - 1); - int nextChunkLength = NEXT_CHUNK_LENGTH[nextChunkLevel]; - - // avoid splitting the buffer into too many chunks - int newChunkLength = Math.max(newChunkLengthAtLeast, nextChunkLength); - - // copy buffer into the last chunk and the new chunk - byte[] newChunk = new byte[newChunkLength]; - if (fitsInLastChunk > 0) { - System.arraycopy( - buffer, - 0, - compressedData[compressedData.length - 1], - posInLastChunk, - fitsInLastChunk + private byte[] ensureCompressedTargetsCapacity(long index, int pos, int required) { + int targetLength = pos + required; + var compressedTargets = targetLists.get(index); + + if (targetLength < 0) { + throw new IllegalArgumentException( + formatWithLocale( + "Encountered numeric overflow in internal buffer. Was at position %d and needed to grow by %d.", + pos, + required + ) ); + } else if (compressedTargets.length <= targetLength) { + int newLength = BitUtil.nextHighestPowerOfTwo(targetLength); +// int newLength = ArrayUtil.oversize(pos + required, Byte.BYTES); + compressedTargets = Arrays.copyOf(compressedTargets, newLength); + this.targetLists.set(index, compressedTargets); } - System.arraycopy(buffer, fitsInLastChunk, newChunk, 0, bufferLength - fitsInLastChunk); - // add new chunk to the targets list - compressedData = Arrays.copyOf(compressedData, compressedData.length + 1); - compressedData[compressedData.length - 1] = newChunk; - dataLists.set(index, compressedData); + return compressedTargets; } - // This method and the one before that ("copyTargetData") are the same except for the underlying primitive array type. - // Unfortunately, writing this method to be generic over the primitive array type requires us to use methods - // that are either reflection based or not optimizable by C2, and it shows on benchmarks. - // Any changes made here should also be done in "copyTargetData". - private static void copyPropertiesData( - long index, - int targetPos, - long[] buffer, - int bufferOffset, - int bufferEnd, - long[] targets, - int bufferLength, - HugeSparseLongArrayArrayList dataLists - ) { - if (bufferEnd - bufferOffset != bufferLength) { - // compact the property buffer and ignore properties from ignored targets - var writeIndex = bufferOffset; - for (int i = bufferOffset; i < bufferEnd; i++) { - if (targets[i] != IGNORE_VALUE) { - if (writeIndex != i) { - buffer[writeIndex] = buffer[i]; - } - writeIndex++; - } - } + private long[] ensurePropertyCapacity(long index, int pos, int required, int propertyIndex) { + int targetLength = pos + required; - assert bufferLength == writeIndex - bufferOffset; - } - - long[][] compressedData = dataLists.get(index); - - // last chunk, write pos = posInLastChunk - int posInLastChunk = targetPos; - for (int targetPage = 0; targetPage < compressedData.length - 1; targetPage++) { - long[] compressedTarget = compressedData[targetPage]; - posInLastChunk -= compressedTarget.length; - } - - // can we fit everything in the last chunk? - if (compressedData.length > 0) { - long[] lastChunk = compressedData[compressedData.length - 1]; - if (lastChunk.length >= posInLastChunk + bufferLength) { - // fits in last chunk - System.arraycopy(buffer, bufferOffset, lastChunk, posInLastChunk, bufferLength); - return; - } - } + var currentProperties = this.properties[propertyIndex].get(index); - // how many bytes do we need to write into the last chunk - int fitsInLastChunk = compressedData.length == 0 - ? 0 - : compressedData[compressedData.length - 1].length - posInLastChunk; - - int newChunkLengthAtLeast = bufferLength - fitsInLastChunk; - - // the next chunk size grows slowly depending on the number of chunks we already have - int nextChunkLevel = Math.min(compressedData.length, NEXT_CHUNK_LENGTH.length - 1); - int nextChunkLength = NEXT_CHUNK_LENGTH[nextChunkLevel] / 2; - - // avoid splitting the buffer into too many chunks - int newChunkLength = Math.max(newChunkLengthAtLeast, nextChunkLength); - - // copy buffer into the last chunk and the new chunk - long[] newChunk = new long[newChunkLength]; - if (fitsInLastChunk > 0) { - System.arraycopy( - buffer, - bufferOffset, - compressedData[compressedData.length - 1], - posInLastChunk, - fitsInLastChunk + if (targetLength < 0) { + throw new IllegalArgumentException( + formatWithLocale( + "Encountered numeric overflow in internal buffer. Was at position %d and needed to grow by %d.", + pos, + required + ) ); + } else if (currentProperties.length <= pos + required) { +// int newLength = ArrayUtil.oversize(pos + required, Long.BYTES); + int newLength = BitUtil.nextHighestPowerOfTwo(pos + required); + currentProperties = Arrays.copyOf(currentProperties, newLength); + this.properties[propertyIndex].set(index, currentProperties); } - System.arraycopy(buffer, fitsInLastChunk + bufferOffset, newChunk, 0, bufferLength - fitsInLastChunk); - // add new chunk to the targets list - compressedData = Arrays.copyOf(compressedData, compressedData.length + 1); - compressedData[compressedData.length - 1] = newChunk; - dataLists.set(index, compressedData); + return currentProperties; } public long capacity() { @@ -356,30 +244,30 @@ public void consume(Consumer consumer) { public interface Consumer { void accept( long sourceId, - byte[][] targets, - long[][][] properties, + byte[] targets, + long[][] properties, int compressedByteSize, int numberOfCompressedTargets ); } private static class CompositeDrainingIterator { - private final DrainingIterator targetListIterator; - private final DrainingIterator.DrainingBatch targetListBatch; + private final DrainingIterator targetListIterator; + private final DrainingIterator.DrainingBatch targetListBatch; private final DrainingIterator positionsListIterator; private final DrainingIterator.DrainingBatch positionsListBatch; private final DrainingIterator lastValuesListIterator; private final DrainingIterator.DrainingBatch lastValuesListBatch; private final DrainingIterator lengthsListIterator; private final DrainingIterator.DrainingBatch lengthsListBatch; - private final List> propertyIterators; - private final List> propertyBatches; + private final List> propertyIterators; + private final List> propertyBatches; - private final long[][][] propertiesBuffer; + private final long[][] propertiesBuffer; CompositeDrainingIterator( - HugeSparseByteArrayArrayList targets, - HugeSparseLongArrayArrayList[] properties, + HugeSparseByteArrayList targets, + HugeSparseLongArrayList[] properties, HugeSparseIntList positions, HugeSparseLongList lastValues, HugeSparseIntList lengths @@ -399,13 +287,13 @@ private static class CompositeDrainingIterator { propertiesBuffer = null; } else { this.propertyIterators = Arrays.stream(properties) - .map(HugeSparseLongArrayArrayList::drainingIterator) + .map(HugeSparseLongArrayList::drainingIterator) .collect(Collectors.toList()); this.propertyBatches = this.propertyIterators .stream() .map(DrainingIterator::drainingBatch) .collect(Collectors.toList()); - propertiesBuffer = new long[properties.length][][]; + propertiesBuffer = new long[properties.length][]; } } @@ -426,21 +314,19 @@ public void consume(Consumer consumer) { for (int indexInPage = 0; indexInPage < targetsPage.length; indexInPage++) { var targets = targetsPage[indexInPage]; - if (targets == EMPTY_BYTES_BYTES) { + if (targets == EMPTY_BYTES) { continue; } - // make targets eligible for GC - targetsPage[indexInPage] = null; - var position = positionsPage[indexInPage]; var length = lengthsPage[indexInPage]; for (int propertyIndex = 0; propertyIndex < propertyBatches.size(); propertyIndex++) { - long[][][] page = propertyBatches.get(propertyIndex).page; - + long[][] page = propertyBatches.get(propertyIndex).page; propertiesBuffer[propertyIndex] = page[indexInPage]; // make properties eligible for GC page[indexInPage] = null; } + // make targets eligible for GC + targetsPage[indexInPage] = null; consumer.accept(offset + indexInPage, targets, propertiesBuffer, position, length); } diff --git a/core/src/test/java/org/neo4j/gds/core/compression/common/AdjacencyCompressionTest.java b/core/src/test/java/org/neo4j/gds/core/compression/common/AdjacencyCompressionTest.java index 11d26bac8c..303b2422a1 100644 --- a/core/src/test/java/org/neo4j/gds/core/compression/common/AdjacencyCompressionTest.java +++ b/core/src/test/java/org/neo4j/gds/core/compression/common/AdjacencyCompressionTest.java @@ -19,10 +19,8 @@ */ package org.neo4j.gds.core.compression.common; -import org.apache.commons.lang3.mutable.MutableInt; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; -import org.junit.jupiter.params.provider.CsvSource; import org.junit.jupiter.params.provider.MethodSource; import org.neo4j.gds.api.compress.LongArrayBuffer; import org.neo4j.gds.core.Aggregation; @@ -31,7 +29,6 @@ import java.util.stream.Stream; import static java.lang.Double.doubleToLongBits; -import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertSame; @@ -55,7 +52,9 @@ void shouldCountRelationships( AdjacencyCompression.applyDeltaEncoding( data, - new long[][][]{new long[][]{unsortedProperties[0]}, new long[][]{unsortedProperties[1]}}, + new long[][]{ + unsortedProperties[0], unsortedProperties[1] + }, sortedProperties, aggregations, false @@ -78,25 +77,6 @@ void shouldCountRelationships( assertEquals(4L, data.buffer[1]); } - @ParameterizedTest - @CsvSource( - { - "0, 0, 0", - "9, 0, 9", - "10, 1, 0", - "41, 1, 31", - "42, 2, 0", - } - ) - void findPosition(int position, int expectedPageIndex, int expectedIndexInPage) { - int[] lengths = {10, 42, 1337}; - var pageIndex = new MutableInt(); - var indexInPage = new MutableInt(); - AdjacencyCompression.findPosition(lengths, position, pageIndex, indexInPage); - assertThat(pageIndex.getValue()).isEqualTo(expectedPageIndex); - assertThat(indexInPage.getValue()).isEqualTo(expectedIndexInPage); - } - static Stream aggregationsWithResults() { return Stream.of( Arguments.of( @@ -106,7 +86,13 @@ static Stream aggregationsWithResults() { Aggregation.COUNT, Aggregation.COUNT }, - new double[][]{{4d, 2d}, {2d, 1d}}, + new double[][]{ + { + 4d, 2d + }, { + 2d, 1d + } + }, "COUNT" ), Arguments.of( @@ -116,7 +102,13 @@ static Stream aggregationsWithResults() { Aggregation.SUM, Aggregation.SUM }, - new double[][]{{16d, 8d}, {27d, 14d}}, + new double[][]{ + { + 16d, 8d + }, { + 27d, 14d + } + }, "SUM" ), Arguments.of( @@ -126,7 +118,13 @@ static Stream aggregationsWithResults() { Aggregation.MIN, Aggregation.MIN }, - new double[][]{{2d, 3d}, {4d, 6d}}, + new double[][]{ + { + 2d, 3d + }, { + 4d, 6d + } + }, "MIN" ), Arguments.of( @@ -136,50 +134,62 @@ static Stream aggregationsWithResults() { Aggregation.MAX, Aggregation.MAX }, - new double[][]{{5d, 5d}, {8d, 8d}}, + new double[][]{ + { + 5d, 5d + }, { + 8d, 8d + } + }, "MAX" ) ); } private static long[][] weights() { - return new long[][]{{ - doubleToLongBits(2), - doubleToLongBits(4), - doubleToLongBits(3), - doubleToLongBits(5), - doubleToLongBits(5), - doubleToLongBits(5) - }, { - doubleToLongBits(4), - doubleToLongBits(7), - doubleToLongBits(6), - doubleToLongBits(8), - doubleToLongBits(8), - doubleToLongBits(8) - }}; + return new long[][]{ + { + doubleToLongBits(2), + doubleToLongBits(4), + doubleToLongBits(3), + doubleToLongBits(5), + doubleToLongBits(5), + doubleToLongBits(5) + }, { + doubleToLongBits(4), + doubleToLongBits(7), + doubleToLongBits(6), + doubleToLongBits(8), + doubleToLongBits(8), + doubleToLongBits(8) + } + }; } private static long[][] countWeights() { - return new long[][]{{ - doubleToLongBits(1.0), - doubleToLongBits(1.0), - doubleToLongBits(1.0), - doubleToLongBits(1.0), - doubleToLongBits(1.0), - doubleToLongBits(1.0) - }, { - doubleToLongBits(1.0), - doubleToLongBits(0.0), - doubleToLongBits(1.0), - doubleToLongBits(0.0), - doubleToLongBits(1.0), - doubleToLongBits(0.0) - }}; + return new long[][]{ + { + doubleToLongBits(1.0), + doubleToLongBits(1.0), + doubleToLongBits(1.0), + doubleToLongBits(1.0), + doubleToLongBits(1.0), + doubleToLongBits(1.0) + }, { + doubleToLongBits(1.0), + doubleToLongBits(0.0), + doubleToLongBits(1.0), + doubleToLongBits(0.0), + doubleToLongBits(1.0), + doubleToLongBits(0.0) + } + }; } private static long[] values() { - return new long[]{1, 1, 5, 5, 1, 1}; + return new long[]{ + 1, 1, 5, 5, 1, 1 + }; } } diff --git a/core/src/test/java/org/neo4j/gds/core/loading/ChunkedAdjacencyListsTest.java b/core/src/test/java/org/neo4j/gds/core/loading/ChunkedAdjacencyListsTest.java index e2af86d88f..c8e353a7a2 100644 --- a/core/src/test/java/org/neo4j/gds/core/loading/ChunkedAdjacencyListsTest.java +++ b/core/src/test/java/org/neo4j/gds/core/loading/ChunkedAdjacencyListsTest.java @@ -23,13 +23,10 @@ import org.neo4j.gds.core.compression.common.AdjacencyCompression; import org.neo4j.gds.core.compression.common.ZigZagLongDecoding; -import java.util.Arrays; - import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.fail; import static org.neo4j.gds.core.compression.common.ZigZagLongDecoding.Identity.INSTANCE; import static org.neo4j.gds.core.loading.AdjacencyPreAggregation.IGNORE_VALUE; -import static org.neo4j.gds.core.loading.ChunkedAdjacencyLists.NEXT_CHUNK_LENGTH; import static org.neo4j.gds.utils.StringFormatting.formatWithLocale; class ChunkedAdjacencyListsTest { @@ -38,19 +35,25 @@ class ChunkedAdjacencyListsTest { void shouldWriteSingleTargetList() { var adjacencyLists = ChunkedAdjacencyLists.of(0, 0); - var input = new long[]{42L, 1337L, 5L}; + var input = new long[]{ + 42L, 1337L, 5L + }; adjacencyLists.add(0, input, 0, 3, 3); - var expectedTargets = new long[]{42L, 1337L, 5L}; + var expectedTargets = new long[]{ + 42L, 1337L, 5L + }; var actualTargets = new long[3]; - adjacencyLists.consume((nodeId, targets, __, position, length) -> AdjacencyCompression.zigZagUncompressFrom( - actualTargets, - targets, - length, - position, - INSTANCE - )); + adjacencyLists.consume( + (nodeId, targets, __, position, length) -> AdjacencyCompression.zigZagUncompressFrom( + actualTargets, + targets, + length, + position, + INSTANCE + ) + ); assertThat(actualTargets).containsExactly(expectedTargets); } @@ -58,109 +61,50 @@ void shouldWriteSingleTargetList() { void shouldWriteMultipleTimesIntoTargetList() { var adjacencyLists = ChunkedAdjacencyLists.of(0, 0); - adjacencyLists.add(0, new long[]{42L, 1337L, 5L}, 0, 3, 3); - adjacencyLists.add(0, new long[]{42L, 1337L, 5L}, 1, 3, 2); + adjacencyLists.add(0, new long[]{ + 42L, 1337L, 5L + }, 0, 3, 3); + adjacencyLists.add(0, new long[]{ + 42L, 1337L, 5L + }, 1, 3, 2); - var expectedTargets = new long[]{42L, 1337L, 5L, 1337L, 5L}; + var expectedTargets = new long[]{ + 42L, 1337L, 5L, 1337L, 5L + }; var actualTargets = new long[5]; - adjacencyLists.consume((nodeId, targets, __, position, length) -> AdjacencyCompression.zigZagUncompressFrom( - actualTargets, - targets, - length, - position, - INSTANCE - )); - assertThat(actualTargets).containsExactly(expectedTargets); - } - - @Test - void shouldWriteLargeAdjacencyListsWithOverflow() { - var adjacencyLists = ChunkedAdjacencyLists.of(1, 0); - - var smallAdjacency = new long[37]; - Arrays.setAll(smallAdjacency, i -> i); - - var largeAdjacency = new long[NEXT_CHUNK_LENGTH[NEXT_CHUNK_LENGTH.length - 1] + 100]; - Arrays.setAll(largeAdjacency, i -> i + 42L); - - var smallProperties = new long[37]; - Arrays.setAll(smallProperties, i -> i); - - var largeProperties = new long[NEXT_CHUNK_LENGTH[NEXT_CHUNK_LENGTH.length - 1] + 100]; - Arrays.setAll(largeProperties, i -> i + 42L); - - adjacencyLists.add( - 0, - Arrays.copyOf(smallAdjacency, smallAdjacency.length), - new long[][]{smallProperties}, - 0, - smallAdjacency.length, - smallAdjacency.length - ); - adjacencyLists.add( - 0, - Arrays.copyOf(smallAdjacency, smallAdjacency.length), - new long[][]{smallProperties}, - 0, - smallAdjacency.length, - smallAdjacency.length - ); - adjacencyLists.add( - 0, - Arrays.copyOf(largeAdjacency, largeAdjacency.length), - new long[][]{largeProperties}, - 0, - largeAdjacency.length, - largeAdjacency.length - ); - - var expectedTargets = new long[smallAdjacency.length * 2 + largeAdjacency.length]; - System.arraycopy(smallAdjacency, 0, expectedTargets, 0, smallAdjacency.length); - System.arraycopy(smallAdjacency, 0, expectedTargets, smallAdjacency.length, smallAdjacency.length); - System.arraycopy(largeAdjacency, 0, expectedTargets, smallAdjacency.length * 2, largeAdjacency.length); - - var expectedProperties = new long[smallProperties.length * 2 + largeProperties.length]; - System.arraycopy(smallProperties, 0, expectedProperties, 0, smallProperties.length); - System.arraycopy(smallProperties, 0, expectedProperties, smallAdjacency.length, smallProperties.length); - System.arraycopy(largeProperties, 0, expectedProperties, smallAdjacency.length * 2, largeProperties.length); - - var actualTargets = new long[smallAdjacency.length * 2 + largeAdjacency.length]; - adjacencyLists.consume((nodeId, targets, properties, position, length) -> { - AdjacencyCompression.zigZagUncompressFrom( + adjacencyLists.consume( + (nodeId, targets, __, position, length) -> AdjacencyCompression.zigZagUncompressFrom( actualTargets, targets, length, position, INSTANCE - ); - - long[] actualProperties = new long[expectedProperties.length]; - var written = 0; - for (long[] propertyChunk : properties[0]) { - var valuesToCopy = Math.min(propertyChunk.length, length - written); - System.arraycopy(propertyChunk, 0, actualProperties, written, valuesToCopy); - written += valuesToCopy; - } - assertThat(Arrays.compare(expectedProperties, actualProperties)).isEqualTo(0); - }); - assertThat(Arrays.compare(expectedTargets, actualTargets)).isEqualTo(0); + ) + ); + assertThat(actualTargets).containsExactly(expectedTargets); } @Test void shouldWriteWithProperties() { var adjacencyLists = ChunkedAdjacencyLists.of(2, 0); - var input = new long[]{42L, 1337L, 5L, 6L}; - var properties = new long[][]{{42L, 1337L, 5L, 6L}, {8L, 8L, 8L, 8L}}; + var input = new long[]{ + 42L, 1337L, 5L, 6L + }; + var properties = new long[][]{ + { + 42L, 1337L, 5L, 6L + }, { + 8L, 8L, 8L, 8L + } + }; adjacencyLists.add(0, input, properties, 0, 4, 4); adjacencyLists.consume((nodeId, targets, actualProperties, position, length) -> { assertThat(actualProperties).hasNumberOfRows(2); - assertThat(actualProperties[0]).hasNumberOfRows(1); - assertThat(actualProperties[1]).hasNumberOfRows(1); - assertThat(actualProperties[0][0]).containsSequence(42L, 1337L, 5L, 6L); - assertThat(actualProperties[1][0]).containsSequence(8L, 8L, 8L, 8L); + assertThat(actualProperties[0]).containsSequence(42L, 1337L, 5L, 6L); + assertThat(actualProperties[1]).containsSequence(8L, 8L, 8L, 8L); }); } @@ -168,17 +112,23 @@ void shouldWriteWithProperties() { void shouldWriteWithPropertiesWithOffset() { var adjacencyLists = ChunkedAdjacencyLists.of(2, 0); - var input = new long[]{13L, 37L, 42L, 1337L, 5L, 6L}; - var properties = new long[][]{{0L, 0L, 42L, 1337L, 5L, 6L}, {0L, 0L, 8L, 8L, 8L, 8L}}; + var input = new long[]{ + 13L, 37L, 42L, 1337L, 5L, 6L + }; + var properties = new long[][]{ + { + 0L, 0L, 42L, 1337L, 5L, 6L + }, { + 0L, 0L, 8L, 8L, 8L, 8L + } + }; adjacencyLists.add(0, input, properties, 2, 6, 4); adjacencyLists.consume((nodeId, targets, actualProperties, position, length) -> { assertThat(actualProperties).hasNumberOfRows(2); - assertThat(actualProperties[0]).hasNumberOfRows(1); - assertThat(actualProperties[1]).hasNumberOfRows(1); - assertThat(actualProperties[0][0]).containsSequence(42L, 1337L, 5L, 6L); - assertThat(actualProperties[1][0]).containsSequence(8L, 8L, 8L, 8L); + assertThat(actualProperties[0]).containsSequence(42L, 1337L, 5L, 6L); + assertThat(actualProperties[1]).containsSequence(8L, 8L, 8L, 8L); }); } @@ -186,12 +136,18 @@ void shouldWriteWithPropertiesWithOffset() { void shouldAllowConsumptionOfAllElements() { var adjacencyLists = ChunkedAdjacencyLists.of(0, 0); - adjacencyLists.add(1, new long[]{42L, 1337L, 5L}, 0, 3, 3); - adjacencyLists.add(8, new long[]{1L, 2L}, 0, 2, 2); + adjacencyLists.add(1, new long[]{ + 42L, 1337L, 5L + }, 0, 3, 3); + adjacencyLists.add(8, new long[]{ + 1L, 2L + }, 0, 2, 2); // Skip 2 pages var largeIndex = 3 * 4096 + 1; - adjacencyLists.add(largeIndex, new long[]{42L, 42L}, 0, 2, 2); + adjacencyLists.add(largeIndex, new long[]{ + 42L, 42L + }, 0, 2, 2); adjacencyLists.consume((id, targets, properties, compressedBytesSize, compressedTargets) -> { assertThat(properties).isNull(); @@ -221,19 +177,25 @@ void shouldAllowConsumptionOfAllElements() { void addWithPreAggregation() { var adjacencyLists = ChunkedAdjacencyLists.of(0, 0); - var input = new long[]{42L, IGNORE_VALUE, IGNORE_VALUE, 1337L, 5L}; + var input = new long[]{ + 42L, IGNORE_VALUE, IGNORE_VALUE, 1337L, 5L + }; adjacencyLists.add(0, input, 0, 5, 3); - var expectedTargets = new long[]{42L, 1337L, 5L}; + var expectedTargets = new long[]{ + 42L, 1337L, 5L + }; var actualTargets = new long[3]; - adjacencyLists.consume((nodeId, targets, __, position, length) -> AdjacencyCompression.zigZagUncompressFrom( - actualTargets, - targets, - length, - position, - INSTANCE - )); + adjacencyLists.consume( + (nodeId, targets, __, position, length) -> AdjacencyCompression.zigZagUncompressFrom( + actualTargets, + targets, + length, + position, + INSTANCE + ) + ); assertThat(actualTargets).containsExactly(expectedTargets); } @@ -241,11 +203,19 @@ void addWithPreAggregation() { void addWithPreAggregatedWeights() { var adjacencyLists = ChunkedAdjacencyLists.of(1, 0); - var input = new long[]{42L, IGNORE_VALUE, 1337L, 5L}; - var properties = new long[][]{{3L, 2L, 3L, 4L}}; + var input = new long[]{ + 42L, IGNORE_VALUE, 1337L, 5L + }; + var properties = new long[][]{ + { + 3L, 2L, 3L, 4L + } + }; adjacencyLists.add(0, input, properties, 0, 4, 3); - var expectedTargets = new long[]{42L, 1337L, 5L}; + var expectedTargets = new long[]{ + 42L, 1337L, 5L + }; adjacencyLists.consume((nodeId, targets, actualProperties, position, length) -> { var actualTargets = new long[3]; @@ -259,8 +229,7 @@ void addWithPreAggregatedWeights() { assertThat(actualTargets).containsExactly(expectedTargets); assertThat(actualProperties).hasNumberOfRows(1); - assertThat(actualProperties[0]).hasNumberOfRows(1); - assertThat(actualProperties[0][0]).containsSequence(3L, 3L, 4L); + assertThat(actualProperties[0]).containsSequence(3L, 3L, 4L); }); } } diff --git a/core/src/test/java/org/neo4j/gds/core/loading/CypherFactoryTest.java b/core/src/test/java/org/neo4j/gds/core/loading/CypherFactoryTest.java index ccaa102680..ca3179597b 100644 --- a/core/src/test/java/org/neo4j/gds/core/loading/CypherFactoryTest.java +++ b/core/src/test/java/org/neo4j/gds/core/loading/CypherFactoryTest.java @@ -78,29 +78,34 @@ void setUp() { void testLoadCypher() { clearDb(); String query = " CREATE (n1 {partition: 6})-[:REL {prop: 1}]->(n2 {foo: 4.0})-[:REL {prop: 2}]->(n3)" + - " CREATE (n1)-[:REL {prop: 3}]->(n3)" + - " RETURN id(n1) AS id1, id(n2) AS id2, id(n3) AS id3"; + " CREATE (n1)-[:REL {prop: 3}]->(n3)" + + " RETURN id(n1) AS id1, id(n2) AS id2, id(n3) AS id3"; runQuery(query); String nodes = "MATCH (n) RETURN id(n) AS id, COALESCE(n.partition, 0) AS partition, COALESCE(n.foo, 5.0) AS foo"; String rels = "MATCH (n)-[r]->(m) WHERE type(r) = 'REL' " + - "RETURN id(n) AS source, id(m) AS target, coalesce(head(collect(r.prop)), 0)"; + "RETURN id(n) AS source, id(m) AS target, coalesce(head(collect(r.prop)), 0)"; - Graph graph = applyInTransaction(db, tx -> new CypherLoaderBuilder().databaseService(db) + Graph graph = applyInTransaction( + db, + tx -> new CypherLoaderBuilder().databaseService(db) .nodeQuery(nodes) .relationshipQuery(rels) .build() .graph() ); - assertGraphEquals(fromGdl( - "(a {partition: 6, foo: 5.0})" + - "(b {partition: 0, foo: 4.0})" + - "(c {partition: 0, foo: 5.0})" + - "(a)-[{w:1.0}]->(b)" + - "(a)-[{w:3.0}]->(c)" + - "(b)-[{w:2.0}]->(c)" - ), graph); + assertGraphEquals( + fromGdl( + "(a {partition: 6, foo: 5.0})" + + "(b {partition: 0, foo: 4.0})" + + "(c {partition: 0, foo: 5.0})" + + "(a)-[{w:1.0}]->(b)" + + "(a)-[{w:3.0}]->(c)" + + "(b)-[{w:2.0}]->(c)" + ), + graph + ); } @Test @@ -164,8 +169,8 @@ void doubleListNodeProperty() { @Test void doubleListWithEmptyList() { var nodeQuery = "WITH [0, 1] AS ids, [[1.3, 3.7], []] AS properties " + - "UNWIND ids AS id " + - "RETURN id, properties[id] AS list"; + "UNWIND ids AS id " + + "RETURN id, properties[id] AS list"; var builder = new CypherLoaderBuilder() .databaseService(db) @@ -180,8 +185,8 @@ void doubleListWithEmptyList() { @Test void longListWithEmptyList() { var nodeQuery = "WITH [0, 1] AS ids, [[1, 3, 3, 7], []] AS properties " + - "UNWIND ids AS id " + - "RETURN id, properties[id] AS list"; + "UNWIND ids AS id " + + "RETURN id, properties[id] AS list"; var builder = new CypherLoaderBuilder() .databaseService(db) @@ -269,9 +274,9 @@ void testMultipleNodeProperties() { clearDb(); runQuery( "CREATE" + - " ({prop1: 1})" + - ", ({prop2: 2})" + - ", ({prop3: 3})" + " ({prop1: 1})" + + ", ({prop2: 2})" + + ", ({prop3: 3})" ); PropertyMapping prop1 = PropertyMapping.of("prop1", 0); PropertyMapping prop2 = PropertyMapping.of("prop2", 0); @@ -282,8 +287,8 @@ void testMultipleNodeProperties() { .graph(); String gdl = "(a {prop1: 1, prop2: 0, prop3: 0})" + - "(b {prop1: 0, prop2: 2, prop3: 0})" + - "(c {prop1: 0, prop2: 0, prop3: 3})"; + "(b {prop1: 0, prop2: 2, prop3: 0})" + + "(c {prop1: 0, prop2: 0, prop3: 3})"; assertGraphEquals(fromGdl(gdl), graph); } @@ -293,11 +298,11 @@ void testMultipleRelationshipProperties() { clearDb(); runQuery( "CREATE" + - " (n1)" + - ", (n2)" + - ", (n1)-[:REL {prop1: 1.0}]->(n2)" + - ", (n1)-[:REL {prop2: 2.0}]->(n2)" + - ", (n1)-[:REL {prop3: 3.0}]->(n2)" + " (n1)" + + ", (n2)" + + ", (n1)-[:REL {prop1: 1.0}]->(n2)" + + ", (n1)-[:REL {prop2: 2.0}]->(n2)" + + ", (n1)-[:REL {prop3: 3.0}]->(n2)" ); PropertyMapping prop1 = PropertyMapping.of("prop1", 0D); PropertyMapping prop2 = PropertyMapping.of("prop2", 0D); @@ -308,23 +313,43 @@ void testMultipleRelationshipProperties() { .withDefaultAggregation(Aggregation.DEFAULT) .graphStore(); - String expectedGraph = - " (a)-[{w: %f}]->(b)" + + String expectedGraph = " (a)-[{w: %f}]->(b)" + ", (a)-[{w: %f}]->(b)" + ", (a)-[{w: %f}]->(b)"; assertGraphEquals( - fromGdl(formatWithLocale(expectedGraph, 1.0f, prop1.defaultValue().doubleValue(), prop1.defaultValue().doubleValue())), + fromGdl( + formatWithLocale( + expectedGraph, + 1.0f, + prop1.defaultValue().doubleValue(), + prop1.defaultValue().doubleValue() + ) + ), graphs.getGraph(ALL_RELATIONSHIPS, Optional.of(prop1.propertyKey())) ); assertGraphEquals( - fromGdl(formatWithLocale(expectedGraph, prop2.defaultValue().doubleValue(), 2.0, prop2.defaultValue().doubleValue())), + fromGdl( + formatWithLocale( + expectedGraph, + prop2.defaultValue().doubleValue(), + 2.0, + prop2.defaultValue().doubleValue() + ) + ), graphs.getGraph(ALL_RELATIONSHIPS, Optional.of(prop2.propertyKey())) ); assertGraphEquals( - fromGdl(formatWithLocale(expectedGraph, prop3.defaultValue().doubleValue(), prop3.defaultValue().doubleValue(), 3.0)), + fromGdl( + formatWithLocale( + expectedGraph, + prop3.defaultValue().doubleValue(), + prop3.defaultValue().doubleValue(), + 3.0 + ) + ), graphs.getGraph(ALL_RELATIONSHIPS, Optional.of(prop3.propertyKey())) ); } @@ -347,15 +372,15 @@ void loadGraphWithParameterizedCypherQuery() { void testLoadingGraphWithLabelInformation() { clearDb(); String query = "CREATE" + - " (a:A)" + - ", (b:B)" + - ", (c:C)" + - ", (ab:A:B)" + - "CREATE" + - " (a)-[:REL]->(b)" + - ", (a)-[:REL]->(c)" + - ", (a)-[:REL]->(ab)" + - ", (c)-[:REL]->(a)"; + " (a:A)" + + ", (b:B)" + + ", (c:C)" + + ", (ab:A:B)" + + "CREATE" + + " (a)-[:REL]->(b)" + + ", (a)-[:REL]->(c)" + + ", (a)-[:REL]->(ab)" + + ", (c)-[:REL]->(a)"; runQuery(query); @@ -438,13 +463,15 @@ void canLoadArrayNodeProperties() { String nodes = "MATCH (n) RETURN id(n) AS id, n.longArray as longArray, n.doubleArray as doubleArray"; String rels = "MATCH (n)" + - "RETURN id(n) AS source, id(n) AS target"; + "RETURN id(n) AS source, id(n) AS target"; - Graph graph = applyInTransaction(db, tx -> new CypherLoaderBuilder().databaseService(db) - .nodeQuery(nodes) - .relationshipQuery(rels) - .build() - .graph() + Graph graph = applyInTransaction( + db, + tx -> new CypherLoaderBuilder().databaseService(db) + .nodeQuery(nodes) + .relationshipQuery(rels) + .build() + .graph() ); assertGraphEquals(fromGdl("(a {longArray: [42L], doubleArray: [42.0D]}), (a)-->(a)"), graph); @@ -524,24 +551,24 @@ private static Stream memoryEstimationVariants() { "Topology Only", "MATCH (n) RETURN id(n) as id", "MATCH (n)-[r]->(m) RETURN id(n) AS source, id(m) AS target", - 1202360, - 1202360 + 1202280, + 1202280 ), Arguments.of( "Node properties", "MATCH (n) RETURN id(n) as id, n.id as idProp", "MATCH (n)-[r]->(m) RETURN id(n) AS source, id(m) AS target", - 1300800, - 1300800 + 1300720, + 1300720 ), Arguments.of( "Relationship properties", "MATCH (n) RETURN id(n) as id", "MATCH (n)-[r]->(m) RETURN id(n) AS source, id(m) AS target, r.prop as prop", - 1692720, - 1692720 + 1692640, + 1692640 ) ); } diff --git a/core/src/test/java/org/neo4j/gds/core/loading/NativeFactoryTest.java b/core/src/test/java/org/neo4j/gds/core/loading/NativeFactoryTest.java index a04f324c73..5f0b456357 100644 --- a/core/src/test/java/org/neo4j/gds/core/loading/NativeFactoryTest.java +++ b/core/src/test/java/org/neo4j/gds/core/loading/NativeFactoryTest.java @@ -47,11 +47,13 @@ void memoryEstimationBitMapDisabled() { .build(); var memoryEstimation = new AtomicReference(); - memoryEstimation.set(NativeFactory.getMemoryEstimation( - NodeProjections.all(), - RelationshipProjections.single(RelationshipType.ALL_RELATIONSHIPS, RelationshipProjection.ALL), - false - )); + memoryEstimation.set( + NativeFactory.getMemoryEstimation( + NodeProjections.all(), + RelationshipProjections.single(RelationshipType.ALL_RELATIONSHIPS, RelationshipProjection.ALL), + false + ) + ); var estimate = memoryEstimation.get().estimate(dimensions, 1); @@ -83,8 +85,8 @@ void memoryEstimationForSingleProjection() { .getMemoryEstimation(nodeProjections, relationshipProjections, true) .estimate(dimensions, 1); - assertEquals(6_828_526_824L, estimate.memoryUsage().min); - assertEquals(7_633_833_192L, estimate.memoryUsage().max); + assertEquals(6_828_526_776L, estimate.memoryUsage().min); + assertEquals(7_633_833_144L, estimate.memoryUsage().max); } @Test @@ -112,8 +114,8 @@ void memoryEstimationForIndexedProjection() { .getMemoryEstimation(nodeProjections, relationshipProjections, true) .estimate(dimensions, 1); - assertEquals(12_056_534_496L, estimate.memoryUsage().min); - assertEquals(13_667_147_232L, estimate.memoryUsage().max); + assertEquals(12_056_534_400L, estimate.memoryUsage().min); + assertEquals(13_667_147_136L, estimate.memoryUsage().max); } @Test @@ -135,7 +137,7 @@ void memoryEstimationForMultipleProjections() { .getMemoryEstimation(nodeProjections, relationshipProjections, true) .estimate(dimensions, 1); - assertEquals(12_056_534_496L, estimate.memoryUsage().min); - assertEquals(13_667_147_232L, estimate.memoryUsage().max); + assertEquals(12_056_534_400L, estimate.memoryUsage().min); + assertEquals(13_667_147_136L, estimate.memoryUsage().max); } } From aeb3eb97181c3bd921bbe261d4e9c74028ad6646 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20Kie=C3=9Fling?= Date: Wed, 21 Jun 2023 12:05:50 +0200 Subject: [PATCH 115/273] Fix memory estimation test --- .../neo4j/gds/catalog/GraphProjectProcEstimateTest.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphProjectProcEstimateTest.java b/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphProjectProcEstimateTest.java index 22d215e805..88b03872e3 100644 --- a/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphProjectProcEstimateTest.java +++ b/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphProjectProcEstimateTest.java @@ -108,8 +108,8 @@ void virtualEstimateHeapPercentage(SoftAssertions softly) { runQueryWithRowConsumer(query, map("relProjection", relProjection), row -> { - softly.assertThat(row.getNumber("bytesMin").longValue()).isEqualTo(68686368); - softly.assertThat(row.getNumber("bytesMax").longValue()).isEqualTo(68686368); + softly.assertThat(row.getNumber("bytesMin").longValue()).isEqualTo(68686120); + softly.assertThat(row.getNumber("bytesMax").longValue()).isEqualTo(68686120); softly.assertThat(row.getNumber("heapPercentageMin").doubleValue()).isEqualTo(expectedPercentage); softly.assertThat(row.getNumber("heapPercentageMax").doubleValue()).isEqualTo(expectedPercentage); } @@ -207,8 +207,8 @@ void computeMemoryEstimationForVirtualGraphWithLargeValues(SoftAssertions softly runQueryWithRowConsumer( query, row -> { - softly.assertThat(row.getNumber("bytesMin").longValue()).isEqualTo(344128414784L); - softly.assertThat(row.getNumber("bytesMax").longValue()).isEqualTo(384930604096L); + softly.assertThat(row.getNumber("bytesMin").longValue()).isEqualTo(344128414632L); + softly.assertThat(row.getNumber("bytesMax").longValue()).isEqualTo(384930603944L); softly.assertThat(row.getNumber("nodeCount").longValue()).isEqualTo(5_000_000_000L); softly.assertThat(row.getNumber("relationshipCount").longValue()).isEqualTo(20_000_000_000L); } From ae763c377b0cbf6e16c071492ce6c5d990c91c67 Mon Sep 17 00:00:00 2001 From: Mats Rydberg Date: Fri, 9 Jun 2023 14:42:34 +0200 Subject: [PATCH 116/273] Sort algo references alphanumerically And move to correct tier sections --- .../algorithm-references.adoc | 454 +++++++++--------- 1 file changed, 227 insertions(+), 227 deletions(-) diff --git a/doc/modules/ROOT/pages/operations-reference/algorithm-references.adoc b/doc/modules/ROOT/pages/operations-reference/algorithm-references.adoc index 9ba4d322bb..b7653fa121 100644 --- a/doc/modules/ROOT/pages/operations-reference/algorithm-references.adoc +++ b/doc/modules/ROOT/pages/operations-reference/algorithm-references.adoc @@ -11,6 +11,106 @@ [opts=header,cols="1, 2"] |=== | Algorithm name | Operation +.8+<.^| xref:algorithms/delta-single-source.adoc[All Shortest Paths Delta-Stepping] +| `gds.allShortestPaths.delta.stream` +| `gds.allShortestPaths.delta.stream.estimate` +| `gds.allShortestPaths.delta.write` +| `gds.allShortestPaths.delta.write.estimate` +| `gds.allShortestPaths.delta.mutate` +| `gds.allShortestPaths.delta.mutate.estimate` +| `gds.allShortestPaths.delta.stats` +| `gds.allShortestPaths.delta.stats.estimate` +.6+<.^| xref:algorithms/dijkstra-single-source.adoc[All Shortest Paths Dijkstra] +| `gds.allShortestPaths.dijkstra.stream` +| `gds.allShortestPaths.dijkstra.stream.estimate` +| `gds.allShortestPaths.dijkstra.write` +| `gds.allShortestPaths.dijkstra.write.estimate` +| `gds.allShortestPaths.dijkstra.mutate` +| `gds.allShortestPaths.dijkstra.mutate.estimate` +.8+<.^|xref:algorithms/article-rank.adoc[ArticleRank] +| `gds.articleRank.mutate` +| `gds.articleRank.mutate.estimate` +| `gds.articleRank.write` +| `gds.articleRank.write.estimate` +| `gds.articleRank.stream` +| `gds.articleRank.stream.estimate` +| `gds.articleRank.stats` +| `gds.articleRank.stats.estimate` +.8+<.^| xref:algorithms/bellman-ford-single-source.adoc[Bellman-Ford] +| `gds.bellmanFord.mutate` +| `gds.bellmanFord.mutate.estimate` +| `gds.bellmanFord.stats` +| `gds.bellmanFord.stats.estimate` +| `gds.bellmanFord.stream` +| `gds.bellmanFord.stream.estimate` +| `gds.bellmanFord.write` +| `gds.bellmanFord.write.estimate` +.8+<.^| xref:algorithms/betweenness-centrality.adoc#algorithms-betweenness-centrality-syntax[Betweenness Centrality] +| `gds.betweenness.stream` +| `gds.betweenness.stream.estimate` +| `gds.betweenness.stats` +| `gds.betweenness.stats.estimate` +| `gds.betweenness.mutate` +| `gds.betweenness.mutate.estimate` +| `gds.betweenness.write` +| `gds.betweenness.write.estimate` +.6+<.^|xref:algorithms/bfs.adoc[Breadth First Search] +| `gds.bfs.mutate` +| `gds.bfs.mutate.estimate` +| `gds.bfs.stream` +| `gds.bfs.stream.estimate` +| `gds.bfs.stats` +| `gds.bfs.stats.estimate` +.8+<.^| xref:algorithms/degree-centrality.adoc[Degree Centrality] +| `gds.degree.mutate` +| `gds.degree.mutate.estimate` +| `gds.degree.stats` +| `gds.degree.stats.estimate` +| `gds.degree.stream` +| `gds.degree.stream.estimate` +| `gds.degree.write` +| `gds.degree.write.estimate` +.4+<.^|xref:algorithms/dfs.adoc[Depth First Search] +| `gds.dfs.mutate` +| `gds.dfs.mutate.estimate` +| `gds.dfs.stream` +| `gds.dfs.stream.estimate` +.8+<.^|xref:algorithms/eigenvector-centrality.adoc[Eigenvector] +| `gds.eigenvector.mutate` +| `gds.eigenvector.mutate.estimate` +| `gds.eigenvector.write` +| `gds.eigenvector.write.estimate` +| `gds.eigenvector.stream` +| `gds.eigenvector.stream.estimate` +| `gds.eigenvector.stats` +| `gds.eigenvector.stats.estimate` +.8+<.^| xref:machine-learning/node-embeddings/fastrp.adoc[Fast Random Projection] +| `gds.fastRP.mutate` +| `gds.fastRP.mutate.estimate` +| `gds.fastRP.stats` +| `gds.fastRP.stats.estimate` +| `gds.fastRP.stream` +| `gds.fastRP.stream.estimate` +| `gds.fastRP.write` +| `gds.fastRP.write.estimate` +.8+<.^| xref:algorithms/k-core.adoc[K-Core Decomposition] +| `gds.kcore.stats` +| `gds.kcore.stats.estimate` +| `gds.kcore.stream` +| `gds.kcore.stream.estimate` +| `gds.kcore.mutate` +| `gds.kcore.mutate.estimate` +| `gds.kcore.write` +| `gds.kcore.write.estimate` +.8+<.^|xref:algorithms/knn.adoc[K-Nearest Neighbors] +| `gds.knn.mutate` +| `gds.knn.mutate.estimate` +| `gds.knn.stats` +| `gds.knn.stats.estimate` +| `gds.knn.stream` +| `gds.knn.stream.estimate` +| `gds.knn.write` +| `gds.knn.write.estimate` .8+<.^|xref:algorithms/label-propagation.adoc#algorithms-label-propagation-syntax[Label Propagation] | `gds.labelPropagation.mutate` | `gds.labelPropagation.mutate.estimate` @@ -20,6 +120,15 @@ | `gds.labelPropagation.stream.estimate` | `gds.labelPropagation.stats` | `gds.labelPropagation.stats.estimate` +.8+<.^| xref:algorithms/local-clustering-coefficient.adoc#algorithms-local-clustering-coefficient-syntax[Local Clustering Coefficient] +| `gds.localClusteringCoefficient.stream` +| `gds.localClusteringCoefficient.stream.estimate` +| `gds.localClusteringCoefficient.stats` +| `gds.localClusteringCoefficient.stats.estimate` +| `gds.localClusteringCoefficient.write` +| `gds.localClusteringCoefficient.write.estimate` +| `gds.localClusteringCoefficient.mutate` +| `gds.localClusteringCoefficient.mutate.estimate` .8+<.^|xref:algorithms/louvain.adoc#algorithms-louvain-syntax[Louvain] | `gds.louvain.mutate` | `gds.louvain.mutate.estimate` @@ -47,87 +156,27 @@ | `gds.pageRank.stream.estimate` | `gds.pageRank.stats` | `gds.pageRank.stats.estimate` -.8+<.^|xref:algorithms/wcc.adoc#algorithms-wcc-syntax[Weakly Connected Components] -| `gds.wcc.mutate` -| `gds.wcc.mutate.estimate` -| `gds.wcc.write` -| `gds.wcc.write.estimate` -| `gds.wcc.stream` -| `gds.wcc.stream.estimate` -| `gds.wcc.stats` -| `gds.wcc.stats.estimate` -.8+<.^| xref:algorithms/triangle-count.adoc#algorithms-triangle-count-syntax[Triangle Count] -| `gds.triangleCount.stream` -| `gds.triangleCount.stream.estimate` -| `gds.triangleCount.stats` -| `gds.triangleCount.stats.estimate` -| `gds.triangleCount.write` -| `gds.triangleCount.write.estimate` -| `gds.triangleCount.mutate` -| `gds.triangleCount.mutate.estimate` -.8+<.^| xref:algorithms/local-clustering-coefficient.adoc#algorithms-local-clustering-coefficient-syntax[Local Clustering Coefficient] -| `gds.localClusteringCoefficient.stream` -| `gds.localClusteringCoefficient.stream.estimate` -| `gds.localClusteringCoefficient.stats` -| `gds.localClusteringCoefficient.stats.estimate` -| `gds.localClusteringCoefficient.write` -| `gds.localClusteringCoefficient.write.estimate` -| `gds.localClusteringCoefficient.mutate` -| `gds.localClusteringCoefficient.mutate.estimate` -.8+<.^| xref:algorithms/betweenness-centrality.adoc#algorithms-betweenness-centrality-syntax[Betweenness Centrality] -| `gds.betweenness.stream` -| `gds.betweenness.stream.estimate` -| `gds.betweenness.stats` -| `gds.betweenness.stats.estimate` -| `gds.betweenness.mutate` -| `gds.betweenness.mutate.estimate` -| `gds.betweenness.write` -| `gds.betweenness.write.estimate` -.8+<.^| xref:machine-learning/node-embeddings/fastrp.adoc[Fast Random Projection] -| `gds.fastRP.mutate` -| `gds.fastRP.mutate.estimate` -| `gds.fastRP.stats` -| `gds.fastRP.stats.estimate` -| `gds.fastRP.stream` -| `gds.fastRP.stream.estimate` -| `gds.fastRP.write` -| `gds.fastRP.write.estimate` -.8+<.^| xref:algorithms/degree-centrality.adoc[Degree Centrality] -| `gds.degree.mutate` -| `gds.degree.mutate.estimate` -| `gds.degree.stats` -| `gds.degree.stats.estimate` -| `gds.degree.stream` -| `gds.degree.stream.estimate` -| `gds.degree.write` -| `gds.degree.write.estimate` -.8+<.^|xref:algorithms/article-rank.adoc[ArticleRank] -| `gds.articleRank.mutate` -| `gds.articleRank.mutate.estimate` -| `gds.articleRank.write` -| `gds.articleRank.write.estimate` -| `gds.articleRank.stream` -| `gds.articleRank.stream.estimate` -| `gds.articleRank.stats` -| `gds.articleRank.stats.estimate` -.8+<.^|xref:algorithms/eigenvector-centrality.adoc[Eigenvector] -| `gds.eigenvector.mutate` -| `gds.eigenvector.mutate.estimate` -| `gds.eigenvector.write` -| `gds.eigenvector.write.estimate` -| `gds.eigenvector.stream` -| `gds.eigenvector.stream.estimate` -| `gds.eigenvector.stats` -| `gds.eigenvector.stats.estimate` -.8+<.^| xref:algorithms/delta-single-source.adoc[All Shortest Paths Delta-Stepping] -| `gds.allShortestPaths.delta.stream` -| `gds.allShortestPaths.delta.stream.estimate` -| `gds.allShortestPaths.delta.write` -| `gds.allShortestPaths.delta.write.estimate` -| `gds.allShortestPaths.delta.mutate` -| `gds.allShortestPaths.delta.mutate.estimate` -| `gds.allShortestPaths.delta.stats` -| `gds.allShortestPaths.delta.stats.estimate` +.4+<.^|xref:algorithms/random-walk.adoc[Random Walk] +| `gds.randomWalk.stats` +| `gds.randomWalk.stats.estimate` +| `gds.randomWalk.stream` +| `gds.randomWalk.stream.estimate` +.8+<.^|xref:algorithms/scale-properties.adoc[Scale Properties] +| `gds.scaleProperties.mutate` +| `gds.scaleProperties.mutate.estimate` +| `gds.scaleProperties.stream` +| `gds.scaleProperties.stream.estimate` +| `gds.scaleProperties.stats` +| `gds.scaleProperties.stats.estimate` +| `gds.scaleProperties.write` +| `gds.scaleProperties.write.estimate` +.6+<.^| xref:algorithms/astar.adoc[Shortest Path AStar] +| `gds.shortestPath.astar.stream` +| `gds.shortestPath.astar.stream.estimate` +| `gds.shortestPath.astar.write` +| `gds.shortestPath.astar.write.estimate` +| `gds.shortestPath.astar.mutate` +| `gds.shortestPath.astar.mutate.estimate` .6+<.^| xref:algorithms/dijkstra-source-target.adoc[Shortest Path Dijkstra] | `gds.shortestPath.dijkstra.stream` | `gds.shortestPath.dijkstra.stream.estimate` @@ -135,13 +184,6 @@ | `gds.shortestPath.dijkstra.write.estimate` | `gds.shortestPath.dijkstra.mutate` | `gds.shortestPath.dijkstra.mutate.estimate` -.6+<.^| xref:algorithms/dijkstra-single-source.adoc[All Shortest Paths Dijkstra] -| `gds.allShortestPaths.dijkstra.stream` -| `gds.allShortestPaths.dijkstra.stream.estimate` -| `gds.allShortestPaths.dijkstra.write` -| `gds.allShortestPaths.dijkstra.write.estimate` -| `gds.allShortestPaths.dijkstra.mutate` -| `gds.allShortestPaths.dijkstra.mutate.estimate` .6+<.^| xref:algorithms/yens.adoc[Shortest Paths Yens] | `gds.shortestPath.yens.stream` | `gds.shortestPath.yens.stream.estimate` @@ -149,55 +191,31 @@ | `gds.shortestPath.yens.write.estimate` | `gds.shortestPath.yens.mutate` | `gds.shortestPath.yens.mutate.estimate` -.6+<.^| xref:algorithms/astar.adoc[Shortest Path AStar] -| `gds.shortestPath.astar.stream` -| `gds.shortestPath.astar.stream.estimate` -| `gds.shortestPath.astar.write` -| `gds.shortestPath.astar.write.estimate` -| `gds.shortestPath.astar.mutate` -| `gds.shortestPath.astar.mutate.estimate` .6+<.^|xref:algorithms/similarity-functions.adoc[Similarity functions] -| `gds.similarity.cosine` -| `gds.similarity.euclidean` -| `gds.similarity.euclideanDistance` -| `gds.similarity.jaccard` -| `gds.similarity.overlap` -| `gds.similarity.pearson` -.8+<.^|xref:algorithms/knn.adoc[K-Nearest Neighbors] -| `gds.knn.mutate` -| `gds.knn.mutate.estimate` -| `gds.knn.stats` -| `gds.knn.stats.estimate` -| `gds.knn.stream` -| `gds.knn.stream.estimate` -| `gds.knn.write` -| `gds.knn.write.estimate` -.6+<.^|xref:algorithms/bfs.adoc[BFS] -| `gds.bfs.mutate` -| `gds.bfs.mutate.estimate` -| `gds.bfs.stream` -| `gds.bfs.stream.estimate` -| `gds.bfs.stats` -| `gds.bfs.stats.estimate` -.4+<.^|xref:algorithms/dfs.adoc[Depth First Search] -| `gds.dfs.mutate` -| `gds.dfs.mutate.estimate` -| `gds.dfs.stream` -| `gds.dfs.stream.estimate` -.4+<.^|xref:algorithms/random-walk.adoc[Random Walk] -| `gds.randomWalk.stats` -| `gds.randomWalk.stats.estimate` -| `gds.randomWalk.stream` -| `gds.randomWalk.stream.estimate` -.8+<.^|xref:algorithms/scale-properties.adoc[Scale Properties] -| `gds.scaleProperties.mutate` -| `gds.scaleProperties.mutate.estimate` -| `gds.scaleProperties.stream` -| `gds.scaleProperties.stream.estimate` -| `gds.scaleProperties.stats` -| `gds.scaleProperties.stats.estimate` -| `gds.scaleProperties.write` -| `gds.scaleProperties.write.estimate` +| `_gds.similarity.cosine_` +| `_gds.similarity.euclidean_` +| `_gds.similarity.euclideanDistance_` +| `_gds.similarity.jaccard_` +| `_gds.similarity.overlap_` +| `_gds.similarity.pearson_` +.8+<.^| xref:algorithms/triangle-count.adoc#algorithms-triangle-count-syntax[Triangle Count] +| `gds.triangleCount.stream` +| `gds.triangleCount.stream.estimate` +| `gds.triangleCount.stats` +| `gds.triangleCount.stats.estimate` +| `gds.triangleCount.write` +| `gds.triangleCount.write.estimate` +| `gds.triangleCount.mutate` +| `gds.triangleCount.mutate.estimate` +.8+<.^|xref:algorithms/wcc.adoc#algorithms-wcc-syntax[Weakly Connected Components] +| `gds.wcc.mutate` +| `gds.wcc.mutate.estimate` +| `gds.wcc.write` +| `gds.wcc.write.estimate` +| `gds.wcc.stream` +| `gds.wcc.stream.estimate` +| `gds.wcc.stats` +| `gds.wcc.stats.estimate` |=== [[beta-tier]] @@ -225,6 +243,20 @@ | `gds.beta.graphSage.write.estimate` | `gds.beta.graphSage.train` | `gds.beta.graphSage.train.estimate` +.4+<.^|xref:machine-learning/node-embeddings/hashgnn.adoc[HashGNN] +| `gds.beta.hashgnn.mutate` +| `gds.beta.hashgnn.mutate.estimate` +| `gds.beta.hashgnn.stream` +| `gds.beta.hashgnn.stream.estimate` +.8+<.^| xref:algorithms/celf.adoc[Influence Maximization - CELF] +| `gds.beta.influenceMaximization.celf.mutate` +| `gds.beta.influenceMaximization.celf.mutate.estimate` +| `gds.beta.influenceMaximization.celf.stats` +| `gds.beta.influenceMaximization.celf.stats.estimate` +| `gds.beta.influenceMaximization.celf.stream` +| `gds.beta.influenceMaximization.celf.stream.estimate` +| `gds.beta.influenceMaximization.celf.write` +| `gds.beta.influenceMaximization.celf.write.estimate` .8+<.^|xref:algorithms/k1coloring.adoc[K1Coloring] | `gds.beta.k1coloring.mutate` | `gds.beta.k1coloring.mutate.estimate` @@ -234,6 +266,29 @@ | `gds.beta.k1coloring.stream.estimate` | `gds.beta.k1coloring.write` | `gds.beta.k1coloring.write.estimate` +.8+<.^| xref:algorithms/kmeans.adoc[Kmeans] +| `gds.beta.kmeans.mutate` +| `gds.beta.kmeans.mutate.estimate` +| `gds.beta.kmeans.stats` +| `gds.beta.kmeans.stats.estimate` +| `gds.beta.kmeans.stream` +| `gds.beta.kmeans.stream.estimate` +| `gds.beta.kmeans.write` +| `gds.beta.kmeans.write.estimate` +.8+<.^| xref:algorithms/leiden.adoc[Leiden] +| `gds.beta.leiden.mutate` +| `gds.beta.leiden.mutate.estimate` +| `gds.beta.leiden.stats` +| `gds.beta.leiden.stats.estimate` +| `gds.beta.leiden.stream` +| `gds.beta.leiden.stream.estimate` +| `gds.beta.leiden.write` +| `gds.beta.leiden.write.estimate` +.4+<.^| xref:algorithms/directed-steiner-tree.adoc[Minimum Directed Steiner Tree] +| `gds.beta.steinerTree.mutate` +| `gds.beta.steinerTree.stats` +| `gds.beta.steinerTree.stream` +| `gds.beta.steinerTree.write` .6+<.^| xref:algorithms/modularity-optimization.adoc[Modularity Optimization] | `gds.beta.modularityOptimization.mutate` | `gds.beta.modularityOptimization.mutate.estimate` @@ -248,15 +303,15 @@ | `gds.beta.node2vec.stream.estimate` | `gds.beta.node2vec.write` | `gds.beta.node2vec.write.estimate` -.8+<.^| xref:algorithms/celf.adoc[Influence Maximization - CELF] -| `gds.beta.influenceMaximization.celf.mutate` -| `gds.beta.influenceMaximization.celf.mutate.estimate` -| `gds.beta.influenceMaximization.celf.stats` -| `gds.beta.influenceMaximization.celf.stats.estimate` -| `gds.beta.influenceMaximization.celf.stream` -| `gds.beta.influenceMaximization.celf.stream.estimate` -| `gds.beta.influenceMaximization.celf.write` -| `gds.beta.influenceMaximization.celf.write.estimate` +.8+<.^|xref:algorithms/minimum-weight-spanning-tree.adoc[Spanning Tree] +| `gds.beta.spanningTree.mutate` +| `gds.beta.spanningTree.mutate.estimate` +| `gds.beta.spanningTree.stats` +| `gds.beta.spanningTree.stats.estimate` +| `gds.beta.spanningTree.stream` +| `gds.beta.spanningTree.stream.estimate` +| `gds.beta.spanningTree.write` +| `gds.beta.spanningTree.write.estimate` |=== [[alpha-tier]] @@ -268,6 +323,7 @@ [opts=header,cols="1, 2"] |=== |Algorithm name | Operation +| xref:alpha-algorithms/adamic-adar.adoc[Adamic Adar] | `_gds.alpha.linkprediction.adamicAdar_` .1+<.^|xref:alpha-algorithms/all-pairs-shortest-path.adoc[All Shortest Paths] | `gds.alpha.allShortestPaths.stream` .4+<.^|xref:algorithms/alpha/approx-max-k-cut.adoc[Approximate Maximum k-cut] @@ -275,6 +331,23 @@ | `gds.alpha.maxkcut.mutate.estimate` | `gds.alpha.maxkcut.stream` | `gds.alpha.maxkcut.stream.estimate` +| xref:alpha-algorithms/common-neighbors.adoc[Common Neighbors] | `_gds.alpha.linkprediction.commonNeighbors_` +.1+<.^| xref:algorithms/alpha/conductance.adoc[Conductance] +| `gds.alpha.conductance.stream` +.4+<.^| xref:algorithms/alpha/filtered-knn.adoc[Filtered KNN] +| `gds.alpha.knn.filtered.mutate` +| `gds.alpha.knn.filtered.stats` +| `gds.alpha.knn.filtered.stream` +| `gds.alpha.knn.filtered.write` +.8+<.^| xref:algorithms/alpha/filtered-node-similarity.adoc[Filtered NodeSimilarity] +| `gds.alpha.nodeSimilarity.filtered.mutate` +| `gds.alpha.nodeSimilarity.filtered.mutate.estimate` +| `gds.alpha.nodeSimilarity.filtered.stats` +| `gds.alpha.nodeSimilarity.filtered.stats.estimate` +| `gds.alpha.nodeSimilarity.filtered.stream` +| `gds.alpha.nodeSimilarity.filtered.stream.estimate` +| `gds.alpha.nodeSimilarity.filtered.write` +| `gds.alpha.nodeSimilarity.filtered.write.estimate` .2+<.^|xref:algorithms/harmonic-centrality.adoc[Harmonic Centrality] | `gds.alpha.closeness.harmonic.stream` | `gds.alpha.closeness.harmonic.write` @@ -287,9 +360,11 @@ | `gds.alpha.hits.stream.estimate` | `gds.alpha.hits.write` | `gds.alpha.hits.write.estimate` -.2+<.^|xref:algorithms/strongly-connected-components.adoc[Strongly Connected Components] -| `gds.alpha.scc.stream` -| `gds.alpha.scc.write` +.1+<.^|xref:alpha-algorithms/k-minimum-weight-spanning-tree.adoc[ k-Spanning Tree] +| `gds.alpha.kSpanningTree.write` +| xref:alpha-algorithms/preferential-attachment.adoc[Preferential Attachment] | `_gds.alpha.linkprediction.preferentialAttachment_` +| xref:alpha-algorithms/resource-allocation.adoc[Resource Allocation] | `_gds.alpha.linkprediction.resourceAllocation_` +| xref:alpha-algorithms/same-community.adoc[Same Community] | `_gds.alpha.linkprediction.sameCommunity_` .8+<.^|xref:algorithms/sllpa.adoc[Speaker-Listener Label Propagation] | `gds.alpha.sllpa.mutate` | `gds.alpha.sllpa.mutate.estimate` @@ -299,93 +374,18 @@ | `gds.alpha.sllpa.stream.estimate` | `gds.alpha.sllpa.write` | `gds.alpha.sllpa.write.estimate` -.8+<.^|xref:algorithms/minimum-weight-spanning-tree.adoc[Spanning Tree] -| `gds.beta.spanningTree.mutate` -| `gds.beta.spanningTree.mutate.estimate` -| `gds.beta.spanningTree.stats` -| `gds.beta.spanningTree.stats.estimate` -| `gds.beta.spanningTree.stream` -| `gds.beta.spanningTree.stream.estimate` -| `gds.beta.spanningTree.write` -| `gds.beta.spanningTree.write.estimate` -.1+<.^|xref:alpha-algorithms/k-minimum-weight-spanning-tree.adoc[ k-Spanning Tree] -| `gds.alpha.kSpanningTree.write` -| xref:alpha-algorithms/adamic-adar.adoc[Adamic Adar] | `_gds.alpha.linkprediction.adamicAdar_` -| xref:alpha-algorithms/common-neighbors.adoc[Common Neighbors] | `_gds.alpha.linkprediction.commonNeighbors_` -| xref:alpha-algorithms/preferential-attachment.adoc[Preferential Attachment] | `_gds.alpha.linkprediction.preferentialAttachment_` -| xref:alpha-algorithms/preferential-attachment.adoc[Preferential Attachment] | `_gds.alpha.linkprediction.resourceAllocation_` -| xref:alpha-algorithms/same-community.adoc[Same Community] | `_gds.alpha.linkprediction.sameCommunity_` -| xref:alpha-algorithms/total-neighbors.adoc[Total Neighbors] | `_gds.alpha.linkprediction.totalNeighbors_` .1+<.^| xref:alpha-algorithms/split-relationships.adoc[Split Relationships] | `gds.alpha.ml.splitRelationships.mutate` +.2+<.^|xref:algorithms/strongly-connected-components.adoc[Strongly Connected Components] +| `gds.alpha.scc.stream` +| `gds.alpha.scc.write` .1+<.^| xref:algorithms/triangle-count.adoc#algorithms-triangle-count-examples-triangles-listing[Triangle Listing] | `gds.alpha.triangles` -.1+<.^| xref:algorithms/alpha/conductance.adoc[Conductance] -| `gds.alpha.conductance.stream` -.4+<.^|xref:machine-learning/node-embeddings/hashgnn.adoc[HashGNN] -| `gds.beta.hashgnn.mutate` -| `gds.beta.hashgnn.mutate.estimate` -| `gds.beta.hashgnn.stream` -| `gds.beta.hashgnn.stream.estimate` -.8+<.^| xref:algorithms/kmeans.adoc[Kmeans] -| `gds.beta.kmeans.mutate` -| `gds.beta.kmeans.mutate.estimate` -| `gds.beta.kmeans.stats` -| `gds.beta.kmeans.stats.estimate` -| `gds.beta.kmeans.stream` -| `gds.beta.kmeans.stream.estimate` -| `gds.beta.kmeans.write` -| `gds.beta.kmeans.write.estimate` -.4+<.^| xref:algorithms/alpha/filtered-knn.adoc[Filtered KNN] -| `gds.alpha.knn.filtered.mutate` -| `gds.alpha.knn.filtered.stats` -| `gds.alpha.knn.filtered.stream` -| `gds.alpha.knn.filtered.write` -.8+<.^| xref:algorithms/leiden.adoc[Leiden] -| `gds.beta.leiden.mutate` -| `gds.beta.leiden.mutate.estimate` -| `gds.beta.leiden.stats` -| `gds.beta.leiden.stats.estimate` -| `gds.beta.leiden.stream` -| `gds.beta.leiden.stream.estimate` -| `gds.beta.leiden.write` -| `gds.beta.leiden.write.estimate` -.8+<.^| xref:algorithms/alpha/filtered-node-similarity.adoc[Filtered NodeSimilarity] -| `gds.alpha.nodeSimilarity.filtered.mutate` -| `gds.alpha.nodeSimilarity.filtered.mutate.estimate` -| `gds.alpha.nodeSimilarity.filtered.stats` -| `gds.alpha.nodeSimilarity.filtered.stats.estimate` -| `gds.alpha.nodeSimilarity.filtered.stream` -| `gds.alpha.nodeSimilarity.filtered.stream.estimate` -| `gds.alpha.nodeSimilarity.filtered.write` -| `gds.alpha.nodeSimilarity.filtered.write.estimate` +| xref:alpha-algorithms/total-neighbors.adoc[Total Neighbors] | `_gds.alpha.linkprediction.totalNeighbors_` .2+<.^| xref:algorithms/alpha/modularity.adoc[Modularity Metric] | `gds.alpha.modularity.stats` | `gds.alpha.modularity.stream` -.4+<.^| xref:algorithms/directed-steiner-tree.adoc[Minimum Directed Steiner Tree] -| `gds.beta.steinerTree.mutate` -| `gds.beta.steinerTree.stats` -| `gds.beta.steinerTree.stream` -| `gds.beta.steinerTree.write` -.8+<.^| xref:algorithms/bellman-ford-single-source.adoc[Bellman-Ford] -| `gds.bellmanFord.mutate` -| `gds.bellmanFord.mutate.estimate` -| `gds.bellmanFord.stats` -| `gds.bellmanFord.stats.estimate` -| `gds.bellmanFord.stream` -| `gds.bellmanFord.stream.estimate` -| `gds.bellmanFord.write` -| `gds.bellmanFord.write.estimate` .2+<.^|xref:algorithms/scale-properties.adoc[Scale Properties] | `gds.alpha.scaleProperties.mutate` (deprecated) | `gds.alpha.scaleProperties.stream` (deprecated) -.8+<.^| xref:algorithms/k-core.adoc[K-Core Decomposition] -| `gds.kcore.stats` -| `gds.kcore.stats.estimate` -| `gds.kcore.stream` -| `gds.kcore.stream.estimate` -| `gds.kcore.mutate` -| `gds.kcore.mutate.estimate` -| `gds.kcore.write` -| `gds.kcore.write.estimate` |=== From 180a2ed5a2d8121e932c4e27df8a5b3c3293cac6 Mon Sep 17 00:00:00 2001 From: Veselin Nikolov Date: Tue, 27 Jun 2023 10:31:06 +0100 Subject: [PATCH 117/273] Bump AuraDS suffix --- gradle/version.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle/version.gradle b/gradle/version.gradle index 94198224a2..30c5c44952 100644 --- a/gradle/version.gradle +++ b/gradle/version.gradle @@ -1,6 +1,6 @@ ext { gdsBaseVersion = '2.4.1' - gdsAuraVersion = '24' + gdsAuraVersion = '25' gdsVersion = gdsBaseVersion + (rootProject.hasProperty('aurads') ? "+${gdsAuraVersion}" : "") } From 8c4ac685b1d0b968dff22bc5c63fece95a6921db Mon Sep 17 00:00:00 2001 From: Veselin Nikolov Date: Tue, 27 Jun 2023 07:33:02 +0100 Subject: [PATCH 118/273] GDS 2.4.1 post release actions --- README.adoc | 16 ++++++++-------- .../pages/management-ops/utility-functions.adoc | 2 +- examples/pregel-bootstrap/build.gradle | 2 +- gradle/version.gradle | 2 +- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/README.adoc b/README.adoc index 4e0e4901c2..668967ccc3 100644 --- a/README.adoc +++ b/README.adoc @@ -94,7 +94,7 @@ For the most basic set of features, like graph loading and the graph representat org.neo4j.gds core - 2.4.0 + 2.4.1 ---- @@ -106,21 +106,21 @@ The algorithms are located in the `algo-common`, `algo` and `alpha-algo` modules org.neo4j.gds algo-common - 2.4.0 + 2.4.1 org.neo4j.gds algo - 2.4.0 + 2.4.1 org.neo4j.gds alpha-algo - 2.4.0 + 2.4.1 ---- @@ -132,28 +132,28 @@ The procedures are located in the `proc-common`, `proc` and `alpha-proc` modules org.neo4j.gds proc-common - 2.4.0 + 2.4.1 org.neo4j.gds proc - 2.4.0 + 2.4.1 org.neo4j.gds alpha-proc - 2.4.0 + 2.4.1 org.neo4j.gds open-write-services - 2.4.0 + 2.4.1 ---- diff --git a/doc/modules/ROOT/pages/management-ops/utility-functions.adoc b/doc/modules/ROOT/pages/management-ops/utility-functions.adoc index 99d1981c85..cb81f3e9f8 100644 --- a/doc/modules/ROOT/pages/management-ops/utility-functions.adoc +++ b/doc/modules/ROOT/pages/management-ops/utility-functions.adoc @@ -27,7 +27,7 @@ RETURN gds.version() AS version [opts="header"] |=== | version -| "2.4.1" +| "2.4.2" |=== -- diff --git a/examples/pregel-bootstrap/build.gradle b/examples/pregel-bootstrap/build.gradle index d0b7e9ef65..5ca090ea53 100644 --- a/examples/pregel-bootstrap/build.gradle +++ b/examples/pregel-bootstrap/build.gradle @@ -7,7 +7,7 @@ plugins { ext { // Make sure these are the same as your installation of GDS and Neo4j - gdsVersion = '2.4.1' + gdsVersion = '2.4.2' neo4jVersion = '5.8.0' // Necessary to generate value classes for Pregel configs diff --git a/gradle/version.gradle b/gradle/version.gradle index 30c5c44952..235d471ea7 100644 --- a/gradle/version.gradle +++ b/gradle/version.gradle @@ -1,5 +1,5 @@ ext { - gdsBaseVersion = '2.4.1' + gdsBaseVersion = '2.4.2' gdsAuraVersion = '25' gdsVersion = gdsBaseVersion + (rootProject.hasProperty('aurads') ? "+${gdsAuraVersion}" : "") From 63ca4e8946b3cb10acf44628b3d4de7a4bf28f65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Wed, 28 Jun 2023 09:59:18 +0200 Subject: [PATCH 119/273] Bump Neo4j 4.4 version used Co-authored-by: Brian Shi --- README.adoc | 4 ++-- gradle/dependencies.gradle | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.adoc b/README.adoc index 4e0e4901c2..b90924259f 100644 --- a/README.adoc +++ b/README.adoc @@ -37,7 +37,7 @@ When installing GDS manually, please refer to the below compatibility matrix: |Neo4j 5.4.0 |Neo4j 5.2.0 |Neo4j 5.1.0 -|Neo4j 4.4.9 - 4.4.21 +|Neo4j 4.4.9 - 4.4.22 .1+.^|Java 11 .9+<.^|GDS 2.3.x |Neo4j 5.8.0 @@ -49,7 +49,7 @@ When installing GDS manually, please refer to the below compatibility matrix: |Neo4j 5.3.0 |Neo4j 5.2.0 |Neo4j 5.1.0 -|Neo4j 4.4.9 - 4.4.21 +|Neo4j 4.4.9 - 4.4.22 .1+.^|Java 11 |=== diff --git a/gradle/dependencies.gradle b/gradle/dependencies.gradle index 30da1f749a..b8aa431080 100644 --- a/gradle/dependencies.gradle +++ b/gradle/dependencies.gradle @@ -1,6 +1,6 @@ ext { neos = [ - '4.4': properties.getOrDefault('neo4jVersion44', '4.4.21'), + '4.4': properties.getOrDefault('neo4jVersion44', '4.4.22'), '5.1': properties.getOrDefault('neo4jVersion51', '5.1.0'), '5.2': properties.getOrDefault('neo4jVersion52', '5.2.0'), '5.3': properties.getOrDefault('neo4jVersion53', '5.3.0'), From d3aac768ab7552c5f87f551b71b12c4ffb493197 Mon Sep 17 00:00:00 2001 From: Veselin Nikolov Date: Thu, 29 Jun 2023 05:11:15 +0100 Subject: [PATCH 120/273] Don't assert the wcc seeded result in docs --- doc/modules/ROOT/pages/algorithms/wcc.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/modules/ROOT/pages/algorithms/wcc.adoc b/doc/modules/ROOT/pages/algorithms/wcc.adoc index 814cc203f0..db4a95364b 100644 --- a/doc/modules/ROOT/pages/algorithms/wcc.adoc +++ b/doc/modules/ROOT/pages/algorithms/wcc.adoc @@ -521,7 +521,7 @@ CALL gds.graph.project( _Step 4_ -[role=query-example, group=seeding] +[role=query-example, group=seeding, no-result=true] -- .The following will run the algorithm in `stream` mode using `seedProperty`: [source, cypher, role=noplay] From 48884189d3d325986341d5208e9f5daeaaeb35cc Mon Sep 17 00:00:00 2001 From: Martin Junghanns Date: Wed, 28 Jun 2023 14:31:45 +0200 Subject: [PATCH 121/273] Assert that we write within allocated memory Co-Authored-By: Paul Horn --- .../gds/core/compression/packed/AdjacencyPacker.java | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/org/neo4j/gds/core/compression/packed/AdjacencyPacker.java b/core/src/main/java/org/neo4j/gds/core/compression/packed/AdjacencyPacker.java index ec6ea1307e..d6b5b118b7 100644 --- a/core/src/main/java/org/neo4j/gds/core/compression/packed/AdjacencyPacker.java +++ b/core/src/main/java/org/neo4j/gds/core/compression/packed/AdjacencyPacker.java @@ -165,6 +165,7 @@ private static long runPacking( Address address = slice.slice(); long ptr = address.address() + slice.offset(); + long initialPtr = ptr; // write header UnsafeUtil.copyMemory(header, BYTE_ARRAY_BASE_OFFSET, null, ptr, headerSize); @@ -177,6 +178,9 @@ private static long runPacking( in += AdjacencyPacking.BLOCK_SIZE; } + if (ptr > initialPtr + allocationSize) + throw new AssertionError("Written more bytes than allocated. ptr=" + ptr + ", initialPtr=" + initialPtr + ", allocationSize=" + allocationSize); + return adjacencyOffset; } @@ -370,6 +374,7 @@ private static long runPackingWithPackedTail( Address address = slice.slice(); long ptr = address.address() + slice.offset(); + long initialPtr = ptr; // write header UnsafeUtil.copyMemory(header, BYTE_ARRAY_BASE_OFFSET, null, ptr, headerSize); @@ -389,9 +394,13 @@ private static long runPackingWithPackedTail( // tail packing if (hasTail) { byte bits = header[header.length - 1]; - AdjacencyPacking.loopPack(bits, values, in, tailLength, ptr); + ptr = AdjacencyPacking.loopPack(bits, values, in, tailLength, ptr); } + if (ptr > initialPtr + allocationSize) + throw new AssertionError("Written more bytes than allocated. ptr=" + ptr + ", initialPtr=" + initialPtr + ", allocationSize=" + allocationSize); + + return adjacencyOffset; } From fd4160ee9a9080d60f71ec8e74f5d7e555ee42e4 Mon Sep 17 00:00:00 2001 From: Martin Junghanns Date: Wed, 28 Jun 2023 14:56:52 +0200 Subject: [PATCH 122/273] Refactor tail bytes computation Co-Authored-By: Paul Horn --- .../neo4j/gds/core/compression/packed/AdjacencyPacker.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/org/neo4j/gds/core/compression/packed/AdjacencyPacker.java b/core/src/main/java/org/neo4j/gds/core/compression/packed/AdjacencyPacker.java index d6b5b118b7..a6a84c8ec0 100644 --- a/core/src/main/java/org/neo4j/gds/core/compression/packed/AdjacencyPacker.java +++ b/core/src/main/java/org/neo4j/gds/core/compression/packed/AdjacencyPacker.java @@ -334,7 +334,7 @@ private static long preparePackingWithPackedTail( if (hasTail) { int bits = bitsNeeded(values, offset, tailLength); memoryTracker.recordHeaderBits(bits); - bytes += BitUtil.ceilDiv((long) bits * tailLength, Long.BYTES); + bytes += bytesNeeded(bits, tailLength); header[blockIdx] = (byte) bits; } @@ -415,4 +415,8 @@ private static int bitsNeeded(long[] values, int offset, int length) { private static int bytesNeeded(int bits) { return BitUtil.ceilDiv(AdjacencyPacking.BLOCK_SIZE * bits, Byte.SIZE); } + + private static int bytesNeeded(int bits, int length) { + return BitUtil.ceilDiv(length * bits, Byte.SIZE); + } } From 90bbc20c17babced01fcf9c9f18b3339fe2222da Mon Sep 17 00:00:00 2001 From: Martin Junghanns Date: Wed, 28 Jun 2023 17:00:49 +0200 Subject: [PATCH 123/273] Avoid writing last word twice if shift is zero Co-Authored-By: Paul Horn --- .../compression/packed/AdjacencyPacking.java | 384 ++++++++++++------ .../core/compression/packed/gen_packing.rs | 14 +- 2 files changed, 265 insertions(+), 133 deletions(-) diff --git a/core/src/main/java/org/neo4j/gds/core/compression/packed/AdjacencyPacking.java b/core/src/main/java/org/neo4j/gds/core/compression/packed/AdjacencyPacking.java index fb162fd1f1..d0a3e61d6b 100644 --- a/core/src/main/java/org/neo4j/gds/core/compression/packed/AdjacencyPacking.java +++ b/core/src/main/java/org/neo4j/gds/core/compression/packed/AdjacencyPacking.java @@ -10884,8 +10884,10 @@ private static long packLoop1(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -10909,8 +10911,10 @@ private static long packLoop2(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -10934,8 +10938,10 @@ private static long packLoop3(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -10959,8 +10965,10 @@ private static long packLoop4(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -10984,8 +10992,10 @@ private static long packLoop5(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -11009,8 +11019,10 @@ private static long packLoop6(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -11034,8 +11046,10 @@ private static long packLoop7(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -11059,8 +11073,10 @@ private static long packLoop8(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -11084,8 +11100,10 @@ private static long packLoop9(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -11109,8 +11127,10 @@ private static long packLoop10(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -11134,8 +11154,10 @@ private static long packLoop11(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -11159,8 +11181,10 @@ private static long packLoop12(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -11184,8 +11208,10 @@ private static long packLoop13(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -11209,8 +11235,10 @@ private static long packLoop14(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -11234,8 +11262,10 @@ private static long packLoop15(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -11259,8 +11289,10 @@ private static long packLoop16(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -11284,8 +11316,10 @@ private static long packLoop17(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -11309,8 +11343,10 @@ private static long packLoop18(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -11334,8 +11370,10 @@ private static long packLoop19(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -11359,8 +11397,10 @@ private static long packLoop20(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -11384,8 +11424,10 @@ private static long packLoop21(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -11409,8 +11451,10 @@ private static long packLoop22(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -11434,8 +11478,10 @@ private static long packLoop23(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -11459,8 +11505,10 @@ private static long packLoop24(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -11484,8 +11532,10 @@ private static long packLoop25(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -11509,8 +11559,10 @@ private static long packLoop26(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -11534,8 +11586,10 @@ private static long packLoop27(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -11559,8 +11613,10 @@ private static long packLoop28(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -11584,8 +11640,10 @@ private static long packLoop29(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -11609,8 +11667,10 @@ private static long packLoop30(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -11634,8 +11694,10 @@ private static long packLoop31(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -11659,8 +11721,10 @@ private static long packLoop32(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -11684,8 +11748,10 @@ private static long packLoop33(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -11709,8 +11775,10 @@ private static long packLoop34(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -11734,8 +11802,10 @@ private static long packLoop35(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -11759,8 +11829,10 @@ private static long packLoop36(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -11784,8 +11856,10 @@ private static long packLoop37(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -11809,8 +11883,10 @@ private static long packLoop38(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -11834,8 +11910,10 @@ private static long packLoop39(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -11859,8 +11937,10 @@ private static long packLoop40(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -11884,8 +11964,10 @@ private static long packLoop41(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -11909,8 +11991,10 @@ private static long packLoop42(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -11934,8 +12018,10 @@ private static long packLoop43(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -11959,8 +12045,10 @@ private static long packLoop44(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -11984,8 +12072,10 @@ private static long packLoop45(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -12009,8 +12099,10 @@ private static long packLoop46(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -12034,8 +12126,10 @@ private static long packLoop47(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -12059,8 +12153,10 @@ private static long packLoop48(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -12084,8 +12180,10 @@ private static long packLoop49(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -12109,8 +12207,10 @@ private static long packLoop50(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -12134,8 +12234,10 @@ private static long packLoop51(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -12159,8 +12261,10 @@ private static long packLoop52(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -12184,8 +12288,10 @@ private static long packLoop53(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -12209,8 +12315,10 @@ private static long packLoop54(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -12234,8 +12342,10 @@ private static long packLoop55(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -12259,8 +12369,10 @@ private static long packLoop56(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -12284,8 +12396,10 @@ private static long packLoop57(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -12309,8 +12423,10 @@ private static long packLoop58(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -12334,8 +12450,10 @@ private static long packLoop59(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -12359,8 +12477,10 @@ private static long packLoop60(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -12384,8 +12504,10 @@ private static long packLoop61(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -12409,8 +12531,10 @@ private static long packLoop62(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -12434,8 +12558,10 @@ private static long packLoop63(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -12459,8 +12585,10 @@ private static long packLoop64(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } } diff --git a/core/src/main/java/org/neo4j/gds/core/compression/packed/gen_packing.rs b/core/src/main/java/org/neo4j/gds/core/compression/packed/gen_packing.rs index 6073e3aebd..ef845efe64 100755 --- a/core/src/main/java/org/neo4j/gds/core/compression/packed/gen_packing.rs +++ b/core/src/main/java/org/neo4j/gds/core/compression/packed/gen_packing.rs @@ -804,19 +804,23 @@ mod java { }); } Inst::PackLoopRemainder => { - let mut if_body = vec![]; - - if_body.push(Stmt::Expr(Expr::Call(Call::new( + let mut then = vec![]; + then.push(Stmt::Expr(Expr::Call(Call::new( Expr::Ident("UnsafeUtil"), "putLong", vec![Arg::new(Expr::Ident(PW)), Arg::new(Expr::Ident(WORD))], )))); - if_body.push(Stmt::assign_op( + then.push(Stmt::assign_op( Expr::Ident(PW), Expr::Literal(8), BinOp::Add, )); - statements.extend(if_body); + + statements.push(Stmt::If { + cond: Expr::bin(Expr::Ident(SHIFT), BinOp::Neq, Expr::Literal(0)), + then: Box::new(Stmt::Block(then)), + ells: None, + }); } Inst::UnpackLoop => { // PIN[offset + i + OFF] From 1bf9b486c04c6c029093448b6a5e0905d800a0b1 Mon Sep 17 00:00:00 2001 From: Paul Horn Date: Wed, 28 Jun 2023 17:09:03 +0200 Subject: [PATCH 124/273] Only add required imports (manually) --- .../core/compression/packed/gen_packing.rs | 30 +++++++++++++------ 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/core/src/main/java/org/neo4j/gds/core/compression/packed/gen_packing.rs b/core/src/main/java/org/neo4j/gds/core/compression/packed/gen_packing.rs index ef845efe64..340c6c624a 100755 --- a/core/src/main/java/org/neo4j/gds/core/compression/packed/gen_packing.rs +++ b/core/src/main/java/org/neo4j/gds/core/compression/packed/gen_packing.rs @@ -571,7 +571,7 @@ fn single_pack(bits: u32, offset: u32) -> Inst { } mod java { - use std::io::Write; + use std::{collections::HashSet, io::Write}; use super::*; use gen_java::*; @@ -1196,7 +1196,7 @@ mod java { } } - fn gen_class(class: Class) -> ClassDef { + fn gen_class(class: Class) -> (ClassDef, Vec) { fn gen_assert(bs: u32) -> Stmt { Stmt::Assert { assertion: Expr::bin(Expr::Ident(BITS), BinOp::Lte, Expr::Literal(bs)), @@ -1208,6 +1208,8 @@ mod java { } } + let mut imports = HashSet::new(); + let mut members = vec![ Member::Method(MethodDef { documentation: None, @@ -1238,6 +1240,8 @@ mod java { ]; if !class.packers.is_empty() { + imports.insert("org.neo4j.internal.unsafe.UnsafeUtil"); + members.extend([ Member::Method(MethodDef { documentation: None, @@ -1296,6 +1300,8 @@ mod java { } if !class.unpackers.is_empty() { + imports.insert("org.neo4j.internal.unsafe.UnsafeUtil"); + members.extend([ Member::Method(MethodDef { documentation: None, @@ -1354,6 +1360,8 @@ mod java { } if !class.loop_packers.is_empty() { + imports.insert("org.neo4j.internal.unsafe.UnsafeUtil"); + members.extend([ Member::Method(MethodDef { documentation: None, @@ -1417,6 +1425,9 @@ mod java { } if !class.loop_unpackers.is_empty() { + imports.insert("org.neo4j.internal.unsafe.UnsafeUtil"); + imports.insert("org.neo4j.gds.mem.BitUtil"); + members.extend([ Member::Method(MethodDef { documentation: None, @@ -1492,18 +1503,22 @@ mod java { let doc = class.documentation.join("\n"); - ClassDef { + let class = ClassDef { documentation: Some(doc), annotations: Vec::new(), modifiers: "public final", typ: "class", name: class.name, members, - } + }; + + let imports = imports.into_iter().map(String::from).collect(); + + (class, imports) } fn gen_file(file: File) -> FileDef { - let class = gen_class(file.class); + let (class, imports) = gen_class(file.class); let mut file = FileDef::new( r#" Copyright (c) "Neo4j" @@ -1525,10 +1540,7 @@ You should have received a copy of the GNU General Public License along with this program. If not, see . "#, file.package, - vec![ - "org.neo4j.internal.unsafe.UnsafeUtil".into(), - "org.neo4j.gds.mem.BitUtil".into(), - ], + imports, class, ); From 1424d6b8db5072fcd19ce466d2ad954ecb119df6 Mon Sep 17 00:00:00 2001 From: Paul Horn Date: Wed, 28 Jun 2023 17:26:22 +0200 Subject: [PATCH 125/273] Sort imports --- .../java/org/neo4j/gds/core/compression/packed/gen_packing.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/org/neo4j/gds/core/compression/packed/gen_packing.rs b/core/src/main/java/org/neo4j/gds/core/compression/packed/gen_packing.rs index 340c6c624a..7b8e7e3ccc 100755 --- a/core/src/main/java/org/neo4j/gds/core/compression/packed/gen_packing.rs +++ b/core/src/main/java/org/neo4j/gds/core/compression/packed/gen_packing.rs @@ -1512,7 +1512,8 @@ mod java { members, }; - let imports = imports.into_iter().map(String::from).collect(); + let mut imports = imports.into_iter().map(String::from).collect::>(); + imports.sort_unstable(); (class, imports) } From c92b8de3ba14685430ef9a6059d277aca0b1c9f0 Mon Sep 17 00:00:00 2001 From: Paul Horn Date: Wed, 28 Jun 2023 17:53:45 +0200 Subject: [PATCH 126/273] Always generate remainder for bits=64 --- .../core/compression/packed/AdjacencyPacking.java | 6 ++---- .../gds/core/compression/packed/gen_packing.rs | 14 +++++++++----- .../compression/packed/AdjacencyPackerTest.java | 2 +- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/core/src/main/java/org/neo4j/gds/core/compression/packed/AdjacencyPacking.java b/core/src/main/java/org/neo4j/gds/core/compression/packed/AdjacencyPacking.java index d0a3e61d6b..3ef7c3d996 100644 --- a/core/src/main/java/org/neo4j/gds/core/compression/packed/AdjacencyPacking.java +++ b/core/src/main/java/org/neo4j/gds/core/compression/packed/AdjacencyPacking.java @@ -12585,10 +12585,8 @@ private static long packLoop64(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - if (shift != 0) { - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; - } + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; return packedPtr; } } diff --git a/core/src/main/java/org/neo4j/gds/core/compression/packed/gen_packing.rs b/core/src/main/java/org/neo4j/gds/core/compression/packed/gen_packing.rs index 7b8e7e3ccc..870b6a5034 100755 --- a/core/src/main/java/org/neo4j/gds/core/compression/packed/gen_packing.rs +++ b/core/src/main/java/org/neo4j/gds/core/compression/packed/gen_packing.rs @@ -816,11 +816,15 @@ mod java { BinOp::Add, )); - statements.push(Stmt::If { - cond: Expr::bin(Expr::Ident(SHIFT), BinOp::Neq, Expr::Literal(0)), - then: Box::new(Stmt::Block(then)), - ells: None, - }); + if method.bits == 64 { + statements.extend(then); + } else { + statements.push(Stmt::If { + cond: Expr::bin(Expr::Ident(SHIFT), BinOp::Neq, Expr::Literal(0)), + then: Box::new(Stmt::Block(then)), + ells: None, + }); + } } Inst::UnpackLoop => { // PIN[offset + i + OFF] diff --git a/core/src/test/java/org/neo4j/gds/core/compression/packed/AdjacencyPackerTest.java b/core/src/test/java/org/neo4j/gds/core/compression/packed/AdjacencyPackerTest.java index 93ce44d10f..5017b92d04 100644 --- a/core/src/test/java/org/neo4j/gds/core/compression/packed/AdjacencyPackerTest.java +++ b/core/src/test/java/org/neo4j/gds/core/compression/packed/AdjacencyPackerTest.java @@ -160,7 +160,7 @@ void loopPack5Test() { @Test void loopPack64Test() { int bits = 64; - long upperBound = (1L << bits) - 1; + long upperBound = -1L; long[] data = LongStream.concat(LongStream.rangeClosed(0, 42), LongStream.of(upperBound)).toArray(); int length = data.length; int requiredBytes = BitUtil.ceilDiv(length * bits, Byte.SIZE); From 3f0729edf57977673f9bce2dd4acd5a4337b6604 Mon Sep 17 00:00:00 2001 From: Mats Rydberg Date: Thu, 29 Jun 2023 15:29:57 +0200 Subject: [PATCH 127/273] Extract GdsVersionInfo to separate class This is used by EstimationCLI. When it was part of ProxyUtil, it would initialise also other static fields in the class, triggering an exception. By moving to a dedicated class, we avoid this situation. --- .../gds/compat/GdsVersionInfoProvider.java | 95 +++++++++++++++++++ .../java/org/neo4j/gds/compat/ProxyUtil.java | 70 +------------- .../graphsage/GraphSageTrainSpec.java | 4 +- .../LinkPredictionPipelineTrainSpec.java | 4 +- .../NodeClassificationPipelineTrainSpec.java | 4 +- .../NodeRegressionPipelineTrainSpec.java | 4 +- 6 files changed, 104 insertions(+), 77 deletions(-) create mode 100644 compatibility/api/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/GdsVersionInfoProvider.java diff --git a/compatibility/api/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/GdsVersionInfoProvider.java b/compatibility/api/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/GdsVersionInfoProvider.java new file mode 100644 index 0000000000..5c47cf293b --- /dev/null +++ b/compatibility/api/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/GdsVersionInfoProvider.java @@ -0,0 +1,95 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat; + +import java.lang.invoke.MethodHandles; +import java.lang.invoke.MethodType; + +public final class GdsVersionInfoProvider { + + private GdsVersionInfoProvider() {} + + public static final ProxyUtil.GdsVersionInfo GDS_VERSION_INFO = loadGdsVersion(); + + private static ProxyUtil.GdsVersionInfo loadGdsVersion() { + var builder = ImmutableGdsVersionInfo.builder(); + try { + // The class that we use to get the GDS version lives in proc-sysinfo, which is part of the released GDS jar, + // but we don't want to depend on that here. One reason is that this class gets generated and re-generated + // on every build and having it at the top of the dependency graph would cause a lot of recompilation. + // Let's do a bit of class loading and reflection to get the version. + var lookup = MethodHandles.lookup(); + + var buildInfoPropertiesClass = Class.forName("org.neo4j.gds.BuildInfoProperties"); + + // equivalent to: BuildInfoProperties.get() + var buildInfoPropertiesHandle = lookup.findStatic( + buildInfoPropertiesClass, + "get", + MethodType.methodType(buildInfoPropertiesClass) + ); + + // equivalent to: buildInfoProperties.gdsVersion() + var gdsVersionHandle = lookup.findVirtual( + buildInfoPropertiesClass, + "gdsVersion", + MethodType.methodType(String.class) + ); + + // var buildInfoProperties = BuildInfoProperties.get() + var buildInfoProperties = buildInfoPropertiesHandle.invoke(); + // var gdsVersion = buildInfoProperties.gdsVersion() + var gdsVersion = gdsVersionHandle.invoke(buildInfoProperties); + + return builder + .gdsVersion(String.valueOf(gdsVersion)) + .build(); + } catch (ClassNotFoundException e) { + builder.error(ImmutableErrorInfo.builder() + .logLevel(ProxyUtil.LogLevel.DEBUG) + .message( + "Could not determine GDS version, BuildInfoProperties is missing. " + + "This is likely due to not running GDS as a plugin, " + + "for example when running tests or using GDS as a Java module dependency." + ) + .reason(e) + .build() + ); + } catch (NoSuchMethodException | IllegalAccessException e) { + builder.error(ImmutableErrorInfo.builder() + .logLevel(ProxyUtil.LogLevel.WARN) + .message( + "Could not determine GDS version, the according methods on BuildInfoProperties could not be found.") + .reason(e) + .build() + ); + } catch (Throwable e) { + builder.error(ImmutableErrorInfo.builder() + .logLevel(ProxyUtil.LogLevel.WARN) + .message("Could not determine GDS version, the according methods on BuildInfoProperties failed.") + .reason(e) + .build() + ); + } + + return builder.gdsVersion("Unknown").build(); + } + +} diff --git a/compatibility/api/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/ProxyUtil.java b/compatibility/api/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/ProxyUtil.java index 527a20a779..0b29152296 100644 --- a/compatibility/api/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/ProxyUtil.java +++ b/compatibility/api/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/ProxyUtil.java @@ -25,8 +25,6 @@ import org.neo4j.gds.annotation.ValueClass; import org.neo4j.logging.Log; -import java.lang.invoke.MethodHandles; -import java.lang.invoke.MethodType; import java.util.ArrayList; import java.util.Collection; import java.util.Locale; @@ -147,7 +145,7 @@ private static > ProxyInfobuilder() .factoryType(factoryClass) .neo4jVersion(NEO4J_VERSION_INFO) - .gdsVersion(GDS_VERSION_INFO) + .gdsVersion(GdsVersionInfoProvider.GDS_VERSION_INFO) .javaInfo(JAVA_INFO); try { @@ -195,72 +193,6 @@ private static Neo4jVersionInfo loadNeo4jVersion() { } } - public static final GdsVersionInfo GDS_VERSION_INFO = loadGdsVersion(); - - private static GdsVersionInfo loadGdsVersion() { - var builder = ImmutableGdsVersionInfo.builder(); - try { - // The class that we use to get the GDS version lives in proc-sysinfo, which is part of the released GDS jar, - // but we don't want to depend on that here. One reason is that this class gets generated and re-generated - // on every build and having it at the top of the dependency graph would cause a lot of recompilation. - // Let's do a bit of class loading and reflection to get the version. - var lookup = MethodHandles.lookup(); - - var buildInfoPropertiesClass = Class.forName("org.neo4j.gds.BuildInfoProperties"); - - // equivalent to: BuildInfoProperties.get() - var buildInfoPropertiesHandle = lookup.findStatic( - buildInfoPropertiesClass, - "get", - MethodType.methodType(buildInfoPropertiesClass) - ); - - // equivalent to: buildInfoProperties.gdsVersion() - var gdsVersionHandle = lookup.findVirtual( - buildInfoPropertiesClass, - "gdsVersion", - MethodType.methodType(String.class) - ); - - // var buildInfoProperties = BuildInfoProperties.get() - var buildInfoProperties = buildInfoPropertiesHandle.invoke(); - // var gdsVersion = buildInfoProperties.gdsVersion() - var gdsVersion = gdsVersionHandle.invoke(buildInfoProperties); - - return builder - .gdsVersion(String.valueOf(gdsVersion)) - .build(); - } catch (ClassNotFoundException e) { - builder.error(ImmutableErrorInfo.builder() - .logLevel(LogLevel.DEBUG) - .message( - "Could not determine GDS version, BuildInfoProperties is missing. " + - "This is likely due to not running GDS as a plugin, " + - "for example when running tests or using GDS as a Java module dependency." - ) - .reason(e) - .build() - ); - } catch (NoSuchMethodException | IllegalAccessException e) { - builder.error(ImmutableErrorInfo.builder() - .logLevel(LogLevel.WARN) - .message( - "Could not determine GDS version, the according methods on BuildInfoProperties could not be found.") - .reason(e) - .build() - ); - } catch (Throwable e) { - builder.error(ImmutableErrorInfo.builder() - .logLevel(LogLevel.WARN) - .message("Could not determine GDS version, the according methods on BuildInfoProperties failed.") - .reason(e) - .build() - ); - } - - return builder.gdsVersion("Unknown").build(); - } - private static final JavaInfo JAVA_INFO = loadJavaInfo(); private static JavaInfo loadJavaInfo() { diff --git a/proc/embeddings/src/main/java/org/neo4j/gds/embeddings/graphsage/GraphSageTrainSpec.java b/proc/embeddings/src/main/java/org/neo4j/gds/embeddings/graphsage/GraphSageTrainSpec.java index 4329b20f5c..df8e24dfaf 100644 --- a/proc/embeddings/src/main/java/org/neo4j/gds/embeddings/graphsage/GraphSageTrainSpec.java +++ b/proc/embeddings/src/main/java/org/neo4j/gds/embeddings/graphsage/GraphSageTrainSpec.java @@ -19,7 +19,7 @@ */ package org.neo4j.gds.embeddings.graphsage; -import org.neo4j.gds.compat.ProxyUtil; +import org.neo4j.gds.compat.GdsVersionInfoProvider; import org.neo4j.gds.core.model.Model; import org.neo4j.gds.embeddings.graphsage.algo.GraphSageTrain; import org.neo4j.gds.embeddings.graphsage.algo.GraphSageTrainAlgorithmFactory; @@ -50,7 +50,7 @@ public String name() { @Override public GraphSageTrainAlgorithmFactory algorithmFactory(ExecutionContext executionContext) { - var gdsVersion = ProxyUtil.GDS_VERSION_INFO.gdsVersion(); + var gdsVersion = GdsVersionInfoProvider.GDS_VERSION_INFO.gdsVersion(); return new GraphSageTrainAlgorithmFactory(gdsVersion); } diff --git a/proc/machine-learning/src/main/java/org/neo4j/gds/ml/linkmodels/pipeline/train/LinkPredictionPipelineTrainSpec.java b/proc/machine-learning/src/main/java/org/neo4j/gds/ml/linkmodels/pipeline/train/LinkPredictionPipelineTrainSpec.java index 198bd05e69..5eef869c93 100644 --- a/proc/machine-learning/src/main/java/org/neo4j/gds/ml/linkmodels/pipeline/train/LinkPredictionPipelineTrainSpec.java +++ b/proc/machine-learning/src/main/java/org/neo4j/gds/ml/linkmodels/pipeline/train/LinkPredictionPipelineTrainSpec.java @@ -20,7 +20,7 @@ package org.neo4j.gds.ml.linkmodels.pipeline.train; import org.neo4j.gds.VerifyThatModelCanBeStored; -import org.neo4j.gds.compat.ProxyUtil; +import org.neo4j.gds.compat.GdsVersionInfoProvider; import org.neo4j.gds.executor.AlgorithmSpec; import org.neo4j.gds.executor.ComputationResultConsumer; import org.neo4j.gds.executor.ExecutionContext; @@ -53,7 +53,7 @@ public String name() { @Override public LinkPredictionTrainPipelineAlgorithmFactory algorithmFactory(ExecutionContext executionContext) { - var gdsVersion = ProxyUtil.GDS_VERSION_INFO.gdsVersion(); + var gdsVersion = GdsVersionInfoProvider.GDS_VERSION_INFO.gdsVersion(); return new LinkPredictionTrainPipelineAlgorithmFactory(executionContext, gdsVersion); } diff --git a/proc/machine-learning/src/main/java/org/neo4j/gds/ml/pipeline/node/classification/predict/NodeClassificationPipelineTrainSpec.java b/proc/machine-learning/src/main/java/org/neo4j/gds/ml/pipeline/node/classification/predict/NodeClassificationPipelineTrainSpec.java index fe8ab7f80d..c7d1fa36a4 100644 --- a/proc/machine-learning/src/main/java/org/neo4j/gds/ml/pipeline/node/classification/predict/NodeClassificationPipelineTrainSpec.java +++ b/proc/machine-learning/src/main/java/org/neo4j/gds/ml/pipeline/node/classification/predict/NodeClassificationPipelineTrainSpec.java @@ -20,7 +20,7 @@ package org.neo4j.gds.ml.pipeline.node.classification.predict; import org.neo4j.gds.VerifyThatModelCanBeStored; -import org.neo4j.gds.compat.ProxyUtil; +import org.neo4j.gds.compat.GdsVersionInfoProvider; import org.neo4j.gds.core.model.Model; import org.neo4j.gds.executor.AlgorithmSpec; import org.neo4j.gds.executor.ComputationResult; @@ -52,7 +52,7 @@ public String name() { @Override public NodeClassificationTrainPipelineAlgorithmFactory algorithmFactory(ExecutionContext executionContext) { - return new NodeClassificationTrainPipelineAlgorithmFactory(executionContext, ProxyUtil.GDS_VERSION_INFO.gdsVersion()); + return new NodeClassificationTrainPipelineAlgorithmFactory(executionContext, GdsVersionInfoProvider.GDS_VERSION_INFO.gdsVersion()); } @Override diff --git a/proc/machine-learning/src/main/java/org/neo4j/gds/ml/pipeline/node/regression/NodeRegressionPipelineTrainSpec.java b/proc/machine-learning/src/main/java/org/neo4j/gds/ml/pipeline/node/regression/NodeRegressionPipelineTrainSpec.java index 5c36952306..ed04db2d0b 100644 --- a/proc/machine-learning/src/main/java/org/neo4j/gds/ml/pipeline/node/regression/NodeRegressionPipelineTrainSpec.java +++ b/proc/machine-learning/src/main/java/org/neo4j/gds/ml/pipeline/node/regression/NodeRegressionPipelineTrainSpec.java @@ -20,7 +20,7 @@ package org.neo4j.gds.ml.pipeline.node.regression; import org.neo4j.gds.VerifyThatModelCanBeStored; -import org.neo4j.gds.compat.ProxyUtil; +import org.neo4j.gds.compat.GdsVersionInfoProvider; import org.neo4j.gds.executor.AlgorithmSpec; import org.neo4j.gds.executor.ComputationResultConsumer; import org.neo4j.gds.executor.ExecutionContext; @@ -54,7 +54,7 @@ public String name() { @Override public NodeRegressionTrainPipelineAlgorithmFactory algorithmFactory(ExecutionContext executionContext) { - var gdsVersion = ProxyUtil.GDS_VERSION_INFO.gdsVersion(); + var gdsVersion = GdsVersionInfoProvider.GDS_VERSION_INFO.gdsVersion(); return new NodeRegressionTrainPipelineAlgorithmFactory(executionContext, gdsVersion); } From 244fcc15abc8c279dd6674d7a7e79143a1265eb8 Mon Sep 17 00:00:00 2001 From: yuval Date: Wed, 5 Jul 2023 13:59:25 +0200 Subject: [PATCH 128/273] updated the compatibility matrix --- .../supported-neo4j-versions.adoc | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/doc/modules/ROOT/pages/installation/supported-neo4j-versions.adoc b/doc/modules/ROOT/pages/installation/supported-neo4j-versions.adoc index c6403d6c75..9def15f78f 100644 --- a/doc/modules/ROOT/pages/installation/supported-neo4j-versions.adoc +++ b/doc/modules/ROOT/pages/installation/supported-neo4j-versions.adoc @@ -10,14 +10,14 @@ If your version of GDS or Neo4j is not listed in the matrix, you should upgrade. [opts=header] |=== | Neo4j version | Neo4j Graph Data Science -| `5.9` | `2.4`, `2.3.9` or later -| `5.8` | `2.4`, `2.3.6` or later -| `5.7` | `2.4`, `2.3.3` or later -| `5.6` | `2.4`, `2.3.2` or later -| `5.5` | `2.4`, `2.3.1` or later -| `5.4` | `2.4`, `2.3` -| `5.3` | `2.4`, `2.3` -| `5.2` | `2.4`, `2.3` -| `5.1` | `2.4`, `2.3` -| `4.4.9` or later | `2.4`, `2.3` +| `5.9` | `2.4`, `2.3.9` or later footnote:eol[This version series is end-of-life and will not receive further patches. Please use a later version.] +| `5.8` | `2.4`, `2.3.6` or later footnote:eol[] +| `5.7` | `2.4`, `2.3.3` or later footnote:eol[] +| `5.6` | `2.4`, `2.3.2` or later footnote:eol[] +| `5.5` | `2.4`, `2.3.1` or later footnote:eol[] +| `5.4` | `2.4`, `2.3` footnote:eol[] +| `5.3` | `2.4`, `2.3` footnote:eol[] +| `5.2` | `2.4`, `2.3` footnote:eol[] +| `5.1` | `2.4`, `2.3` footnote:eol[] +| `4.4.9` or later | `2.4`, `2.3` footnote:eol[] |=== From 61484eb953bd2620b0d9ef16d706908a510e7e9d Mon Sep 17 00:00:00 2001 From: Martin Junghanns Date: Mon, 3 Jul 2023 14:26:35 +0200 Subject: [PATCH 129/273] Support `threadSafe` parameter for procedure signature --- .../main/java/org/neo4j/gds/compat/_44/Neo4jProxyImpl.java | 3 ++- .../java17/org/neo4j/gds/compat/_51/Neo4jProxyImpl.java | 3 ++- .../java17/org/neo4j/gds/compat/_52/Neo4jProxyImpl.java | 3 ++- .../java17/org/neo4j/gds/compat/_53/Neo4jProxyImpl.java | 3 ++- .../java17/org/neo4j/gds/compat/_54/Neo4jProxyImpl.java | 3 ++- .../java17/org/neo4j/gds/compat/_55/Neo4jProxyImpl.java | 3 ++- .../java17/org/neo4j/gds/compat/_56/Neo4jProxyImpl.java | 3 ++- .../java17/org/neo4j/gds/compat/_57/Neo4jProxyImpl.java | 3 ++- .../java17/org/neo4j/gds/compat/_58/Neo4jProxyImpl.java | 3 ++- .../java17/org/neo4j/gds/compat/_59/Neo4jProxyImpl.java | 3 ++- .../src/main/java/org/neo4j/gds/compat/Neo4jProxyApi.java | 3 ++- .../src/main/java/org/neo4j/gds/compat/Neo4jProxy.java | 6 ++++-- 12 files changed, 26 insertions(+), 13 deletions(-) diff --git a/compatibility/4.4/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_44/Neo4jProxyImpl.java b/compatibility/4.4/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_44/Neo4jProxyImpl.java index 946ecd4b7e..84e4af6d80 100644 --- a/compatibility/4.4/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_44/Neo4jProxyImpl.java +++ b/compatibility/4.4/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_44/Neo4jProxyImpl.java @@ -511,7 +511,8 @@ public ProcedureSignature procedureSignature( boolean caseInsensitive, boolean systemProcedure, boolean internal, - boolean allowExpiredCredentials + boolean allowExpiredCredentials, + boolean threadSafe ) { return new ProcedureSignature( name, diff --git a/compatibility/5.1/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_51/Neo4jProxyImpl.java b/compatibility/5.1/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_51/Neo4jProxyImpl.java index d42c6df7c5..fa9f9292d5 100644 --- a/compatibility/5.1/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_51/Neo4jProxyImpl.java +++ b/compatibility/5.1/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_51/Neo4jProxyImpl.java @@ -605,7 +605,8 @@ public ProcedureSignature procedureSignature( boolean caseInsensitive, boolean systemProcedure, boolean internal, - boolean allowExpiredCredentials + boolean allowExpiredCredentials, + boolean threadSafe ) { return new ProcedureSignature( name, diff --git a/compatibility/5.2/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_52/Neo4jProxyImpl.java b/compatibility/5.2/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_52/Neo4jProxyImpl.java index 017e4b313b..222a7b596d 100644 --- a/compatibility/5.2/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_52/Neo4jProxyImpl.java +++ b/compatibility/5.2/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_52/Neo4jProxyImpl.java @@ -603,7 +603,8 @@ public ProcedureSignature procedureSignature( boolean caseInsensitive, boolean systemProcedure, boolean internal, - boolean allowExpiredCredentials + boolean allowExpiredCredentials, + boolean threadSafe ) { return new ProcedureSignature( name, diff --git a/compatibility/5.3/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_53/Neo4jProxyImpl.java b/compatibility/5.3/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_53/Neo4jProxyImpl.java index 994098ea39..e326af19cc 100644 --- a/compatibility/5.3/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_53/Neo4jProxyImpl.java +++ b/compatibility/5.3/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_53/Neo4jProxyImpl.java @@ -603,7 +603,8 @@ public ProcedureSignature procedureSignature( boolean caseInsensitive, boolean systemProcedure, boolean internal, - boolean allowExpiredCredentials + boolean allowExpiredCredentials, + boolean threadSafe ) { return new ProcedureSignature( name, diff --git a/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/Neo4jProxyImpl.java b/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/Neo4jProxyImpl.java index 4a74f5600a..680bf0acec 100644 --- a/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/Neo4jProxyImpl.java +++ b/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/Neo4jProxyImpl.java @@ -603,7 +603,8 @@ public ProcedureSignature procedureSignature( boolean caseInsensitive, boolean systemProcedure, boolean internal, - boolean allowExpiredCredentials + boolean allowExpiredCredentials, + boolean threadSafe ) { return new ProcedureSignature( name, diff --git a/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/Neo4jProxyImpl.java b/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/Neo4jProxyImpl.java index 978f9fe60b..64e14d50a5 100644 --- a/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/Neo4jProxyImpl.java +++ b/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/Neo4jProxyImpl.java @@ -604,7 +604,8 @@ public ProcedureSignature procedureSignature( boolean caseInsensitive, boolean systemProcedure, boolean internal, - boolean allowExpiredCredentials + boolean allowExpiredCredentials, + boolean threadSafe ) { return new ProcedureSignature( name, diff --git a/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/Neo4jProxyImpl.java b/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/Neo4jProxyImpl.java index 9e67bb68ab..420eeb3474 100644 --- a/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/Neo4jProxyImpl.java +++ b/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/Neo4jProxyImpl.java @@ -604,7 +604,8 @@ public ProcedureSignature procedureSignature( boolean caseInsensitive, boolean systemProcedure, boolean internal, - boolean allowExpiredCredentials + boolean allowExpiredCredentials, + boolean threadSafe ) { return new ProcedureSignature( name, diff --git a/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/Neo4jProxyImpl.java b/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/Neo4jProxyImpl.java index 05c7cef633..03dd860406 100644 --- a/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/Neo4jProxyImpl.java +++ b/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/Neo4jProxyImpl.java @@ -599,7 +599,8 @@ public ProcedureSignature procedureSignature( boolean caseInsensitive, boolean systemProcedure, boolean internal, - boolean allowExpiredCredentials + boolean allowExpiredCredentials, + boolean threadSafe ) { return new ProcedureSignature( name, diff --git a/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/Neo4jProxyImpl.java b/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/Neo4jProxyImpl.java index fa8898dc9f..a9db437b3e 100644 --- a/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/Neo4jProxyImpl.java +++ b/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/Neo4jProxyImpl.java @@ -599,7 +599,8 @@ public ProcedureSignature procedureSignature( boolean caseInsensitive, boolean systemProcedure, boolean internal, - boolean allowExpiredCredentials + boolean allowExpiredCredentials, + boolean threadSafe ) { return new ProcedureSignature( name, diff --git a/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/Neo4jProxyImpl.java b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/Neo4jProxyImpl.java index 43d25ade58..243d4e32cd 100644 --- a/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/Neo4jProxyImpl.java +++ b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/Neo4jProxyImpl.java @@ -599,7 +599,8 @@ public ProcedureSignature procedureSignature( boolean caseInsensitive, boolean systemProcedure, boolean internal, - boolean allowExpiredCredentials + boolean allowExpiredCredentials, + boolean threadSafe ) { return new ProcedureSignature( name, diff --git a/compatibility/api/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/Neo4jProxyApi.java b/compatibility/api/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/Neo4jProxyApi.java index 9e4b754704..81a7b2f6d7 100644 --- a/compatibility/api/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/Neo4jProxyApi.java +++ b/compatibility/api/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/Neo4jProxyApi.java @@ -230,7 +230,8 @@ ProcedureSignature procedureSignature( boolean caseInsensitive, boolean systemProcedure, boolean internal, - boolean allowExpiredCredentials + boolean allowExpiredCredentials, + boolean threadSafe ); long getHighestPossibleNodeCount(Read read, @Nullable IdGeneratorFactory idGeneratorFactory); diff --git a/compatibility/common/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/Neo4jProxy.java b/compatibility/common/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/Neo4jProxy.java index ee819542b0..5eee84233e 100644 --- a/compatibility/common/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/Neo4jProxy.java +++ b/compatibility/common/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/Neo4jProxy.java @@ -318,7 +318,8 @@ public static ProcedureSignature procedureSignature( boolean caseInsensitive, boolean systemProcedure, boolean internal, - boolean allowExpiredCredentials + boolean allowExpiredCredentials, + boolean threadSafe ) { return IMPL.procedureSignature( name, @@ -333,7 +334,8 @@ public static ProcedureSignature procedureSignature( caseInsensitive, systemProcedure, internal, - allowExpiredCredentials + allowExpiredCredentials, + threadSafe ); } From 256a5dd4c95e99280e3412d9fca9640d82fb66ac Mon Sep 17 00:00:00 2001 From: Martin Junghanns Date: Mon, 3 Jul 2023 15:14:58 +0200 Subject: [PATCH 130/273] Make ElementVisitor#reset public CoreDB introduced the same method on the interface level: https://github.com/neo-technology/neo4j/commit/cbffcc77baeba7688af3e76c21888e33bad25722 In order to not conflict with visibilty level, we are making our method abstract and avoid moving the code into compat layers. --- .../main/java/org/neo4j/gds/core/io/file/ElementVisitor.java | 2 +- .../src/main/java/org/neo4j/gds/core/io/file/NodeVisitor.java | 2 +- .../java/org/neo4j/gds/core/io/file/RelationshipVisitor.java | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/io/core/src/main/java/org/neo4j/gds/core/io/file/ElementVisitor.java b/io/core/src/main/java/org/neo4j/gds/core/io/file/ElementVisitor.java index 6a29089ead..dc513f170c 100644 --- a/io/core/src/main/java/org/neo4j/gds/core/io/file/ElementVisitor.java +++ b/io/core/src/main/java/org/neo4j/gds/core/io/file/ElementVisitor.java @@ -56,7 +56,7 @@ abstract class ElementVisitor extends In protected abstract List getPropertySchema(); - abstract void reset(); + public abstract void reset(); @Override public boolean property(String key, Object value) { diff --git a/io/core/src/main/java/org/neo4j/gds/core/io/file/NodeVisitor.java b/io/core/src/main/java/org/neo4j/gds/core/io/file/NodeVisitor.java index 7c97af5307..5cd311e16e 100644 --- a/io/core/src/main/java/org/neo4j/gds/core/io/file/NodeVisitor.java +++ b/io/core/src/main/java/org/neo4j/gds/core/io/file/NodeVisitor.java @@ -106,7 +106,7 @@ protected List getPropertySchema() { } @Override - void reset() { + public void reset() { currentId = -1; currentLabels = EMPTY_LABELS; labelIdentifier = ""; diff --git a/io/core/src/main/java/org/neo4j/gds/core/io/file/RelationshipVisitor.java b/io/core/src/main/java/org/neo4j/gds/core/io/file/RelationshipVisitor.java index ce848e7e5b..3ada3cdcd7 100644 --- a/io/core/src/main/java/org/neo4j/gds/core/io/file/RelationshipVisitor.java +++ b/io/core/src/main/java/org/neo4j/gds/core/io/file/RelationshipVisitor.java @@ -99,7 +99,7 @@ protected List getPropertySchema() { } @Override - void reset() { + public void reset() { currentStartNode = -1; currentEndNode = -1; } From 15c4f4f239f7df6c55085e3fd081c55687c785cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Thu, 6 Jul 2023 15:10:50 +0200 Subject: [PATCH 131/273] Document 4.4.23 as supported --- README.adoc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.adoc b/README.adoc index 1e7648d88f..bdb16c96ee 100644 --- a/README.adoc +++ b/README.adoc @@ -37,7 +37,7 @@ When installing GDS manually, please refer to the below compatibility matrix: |Neo4j 5.4.0 |Neo4j 5.2.0 |Neo4j 5.1.0 -|Neo4j 4.4.9 - 4.4.22 +|Neo4j 4.4.9 - 4.4.23 .1+.^|Java 11 .9+<.^|GDS 2.3.x |Neo4j 5.8.0 @@ -49,7 +49,7 @@ When installing GDS manually, please refer to the below compatibility matrix: |Neo4j 5.3.0 |Neo4j 5.2.0 |Neo4j 5.1.0 -|Neo4j 4.4.9 - 4.4.22 +|Neo4j 4.4.9 - 4.4.23 .1+.^|Java 11 |=== From 7ae99bf7a336f31adcf57b5335f4663e92a24a42 Mon Sep 17 00:00:00 2001 From: Veselin Nikolov Date: Fri, 7 Jul 2023 08:55:53 +0100 Subject: [PATCH 132/273] Update AuraDS version Co-authored-by: Ioannis Panagiotas --- gradle/version.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle/version.gradle b/gradle/version.gradle index 235d471ea7..73ac1df4ff 100644 --- a/gradle/version.gradle +++ b/gradle/version.gradle @@ -1,6 +1,6 @@ ext { gdsBaseVersion = '2.4.2' - gdsAuraVersion = '25' + gdsAuraVersion = '26' gdsVersion = gdsBaseVersion + (rootProject.hasProperty('aurads') ? "+${gdsAuraVersion}" : "") } From 99b645b9d143ef242090c5eee5ce3f76b7862905 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B6ren=20Reichardt?= Date: Mon, 26 Jun 2023 11:56:24 +0200 Subject: [PATCH 133/273] Add option to stream node labels with properties --- .../GraphStreamNodePropertiesProc.java | 23 ++++++--- .../UserInputAsStringOrListOfString.java | 2 +- .../GraphExportNodePropertiesConfig.java | 9 +++- .../GraphStreamNodePropertiesProcTest.java | 51 +++++++++++++------ 4 files changed, 61 insertions(+), 24 deletions(-) diff --git a/proc/catalog/src/main/java/org/neo4j/gds/catalog/GraphStreamNodePropertiesProc.java b/proc/catalog/src/main/java/org/neo4j/gds/catalog/GraphStreamNodePropertiesProc.java index d8f3ebf331..0cdf5ea762 100644 --- a/proc/catalog/src/main/java/org/neo4j/gds/catalog/GraphStreamNodePropertiesProc.java +++ b/proc/catalog/src/main/java/org/neo4j/gds/catalog/GraphStreamNodePropertiesProc.java @@ -33,9 +33,11 @@ import org.neo4j.procedure.Procedure; import java.util.Collection; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.LongStream; import java.util.stream.Stream; @@ -93,7 +95,7 @@ public Stream streamNodeProperty( configuration, List.of(nodeProperty), nodeLabels, - (nodeId, propertyName, propertyValue) -> new PropertyResult(nodeId, propertyValue) + (nodeId, propertyName, propertyValue, nodeLabelList) -> new PropertyResult(nodeId, propertyValue, nodeLabelList) ); } @@ -111,7 +113,7 @@ public Stream streamProperty( configuration, List.of(nodeProperty), nodeLabels, - (nodeId, propertyName, propertyValue) -> new PropertyResult(nodeId, propertyValue), + (nodeId, propertyName, propertyValue, nodeLabelList) -> new PropertyResult(nodeId, propertyValue, nodeLabelList), Optional.of(deprecationWarning) ); } @@ -173,6 +175,10 @@ private Stream streamNodeProperties( deprecationWarning.ifPresent(taskProgressTracker::logWarning); + Function> nodeLabelsFn = config.listNodeLabels() + ? nodeId -> subGraph.nodeLabels(nodeId).stream().map(NodeLabel::name).collect(Collectors.toList()) + : nodeId -> Collections.emptyList(); + var resultStream = LongStream .range(0, subGraph.nodeCount()) .boxed() @@ -184,7 +190,8 @@ private Stream streamNodeProperties( return producer.produce( originalId, usesPropertyNameColumn ? propertyKeyAndValues.getKey() : null, - propertyKeyAndValues.getValue().getObject(nodeId) + propertyKeyAndValues.getValue().getObject(nodeId), + nodeLabelsFn.apply(nodeId) ); }); }) @@ -201,11 +208,13 @@ public static class PropertiesResult { public final long nodeId; public final String nodeProperty; public final Object propertyValue; + public final List nodeLabels; - PropertiesResult(long nodeId, String nodeProperty, Object propertyValue) { + PropertiesResult(long nodeId, String nodeProperty, Object propertyValue, List nodeLabels) { this.nodeId = nodeId; this.nodeProperty = nodeProperty; this.propertyValue = propertyValue; + this.nodeLabels = nodeLabels; } } @@ -213,15 +222,17 @@ public static class PropertiesResult { public static class PropertyResult { public final long nodeId; public final Object propertyValue; + public final List nodeLabels; - PropertyResult(long nodeId, Object propertyValue) { + PropertyResult(long nodeId, Object propertyValue, List nodeLabels) { this.nodeId = nodeId; this.propertyValue = propertyValue; + this.nodeLabels = nodeLabels; } } interface ResultProducer { - R produce(long nodeId, String propertyName, Object propertyValue); + R produce(long nodeId, String propertyName, Object propertyValue, List nodeLabels); } } diff --git a/proc/catalog/src/main/java/org/neo4j/gds/catalog/UserInputAsStringOrListOfString.java b/proc/catalog/src/main/java/org/neo4j/gds/catalog/UserInputAsStringOrListOfString.java index 71ae76da9f..4f02d87dd7 100644 --- a/proc/catalog/src/main/java/org/neo4j/gds/catalog/UserInputAsStringOrListOfString.java +++ b/proc/catalog/src/main/java/org/neo4j/gds/catalog/UserInputAsStringOrListOfString.java @@ -36,7 +36,7 @@ private UserInputAsStringOrListOfString() {} public static List parse(Object userInput, String configurationKey) { if (userInput instanceof Iterable) { var result = new ArrayList(); - for (Object item : (Iterable) userInput) { + for (Object item : (Iterable) userInput) { result.add(parseOne(item, configurationKey)); } return result; diff --git a/proc/catalog/src/main/java/org/neo4j/gds/config/GraphExportNodePropertiesConfig.java b/proc/catalog/src/main/java/org/neo4j/gds/config/GraphExportNodePropertiesConfig.java index 8d61253784..0d190e676f 100644 --- a/proc/catalog/src/main/java/org/neo4j/gds/config/GraphExportNodePropertiesConfig.java +++ b/proc/catalog/src/main/java/org/neo4j/gds/config/GraphExportNodePropertiesConfig.java @@ -19,6 +19,7 @@ */ package org.neo4j.gds.config; +import org.immutables.value.Value; import org.neo4j.gds.ElementProjection; import org.neo4j.gds.NodeLabel; import org.neo4j.gds.annotation.Configuration; @@ -38,8 +39,14 @@ public interface GraphExportNodePropertiesConfig extends GraphNodePropertiesConf @Configuration.ConvertWith(method = "org.neo4j.gds.config.GraphExportNodePropertiesConfig#parseNodeProperties") List nodeProperties(); + @Value.Default + @Value.Parameter(false) + @Configuration.Key("listNodeLabels") + default boolean listNodeLabels() { + return false; + } + static List parseNodeProperties(Object userInput) { - return UserInputAsStringOrListOfString.parse(userInput, "nodeProperties"); } diff --git a/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphStreamNodePropertiesProcTest.java b/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphStreamNodePropertiesProcTest.java index d1426b048b..3aa9524a97 100644 --- a/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphStreamNodePropertiesProcTest.java +++ b/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphStreamNodePropertiesProcTest.java @@ -44,6 +44,8 @@ import org.neo4j.test.TestDatabaseManagementServiceBuilder; import org.neo4j.test.extension.ExtensionCallback; +import java.util.Collections; +import java.util.List; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; @@ -129,32 +131,32 @@ void tearDown() { "CALL gds.graph.nodeProperties.stream(" + " '%s', " + " ['newNodeProp1', 'newNodeProp2']" + - ") YIELD nodeId, nodeProperty, propertyValue " + - "RETURN nodeId AS id, nodeProperty, propertyValue", + ") YIELD nodeId, nodeProperty, propertyValue, nodeLabels " + + "RETURN nodeId AS id, nodeProperty, propertyValue, nodeLabels", // explicit PROJECT_ALL "CALL gds.graph.nodeProperties.stream(" + " '%s', " + " ['newNodeProp1', 'newNodeProp2'], " + " ['*']" + - ") YIELD nodeId, nodeProperty, propertyValue " + - "RETURN nodeId AS id, nodeProperty, propertyValue" + ") YIELD nodeId, nodeProperty, propertyValue, nodeLabels " + + "RETURN nodeId AS id, nodeProperty, propertyValue, nodeLabels" }) void streamLoadedNodeProperties(String graphWriteQueryTemplate) { String graphWriteQuery = formatWithLocale(graphWriteQueryTemplate, TEST_GRAPH_SAME_PROPERTIES); assertCypherResult(graphWriteQuery, asList( - map("id", idFunction.of("a"), "nodeProperty", "newNodeProp1", "propertyValue", 0D), - map("id", idFunction.of("a"), "nodeProperty", "newNodeProp2", "propertyValue", 42L), - map("id", idFunction.of("b"), "nodeProperty", "newNodeProp1", "propertyValue", 1D), - map("id", idFunction.of("b"), "nodeProperty", "newNodeProp2", "propertyValue", 43L), - map("id", idFunction.of("c"), "nodeProperty", "newNodeProp1", "propertyValue", 2D), - map("id", idFunction.of("c"), "nodeProperty", "newNodeProp2", "propertyValue", 44L), - map("id", idFunction.of("d"), "nodeProperty", "newNodeProp1", "propertyValue", 3D), - map("id", idFunction.of("d"), "nodeProperty", "newNodeProp2", "propertyValue", 45L), - map("id", idFunction.of("e"), "nodeProperty", "newNodeProp1", "propertyValue", 4D), - map("id", idFunction.of("e"), "nodeProperty", "newNodeProp2", "propertyValue", 46L), - map("id", idFunction.of("f"), "nodeProperty", "newNodeProp1", "propertyValue", 5D), - map("id", idFunction.of("f"), "nodeProperty", "newNodeProp2", "propertyValue", 47L) + map("id", idFunction.of("a"), "nodeProperty", "newNodeProp1", "propertyValue", 0D, "nodeLabels", Collections.emptyList()), + map("id", idFunction.of("a"), "nodeProperty", "newNodeProp2", "propertyValue", 42L, "nodeLabels", Collections.emptyList()), + map("id", idFunction.of("b"), "nodeProperty", "newNodeProp1", "propertyValue", 1D, "nodeLabels", Collections.emptyList()), + map("id", idFunction.of("b"), "nodeProperty", "newNodeProp2", "propertyValue", 43L, "nodeLabels", Collections.emptyList()), + map("id", idFunction.of("c"), "nodeProperty", "newNodeProp1", "propertyValue", 2D, "nodeLabels", Collections.emptyList()), + map("id", idFunction.of("c"), "nodeProperty", "newNodeProp2", "propertyValue", 44L, "nodeLabels", Collections.emptyList()), + map("id", idFunction.of("d"), "nodeProperty", "newNodeProp1", "propertyValue", 3D, "nodeLabels", Collections.emptyList()), + map("id", idFunction.of("d"), "nodeProperty", "newNodeProp2", "propertyValue", 45L, "nodeLabels", Collections.emptyList()), + map("id", idFunction.of("e"), "nodeProperty", "newNodeProp1", "propertyValue", 4D, "nodeLabels", Collections.emptyList()), + map("id", idFunction.of("e"), "nodeProperty", "newNodeProp2", "propertyValue", 46L, "nodeLabels", Collections.emptyList()), + map("id", idFunction.of("f"), "nodeProperty", "newNodeProp1", "propertyValue", 5D, "nodeLabels", Collections.emptyList()), + map("id", idFunction.of("f"), "nodeProperty", "newNodeProp2", "propertyValue", 47L, "nodeLabels", Collections.emptyList()) )); } @@ -240,6 +242,23 @@ void streamMutatedNodeProperties() { )); } + @Test + void shouldListLabelsIfConfigIsSet() { + assertCypherResult( + "CALL gds.graph.nodeProperties.stream($graph, 'newNodeProp1', ['*'], {listNodeLabels: true}) " + + " YIELD nodeId, nodeProperty, propertyValue, nodeLabels " + + " RETURN nodeId AS id, nodeProperty, propertyValue, nodeLabels", + Map.of("graph", TEST_GRAPH_DIFFERENT_PROPERTIES), + asList( + Map.of("id", idFunction.of("a"), "nodeProperty", "newNodeProp1", "propertyValue", 0D, "nodeLabels", List.of("A")), + Map.of("id", idFunction.of("b"), "nodeProperty", "newNodeProp1", "propertyValue", 1D, "nodeLabels", List.of("A")), + Map.of("id", idFunction.of("c"), "nodeProperty", "newNodeProp1", "propertyValue", 2D, "nodeLabels", List.of("A")), + Map.of("id", idFunction.of("d"), "nodeProperty", "newNodeProp1", "propertyValue", 3D, "nodeLabels", List.of("B")), + Map.of("id", idFunction.of("e"), "nodeProperty", "newNodeProp1", "propertyValue", 4D, "nodeLabels", List.of("B")), + Map.of("id", idFunction.of("f"), "nodeProperty", "newNodeProp1", "propertyValue", 5D, "nodeLabels", List.of("B")) + )); + } + @Test void shouldFailOnNonExistingNodeProperties() { assertError( From afafe8ef529336fd15f380ce96b203ca8421e8de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B6ren=20Reichardt?= Date: Wed, 5 Jul 2023 12:02:13 +0200 Subject: [PATCH 134/273] Stream nodeLabels with properties in arrow --- .../org/neo4j/gds/config/GraphExportNodePropertiesConfig.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/proc/catalog/src/main/java/org/neo4j/gds/config/GraphExportNodePropertiesConfig.java b/proc/catalog/src/main/java/org/neo4j/gds/config/GraphExportNodePropertiesConfig.java index 0d190e676f..648af2657e 100644 --- a/proc/catalog/src/main/java/org/neo4j/gds/config/GraphExportNodePropertiesConfig.java +++ b/proc/catalog/src/main/java/org/neo4j/gds/config/GraphExportNodePropertiesConfig.java @@ -35,6 +35,8 @@ public interface GraphExportNodePropertiesConfig extends GraphNodePropertiesConfig { + boolean LIST_NODE_LABELS_DEFAULT_VALUE = false; + @Configuration.Parameter @Configuration.ConvertWith(method = "org.neo4j.gds.config.GraphExportNodePropertiesConfig#parseNodeProperties") List nodeProperties(); @@ -43,7 +45,7 @@ public interface GraphExportNodePropertiesConfig extends GraphNodePropertiesConf @Value.Parameter(false) @Configuration.Key("listNodeLabels") default boolean listNodeLabels() { - return false; + return LIST_NODE_LABELS_DEFAULT_VALUE; } static List parseNodeProperties(Object userInput) { From f7e9756f625bf41c4c8497fed2f200dae6e80c53 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B6ren=20Reichardt?= Date: Wed, 5 Jul 2023 14:41:05 +0200 Subject: [PATCH 135/273] Document listNodeLabels flag --- .../pages/graph-catalog-apache-arrow-ops.adoc | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/doc/modules/ROOT/pages/graph-catalog-apache-arrow-ops.adoc b/doc/modules/ROOT/pages/graph-catalog-apache-arrow-ops.adoc index 4023f7a5b5..a6f763b7a5 100644 --- a/doc/modules/ROOT/pages/graph-catalog-apache-arrow-ops.adoc +++ b/doc/modules/ROOT/pages/graph-catalog-apache-arrow-ops.adoc @@ -47,7 +47,8 @@ To stream a single node property, the client needs to encode that information in procedure_name: "gds.graph.nodeProperty.stream", configuration: { node_labels: ["*"], - node_property: "foo" + node_property: "foo", + list_node_labels: true } } ---- @@ -61,20 +62,22 @@ The specific configuration needs to include the following keys: | Name | Type | Description | node_labels | String or List of Strings | Stream only properties for nodes with the given labels. | node_property | String | The node property in the graph to stream. +| list_node_labels | Boolean | Whether to include node labels of the respective nodes in the result. |=== The schema of the result records is identical to the corresponding procedure: .Results -[opts="header",cols="2,3,5"] +[opts="header",cols="2,3,5, 1"] |=== -| Name | Type | Description -|nodeId | Integer | The id of the node. +| Name | Type | Description | Optional +| nodeId | Integer | The id of the node. | false .^|propertyValue a| * Integer * Float * List of Integer -* List of Float .^| The stored property value. +* List of Float .^| The stored property value. | false +| labels | List of Strings | The labels of the node. If the `list_node_labels` option was set | true |=== @@ -89,7 +92,8 @@ To stream multiple node properties, the client needs to encode that information procedure_name: "gds.graph.streamNodeProperties", configuration: { node_labels: ["*"], - node_properties: ["foo", "bar", "baz"] + node_properties: ["foo", "bar", "baz"], + list_node_labels: true } } ---- @@ -103,6 +107,7 @@ The specific configuration needs to include the following keys: | Name | Type | Description | node_labels | String or List of Strings | Stream only properties for nodes with the given labels. | node_properties | String or List of Strings | The node properties in the graph to stream. +| list_node_labels | Boolean | Whether to include node labels of the respective nodes in the result. |=== Note that the schema of the result records is not identical to the corresponding procedure. From e7e9c0076cc78eab9671eb270fc0d367add1520d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Mon, 10 Jul 2023 11:04:32 +0200 Subject: [PATCH 136/273] Close progress tracker on failure --- .../io/file/GraphStoreToFileExporter.java | 15 +++++--- .../gds/catalog/GraphStoreExportProc.java | 36 +++++++++++-------- .../java/org/neo4j/gds/TestTaskStore.java | 1 + 3 files changed, 34 insertions(+), 18 deletions(-) diff --git a/io/core/src/main/java/org/neo4j/gds/core/io/file/GraphStoreToFileExporter.java b/io/core/src/main/java/org/neo4j/gds/core/io/file/GraphStoreToFileExporter.java index 1d854e994b..e3096445ed 100644 --- a/io/core/src/main/java/org/neo4j/gds/core/io/file/GraphStoreToFileExporter.java +++ b/io/core/src/main/java/org/neo4j/gds/core/io/file/GraphStoreToFileExporter.java @@ -104,10 +104,17 @@ protected void export(GraphStoreInput graphStoreInput) { exportGraphCapabilities(graphStoreInput); } var progressTracker = createProgressTracker(graphStoreInput); - progressTracker.beginSubTask(); - exportNodes(graphStoreInput, progressTracker); - exportRelationships(graphStoreInput, progressTracker); - exportGraphProperties(graphStoreInput, progressTracker); + + try { + progressTracker.beginSubTask(); + exportNodes(graphStoreInput, progressTracker); + exportRelationships(graphStoreInput, progressTracker); + exportGraphProperties(graphStoreInput, progressTracker); + } catch (Exception e) { + // as the tracker is created in this method + progressTracker.endSubTaskWithFailure(); + throw e; + } progressTracker.endSubTask(); } diff --git a/proc/catalog/src/main/java/org/neo4j/gds/catalog/GraphStoreExportProc.java b/proc/catalog/src/main/java/org/neo4j/gds/catalog/GraphStoreExportProc.java index ba8250e135..59aa437545 100644 --- a/proc/catalog/src/main/java/org/neo4j/gds/catalog/GraphStoreExportProc.java +++ b/proc/catalog/src/main/java/org/neo4j/gds/catalog/GraphStoreExportProc.java @@ -92,20 +92,28 @@ public Stream database( progressTracker ); - var start = System.nanoTime(); - var exportedProperties = exporter.run(); - var end = System.nanoTime(); - - return new DatabaseExportResult( - graphName, - exportConfig.dbName(), - graphStore.nodeCount(), - graphStore.relationshipCount(), - graphStore.relationshipTypes().size(), - exportedProperties.nodePropertyCount(), - exportedProperties.relationshipPropertyCount(), - java.util.concurrent.TimeUnit.NANOSECONDS.toMillis(end - start) - ); + try { + var start = System.nanoTime(); + var exportedProperties = exporter.run(); + var end = System.nanoTime(); + + return new DatabaseExportResult( + graphName, + exportConfig.dbName(), + graphStore.nodeCount(), + graphStore.relationshipCount(), + graphStore.relationshipTypes().size(), + exportedProperties.nodePropertyCount(), + exportedProperties.relationshipPropertyCount(), + java.util.concurrent.TimeUnit.NANOSECONDS.toMillis(end - start) + ); + } catch (Exception e) { + // Ideally we should not have this logic on the proc level + // the progress tracker is instantiated in this proc + // so we need to make sure it closes as well + progressTracker.endSubTaskWithFailure(); + throw e; + } } ); diff --git a/test-utils/src/main/java/org/neo4j/gds/TestTaskStore.java b/test-utils/src/main/java/org/neo4j/gds/TestTaskStore.java index 339f7f8e84..30d2221511 100644 --- a/test-utils/src/main/java/org/neo4j/gds/TestTaskStore.java +++ b/test-utils/src/main/java/org/neo4j/gds/TestTaskStore.java @@ -43,6 +43,7 @@ public void store(String username, JobId jobId, Task task) { @Override public void remove(String username, JobId jobId) { + super.remove(username, jobId); tasks.remove(jobId); } From 65b7cb3146548f9ef10fc94169d009a3e6a92b71 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonatan=20J=C3=A4derberg?= Date: Tue, 11 Jul 2023 14:27:07 +0200 Subject: [PATCH 137/273] Publish compat layer for DB V5.10 Co-Authored-By: Ioannis Panagiotas Co-Authored-By: Veselin Nikolov --- .../5.10/neo4j-kernel-adapter/build.gradle | 65 ++ .../compat/_510/Neo4jProxyFactoryImpl.java | 44 + .../compat/_510/SettingProxyFactoryImpl.java | 44 + .../_510/BoltTransactionRunnerImpl.java | 97 ++ .../compat/_510/CallableProcedureImpl.java | 53 + .../CallableUserAggregationFunctionImpl.java | 77 ++ .../gds/compat/_510/CompatAccessModeImpl.java | 44 + .../_510/CompatGraphDatabaseAPIImpl.java | 74 ++ .../gds/compat/_510/CompatIndexQueryImpl.java | 31 + .../_510/CompatUsernameAuthSubjectImpl.java | 35 + .../compat/_510/CompositeNodeCursorImpl.java | 32 + .../compat/_510/GdsDatabaseLayoutImpl.java | 50 + ...sDatabaseManagementServiceBuilderImpl.java | 54 + .../compat/_510/Neo4jProxyFactoryImpl.java | 44 + .../neo4j/gds/compat/_510/Neo4jProxyImpl.java | 919 ++++++++++++++++++ .../compat/_510/NodeLabelIndexLookupImpl.java | 68 ++ .../gds/compat/_510/PartitionedStoreScan.java | 58 ++ .../_510/ReferencePropertyReference.java | 49 + .../compat/_510/ScanBasedStoreScanImpl.java | 40 + .../compat/_510/SettingProxyFactoryImpl.java | 44 + .../gds/compat/_510/SettingProxyImpl.java | 87 ++ .../neo4j/gds/compat/_510/TestLogImpl.java | 146 +++ .../compat/_510/VirtualRelationshipImpl.java | 41 + .../5.10/storage-engine-adapter/build.gradle | 68 ++ .../_510/InMemoryStorageEngineFactory.java | 268 +++++ .../_510/StorageEngineProxyFactoryImpl.java | 44 + .../InMemoryCommandCreationContextImpl.java | 107 ++ .../compat/_510/InMemoryCountsStoreImpl.java | 112 +++ .../_510/InMemoryMetaDataProviderImpl.java | 201 ++++ .../gds/compat/_510/InMemoryNodeCursor.java | 82 ++ .../_510/InMemoryNodePropertyCursor.java | 45 + .../compat/_510/InMemoryPropertyCursor.java | 71 ++ .../_510/InMemoryPropertySelectionImpl.java | 55 ++ .../InMemoryRelationshipPropertyCursor.java | 60 ++ .../_510/InMemoryRelationshipScanCursor.java | 61 ++ .../InMemoryRelationshipTraversalCursor.java | 47 + .../_510/InMemoryStorageEngineFactory.java | 568 +++++++++++ .../_510/InMemoryStorageEngineImpl.java | 347 +++++++ .../compat/_510/InMemoryStorageLocksImpl.java | 86 ++ .../gds/compat/_510/InMemoryStoreVersion.java | 70 ++ .../_510/InMemoryTransactionIdStoreImpl.java | 117 +++ .../gds/compat/_510/InMemoryVersionCheck.java | 61 ++ .../_510/StorageEngineProxyFactoryImpl.java | 44 + .../compat/_510/StorageEngineProxyImpl.java | 156 +++ .../InMemoryLogVersionRepository510.java | 71 ++ ...nMemoryStorageCommandReaderFactory510.java | 43 + .../InMemoryStorageReader510.java | 332 +++++++ gradle/dependencies.gradle | 1 + .../org/neo4j/gds/compat/Neo4jVersion.java | 7 +- .../neo4j/gds/compat/Neo4jVersionTest.java | 3 +- .../java/org/neo4j/gds/SysInfoProcTest.java | 13 + 51 files changed, 5334 insertions(+), 2 deletions(-) create mode 100644 compatibility/5.10/neo4j-kernel-adapter/build.gradle create mode 100644 compatibility/5.10/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_510/Neo4jProxyFactoryImpl.java create mode 100644 compatibility/5.10/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_510/SettingProxyFactoryImpl.java create mode 100644 compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/BoltTransactionRunnerImpl.java create mode 100644 compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/CallableProcedureImpl.java create mode 100644 compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/CallableUserAggregationFunctionImpl.java create mode 100644 compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/CompatAccessModeImpl.java create mode 100644 compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/CompatGraphDatabaseAPIImpl.java create mode 100644 compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/CompatIndexQueryImpl.java create mode 100644 compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/CompatUsernameAuthSubjectImpl.java create mode 100644 compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/CompositeNodeCursorImpl.java create mode 100644 compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/GdsDatabaseLayoutImpl.java create mode 100644 compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/GdsDatabaseManagementServiceBuilderImpl.java create mode 100644 compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/Neo4jProxyFactoryImpl.java create mode 100644 compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/Neo4jProxyImpl.java create mode 100644 compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/NodeLabelIndexLookupImpl.java create mode 100644 compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/PartitionedStoreScan.java create mode 100644 compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/ReferencePropertyReference.java create mode 100644 compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/ScanBasedStoreScanImpl.java create mode 100644 compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/SettingProxyFactoryImpl.java create mode 100644 compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/SettingProxyImpl.java create mode 100644 compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/TestLogImpl.java create mode 100644 compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/VirtualRelationshipImpl.java create mode 100644 compatibility/5.10/storage-engine-adapter/build.gradle create mode 100644 compatibility/5.10/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_510/InMemoryStorageEngineFactory.java create mode 100644 compatibility/5.10/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_510/StorageEngineProxyFactoryImpl.java create mode 100644 compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/InMemoryCommandCreationContextImpl.java create mode 100644 compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/InMemoryCountsStoreImpl.java create mode 100644 compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/InMemoryMetaDataProviderImpl.java create mode 100644 compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/InMemoryNodeCursor.java create mode 100644 compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/InMemoryNodePropertyCursor.java create mode 100644 compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/InMemoryPropertyCursor.java create mode 100644 compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/InMemoryPropertySelectionImpl.java create mode 100644 compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/InMemoryRelationshipPropertyCursor.java create mode 100644 compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/InMemoryRelationshipScanCursor.java create mode 100644 compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/InMemoryRelationshipTraversalCursor.java create mode 100644 compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/InMemoryStorageEngineFactory.java create mode 100644 compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/InMemoryStorageEngineImpl.java create mode 100644 compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/InMemoryStorageLocksImpl.java create mode 100644 compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/InMemoryStoreVersion.java create mode 100644 compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/InMemoryTransactionIdStoreImpl.java create mode 100644 compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/InMemoryVersionCheck.java create mode 100644 compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/StorageEngineProxyFactoryImpl.java create mode 100644 compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/StorageEngineProxyImpl.java create mode 100644 compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository510.java create mode 100644 compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory510.java create mode 100644 compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageReader510.java diff --git a/compatibility/5.10/neo4j-kernel-adapter/build.gradle b/compatibility/5.10/neo4j-kernel-adapter/build.gradle new file mode 100644 index 0000000000..7b8bcc5224 --- /dev/null +++ b/compatibility/5.10/neo4j-kernel-adapter/build.gradle @@ -0,0 +1,65 @@ +apply plugin: 'java-library' +apply plugin: 'me.champeau.mrjar' + +description = 'Neo4j Graph Data Science :: Neo4j Kernel Adapter 5.10' + +group = 'org.neo4j.gds' + +// for all 5.x versions +if (ver.'neo4j'.startsWith('5.')) { + sourceSets { + main { + java { + srcDirs = ['src/main/java17'] + } + } + } + + dependencies { + annotationProcessor project(':annotations') + annotationProcessor group: 'org.immutables', name: 'value', version: ver.'immutables' + annotationProcessor group: 'org.neo4j', name: 'annotations', version: neos.'5.10' + + compileOnly project(':annotations') + compileOnly group: 'com.github.spotbugs', name: 'spotbugs-annotations', version: ver.'spotbugsToolVersion' + compileOnly group: 'org.immutables', name: 'value-annotations', version: ver.'immutables' + compileOnly group: 'org.neo4j', name: 'annotations', version: neos.'5.10' + compileOnly group: 'org.neo4j', name: 'neo4j', version: neos.'5.10' + compileOnly group: 'org.neo4j', name: 'neo4j-record-storage-engine', version: neos.'5.10' + compileOnly group: 'org.neo4j.community', name: 'it-test-support', version: neos.'5.10' + + implementation project(':neo4j-kernel-adapter-api') + } +} else { + multiRelease { + targetVersions 11, 17 + } + + if (!project.hasProperty('no-forbidden-apis')) { + forbiddenApisJava17 { + exclude('**') + } + } + + dependencies { + annotationProcessor group: 'org.neo4j', name: 'annotations', version: ver.'neo4j' + + compileOnly project(':annotations') + compileOnly group: 'org.neo4j', name: 'annotations', version: ver.'neo4j' + + implementation project(':neo4j-kernel-adapter-api') + + java17AnnotationProcessor project(':annotations') + java17AnnotationProcessor group: 'org.immutables', name: 'value', version: ver.'immutables' + java17AnnotationProcessor group: 'org.neo4j', name: 'annotations', version: neos.'5.10' + + java17CompileOnly project(':annotations') + java17CompileOnly group: 'org.immutables', name: 'value-annotations', version: ver.'immutables' + java17CompileOnly group: 'org.neo4j', name: 'neo4j', version: neos.'5.10' + java17CompileOnly group: 'org.neo4j', name: 'neo4j-record-storage-engine', version: neos.'5.10' + java17CompileOnly group: 'org.neo4j.community', name: 'it-test-support', version: neos.'5.10' + java17CompileOnly group: 'com.github.spotbugs', name: 'spotbugs-annotations', version: ver.'spotbugsToolVersion' + + java17Implementation project(':neo4j-kernel-adapter-api') + } +} diff --git a/compatibility/5.10/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_510/Neo4jProxyFactoryImpl.java b/compatibility/5.10/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_510/Neo4jProxyFactoryImpl.java new file mode 100644 index 0000000000..119c57cae4 --- /dev/null +++ b/compatibility/5.10/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_510/Neo4jProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._510; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jProxyApi; +import org.neo4j.gds.compat.Neo4jProxyFactory; +import org.neo4j.gds.compat.Neo4jVersion; + +@ServiceProvider +public final class Neo4jProxyFactoryImpl implements Neo4jProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return false; + } + + @Override + public Neo4jProxyApi load() { + throw new UnsupportedOperationException("5.10 compatibility requires JDK17"); + } + + @Override + public String description() { + return "Neo4j 5.10 (placeholder)"; + } +} diff --git a/compatibility/5.10/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_510/SettingProxyFactoryImpl.java b/compatibility/5.10/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_510/SettingProxyFactoryImpl.java new file mode 100644 index 0000000000..9a92eb7c20 --- /dev/null +++ b/compatibility/5.10/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_510/SettingProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._510; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jVersion; +import org.neo4j.gds.compat.SettingProxyApi; +import org.neo4j.gds.compat.SettingProxyFactory; + +@ServiceProvider +public final class SettingProxyFactoryImpl implements SettingProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return false; + } + + @Override + public SettingProxyApi load() { + throw new UnsupportedOperationException("5.10 compatibility requires JDK17"); + } + + @Override + public String description() { + return "Neo4j Settings 5.10 (placeholder)"; + } +} diff --git a/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/BoltTransactionRunnerImpl.java b/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/BoltTransactionRunnerImpl.java new file mode 100644 index 0000000000..c507ade688 --- /dev/null +++ b/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/BoltTransactionRunnerImpl.java @@ -0,0 +1,97 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._510; + +import org.neo4j.bolt.dbapi.BoltGraphDatabaseServiceSPI; +import org.neo4j.bolt.dbapi.BoltTransaction; +import org.neo4j.bolt.protocol.common.message.AccessMode; +import org.neo4j.bolt.protocol.common.message.request.connection.RoutingContext; +import org.neo4j.bolt.tx.statement.StatementQuerySubscriber; +import org.neo4j.exceptions.KernelException; +import org.neo4j.gds.compat.BoltQuerySubscriber; +import org.neo4j.gds.compat.BoltTransactionRunner; +import org.neo4j.graphdb.QueryStatistics; +import org.neo4j.internal.kernel.api.connectioninfo.ClientConnectionInfo; +import org.neo4j.internal.kernel.api.security.LoginContext; +import org.neo4j.kernel.api.KernelTransaction; +import org.neo4j.kernel.impl.query.QueryExecutionConfiguration; +import org.neo4j.kernel.impl.query.QueryExecutionKernelException; +import org.neo4j.values.virtual.MapValue; + +import java.time.Duration; +import java.util.List; +import java.util.Map; + +public class BoltTransactionRunnerImpl extends BoltTransactionRunner { + + @Override + protected BoltQuerySubscriber boltQuerySubscriber() { + var subscriber = new StatementQuerySubscriber(); + return new BoltQuerySubscriber<>() { + @Override + public void assertSucceeded() throws KernelException { + subscriber.assertSuccess(); + } + + @Override + public QueryStatistics queryStatistics() { + return subscriber.getStatistics(); + } + + @Override + public StatementQuerySubscriber innerSubscriber() { + return subscriber; + } + }; + } + + @Override + protected void executeQuery( + BoltTransaction boltTransaction, + String query, + MapValue parameters, + StatementQuerySubscriber querySubscriber + ) throws QueryExecutionKernelException { + boltTransaction.executeQuery(query, parameters, true, querySubscriber); + } + + @Override + protected BoltTransaction beginBoltWriteTransaction( + BoltGraphDatabaseServiceSPI fabricDb, + LoginContext loginContext, + KernelTransaction.Type kernelTransactionType, + ClientConnectionInfo clientConnectionInfo, + List bookmarks, + Duration txTimeout, + Map txMetadata + ) { + return fabricDb.beginTransaction( + kernelTransactionType, + loginContext, + clientConnectionInfo, + bookmarks, + txTimeout, + AccessMode.WRITE, + txMetadata, + new RoutingContext(true, Map.of()), + QueryExecutionConfiguration.DEFAULT_CONFIG + ); + } +} diff --git a/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/CallableProcedureImpl.java b/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/CallableProcedureImpl.java new file mode 100644 index 0000000000..b273fb43e8 --- /dev/null +++ b/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/CallableProcedureImpl.java @@ -0,0 +1,53 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._510; + +import org.neo4j.collection.RawIterator; +import org.neo4j.gds.annotation.SuppressForbidden; +import org.neo4j.gds.compat.CompatCallableProcedure; +import org.neo4j.internal.kernel.api.exceptions.ProcedureException; +import org.neo4j.internal.kernel.api.procs.ProcedureSignature; +import org.neo4j.kernel.api.ResourceMonitor; +import org.neo4j.kernel.api.procedure.CallableProcedure; +import org.neo4j.kernel.api.procedure.Context; +import org.neo4j.values.AnyValue; + +@SuppressForbidden(reason = "This is the compat API") +public final class CallableProcedureImpl implements CallableProcedure { + private final CompatCallableProcedure procedure; + + CallableProcedureImpl(CompatCallableProcedure procedure) { + this.procedure = procedure; + } + + @Override + public ProcedureSignature signature() { + return this.procedure.signature(); + } + + @Override + public RawIterator apply( + Context ctx, + AnyValue[] input, + ResourceMonitor resourceMonitor + ) throws ProcedureException { + return this.procedure.apply(ctx, input); + } +} diff --git a/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/CallableUserAggregationFunctionImpl.java b/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/CallableUserAggregationFunctionImpl.java new file mode 100644 index 0000000000..d120d4b831 --- /dev/null +++ b/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/CallableUserAggregationFunctionImpl.java @@ -0,0 +1,77 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._510; + +import org.neo4j.gds.annotation.SuppressForbidden; +import org.neo4j.gds.compat.CompatUserAggregationFunction; +import org.neo4j.gds.compat.CompatUserAggregator; +import org.neo4j.internal.kernel.api.exceptions.ProcedureException; +import org.neo4j.internal.kernel.api.procs.UserAggregationReducer; +import org.neo4j.internal.kernel.api.procs.UserAggregationUpdater; +import org.neo4j.internal.kernel.api.procs.UserFunctionSignature; +import org.neo4j.kernel.api.procedure.CallableUserAggregationFunction; +import org.neo4j.kernel.api.procedure.Context; +import org.neo4j.values.AnyValue; + +@SuppressForbidden(reason = "This is the compat API") +public final class CallableUserAggregationFunctionImpl implements CallableUserAggregationFunction { + private final CompatUserAggregationFunction function; + + CallableUserAggregationFunctionImpl(CompatUserAggregationFunction function) { + this.function = function; + } + + @Override + public UserFunctionSignature signature() { + return this.function.signature(); + } + + @Override + public UserAggregationReducer createReducer(Context ctx) throws ProcedureException { + return new UserAggregatorImpl(this.function.create(ctx)); + } + + private static final class UserAggregatorImpl implements UserAggregationReducer, UserAggregationUpdater { + private final CompatUserAggregator aggregator; + + private UserAggregatorImpl(CompatUserAggregator aggregator) { + this.aggregator = aggregator; + } + + @Override + public UserAggregationUpdater newUpdater() { + return this; + } + + @Override + public void update(AnyValue[] input) throws ProcedureException { + this.aggregator.update(input); + } + + @Override + public void applyUpdates() { + } + + @Override + public AnyValue result() throws ProcedureException { + return this.aggregator.result(); + } + } +} diff --git a/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/CompatAccessModeImpl.java b/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/CompatAccessModeImpl.java new file mode 100644 index 0000000000..2152447a12 --- /dev/null +++ b/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/CompatAccessModeImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._510; + +import org.neo4j.gds.compat.CompatAccessMode; +import org.neo4j.gds.compat.CustomAccessMode; +import org.neo4j.internal.kernel.api.RelTypeSupplier; +import org.neo4j.internal.kernel.api.TokenSet; + +import java.util.function.Supplier; + +public final class CompatAccessModeImpl extends CompatAccessMode { + + CompatAccessModeImpl(CustomAccessMode custom) { + super(custom); + } + + @Override + public boolean allowsReadNodeProperty(Supplier labels, int propertyKey) { + return custom.allowsReadNodeProperty(propertyKey); + } + + @Override + public boolean allowsReadRelationshipProperty(RelTypeSupplier relType, int propertyKey) { + return custom.allowsReadRelationshipProperty(propertyKey); + } +} diff --git a/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/CompatGraphDatabaseAPIImpl.java b/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/CompatGraphDatabaseAPIImpl.java new file mode 100644 index 0000000000..c7c19eda5e --- /dev/null +++ b/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/CompatGraphDatabaseAPIImpl.java @@ -0,0 +1,74 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._510; + +import org.neo4j.dbms.api.DatabaseManagementService; +import org.neo4j.dbms.systemgraph.TopologyGraphDbmsModel; +import org.neo4j.gds.compat.GdsGraphDatabaseAPI; +import org.neo4j.internal.kernel.api.connectioninfo.ClientConnectionInfo; +import org.neo4j.internal.kernel.api.security.LoginContext; +import org.neo4j.kernel.api.KernelTransaction; +import org.neo4j.kernel.api.exceptions.Status; +import org.neo4j.kernel.impl.coreapi.InternalTransaction; +import org.neo4j.kernel.impl.coreapi.TransactionExceptionMapper; + +import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; + +final class CompatGraphDatabaseAPIImpl extends GdsGraphDatabaseAPI { + + CompatGraphDatabaseAPIImpl(DatabaseManagementService dbms) { + super(dbms); + } + + @Override + public boolean isAvailable() { + return api.isAvailable(); + } + + @Override + public TopologyGraphDbmsModel.HostedOnMode mode() { + // NOTE: This means we can never start clusters locally, which is probably fine since: + // 1) We never did this before + // 2) We only use this for tests and benchmarks + return TopologyGraphDbmsModel.HostedOnMode.SINGLE; + } + + @Override + public InternalTransaction beginTransaction( + KernelTransaction.Type type, + LoginContext loginContext, + ClientConnectionInfo clientInfo, + long timeout, + TimeUnit unit, + Consumer terminationCallback, + TransactionExceptionMapper transactionExceptionMapper + ) { + return api.beginTransaction( + type, + loginContext, + clientInfo, + timeout, + unit, + terminationCallback, + transactionExceptionMapper + ); + } +} diff --git a/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/CompatIndexQueryImpl.java b/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/CompatIndexQueryImpl.java new file mode 100644 index 0000000000..35211b8855 --- /dev/null +++ b/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/CompatIndexQueryImpl.java @@ -0,0 +1,31 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._510; + +import org.neo4j.gds.compat.CompatIndexQuery; +import org.neo4j.internal.kernel.api.PropertyIndexQuery; + +final class CompatIndexQueryImpl implements CompatIndexQuery { + final PropertyIndexQuery indexQuery; + + CompatIndexQueryImpl(PropertyIndexQuery indexQuery) { + this.indexQuery = indexQuery; + } +} diff --git a/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/CompatUsernameAuthSubjectImpl.java b/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/CompatUsernameAuthSubjectImpl.java new file mode 100644 index 0000000000..d4521de6ed --- /dev/null +++ b/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/CompatUsernameAuthSubjectImpl.java @@ -0,0 +1,35 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._510; + +import org.neo4j.gds.compat.CompatUsernameAuthSubject; +import org.neo4j.internal.kernel.api.security.AuthSubject; + +final class CompatUsernameAuthSubjectImpl extends CompatUsernameAuthSubject { + + CompatUsernameAuthSubjectImpl(String username, AuthSubject authSubject) { + super(username, authSubject); + } + + @Override + public String executingUser() { + return username; + } +} diff --git a/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/CompositeNodeCursorImpl.java b/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/CompositeNodeCursorImpl.java new file mode 100644 index 0000000000..95fab00f46 --- /dev/null +++ b/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/CompositeNodeCursorImpl.java @@ -0,0 +1,32 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._510; + +import org.neo4j.gds.compat.CompositeNodeCursor; +import org.neo4j.internal.kernel.api.NodeLabelIndexCursor; + +import java.util.List; + +public final class CompositeNodeCursorImpl extends CompositeNodeCursor { + + CompositeNodeCursorImpl(List cursors, int[] labelIds) { + super(cursors, labelIds); + } +} diff --git a/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/GdsDatabaseLayoutImpl.java b/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/GdsDatabaseLayoutImpl.java new file mode 100644 index 0000000000..e656e8bc79 --- /dev/null +++ b/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/GdsDatabaseLayoutImpl.java @@ -0,0 +1,50 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._510; + +import org.neo4j.gds.compat.GdsDatabaseLayout; +import org.neo4j.io.layout.DatabaseLayout; + +import java.nio.file.Path; + +public class GdsDatabaseLayoutImpl implements GdsDatabaseLayout { + private final DatabaseLayout databaseLayout; + + public GdsDatabaseLayoutImpl(DatabaseLayout databaseLayout) {this.databaseLayout = databaseLayout;} + + @Override + public Path databaseDirectory() { + return databaseLayout.databaseDirectory(); + } + + @Override + public Path getTransactionLogsDirectory() { + return databaseLayout.getTransactionLogsDirectory(); + } + + @Override + public Path metadataStore() { + return databaseLayout.metadataStore(); + } + + public DatabaseLayout databaseLayout() { + return databaseLayout; + } +} diff --git a/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/GdsDatabaseManagementServiceBuilderImpl.java b/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/GdsDatabaseManagementServiceBuilderImpl.java new file mode 100644 index 0000000000..3fd29287f7 --- /dev/null +++ b/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/GdsDatabaseManagementServiceBuilderImpl.java @@ -0,0 +1,54 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._510; + +import org.neo4j.dbms.api.DatabaseManagementService; +import org.neo4j.dbms.api.DatabaseManagementServiceBuilderImplementation; +import org.neo4j.gds.compat.GdsDatabaseManagementServiceBuilder; +import org.neo4j.graphdb.config.Setting; + +import java.nio.file.Path; +import java.util.Map; + +public class GdsDatabaseManagementServiceBuilderImpl implements GdsDatabaseManagementServiceBuilder { + + private final DatabaseManagementServiceBuilderImplementation dbmsBuilder; + + GdsDatabaseManagementServiceBuilderImpl(Path storeDir) { + this.dbmsBuilder = new DatabaseManagementServiceBuilderImplementation(storeDir); + } + + @Override + public GdsDatabaseManagementServiceBuilder setConfigRaw(Map configMap) { + dbmsBuilder.setConfigRaw(configMap); + return this; + } + + @Override + public GdsDatabaseManagementServiceBuilder setConfig(Setting setting, S value) { + dbmsBuilder.setConfig(setting, value); + return this; + } + + @Override + public DatabaseManagementService build() { + return dbmsBuilder.build(); + } +} diff --git a/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/Neo4jProxyFactoryImpl.java b/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/Neo4jProxyFactoryImpl.java new file mode 100644 index 0000000000..f60c8eda56 --- /dev/null +++ b/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/Neo4jProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._510; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jProxyApi; +import org.neo4j.gds.compat.Neo4jProxyFactory; +import org.neo4j.gds.compat.Neo4jVersion; + +@ServiceProvider +public final class Neo4jProxyFactoryImpl implements Neo4jProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return version == Neo4jVersion.V_5_10; + } + + @Override + public Neo4jProxyApi load() { + return new Neo4jProxyImpl(); + } + + @Override + public String description() { + return "Neo4j 5.10"; + } +} diff --git a/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/Neo4jProxyImpl.java b/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/Neo4jProxyImpl.java new file mode 100644 index 0000000000..48822655c0 --- /dev/null +++ b/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/Neo4jProxyImpl.java @@ -0,0 +1,919 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._510; + +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import org.neo4j.common.DependencyResolver; +import org.neo4j.common.EntityType; +import org.neo4j.configuration.BootloaderSettings; +import org.neo4j.configuration.Config; +import org.neo4j.configuration.GraphDatabaseSettings; +import org.neo4j.configuration.SettingValueParsers; +import org.neo4j.configuration.connectors.ConnectorPortRegister; +import org.neo4j.configuration.connectors.ConnectorType; +import org.neo4j.configuration.helpers.DatabaseNameValidator; +import org.neo4j.dbms.api.DatabaseManagementService; +import org.neo4j.exceptions.KernelException; +import org.neo4j.fabric.FabricDatabaseManager; +import org.neo4j.gds.annotation.SuppressForbidden; +import org.neo4j.gds.compat.BoltTransactionRunner; +import org.neo4j.gds.compat.CompatCallableProcedure; +import org.neo4j.gds.compat.CompatExecutionMonitor; +import org.neo4j.gds.compat.CompatIndexQuery; +import org.neo4j.gds.compat.CompatInput; +import org.neo4j.gds.compat.CompatUserAggregationFunction; +import org.neo4j.gds.compat.CompositeNodeCursor; +import org.neo4j.gds.compat.CustomAccessMode; +import org.neo4j.gds.compat.GdsDatabaseLayout; +import org.neo4j.gds.compat.GdsDatabaseManagementServiceBuilder; +import org.neo4j.gds.compat.GdsGraphDatabaseAPI; +import org.neo4j.gds.compat.GraphDatabaseApiProxy; +import org.neo4j.gds.compat.InputEntityIdVisitor; +import org.neo4j.gds.compat.Neo4jProxyApi; +import org.neo4j.gds.compat.PropertyReference; +import org.neo4j.gds.compat.StoreScan; +import org.neo4j.gds.compat.TestLog; +import org.neo4j.graphdb.GraphDatabaseService; +import org.neo4j.graphdb.Node; +import org.neo4j.graphdb.Relationship; +import org.neo4j.graphdb.RelationshipType; +import org.neo4j.graphdb.config.Setting; +import org.neo4j.internal.batchimport.AdditionalInitialIds; +import org.neo4j.internal.batchimport.BatchImporter; +import org.neo4j.internal.batchimport.BatchImporterFactory; +import org.neo4j.internal.batchimport.Configuration; +import org.neo4j.internal.batchimport.IndexConfig; +import org.neo4j.internal.batchimport.InputIterable; +import org.neo4j.internal.batchimport.Monitor; +import org.neo4j.internal.batchimport.input.Collector; +import org.neo4j.internal.batchimport.input.IdType; +import org.neo4j.internal.batchimport.input.Input; +import org.neo4j.internal.batchimport.input.InputEntityVisitor; +import org.neo4j.internal.batchimport.input.PropertySizeCalculator; +import org.neo4j.internal.batchimport.input.ReadableGroups; +import org.neo4j.internal.batchimport.staging.ExecutionMonitor; +import org.neo4j.internal.batchimport.staging.StageExecution; +import org.neo4j.internal.helpers.HostnamePort; +import org.neo4j.internal.id.IdGenerator; +import org.neo4j.internal.id.IdGeneratorFactory; +import org.neo4j.internal.kernel.api.Cursor; +import org.neo4j.internal.kernel.api.IndexQueryConstraints; +import org.neo4j.internal.kernel.api.IndexReadSession; +import org.neo4j.internal.kernel.api.NodeCursor; +import org.neo4j.internal.kernel.api.NodeLabelIndexCursor; +import org.neo4j.internal.kernel.api.NodeValueIndexCursor; +import org.neo4j.internal.kernel.api.PropertyCursor; +import org.neo4j.internal.kernel.api.PropertyIndexQuery; +import org.neo4j.internal.kernel.api.QueryContext; +import org.neo4j.internal.kernel.api.Read; +import org.neo4j.internal.kernel.api.RelationshipScanCursor; +import org.neo4j.internal.kernel.api.Scan; +import org.neo4j.internal.kernel.api.TokenPredicate; +import org.neo4j.internal.kernel.api.connectioninfo.ClientConnectionInfo; +import org.neo4j.internal.kernel.api.procs.FieldSignature; +import org.neo4j.internal.kernel.api.procs.Neo4jTypes; +import org.neo4j.internal.kernel.api.procs.ProcedureSignature; +import org.neo4j.internal.kernel.api.procs.QualifiedName; +import org.neo4j.internal.kernel.api.procs.UserFunctionSignature; +import org.neo4j.internal.kernel.api.security.AccessMode; +import org.neo4j.internal.kernel.api.security.AuthSubject; +import org.neo4j.internal.kernel.api.security.SecurityContext; +import org.neo4j.internal.recordstorage.RecordIdType; +import org.neo4j.internal.schema.IndexCapability; +import org.neo4j.internal.schema.IndexDescriptor; +import org.neo4j.internal.schema.IndexOrder; +import org.neo4j.internal.schema.SchemaDescriptors; +import org.neo4j.io.fs.FileSystemAbstraction; +import org.neo4j.io.layout.DatabaseLayout; +import org.neo4j.io.layout.Neo4jLayout; +import org.neo4j.io.layout.recordstorage.RecordDatabaseLayout; +import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.context.CursorContextFactory; +import org.neo4j.io.pagecache.context.EmptyVersionContextSupplier; +import org.neo4j.io.pagecache.tracing.PageCacheTracer; +import org.neo4j.kernel.api.KernelTransaction; +import org.neo4j.kernel.api.KernelTransactionHandle; +import org.neo4j.kernel.api.procedure.CallableProcedure; +import org.neo4j.kernel.api.procedure.CallableUserAggregationFunction; +import org.neo4j.kernel.database.NormalizedDatabaseName; +import org.neo4j.kernel.impl.coreapi.InternalTransaction; +import org.neo4j.kernel.impl.index.schema.IndexImporterFactoryImpl; +import org.neo4j.kernel.impl.query.QueryExecutionConfiguration; +import org.neo4j.kernel.impl.query.TransactionalContext; +import org.neo4j.kernel.impl.query.TransactionalContextFactory; +import org.neo4j.kernel.impl.store.RecordStore; +import org.neo4j.kernel.impl.store.format.RecordFormatSelector; +import org.neo4j.kernel.impl.store.format.RecordFormats; +import org.neo4j.kernel.impl.store.record.AbstractBaseRecord; +import org.neo4j.kernel.impl.transaction.log.EmptyLogTailMetadata; +import org.neo4j.kernel.impl.transaction.log.files.TransactionLogInitializer; +import org.neo4j.logging.Log; +import org.neo4j.logging.internal.LogService; +import org.neo4j.memory.EmptyMemoryTracker; +import org.neo4j.procedure.Mode; +import org.neo4j.scheduler.JobScheduler; +import org.neo4j.ssl.config.SslPolicyLoader; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.StorageEngineFactory; +import org.neo4j.util.Bits; +import org.neo4j.values.storable.TextArray; +import org.neo4j.values.storable.ValueCategory; +import org.neo4j.values.storable.Values; +import org.neo4j.values.virtual.MapValue; +import org.neo4j.values.virtual.NodeValue; +import org.neo4j.values.virtual.VirtualValues; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Locale; +import java.util.Optional; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +import static java.lang.String.format; +import static org.neo4j.gds.compat.InternalReadOps.countByIdGenerator; +import static org.neo4j.io.pagecache.context.EmptyVersionContextSupplier.EMPTY; + +public final class Neo4jProxyImpl implements Neo4jProxyApi { + + @Override + public GdsGraphDatabaseAPI newDb(DatabaseManagementService dbms) { + return new CompatGraphDatabaseAPIImpl(dbms); + } + + @Override + public String validateExternalDatabaseName(String databaseName) { + var normalizedName = new NormalizedDatabaseName(databaseName); + DatabaseNameValidator.validateExternalDatabaseName(normalizedName); + return normalizedName.name(); + } + + @Override + public AccessMode accessMode(CustomAccessMode customAccessMode) { + return new CompatAccessModeImpl(customAccessMode); + } + + @Override + public String username(AuthSubject subject) { + return subject.executingUser(); + } + + @Override + public SecurityContext securityContext( + String username, + AuthSubject authSubject, + AccessMode mode, + String databaseName + ) { + return new SecurityContext( + new CompatUsernameAuthSubjectImpl(username, authSubject), + mode, + // GDS is always operating from an embedded context + ClientConnectionInfo.EMBEDDED_CONNECTION, + databaseName + ); + } + + @Override + public long getHighId(RecordStore recordStore) { + return recordStore.getIdGenerator().getHighId(); + } + + @Override + public List> entityCursorScan( + KernelTransaction transaction, + int[] labelIds, + int batchSize, + boolean allowPartitionedScan + ) { + if (allowPartitionedScan) { + return partitionedNodeLabelIndexScan(transaction, batchSize, labelIds); + } else { + var read = transaction.dataRead(); + return Arrays + .stream(labelIds) + .mapToObj(read::nodeLabelScan) + .map(scan -> scanToStoreScan(scan, batchSize)) + .collect(Collectors.toList()); + } + } + + @Override + public PropertyCursor allocatePropertyCursor(KernelTransaction kernelTransaction) { + return kernelTransaction + .cursors() + .allocatePropertyCursor(kernelTransaction.cursorContext(), kernelTransaction.memoryTracker()); + } + + @Override + public PropertyReference propertyReference(NodeCursor nodeCursor) { + return ReferencePropertyReference.of(nodeCursor.propertiesReference()); + } + + @Override + public PropertyReference propertyReference(RelationshipScanCursor relationshipScanCursor) { + return ReferencePropertyReference.of(relationshipScanCursor.propertiesReference()); + } + + @Override + public PropertyReference noPropertyReference() { + return ReferencePropertyReference.empty(); + } + + @Override + public void nodeProperties( + KernelTransaction kernelTransaction, + long nodeReference, + PropertyReference reference, + PropertyCursor cursor + ) { + var neoReference = ((ReferencePropertyReference) reference).reference; + kernelTransaction + .dataRead() + .nodeProperties(nodeReference, neoReference, PropertySelection.ALL_PROPERTIES, cursor); + } + + @Override + public void relationshipProperties( + KernelTransaction kernelTransaction, + long relationshipReference, + PropertyReference reference, + PropertyCursor cursor + ) { + var neoReference = ((ReferencePropertyReference) reference).reference; + kernelTransaction + .dataRead() + .relationshipProperties(relationshipReference, neoReference, PropertySelection.ALL_PROPERTIES, cursor); + } + + @Override + public NodeCursor allocateNodeCursor(KernelTransaction kernelTransaction) { + return kernelTransaction.cursors().allocateNodeCursor(kernelTransaction.cursorContext()); + } + + @Override + public RelationshipScanCursor allocateRelationshipScanCursor(KernelTransaction kernelTransaction) { + return kernelTransaction.cursors().allocateRelationshipScanCursor(kernelTransaction.cursorContext()); + } + + @Override + public NodeLabelIndexCursor allocateNodeLabelIndexCursor(KernelTransaction kernelTransaction) { + return kernelTransaction.cursors().allocateNodeLabelIndexCursor(kernelTransaction.cursorContext()); + } + + @Override + public NodeValueIndexCursor allocateNodeValueIndexCursor(KernelTransaction kernelTransaction) { + return kernelTransaction + .cursors() + .allocateNodeValueIndexCursor(kernelTransaction.cursorContext(), kernelTransaction.memoryTracker()); + } + + @Override + public boolean hasNodeLabelIndex(KernelTransaction kernelTransaction) { + return NodeLabelIndexLookupImpl.hasNodeLabelIndex(kernelTransaction); + } + + @Override + public StoreScan nodeLabelIndexScan( + KernelTransaction transaction, + int labelId, + int batchSize, + boolean allowPartitionedScan + ) { + if (allowPartitionedScan) { + return partitionedNodeLabelIndexScan(transaction, batchSize, labelId).get(0); + } else { + var read = transaction.dataRead(); + return scanToStoreScan(read.nodeLabelScan(labelId), batchSize); + } + } + + @Override + public StoreScan scanToStoreScan(Scan scan, int batchSize) { + return new ScanBasedStoreScanImpl<>(scan, batchSize); + } + + private List> partitionedNodeLabelIndexScan( + KernelTransaction transaction, + int batchSize, + int... labelIds + ) { + var indexDescriptor = NodeLabelIndexLookupImpl.findUsableMatchingIndex( + transaction, + SchemaDescriptors.forAnyEntityTokens(EntityType.NODE) + ); + + if (indexDescriptor == IndexDescriptor.NO_INDEX) { + throw new IllegalStateException("There is no index that can back a node label scan."); + } + + var read = transaction.dataRead(); + + // Our current strategy is to select the token with the highest count + // and use that one as the driving partitioned index scan. The partitions + // of all other partitioned index scans will be aligned to that one. + int maxToken = labelIds[0]; + long maxCount = read.countsForNodeWithoutTxState(labelIds[0]); + + for (int i = 1; i < labelIds.length; i++) { + long count = read.countsForNodeWithoutTxState(labelIds[i]); + if (count > maxCount) { + maxCount = count; + maxToken = labelIds[i]; + } + } + + int numberOfPartitions = PartitionedStoreScan.getNumberOfPartitions(maxCount, batchSize); + + try { + var session = read.tokenReadSession(indexDescriptor); + + var partitionedScan = read.nodeLabelScan( + session, + numberOfPartitions, + transaction.cursorContext(), + new TokenPredicate(maxToken) + ); + + var scans = new ArrayList>(labelIds.length); + scans.add(new PartitionedStoreScan(partitionedScan)); + + // Initialize the remaining index scans with the partitioning of the first scan. + for (int labelToken : labelIds) { + if (labelToken != maxToken) { + var scan = read.nodeLabelScan(session, partitionedScan, new TokenPredicate(labelToken)); + scans.add(new PartitionedStoreScan(scan)); + } + } + + return scans; + } catch (KernelException e) { + // should not happen, we check for the index existence and applicability + // before reading it + throw new RuntimeException("Unexpected error while initialising reading from node label index", e); + } + } + + @Override + public CompatIndexQuery rangeIndexQuery( + int propertyKeyId, + double from, + boolean fromInclusive, + double to, + boolean toInclusive + ) { + return new CompatIndexQueryImpl(PropertyIndexQuery.range(propertyKeyId, from, fromInclusive, to, toInclusive)); + } + + @Override + public CompatIndexQuery rangeAllIndexQuery(int propertyKeyId) { + var rangePredicate = PropertyIndexQuery.range( + propertyKeyId, + Values.doubleValue(Double.NEGATIVE_INFINITY), + true, + Values.doubleValue(Double.POSITIVE_INFINITY), + true + ); + return new CompatIndexQueryImpl(rangePredicate); + } + + @Override + public void nodeIndexSeek( + Read dataRead, + IndexReadSession index, + NodeValueIndexCursor cursor, + IndexOrder indexOrder, + boolean needsValues, + CompatIndexQuery query + ) throws KernelException { + var indexQueryConstraints = indexOrder == IndexOrder.NONE + ? IndexQueryConstraints.unordered(needsValues) + : IndexQueryConstraints.constrained(indexOrder, needsValues); + + dataRead.nodeIndexSeek( + (QueryContext) dataRead, + index, + cursor, + indexQueryConstraints, + ((CompatIndexQueryImpl) query).indexQuery + ); + } + + @Override + public CompositeNodeCursor compositeNodeCursor(List cursors, int[] labelIds) { + return new CompositeNodeCursorImpl(cursors, labelIds); + } + + @Override + public Configuration batchImporterConfig( + int batchSize, + int writeConcurrency, + Optional pageCacheMemory, + boolean highIO, + IndexConfig indexConfig + ) { + return new org.neo4j.internal.batchimport.Configuration() { + @Override + public int batchSize() { + return batchSize; + } + + @Override + public int maxNumberOfWorkerThreads() { + return writeConcurrency; + } + + @Override + public boolean highIO() { + return highIO; + } + + @Override + public IndexConfig indexConfig() { + return indexConfig; + } + }; + } + + @Override + public int writeConcurrency(Configuration batchImportConfiguration) { + return batchImportConfiguration.maxNumberOfWorkerThreads(); + } + + @Override + public BatchImporter instantiateBatchImporter( + BatchImporterFactory factory, + GdsDatabaseLayout directoryStructure, + FileSystemAbstraction fileSystem, + PageCacheTracer pageCacheTracer, + Configuration configuration, + LogService logService, + ExecutionMonitor executionMonitor, + AdditionalInitialIds additionalInitialIds, + Config dbConfig, + RecordFormats recordFormats, + JobScheduler jobScheduler, + Collector badCollector + ) { + dbConfig.set(GraphDatabaseSettings.db_format, recordFormats.name()); + var databaseLayout = ((GdsDatabaseLayoutImpl) directoryStructure).databaseLayout(); + return factory.instantiate( + databaseLayout, + fileSystem, + pageCacheTracer, + configuration, + logService, + executionMonitor, + additionalInitialIds, + new EmptyLogTailMetadata(dbConfig), + dbConfig, + Monitor.NO_MONITOR, + jobScheduler, + badCollector, + TransactionLogInitializer.getLogFilesInitializer(), + new IndexImporterFactoryImpl(), + EmptyMemoryTracker.INSTANCE, + new CursorContextFactory(PageCacheTracer.NULL, EmptyVersionContextSupplier.EMPTY) + ); + } + + @Override + public Input batchInputFrom(CompatInput compatInput) { + return new InputFromCompatInput(compatInput); + } + + @Override + public InputEntityIdVisitor.Long inputEntityLongIdVisitor(IdType idType, ReadableGroups groups) { + switch (idType) { + case ACTUAL -> { + return new InputEntityIdVisitor.Long() { + @Override + public void visitNodeId(InputEntityVisitor visitor, long id) { + visitor.id(id); + } + + @Override + public void visitSourceId(InputEntityVisitor visitor, long id) { + visitor.startId(id); + } + + @Override + public void visitTargetId(InputEntityVisitor visitor, long id) { + visitor.endId(id); + } + }; + } + case INTEGER -> { + var globalGroup = groups.get(null); + + return new InputEntityIdVisitor.Long() { + @Override + public void visitNodeId(InputEntityVisitor visitor, long id) { + visitor.id(id, globalGroup); + } + + @Override + public void visitSourceId(InputEntityVisitor visitor, long id) { + visitor.startId(id, globalGroup); + } + + @Override + public void visitTargetId(InputEntityVisitor visitor, long id) { + visitor.endId(id, globalGroup); + } + }; + } + default -> throw new IllegalStateException("Unexpected value: " + idType); + } + } + + @Override + public InputEntityIdVisitor.String inputEntityStringIdVisitor(ReadableGroups groups) { + var globalGroup = groups.get(null); + + return new InputEntityIdVisitor.String() { + @Override + public void visitNodeId(InputEntityVisitor visitor, String id) { + visitor.id(id, globalGroup); + } + + @Override + public void visitSourceId(InputEntityVisitor visitor, String id) { + visitor.startId(id, globalGroup); + } + + @Override + public void visitTargetId(InputEntityVisitor visitor, String id) { + visitor.endId(id, globalGroup); + } + }; + } + + @Override + public Setting additionalJvm() { + return BootloaderSettings.additional_jvm; + } + + @Override + public Setting pageCacheMemory() { + return GraphDatabaseSettings.pagecache_memory; + } + + @Override + public Long pageCacheMemoryValue(String value) { + return SettingValueParsers.BYTES.parse(value); + } + + @Override + public ProcedureSignature procedureSignature( + QualifiedName name, + List inputSignature, + List outputSignature, + Mode mode, + boolean admin, + String deprecated, + String description, + String warning, + boolean eager, + boolean caseInsensitive, + boolean systemProcedure, + boolean internal, + boolean allowExpiredCredentials, + boolean threadSafe + ) { + return new ProcedureSignature( + name, + inputSignature, + outputSignature, + mode, + admin, + deprecated, + description, + warning, + eager, + caseInsensitive, + systemProcedure, + internal, + allowExpiredCredentials, + threadSafe + ); + } + + @Override + public long getHighestPossibleNodeCount( + Read read, IdGeneratorFactory idGeneratorFactory + ) { + return countByIdGenerator(idGeneratorFactory, RecordIdType.NODE).orElseGet(read::nodesGetCount); + } + + @Override + public long getHighestPossibleRelationshipCount( + Read read, IdGeneratorFactory idGeneratorFactory + ) { + return countByIdGenerator(idGeneratorFactory, RecordIdType.RELATIONSHIP).orElseGet(read::relationshipsGetCount); + } + + @Override + public String versionLongToString(long storeVersion) { + // copied from org.neo4j.kernel.impl.store.LegacyMetadataHandler.versionLongToString which is private + if (storeVersion == -1) { + return "Unknown"; + } + Bits bits = Bits.bitsFromLongs(new long[]{storeVersion}); + int length = bits.getShort(8); + if (length == 0 || length > 7) { + throw new IllegalArgumentException(format(Locale.ENGLISH, "The read version string length %d is not proper.", length)); + } + char[] result = new char[length]; + for (int i = 0; i < length; i++) { + result[i] = (char) bits.getShort(8); + } + return new String(result); + } + + private static final class InputFromCompatInput implements Input { + private final CompatInput delegate; + + private InputFromCompatInput(CompatInput delegate) { + this.delegate = delegate; + } + + @Override + public InputIterable nodes(Collector badCollector) { + return delegate.nodes(badCollector); + } + + @Override + public InputIterable relationships(Collector badCollector) { + return delegate.relationships(badCollector); + } + + @Override + public IdType idType() { + return delegate.idType(); + } + + @Override + public ReadableGroups groups() { + return delegate.groups(); + } + + @Override + public Estimates calculateEstimates(PropertySizeCalculator propertySizeCalculator) throws IOException { + return delegate.calculateEstimates((values, kernelTransaction) -> propertySizeCalculator.calculateSize( + values, + kernelTransaction.cursorContext(), + kernelTransaction.memoryTracker() + )); + } + } + + @Override + public TestLog testLog() { + return new TestLogImpl(); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public Log getUserLog(LogService logService, Class loggingClass) { + return logService.getUserLog(loggingClass); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public Log getInternalLog(LogService logService, Class loggingClass) { + return logService.getInternalLog(loggingClass); + } + + @Override + public NodeValue nodeValue(long id, TextArray labels, MapValue properties) { + return VirtualValues.nodeValue(id, String.valueOf(id), labels, properties); + } + + @Override + public Relationship virtualRelationship(long id, Node startNode, Node endNode, RelationshipType type) { + return new VirtualRelationshipImpl(id, startNode, endNode, type); + } + + @Override + public GdsDatabaseManagementServiceBuilder databaseManagementServiceBuilder(Path storeDir) { + return new GdsDatabaseManagementServiceBuilderImpl(storeDir); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public RecordFormats selectRecordFormatForStore( + DatabaseLayout databaseLayout, + FileSystemAbstraction fs, + PageCache pageCache, + LogService logService, + PageCacheTracer pageCacheTracer + ) { + return RecordFormatSelector.selectForStore( + (RecordDatabaseLayout) databaseLayout, + fs, + pageCache, + logService.getInternalLogProvider(), + new CursorContextFactory(pageCacheTracer, EMPTY) + ); + } + + @Override + public boolean isNotNumericIndex(IndexCapability indexCapability) { + return !indexCapability.areValueCategoriesAccepted(ValueCategory.NUMBER); + } + + @Override + public void setAllowUpgrades(Config.Builder configBuilder, boolean value) { + } + + @Override + public String defaultRecordFormatSetting() { + return GraphDatabaseSettings.db_format.defaultValue(); + } + + @Override + public void configureRecordFormat(Config.Builder configBuilder, String recordFormat) { + var databaseRecordFormat = recordFormat.toLowerCase(Locale.ENGLISH); + configBuilder.set(GraphDatabaseSettings.db_format, databaseRecordFormat); + } + + @Override + public GdsDatabaseLayout databaseLayout(Config config, String databaseName) { + var storageEngineFactory = StorageEngineFactory.selectStorageEngine(config); + var dbLayout = neo4jLayout(config).databaseLayout(databaseName); + var databaseLayout = storageEngineFactory.formatSpecificDatabaseLayout(dbLayout); + return new GdsDatabaseLayoutImpl(databaseLayout); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public Neo4jLayout neo4jLayout(Config config) { + return Neo4jLayout.of(config); + } + + @Override + public BoltTransactionRunner boltTransactionRunner() { + return new BoltTransactionRunnerImpl(); + } + + @Override + public HostnamePort getLocalBoltAddress(ConnectorPortRegister connectorPortRegister) { + return connectorPortRegister.getLocalAddress(ConnectorType.BOLT); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public SslPolicyLoader createSllPolicyLoader( + FileSystemAbstraction fileSystem, + Config config, + LogService logService + ) { + return SslPolicyLoader.create(fileSystem, config, logService.getInternalLogProvider()); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public RecordFormats recordFormatSelector( + String databaseName, + Config databaseConfig, + FileSystemAbstraction fs, + LogService logService, + GraphDatabaseService databaseService + ) { + var neo4jLayout = Neo4jLayout.of(databaseConfig); + var recordDatabaseLayout = RecordDatabaseLayout.of(neo4jLayout, databaseName); + return RecordFormatSelector.selectForStoreOrConfigForNewDbs( + databaseConfig, + recordDatabaseLayout, + fs, + GraphDatabaseApiProxy.resolveDependency(databaseService, PageCache.class), + logService.getInternalLogProvider(), + GraphDatabaseApiProxy.resolveDependency(databaseService, CursorContextFactory.class) + ); + } + + @Override + public ExecutionMonitor executionMonitor(CompatExecutionMonitor compatExecutionMonitor) { + return new ExecutionMonitor.Adapter( + compatExecutionMonitor.checkIntervalMillis(), + TimeUnit.MILLISECONDS + ) { + + @Override + public void initialize(DependencyResolver dependencyResolver) { + compatExecutionMonitor.initialize(dependencyResolver); + } + + @Override + public void start(StageExecution execution) { + compatExecutionMonitor.start(execution); + } + + @Override + public void end(StageExecution execution, long totalTimeMillis) { + compatExecutionMonitor.end(execution, totalTimeMillis); + } + + @Override + public void done(boolean successful, long totalTimeMillis, String additionalInformation) { + compatExecutionMonitor.done(successful, totalTimeMillis, additionalInformation); + } + + @Override + public void check(StageExecution execution) { + compatExecutionMonitor.check(execution); + } + }; + } + + @Override + @SuppressFBWarnings("NP_LOAD_OF_KNOWN_NULL_VALUE") // We assign nulls because it makes the code more readable + public UserFunctionSignature userFunctionSignature( + QualifiedName name, + List inputSignature, + Neo4jTypes.AnyType type, + String description, + boolean internal, + boolean threadSafe, + Optional deprecatedBy + ) { + String category = null; // No predefined categpry (like temporal or math) + var caseInsensitive = false; // case sensitive name match + var isBuiltIn = false; // is built in; never true for GDS + + return new UserFunctionSignature( + name, + inputSignature, + type, + deprecatedBy.orElse(null), + description, + category, + caseInsensitive, + isBuiltIn, + internal, + threadSafe + ); + } + + @Override + @SuppressForbidden(reason = "This is the compat API") + public CallableProcedure callableProcedure(CompatCallableProcedure procedure) { + return new CallableProcedureImpl(procedure); + } + + @Override + @SuppressForbidden(reason = "This is the compat API") + public CallableUserAggregationFunction callableUserAggregationFunction(CompatUserAggregationFunction function) { + return new CallableUserAggregationFunctionImpl(function); + } + + @Override + public long transactionId(KernelTransactionHandle kernelTransactionHandle) { + return kernelTransactionHandle.getTransactionSequenceNumber(); + } + + @Override + public void reserveNeo4jIds(IdGeneratorFactory generatorFactory, int size, CursorContext cursorContext) { + IdGenerator idGenerator = generatorFactory.get(RecordIdType.NODE); + + idGenerator.nextConsecutiveIdRange(size, false, cursorContext); + } + + @Override + public TransactionalContext newQueryContext( + TransactionalContextFactory contextFactory, + InternalTransaction tx, + String queryText, + MapValue queryParameters + ) { + return contextFactory.newContext(tx, queryText, queryParameters, QueryExecutionConfiguration.DEFAULT_CONFIG); + } + + @Override + public boolean isCompositeDatabase(GraphDatabaseService databaseService) { + var databaseManager = GraphDatabaseApiProxy.resolveDependency(databaseService, FabricDatabaseManager.class); + return databaseManager.isFabricDatabase(GraphDatabaseApiProxy.databaseId(databaseService)); + } +} diff --git a/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/NodeLabelIndexLookupImpl.java b/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/NodeLabelIndexLookupImpl.java new file mode 100644 index 0000000000..aaf6b0ee05 --- /dev/null +++ b/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/NodeLabelIndexLookupImpl.java @@ -0,0 +1,68 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._510; + +import org.neo4j.common.EntityType; +import org.neo4j.internal.kernel.api.InternalIndexState; +import org.neo4j.internal.kernel.api.SchemaRead; +import org.neo4j.internal.kernel.api.exceptions.schema.IndexNotFoundKernelException; +import org.neo4j.internal.schema.IndexDescriptor; +import org.neo4j.internal.schema.IndexType; +import org.neo4j.internal.schema.SchemaDescriptor; +import org.neo4j.internal.schema.SchemaDescriptors; +import org.neo4j.kernel.api.KernelTransaction; + +final class NodeLabelIndexLookupImpl { + + static boolean hasNodeLabelIndex(KernelTransaction transaction) { + return NodeLabelIndexLookupImpl.findUsableMatchingIndex( + transaction, + SchemaDescriptors.forAnyEntityTokens(EntityType.NODE) + ) != IndexDescriptor.NO_INDEX; + } + + static IndexDescriptor findUsableMatchingIndex( + KernelTransaction transaction, + SchemaDescriptor schemaDescriptor + ) { + var schemaRead = transaction.schemaRead(); + var iterator = schemaRead.index(schemaDescriptor); + while (iterator.hasNext()) { + var index = iterator.next(); + if (index.getIndexType() == IndexType.LOOKUP && indexIsOnline(schemaRead, index)) { + return index; + } + } + return IndexDescriptor.NO_INDEX; + } + + private static boolean indexIsOnline(SchemaRead schemaRead, IndexDescriptor index) { + var state = InternalIndexState.FAILED; + try { + state = schemaRead.indexGetState(index); + } catch (IndexNotFoundKernelException e) { + // Well the index should always exist here, but if we didn't find it while checking the state, + // then we obviously don't want to use it. + } + return state == InternalIndexState.ONLINE; + } + + private NodeLabelIndexLookupImpl() {} +} diff --git a/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/PartitionedStoreScan.java b/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/PartitionedStoreScan.java new file mode 100644 index 0000000000..5d542426fd --- /dev/null +++ b/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/PartitionedStoreScan.java @@ -0,0 +1,58 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._510; + +import org.neo4j.gds.compat.StoreScan; +import org.neo4j.internal.kernel.api.NodeLabelIndexCursor; +import org.neo4j.internal.kernel.api.PartitionedScan; +import org.neo4j.kernel.api.KernelTransaction; + +final class PartitionedStoreScan implements StoreScan { + private final PartitionedScan scan; + + PartitionedStoreScan(PartitionedScan scan) { + this.scan = scan; + } + + static int getNumberOfPartitions(long nodeCount, int batchSize) { + int numberOfPartitions; + if (nodeCount > 0) { + // ceil div to try to get enough partitions so a single one does + // not include more nodes than batchSize + long partitions = ((nodeCount - 1) / batchSize) + 1; + + // value must be positive + if (partitions < 1) { + partitions = 1; + } + + numberOfPartitions = (int) Long.min(Integer.MAX_VALUE, partitions); + } else { + // we have no partitions to scan, but the value must still be positive + numberOfPartitions = 1; + } + return numberOfPartitions; + } + + @Override + public boolean reserveBatch(NodeLabelIndexCursor cursor, KernelTransaction ktx) { + return scan.reservePartition(cursor, ktx.cursorContext(), ktx.securityContext().mode()); + } +} diff --git a/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/ReferencePropertyReference.java b/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/ReferencePropertyReference.java new file mode 100644 index 0000000000..c94d0d621d --- /dev/null +++ b/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/ReferencePropertyReference.java @@ -0,0 +1,49 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._510; + +import org.neo4j.gds.compat.PropertyReference; +import org.neo4j.storageengine.api.Reference; + +import java.util.Objects; + +public final class ReferencePropertyReference implements PropertyReference { + + private static final PropertyReference EMPTY = new ReferencePropertyReference(null); + + public final Reference reference; + + private ReferencePropertyReference(Reference reference) { + this.reference = reference; + } + + public static PropertyReference of(Reference reference) { + return new ReferencePropertyReference(Objects.requireNonNull(reference)); + } + + public static PropertyReference empty() { + return EMPTY; + } + + @Override + public boolean isEmpty() { + return reference == null; + } +} diff --git a/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/ScanBasedStoreScanImpl.java b/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/ScanBasedStoreScanImpl.java new file mode 100644 index 0000000000..f26affc5f0 --- /dev/null +++ b/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/ScanBasedStoreScanImpl.java @@ -0,0 +1,40 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._510; + +import org.neo4j.gds.compat.StoreScan; +import org.neo4j.internal.kernel.api.Cursor; +import org.neo4j.internal.kernel.api.Scan; +import org.neo4j.kernel.api.KernelTransaction; + +public final class ScanBasedStoreScanImpl implements StoreScan { + private final Scan scan; + private final int batchSize; + + public ScanBasedStoreScanImpl(Scan scan, int batchSize) { + this.scan = scan; + this.batchSize = batchSize; + } + + @Override + public boolean reserveBatch(C cursor, KernelTransaction ktx) { + return scan.reserveBatch(cursor, batchSize, ktx.cursorContext(), ktx.securityContext().mode()); + } +} diff --git a/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/SettingProxyFactoryImpl.java b/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/SettingProxyFactoryImpl.java new file mode 100644 index 0000000000..69239b3098 --- /dev/null +++ b/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/SettingProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._510; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jVersion; +import org.neo4j.gds.compat.SettingProxyApi; +import org.neo4j.gds.compat.SettingProxyFactory; + +@ServiceProvider +public final class SettingProxyFactoryImpl implements SettingProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return version == Neo4jVersion.V_5_10; + } + + @Override + public SettingProxyApi load() { + return new SettingProxyImpl(); + } + + @Override + public String description() { + return "Neo4j Settings 5.10"; + } +} diff --git a/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/SettingProxyImpl.java b/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/SettingProxyImpl.java new file mode 100644 index 0000000000..26ca820cef --- /dev/null +++ b/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/SettingProxyImpl.java @@ -0,0 +1,87 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._510; + +import org.neo4j.configuration.Config; +import org.neo4j.configuration.SettingBuilder; +import org.neo4j.dbms.systemgraph.TopologyGraphDbmsModel; +import org.neo4j.gds.compat.DatabaseMode; +import org.neo4j.gds.compat.SettingProxyApi; +import org.neo4j.graphdb.GraphDatabaseService; +import org.neo4j.graphdb.config.Setting; +import org.neo4j.kernel.impl.factory.GraphDatabaseFacade; +import org.neo4j.kernel.internal.GraphDatabaseAPI; + +public class SettingProxyImpl implements SettingProxyApi { + + @Override + public Setting setting(org.neo4j.gds.compat.Setting setting) { + var builder = SettingBuilder.newBuilder(setting.name(), setting.parser(), setting.defaultValue()); + if (setting.dynamic()) { + builder = builder.dynamic(); + } + if (setting.immutable()) { + builder = builder.immutable(); + } + setting.dependency().ifPresent(builder::setDependency); + setting.constraints().forEach(builder::addConstraint); + return builder.build(); + } + + @Override + public DatabaseMode databaseMode(Config config, GraphDatabaseService databaseService) { + return switch (((GraphDatabaseAPI) databaseService).mode()) { + case RAFT -> DatabaseMode.CORE; + case REPLICA -> DatabaseMode.READ_REPLICA; + case SINGLE -> DatabaseMode.SINGLE; + case VIRTUAL -> throw new UnsupportedOperationException("What's a virtual database anyway?"); + }; + } + + @Override + public void setDatabaseMode(Config config, DatabaseMode databaseMode, GraphDatabaseService databaseService) { + // super hacky, there is no way to set the mode of a database without restarting it + if (!(databaseService instanceof GraphDatabaseFacade db)) { + throw new IllegalArgumentException( + "Cannot set database mode on a database that is not a GraphDatabaseFacade"); + } + try { + var modeField = GraphDatabaseFacade.class.getDeclaredField("mode"); + modeField.setAccessible(true); + modeField.set(db, switch (databaseMode) { + case CORE -> TopologyGraphDbmsModel.HostedOnMode.RAFT; + case READ_REPLICA -> TopologyGraphDbmsModel.HostedOnMode.REPLICA; + case SINGLE -> TopologyGraphDbmsModel.HostedOnMode.SINGLE; + }); + } catch (NoSuchFieldException e) { + throw new RuntimeException( + "Could not set the mode field because it no longer exists. This compat layer needs to be updated.", + e + ); + } catch (IllegalAccessException e) { + throw new RuntimeException("Could not get the permissions to set the mode field.", e); + } + } + + @Override + public String secondaryModeName() { + return "Secondary"; + } +} diff --git a/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/TestLogImpl.java b/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/TestLogImpl.java new file mode 100644 index 0000000000..cd67346d60 --- /dev/null +++ b/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/TestLogImpl.java @@ -0,0 +1,146 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._510; + +import org.neo4j.gds.annotation.SuppressForbidden; +import org.neo4j.gds.compat.TestLog; + +import java.util.ArrayList; +import java.util.Locale; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.ConcurrentMap; + +public class TestLogImpl implements TestLog { + + private final ConcurrentMap> messages; + + TestLogImpl() { + messages = new ConcurrentHashMap<>(3); + } + + @Override + public void assertContainsMessage(String level, String fragment) { + if (!containsMessage(level, fragment)) { + throw new RuntimeException( + String.format( + Locale.US, + "Expected log output to contain `%s` for log level `%s`%nLog messages:%n%s", + fragment, + level, + String.join("\n", messages.get(level)) + ) + ); + } + } + + @Override + public boolean containsMessage(String level, String fragment) { + ConcurrentLinkedQueue messageList = messages.getOrDefault(level, new ConcurrentLinkedQueue<>()); + return messageList.stream().anyMatch((message) -> message.contains(fragment)); + } + + @Override + public boolean hasMessages(String level) { + return !messages.getOrDefault(level, new ConcurrentLinkedQueue<>()).isEmpty(); + } + + @Override + public ArrayList getMessages(String level) { + return new ArrayList<>(messages.getOrDefault(level, new ConcurrentLinkedQueue<>())); + } + + @SuppressForbidden(reason = "test log can print") + public void printMessages() { + System.out.println("TestLog Messages: " + messages); + } + + @Override + public boolean isDebugEnabled() { + return true; + } + + @Override + public void debug(String message) { + logMessage(DEBUG, message); + } + + @Override + public void debug(String message, Throwable throwable) { + debug(String.format(Locale.US, "%s - %s", message, throwable.getMessage())); + } + + @Override + public void debug(String format, Object... arguments) { + debug(String.format(Locale.US, format, arguments)); + } + + @Override + public void info(String message) { + logMessage(INFO, message); + } + + @Override + public void info(String message, Throwable throwable) { + info(String.format(Locale.US, "%s - %s", message, throwable.getMessage())); + } + + @Override + public void info(String format, Object... arguments) { + info(String.format(Locale.US, format, arguments)); + } + + @Override + public void warn(String message) { + logMessage(WARN, message); + } + + @Override + public void warn(String message, Throwable throwable) { + warn(String.format(Locale.US, "%s - %s", message, throwable.getMessage())); + } + + @Override + public void warn(String format, Object... arguments) { + warn(String.format(Locale.US, format, arguments)); + } + + @Override + public void error(String message) { + logMessage(ERROR, message); + } + + @Override + public void error(String message, Throwable throwable) { + error(String.format(Locale.US, "%s - %s", message, throwable.getMessage())); + } + + @Override + public void error(String format, Object... arguments) { + error(String.format(Locale.US, format, arguments)); + } + + private void logMessage(String level, String message) { + messages.computeIfAbsent( + level, + (ignore) -> new ConcurrentLinkedQueue<>() + ).add(message); + } +} diff --git a/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/VirtualRelationshipImpl.java b/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/VirtualRelationshipImpl.java new file mode 100644 index 0000000000..d7a9c41076 --- /dev/null +++ b/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/VirtualRelationshipImpl.java @@ -0,0 +1,41 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._510; + +import org.neo4j.gds.compat.VirtualRelationship; +import org.neo4j.graphdb.Node; +import org.neo4j.graphdb.RelationshipType; + +public class VirtualRelationshipImpl extends VirtualRelationship { + + VirtualRelationshipImpl( + long id, + Node startNode, + Node endNode, + RelationshipType type + ) { + super(id, startNode, endNode, type); + } + + @Override + public String getElementId() { + return Long.toString(getId()); + } +} diff --git a/compatibility/5.10/storage-engine-adapter/build.gradle b/compatibility/5.10/storage-engine-adapter/build.gradle new file mode 100644 index 0000000000..4ab26345d2 --- /dev/null +++ b/compatibility/5.10/storage-engine-adapter/build.gradle @@ -0,0 +1,68 @@ +apply plugin: 'java-library' +apply plugin: 'me.champeau.mrjar' + +description = 'Neo4j Graph Data Science :: Storage Engine Adapter 5.10' + +group = 'org.neo4j.gds' + +// for all 5.x versions +if (ver.'neo4j'.startsWith('5.')) { + sourceSets { + main { + java { + srcDirs = ['src/main/java17'] + } + } + } + + dependencies { + annotationProcessor project(':annotations') + annotationProcessor group: 'org.immutables', name: 'value', version: ver.'immutables' + annotationProcessor group: 'org.neo4j', name: 'annotations', version: neos.'5.10' + + compileOnly project(':annotations') + compileOnly project(':progress-tracking') + compileOnly group: 'org.immutables', name: 'value-annotations', version: ver.'immutables' + compileOnly group: 'org.neo4j', name: 'neo4j', version: neos.'5.10' + compileOnly group: 'org.neo4j', name: 'neo4j-record-storage-engine', version: neos.'5.10' + + implementation project(':core') + implementation project(':storage-engine-adapter-api') + implementation project(':config-api') + implementation project(':string-formatting') + } +} else { + multiRelease { + targetVersions 11, 17 + } + + if (!project.hasProperty('no-forbidden-apis')) { + forbiddenApisJava17 { + exclude('**') + } + } + + dependencies { + annotationProcessor group: 'org.neo4j', name: 'annotations', version: ver.'neo4j' + compileOnly group: 'org.neo4j', name: 'annotations', version: ver.'neo4j' + compileOnly group: 'org.neo4j', name: 'neo4j-kernel-api', version: ver.'neo4j' + + implementation project(':neo4j-adapter') + implementation project(':storage-engine-adapter-api') + + java17AnnotationProcessor project(':annotations') + java17AnnotationProcessor group: 'org.immutables', name: 'value', version: ver.'immutables' + java17AnnotationProcessor group: 'org.neo4j', name: 'annotations', version: neos.'5.10' + + java17CompileOnly project(':annotations') + java17CompileOnly project(':progress-tracking') + java17CompileOnly group: 'org.immutables', name: 'value-annotations', version: ver.'immutables' + java17CompileOnly group: 'org.neo4j', name: 'neo4j', version: neos.'5.10' + java17CompileOnly group: 'org.neo4j', name: 'neo4j-record-storage-engine', version: neos.'5.10' + + java17Implementation project(':core') + java17Implementation project(':storage-engine-adapter-api') + java17Implementation project(':config-api') + java17Implementation project(':string-formatting') + } +} diff --git a/compatibility/5.10/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_510/InMemoryStorageEngineFactory.java b/compatibility/5.10/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_510/InMemoryStorageEngineFactory.java new file mode 100644 index 0000000000..ec941c14b1 --- /dev/null +++ b/compatibility/5.10/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_510/InMemoryStorageEngineFactory.java @@ -0,0 +1,268 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._510; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.configuration.Config; +import org.neo4j.dbms.database.readonly.DatabaseReadOnlyChecker; +import org.neo4j.index.internal.gbptree.RecoveryCleanupWorkCollector; +import org.neo4j.internal.id.IdController; +import org.neo4j.internal.id.IdGeneratorFactory; +import org.neo4j.internal.schema.IndexConfigCompleter; +import org.neo4j.internal.schema.SchemaRule; +import org.neo4j.internal.schema.SchemaState; +import org.neo4j.io.fs.FileSystemAbstraction; +import org.neo4j.io.layout.DatabaseLayout; +import org.neo4j.io.layout.Neo4jLayout; +import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.tracing.PageCacheTracer; +import org.neo4j.lock.LockService; +import org.neo4j.logging.LogProvider; +import org.neo4j.logging.internal.LogService; +import org.neo4j.memory.MemoryTracker; +import org.neo4j.monitoring.DatabaseHealth; +import org.neo4j.scheduler.JobScheduler; +import org.neo4j.storageengine.api.CommandReaderFactory; +import org.neo4j.storageengine.api.ConstraintRuleAccessor; +import org.neo4j.storageengine.api.LogVersionRepository; +import org.neo4j.storageengine.api.MetadataProvider; +import org.neo4j.storageengine.api.StorageEngine; +import org.neo4j.storageengine.api.StorageEngineFactory; +import org.neo4j.storageengine.api.StorageFilesState; +import org.neo4j.storageengine.api.StoreId; +import org.neo4j.storageengine.api.StoreVersion; +import org.neo4j.storageengine.api.StoreVersionCheck; +import org.neo4j.storageengine.api.TransactionIdStore; +import org.neo4j.storageengine.migration.RollingUpgradeCompatibility; +import org.neo4j.storageengine.migration.SchemaRuleMigrationAccess; +import org.neo4j.storageengine.migration.StoreMigrationParticipant; +import org.neo4j.token.TokenHolders; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.List; +import java.util.Optional; +import java.util.UUID; + +@ServiceProvider +public class InMemoryStorageEngineFactory implements StorageEngineFactory { + + @Override + public String name() { + return "unsupported510"; + } + + @Override + public StoreVersionCheck versionCheck( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + LogService logService, + PageCacheTracer pageCacheTracer + ) { + throw new UnsupportedOperationException("5.10 storage engine requires JDK17"); + } + + @Override + public StoreVersion versionInformation(String storeVersion) { + throw new UnsupportedOperationException("5.10 storage engine requires JDK17"); + } + + @Override + public StoreVersion versionInformation(StoreId storeId) { + throw new UnsupportedOperationException("5.10 storage engine requires JDK17"); + } + + @Override + public RollingUpgradeCompatibility rollingUpgradeCompatibility() { + throw new UnsupportedOperationException("5.10 storage engine requires JDK17"); + } + + @Override + public List migrationParticipants( + FileSystemAbstraction fs, + Config config, + PageCache pageCache, + JobScheduler jobScheduler, + LogService logService, + PageCacheTracer cacheTracer, + MemoryTracker memoryTracker + ) { + throw new UnsupportedOperationException("5.10 storage engine requires JDK17"); + } + + @Override + public StorageEngine instantiate( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + TokenHolders tokenHolders, + SchemaState schemaState, + ConstraintRuleAccessor constraintSemantics, + IndexConfigCompleter indexConfigCompleter, + LockService lockService, + IdGeneratorFactory idGeneratorFactory, + IdController idController, + DatabaseHealth databaseHealth, + LogProvider internalLogProvider, + LogProvider userLogProvider, + RecoveryCleanupWorkCollector recoveryCleanupWorkCollector, + PageCacheTracer cacheTracer, + boolean createStoreIfNotExists, + DatabaseReadOnlyChecker readOnlyChecker, + MemoryTracker memoryTracker + ) { + throw new UnsupportedOperationException("5.10 storage engine requires JDK17"); + } + + @Override + public List listStorageFiles(FileSystemAbstraction fileSystem, DatabaseLayout databaseLayout) throws + IOException { + throw new UnsupportedOperationException("5.10 storage engine requires JDK17"); + } + + @Override + public boolean storageExists(FileSystemAbstraction fileSystem, DatabaseLayout databaseLayout, PageCache pageCache) { + return false; + } + + @Override + public TransactionIdStore readOnlyTransactionIdStore( + FileSystemAbstraction filySystem, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) throws IOException { + throw new UnsupportedOperationException("5.10 storage engine requires JDK17"); + } + + @Override + public LogVersionRepository readOnlyLogVersionRepository( + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) throws IOException { + throw new UnsupportedOperationException("5.10 storage engine requires JDK17"); + } + + @Override + public MetadataProvider transactionMetaDataStore( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + PageCacheTracer cacheTracer, + DatabaseReadOnlyChecker readOnlyChecker + ) throws IOException { + throw new UnsupportedOperationException("5.10 storage engine requires JDK17"); + } + + @Override + public StoreId storeId( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) throws IOException { + throw new UnsupportedOperationException("5.10 storage engine requires JDK17"); + } + + @Override + public void setStoreId( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext, + StoreId storeId, + long upgradeTxChecksum, + long upgradeTxCommitTimestamp + ) throws IOException { + throw new UnsupportedOperationException("5.10 storage engine requires JDK17"); + } + + @Override + public void setExternalStoreUUID( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext, + UUID externalStoreId + ) throws IOException { + throw new UnsupportedOperationException("5.10 storage engine requires JDK17"); + } + + @Override + public Optional databaseIdUuid( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) { + throw new UnsupportedOperationException("5.10 storage engine requires JDK17"); + } + + @Override + public SchemaRuleMigrationAccess schemaRuleMigrationAccess( + FileSystemAbstraction fs, + PageCache pageCache, + Config config, + DatabaseLayout databaseLayout, + LogService logService, + String recordFormats, + PageCacheTracer cacheTracer, + CursorContext cursorContext, + MemoryTracker memoryTracker + ) { + throw new UnsupportedOperationException("5.10 storage engine requires JDK17"); + } + + @Override + public List loadSchemaRules( + FileSystemAbstraction fs, + PageCache pageCache, + Config config, + DatabaseLayout databaseLayout, + CursorContext cursorContext + ) { + throw new UnsupportedOperationException("5.10 storage engine requires JDK17"); + } + + @Override + public StorageFilesState checkStoreFileState( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache + ) { + throw new UnsupportedOperationException("5.10 storage engine requires JDK17"); + } + + @Override + public CommandReaderFactory commandReaderFactory() { + throw new UnsupportedOperationException("5.10 storage engine requires JDK17"); + } + + @Override + public DatabaseLayout databaseLayout(Neo4jLayout neo4jLayout, String databaseName) { + throw new UnsupportedOperationException("5.10 storage engine requires JDK17"); + } +} diff --git a/compatibility/5.10/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_510/StorageEngineProxyFactoryImpl.java b/compatibility/5.10/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_510/StorageEngineProxyFactoryImpl.java new file mode 100644 index 0000000000..6267c704fe --- /dev/null +++ b/compatibility/5.10/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_510/StorageEngineProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._510; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jVersion; +import org.neo4j.gds.compat.StorageEngineProxyApi; +import org.neo4j.gds.compat.StorageEngineProxyFactory; + +@ServiceProvider +public class StorageEngineProxyFactoryImpl implements StorageEngineProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return false; + } + + @Override + public StorageEngineProxyApi load() { + throw new UnsupportedOperationException("5.10 storage engine requires JDK17"); + } + + @Override + public String description() { + return "Storage Engine 5.10"; + } +} diff --git a/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/InMemoryCommandCreationContextImpl.java b/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/InMemoryCommandCreationContextImpl.java new file mode 100644 index 0000000000..7458280206 --- /dev/null +++ b/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/InMemoryCommandCreationContextImpl.java @@ -0,0 +1,107 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._510; + +import org.neo4j.configuration.Config; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.kernel.KernelVersion; +import org.neo4j.kernel.KernelVersionProvider; +import org.neo4j.lock.LockTracer; +import org.neo4j.lock.ResourceLocker; +import org.neo4j.storageengine.api.CommandCreationContext; +import org.neo4j.storageengine.api.cursor.StoreCursors; + +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Supplier; + +public class InMemoryCommandCreationContextImpl implements CommandCreationContext { + + private final AtomicLong schemaTokens; + private final AtomicInteger propertyTokens; + private final AtomicInteger labelTokens; + private final AtomicInteger typeTokens; + + InMemoryCommandCreationContextImpl() { + this.schemaTokens = new AtomicLong(0); + this.propertyTokens = new AtomicInteger(0); + this.labelTokens = new AtomicInteger(0); + this.typeTokens = new AtomicInteger(0); + } + + @Override + public long reserveNode() { + throw new UnsupportedOperationException("Creating nodes is not supported"); + } + + @Override + public long reserveRelationship( + long sourceNode, + long targetNode, + int relationshipType, + boolean sourceNodeAddedInTx, + boolean targetNodeAddedInTx + ) { + throw new UnsupportedOperationException("Creating relationships is not supported"); + } + + @Override + public long reserveSchema() { + return schemaTokens.getAndIncrement(); + } + + @Override + public int reserveLabelTokenId() { + return labelTokens.getAndIncrement(); + } + + @Override + public int reservePropertyKeyTokenId() { + return propertyTokens.getAndIncrement(); + } + + @Override + public int reserveRelationshipTypeTokenId() { + return typeTokens.getAndIncrement(); + } + + @Override + public void close() { + + } + + @Override + public void initialize( + KernelVersionProvider kernelVersionProvider, + CursorContext cursorContext, + StoreCursors storeCursors, + Supplier oldestActiveTransactionSequenceNumber, + ResourceLocker locks, + Supplier lockTracer + ) { + + } + + @Override + public KernelVersion kernelVersion() { + // NOTE: Double-check if this is still correct when you copy this into a new compat layer + return KernelVersion.getLatestVersion(Config.newBuilder().build()); + } +} diff --git a/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/InMemoryCountsStoreImpl.java b/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/InMemoryCountsStoreImpl.java new file mode 100644 index 0000000000..1d31cd83d7 --- /dev/null +++ b/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/InMemoryCountsStoreImpl.java @@ -0,0 +1,112 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._510; + +import org.neo4j.annotations.documented.ReporterFactory; +import org.neo4j.counts.CountsAccessor; +import org.neo4j.counts.CountsStorage; +import org.neo4j.counts.CountsVisitor; +import org.neo4j.gds.NodeLabel; +import org.neo4j.gds.api.GraphStore; +import org.neo4j.internal.helpers.progress.ProgressMonitorFactory; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.context.CursorContextFactory; +import org.neo4j.io.pagecache.tracing.FileFlushEvent; +import org.neo4j.memory.MemoryTracker; +import org.neo4j.storageengine.api.cursor.StoreCursors; +import org.neo4j.token.TokenHolders; +import org.neo4j.token.api.TokenNotFoundException; + +public class InMemoryCountsStoreImpl implements CountsStorage, CountsAccessor { + + private final GraphStore graphStore; + private final TokenHolders tokenHolders; + + public InMemoryCountsStoreImpl( + GraphStore graphStore, + TokenHolders tokenHolders + ) { + + this.graphStore = graphStore; + this.tokenHolders = tokenHolders; + } + + @Override + public void start( + CursorContext cursorContext, StoreCursors storeCursors, MemoryTracker memoryTracker + ) { + + } + + @Override + public void checkpoint(FileFlushEvent fileFlushEvent, CursorContext cursorContext) { + + } + + @Override + public long nodeCount(int labelId, CursorContext cursorContext) { + if (labelId == -1) { + return graphStore.nodeCount(); + } + + String nodeLabel; + try { + nodeLabel = tokenHolders.labelTokens().getTokenById(labelId).name(); + } catch (TokenNotFoundException e) { + throw new RuntimeException(e); + } + return graphStore.nodes().nodeCount(NodeLabel.of(nodeLabel)); + } + + @Override + public long relationshipCount(int startLabelId, int typeId, int endLabelId, CursorContext cursorContext) { + // TODO: this is quite wrong + return graphStore.relationshipCount(); + } + + @Override + public boolean consistencyCheck( + ReporterFactory reporterFactory, + CursorContextFactory contextFactory, + int numThreads, + ProgressMonitorFactory progressMonitorFactory + ) { + return true; + } + + @Override + public CountsAccessor.Updater apply(long txId, boolean isLast, CursorContext cursorContext) { + throw new UnsupportedOperationException("Updates are not supported"); + } + + @Override + public void close() { + + } + + @Override + public void accept(CountsVisitor visitor, CursorContext cursorContext) { + tokenHolders.labelTokens().getAllTokens().forEach(labelToken -> { + visitor.visitNodeCount(labelToken.id(), nodeCount(labelToken.id(), cursorContext)); + }); + + visitor.visitRelationshipCount(-1, -1, -1, graphStore.relationshipCount()); + } +} diff --git a/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/InMemoryMetaDataProviderImpl.java b/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/InMemoryMetaDataProviderImpl.java new file mode 100644 index 0000000000..884eff2c37 --- /dev/null +++ b/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/InMemoryMetaDataProviderImpl.java @@ -0,0 +1,201 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._510; + +import org.neo4j.internal.recordstorage.InMemoryLogVersionRepository510; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.context.TransactionIdSnapshot; +import org.neo4j.storageengine.api.ClosedTransactionMetadata; +import org.neo4j.storageengine.api.ExternalStoreId; +import org.neo4j.storageengine.api.MetadataProvider; +import org.neo4j.storageengine.api.StoreId; +import org.neo4j.storageengine.api.TransactionId; + +import java.io.IOException; +import java.util.Optional; +import java.util.UUID; + +public class InMemoryMetaDataProviderImpl implements MetadataProvider { + + private final ExternalStoreId externalStoreId; + private final InMemoryLogVersionRepository510 logVersionRepository; + private final InMemoryTransactionIdStoreImpl transactionIdStore; + + InMemoryMetaDataProviderImpl() { + this.logVersionRepository = new InMemoryLogVersionRepository510(); + this.externalStoreId = new ExternalStoreId(UUID.randomUUID()); + this.transactionIdStore = new InMemoryTransactionIdStoreImpl(); + } + + @Override + public ExternalStoreId getExternalStoreId() { + return this.externalStoreId; + } + + @Override + public ClosedTransactionMetadata getLastClosedTransaction() { + return this.transactionIdStore.getLastClosedTransaction(); + } + + @Override + public void setCurrentLogVersion(long version) { + logVersionRepository.setCurrentLogVersion(version); + } + + @Override + public long incrementAndGetVersion() { + return logVersionRepository.incrementAndGetVersion(); + } + + @Override + public void setCheckpointLogVersion(long version) { + logVersionRepository.setCheckpointLogVersion(version); + } + + @Override + public long incrementAndGetCheckpointLogVersion() { + return logVersionRepository.incrementAndGetCheckpointLogVersion(); + } + + @Override + public void transactionCommitted(long transactionId, int checksum, long commitTimestamp, long consensusIndex) { + transactionIdStore.transactionCommitted(transactionId, checksum, commitTimestamp, consensusIndex); + } + + @Override + public void setLastCommittedAndClosedTransactionId( + long transactionId, + int checksum, + long commitTimestamp, + long consensusIndex, + long byteOffset, + long logVersion + ) { + transactionIdStore.setLastCommittedAndClosedTransactionId( + transactionId, + checksum, + commitTimestamp, + consensusIndex, + byteOffset, + logVersion + ); + } + + @Override + public void transactionClosed( + long transactionId, + long logVersion, + long byteOffset, + int checksum, + long commitTimestamp, + long consensusIndex + ) { + this.transactionIdStore.transactionClosed( + transactionId, + logVersion, + byteOffset, + checksum, + commitTimestamp, + consensusIndex + ); + } + + @Override + public void resetLastClosedTransaction( + long transactionId, + long logVersion, + long byteOffset, + int checksum, + long commitTimestamp, + long consensusIndex + ) { + this.transactionIdStore.resetLastClosedTransaction( + transactionId, + logVersion, + byteOffset, + checksum, + commitTimestamp, + consensusIndex + ); + } + + @Override + public TransactionIdSnapshot getClosedTransactionSnapshot() { + return new TransactionIdSnapshot(this.getLastClosedTransactionId()); + } + + @Override + public void regenerateMetadata(StoreId storeId, UUID externalStoreUUID, CursorContext cursorContext) { + } + + @Override + public StoreId getStoreId() { + return StoreId.UNKNOWN; + } + + @Override + public void close() throws IOException { + } + + @Override + public long getCurrentLogVersion() { + return this.logVersionRepository.getCurrentLogVersion(); + } + + @Override + public long getCheckpointLogVersion() { + return this.logVersionRepository.getCheckpointLogVersion(); + } + + @Override + public long nextCommittingTransactionId() { + return this.transactionIdStore.nextCommittingTransactionId(); + } + + @Override + public long committingTransactionId() { + return this.transactionIdStore.committingTransactionId(); + } + + @Override + public long getLastCommittedTransactionId() { + return this.transactionIdStore.getLastCommittedTransactionId(); + } + + @Override + public TransactionId getLastCommittedTransaction() { + return this.transactionIdStore.getLastCommittedTransaction(); + } + + @Override + public long getLastClosedTransactionId() { + return this.transactionIdStore.getLastClosedTransactionId(); + } + + @Override + public Optional getDatabaseIdUuid(CursorContext cursorTracer) { + throw new IllegalStateException("Not supported"); + } + + @Override + public void setDatabaseIdUuid(UUID uuid, CursorContext cursorContext) { + throw new IllegalStateException("Not supported"); + } +} diff --git a/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/InMemoryNodeCursor.java b/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/InMemoryNodeCursor.java new file mode 100644 index 0000000000..3b1e13f718 --- /dev/null +++ b/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/InMemoryNodeCursor.java @@ -0,0 +1,82 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._510; + +import org.neo4j.gds.api.GraphStore; +import org.neo4j.gds.compat.AbstractInMemoryNodeCursor; +import org.neo4j.storageengine.api.AllNodeScan; +import org.neo4j.storageengine.api.Degrees; +import org.neo4j.storageengine.api.LongReference; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.storageengine.api.RelationshipSelection; +import org.neo4j.storageengine.api.StoragePropertyCursor; +import org.neo4j.storageengine.api.StorageRelationshipTraversalCursor; +import org.neo4j.token.TokenHolders; + +public class InMemoryNodeCursor extends AbstractInMemoryNodeCursor { + + public InMemoryNodeCursor(GraphStore graphStore, TokenHolders tokenHolders) { + super(graphStore, tokenHolders); + } + + @Override + public boolean hasLabel() { + return hasAtLeastOneLabelForCurrentNode(); + } + + @Override + public Reference propertiesReference() { + return LongReference.longReference(getId()); + } + + @Override + public void properties(StoragePropertyCursor propertyCursor, PropertySelection selection) { + propertyCursor.initNodeProperties(propertiesReference(), selection); + } + + @Override + public void properties(StoragePropertyCursor propertyCursor) { + properties(propertyCursor, PropertySelection.ALL_PROPERTIES); + } + + @Override + public boolean supportsFastRelationshipsTo() { + return false; + } + + @Override + public void relationshipsTo( + StorageRelationshipTraversalCursor storageRelationshipTraversalCursor, + RelationshipSelection relationshipSelection, + long neighbourNodeReference + ) { + throw new UnsupportedOperationException(); + } + + @Override + public void degrees(RelationshipSelection selection, Degrees.Mutator mutator) { + } + + @Override + public boolean scanBatch(AllNodeScan allNodeScan, long sizeHint) { + return super.scanBatch(allNodeScan, (int) sizeHint); + } +} diff --git a/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/InMemoryNodePropertyCursor.java b/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/InMemoryNodePropertyCursor.java new file mode 100644 index 0000000000..41133f5453 --- /dev/null +++ b/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/InMemoryNodePropertyCursor.java @@ -0,0 +1,45 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._510; + +import org.neo4j.gds.compat.AbstractInMemoryNodePropertyCursor; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.storageengine.api.LongReference; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.token.TokenHolders; + +public class InMemoryNodePropertyCursor extends AbstractInMemoryNodePropertyCursor { + + public InMemoryNodePropertyCursor(CypherGraphStore graphStore, TokenHolders tokenHolders) { + super(graphStore, tokenHolders); + } + + @Override + public void initNodeProperties(Reference reference, PropertySelection selection, long ownerReference) { + reset(); + setId(((LongReference) reference).id); + setPropertySelection(new InMemoryPropertySelectionImpl(selection)); + } + + @Override + public void initRelationshipProperties(Reference reference, PropertySelection selection, long ownerReference) { + } +} diff --git a/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/InMemoryPropertyCursor.java b/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/InMemoryPropertyCursor.java new file mode 100644 index 0000000000..2478a7ca9f --- /dev/null +++ b/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/InMemoryPropertyCursor.java @@ -0,0 +1,71 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._510; + +import org.neo4j.gds.compat.AbstractInMemoryPropertyCursor; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.storageengine.api.StorageNodeCursor; +import org.neo4j.storageengine.api.StorageRelationshipCursor; +import org.neo4j.token.TokenHolders; + +public class InMemoryPropertyCursor extends AbstractInMemoryPropertyCursor { + + public InMemoryPropertyCursor(CypherGraphStore graphStore, TokenHolders tokenHolders) { + super(graphStore, tokenHolders); + } + + @Override + public void initNodeProperties(Reference reference, PropertySelection selection, long ownerReference) { + if (this.delegate == null || !(this.delegate instanceof InMemoryNodePropertyCursor)) { + this.delegate = new InMemoryNodePropertyCursor(graphStore, tokenHolders); + } + + ((InMemoryNodePropertyCursor) delegate).initNodeProperties(reference, selection); + } + + @Override + public void initNodeProperties(StorageNodeCursor nodeCursor, PropertySelection selection) { + if (this.delegate == null || !(this.delegate instanceof InMemoryNodePropertyCursor)) { + this.delegate = new InMemoryNodePropertyCursor(graphStore, tokenHolders); + } + + ((InMemoryNodePropertyCursor) delegate).initNodeProperties(nodeCursor, selection); + } + + @Override + public void initRelationshipProperties(StorageRelationshipCursor relationshipCursor, PropertySelection selection) { + if (this.delegate == null || !(this.delegate instanceof InMemoryRelationshipPropertyCursor)) { + this.delegate = new InMemoryRelationshipPropertyCursor(graphStore, tokenHolders); + } + + ((InMemoryRelationshipPropertyCursor) delegate).initRelationshipProperties(relationshipCursor, selection); + } + + @Override + public void initRelationshipProperties(Reference reference, PropertySelection selection, long ownerReference) { + if (this.delegate == null || !(this.delegate instanceof InMemoryRelationshipPropertyCursor)) { + this.delegate = new InMemoryRelationshipPropertyCursor(graphStore, tokenHolders); + } + + ((InMemoryRelationshipPropertyCursor) delegate).initRelationshipProperties(reference, selection); + } +} diff --git a/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/InMemoryPropertySelectionImpl.java b/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/InMemoryPropertySelectionImpl.java new file mode 100644 index 0000000000..d48bf99cb5 --- /dev/null +++ b/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/InMemoryPropertySelectionImpl.java @@ -0,0 +1,55 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._510; + +import org.neo4j.gds.compat.InMemoryPropertySelection; +import org.neo4j.storageengine.api.PropertySelection; + +public class InMemoryPropertySelectionImpl implements InMemoryPropertySelection { + + private final PropertySelection propertySelection; + + public InMemoryPropertySelectionImpl(PropertySelection propertySelection) {this.propertySelection = propertySelection;} + + @Override + public boolean isLimited() { + return propertySelection.isLimited(); + } + + @Override + public int numberOfKeys() { + return propertySelection.numberOfKeys(); + } + + @Override + public int key(int index) { + return propertySelection.key(index); + } + + @Override + public boolean test(int key) { + return propertySelection.test(key); + } + + @Override + public boolean isKeysOnly() { + return propertySelection.isKeysOnly(); + } +} diff --git a/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/InMemoryRelationshipPropertyCursor.java b/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/InMemoryRelationshipPropertyCursor.java new file mode 100644 index 0000000000..dc4ad73301 --- /dev/null +++ b/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/InMemoryRelationshipPropertyCursor.java @@ -0,0 +1,60 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._510; + +import org.neo4j.gds.compat.AbstractInMemoryRelationshipPropertyCursor; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.gds.storageengine.InMemoryRelationshipCursor; +import org.neo4j.storageengine.api.LongReference; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.storageengine.api.StorageRelationshipCursor; +import org.neo4j.token.TokenHolders; + +public class InMemoryRelationshipPropertyCursor extends AbstractInMemoryRelationshipPropertyCursor { + + InMemoryRelationshipPropertyCursor(CypherGraphStore graphStore, TokenHolders tokenHolders) { + super(graphStore, tokenHolders); + } + + @Override + public void initNodeProperties( + Reference reference, PropertySelection propertySelection, long ownerReference + ) { + + } + + @Override + public void initRelationshipProperties( + Reference reference, PropertySelection propertySelection, long ownerReference + ) { + var relationshipId = ((LongReference) reference).id; + var relationshipCursor = new InMemoryRelationshipScanCursor(graphStore, tokenHolders); + relationshipCursor.single(relationshipId); + relationshipCursor.next(); + relationshipCursor.properties(this, new InMemoryPropertySelectionImpl(propertySelection)); + } + + @Override + public void initRelationshipProperties(StorageRelationshipCursor relationshipCursor, PropertySelection selection) { + var inMemoryRelationshipCursor = (InMemoryRelationshipCursor) relationshipCursor; + inMemoryRelationshipCursor.properties(this, selection); + } +} diff --git a/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/InMemoryRelationshipScanCursor.java b/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/InMemoryRelationshipScanCursor.java new file mode 100644 index 0000000000..7ab410e666 --- /dev/null +++ b/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/InMemoryRelationshipScanCursor.java @@ -0,0 +1,61 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._510; + +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.internal.recordstorage.AbstractInMemoryRelationshipScanCursor; +import org.neo4j.storageengine.api.AllRelationshipsScan; +import org.neo4j.storageengine.api.LongReference; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.storageengine.api.StoragePropertyCursor; +import org.neo4j.token.TokenHolders; + +public class InMemoryRelationshipScanCursor extends AbstractInMemoryRelationshipScanCursor { + + public InMemoryRelationshipScanCursor( + CypherGraphStore graphStore, + TokenHolders tokenHolders + ) { + super(graphStore, tokenHolders); + } + + @Override + public void single(long reference, long sourceNodeReference, int type, long targetNodeReference) { + single(reference); + } + + @Override + public Reference propertiesReference() { + return LongReference.longReference(getId()); + } + + @Override + public void properties( + StoragePropertyCursor storagePropertyCursor, PropertySelection propertySelection + ) { + properties(storagePropertyCursor, new InMemoryPropertySelectionImpl(propertySelection)); + } + + @Override + public boolean scanBatch(AllRelationshipsScan allRelationshipsScan, long sizeHint) { + return super.scanBatch(allRelationshipsScan, (int) sizeHint); + } +} diff --git a/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/InMemoryRelationshipTraversalCursor.java b/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/InMemoryRelationshipTraversalCursor.java new file mode 100644 index 0000000000..19d93b86a4 --- /dev/null +++ b/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/InMemoryRelationshipTraversalCursor.java @@ -0,0 +1,47 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._510; + +import org.neo4j.gds.compat.AbstractInMemoryRelationshipTraversalCursor; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.storageengine.api.LongReference; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.storageengine.api.StoragePropertyCursor; +import org.neo4j.token.TokenHolders; + +public class InMemoryRelationshipTraversalCursor extends AbstractInMemoryRelationshipTraversalCursor { + + public InMemoryRelationshipTraversalCursor(CypherGraphStore graphStore, TokenHolders tokenHolders) { + super(graphStore, tokenHolders); + } + + @Override + public Reference propertiesReference() { + return LongReference.longReference(getId()); + } + + @Override + public void properties( + StoragePropertyCursor propertyCursor, PropertySelection selection + ) { + properties(propertyCursor, new InMemoryPropertySelectionImpl(selection)); + } +} diff --git a/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/InMemoryStorageEngineFactory.java b/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/InMemoryStorageEngineFactory.java new file mode 100644 index 0000000000..e1c2e09a95 --- /dev/null +++ b/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/InMemoryStorageEngineFactory.java @@ -0,0 +1,568 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._510; + +import org.eclipse.collections.api.factory.Sets; +import org.eclipse.collections.api.set.ImmutableSet; +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.configuration.Config; +import org.neo4j.consistency.checking.ConsistencyFlags; +import org.neo4j.consistency.report.ConsistencySummaryStatistics; +import org.neo4j.dbms.database.readonly.DatabaseReadOnlyChecker; +import org.neo4j.function.ThrowingSupplier; +import org.neo4j.gds.annotation.SuppressForbidden; +import org.neo4j.gds.compat.Neo4jVersion; +import org.neo4j.gds.compat.StorageEngineFactoryIdProvider; +import org.neo4j.gds.compat.StorageEngineProxyApi; +import org.neo4j.index.internal.gbptree.RecoveryCleanupWorkCollector; +import org.neo4j.internal.batchimport.AdditionalInitialIds; +import org.neo4j.internal.batchimport.BatchImporter; +import org.neo4j.internal.batchimport.Configuration; +import org.neo4j.internal.batchimport.IncrementalBatchImporter; +import org.neo4j.internal.batchimport.IndexImporterFactory; +import org.neo4j.internal.batchimport.Monitor; +import org.neo4j.internal.batchimport.ReadBehaviour; +import org.neo4j.internal.batchimport.input.Collector; +import org.neo4j.internal.batchimport.input.Input; +import org.neo4j.internal.batchimport.input.LenientStoreInput; +import org.neo4j.internal.id.IdGeneratorFactory; +import org.neo4j.internal.id.ScanOnOpenReadOnlyIdGeneratorFactory; +import org.neo4j.internal.recordstorage.InMemoryStorageCommandReaderFactory510; +import org.neo4j.internal.recordstorage.StoreTokens; +import org.neo4j.internal.schema.IndexConfigCompleter; +import org.neo4j.internal.schema.SchemaRule; +import org.neo4j.internal.schema.SchemaState; +import org.neo4j.io.fs.FileSystemAbstraction; +import org.neo4j.io.layout.DatabaseLayout; +import org.neo4j.io.layout.Neo4jLayout; +import org.neo4j.io.layout.recordstorage.RecordDatabaseLayout; +import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.context.CursorContextFactory; +import org.neo4j.io.pagecache.tracing.PageCacheTracer; +import org.neo4j.kernel.KernelVersionRepository; +import org.neo4j.kernel.api.index.IndexProvidersAccess; +import org.neo4j.kernel.impl.api.index.IndexProviderMap; +import org.neo4j.kernel.impl.locking.LockManager; +import org.neo4j.kernel.impl.store.MetaDataStore; +import org.neo4j.kernel.impl.store.NeoStores; +import org.neo4j.kernel.impl.store.StoreFactory; +import org.neo4j.kernel.impl.store.StoreType; +import org.neo4j.kernel.impl.store.cursor.CachedStoreCursors; +import org.neo4j.kernel.impl.transaction.log.LogTailLogVersionsMetadata; +import org.neo4j.kernel.impl.transaction.log.LogTailMetadata; +import org.neo4j.lock.LockService; +import org.neo4j.logging.InternalLog; +import org.neo4j.logging.InternalLogProvider; +import org.neo4j.logging.NullLogProvider; +import org.neo4j.logging.internal.LogService; +import org.neo4j.memory.MemoryTracker; +import org.neo4j.monitoring.DatabaseHealth; +import org.neo4j.scheduler.JobScheduler; +import org.neo4j.storageengine.api.CommandReaderFactory; +import org.neo4j.storageengine.api.ConstraintRuleAccessor; +import org.neo4j.storageengine.api.LogFilesInitializer; +import org.neo4j.storageengine.api.MetadataProvider; +import org.neo4j.storageengine.api.SchemaRule44; +import org.neo4j.storageengine.api.StorageEngine; +import org.neo4j.storageengine.api.StorageEngineFactory; +import org.neo4j.storageengine.api.StorageFilesState; +import org.neo4j.storageengine.api.StoreId; +import org.neo4j.storageengine.api.StoreVersion; +import org.neo4j.storageengine.api.StoreVersionCheck; +import org.neo4j.storageengine.api.StoreVersionIdentifier; +import org.neo4j.storageengine.migration.SchemaRuleMigrationAccessExtended; +import org.neo4j.storageengine.migration.StoreMigrationParticipant; +import org.neo4j.time.SystemNanoClock; +import org.neo4j.token.DelegatingTokenHolder; +import org.neo4j.token.ReadOnlyTokenCreator; +import org.neo4j.token.TokenHolders; +import org.neo4j.token.api.NamedToken; +import org.neo4j.token.api.TokenHolder; +import org.neo4j.token.api.TokensLoader; + +import java.io.IOException; +import java.io.OutputStream; +import java.io.PrintStream; +import java.io.UncheckedIOException; +import java.nio.file.OpenOption; +import java.nio.file.Path; +import java.time.Clock; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.UUID; +import java.util.function.Function; + +@ServiceProvider +public class InMemoryStorageEngineFactory implements StorageEngineFactory { + + static final String IN_MEMORY_STORAGE_ENGINE_NAME = "in-memory-510"; + + public InMemoryStorageEngineFactory() { + StorageEngineProxyApi.requireNeo4jVersion(Neo4jVersion.V_5_10, StorageEngineFactory.class); + } + + @Override + public byte id() { + return StorageEngineFactoryIdProvider.ID; + } + + @Override + public boolean storageExists(FileSystemAbstraction fileSystem, DatabaseLayout databaseLayout) { + return false; + } + + @Override + public StorageEngine instantiate( + FileSystemAbstraction fs, + Clock clock, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + TokenHolders tokenHolders, + SchemaState schemaState, + ConstraintRuleAccessor constraintSemantics, + IndexConfigCompleter indexConfigCompleter, + LockService lockService, + IdGeneratorFactory idGeneratorFactory, + DatabaseHealth databaseHealth, + InternalLogProvider internalLogProvider, + InternalLogProvider userLogProvider, + RecoveryCleanupWorkCollector recoveryCleanupWorkCollector, + LogTailMetadata logTailMetadata, + KernelVersionRepository kernelVersionRepository, + MemoryTracker memoryTracker, + CursorContextFactory contextFactory, + PageCacheTracer pageCacheTracer + ) { + StoreFactory factory = new StoreFactory( + databaseLayout, + config, + idGeneratorFactory, + pageCache, + pageCacheTracer, + fs, + internalLogProvider, + contextFactory, + false, + logTailMetadata + ); + + factory.openNeoStores(StoreType.LABEL_TOKEN).close(); + + return new InMemoryStorageEngineImpl( + databaseLayout, + tokenHolders + ); + } + + @Override + public Optional databaseIdUuid( + FileSystemAbstraction fs, DatabaseLayout databaseLayout, PageCache pageCache, CursorContext cursorContext + ) { + var fieldAccess = MetaDataStore.getFieldAccess( + pageCache, + RecordDatabaseLayout.convert(databaseLayout).metadataStore(), + databaseLayout.getDatabaseName(), + cursorContext + ); + + try { + return fieldAccess.readDatabaseUUID(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + @Override + public List migrationParticipants( + FileSystemAbstraction fileSystemAbstraction, + Config config, + PageCache pageCache, + JobScheduler jobScheduler, + LogService logService, + MemoryTracker memoryTracker, + PageCacheTracer pageCacheTracer, + CursorContextFactory cursorContextFactory, + boolean b + ) { + return List.of(); + } + + @Override + public DatabaseLayout databaseLayout( + Neo4jLayout neo4jLayout, String databaseName + ) { + return RecordDatabaseLayout.of(neo4jLayout, databaseName); + } + + @Override + public DatabaseLayout formatSpecificDatabaseLayout(DatabaseLayout plainLayout) { + return databaseLayout(plainLayout.getNeo4jLayout(), plainLayout.getDatabaseName()); + } + + @Override + public SchemaRuleMigrationAccessExtended schemaRuleMigrationAccess( + FileSystemAbstraction fs, + PageCache pageCache, + PageCacheTracer pageCacheTracer, + Config config, + DatabaseLayout databaseLayout, + CursorContextFactory contextFactory, + MemoryTracker memoryTracker + ) throws IOException { + return null; // Store copy that uses this method doesn't accept GDS in memory store formats target + } + + @SuppressForbidden(reason = "This is the compat layer and we don't really need to go through the proxy") + @Override + public BatchImporter batchImporter( + DatabaseLayout databaseLayout, + FileSystemAbstraction fileSystemAbstraction, + PageCacheTracer pageCacheTracer, + Configuration configuration, + LogService logService, + PrintStream printStream, + boolean b, + AdditionalInitialIds additionalInitialIds, + Config config, + Monitor monitor, + JobScheduler jobScheduler, + Collector collector, + LogFilesInitializer logFilesInitializer, + IndexImporterFactory indexImporterFactory, + MemoryTracker memoryTracker, + CursorContextFactory cursorContextFactory + ) { + throw new UnsupportedOperationException("Batch Import into GDS is not supported"); + } + + @Override + public Input asBatchImporterInput( + DatabaseLayout databaseLayout, + FileSystemAbstraction fileSystemAbstraction, + PageCache pageCache, + PageCacheTracer pageCacheTracer, + Config config, + MemoryTracker memoryTracker, + ReadBehaviour readBehaviour, + boolean b, + CursorContextFactory cursorContextFactory, + LogTailMetadata logTailMetadata + ) { + NeoStores neoStores = (new StoreFactory( + databaseLayout, + config, + new ScanOnOpenReadOnlyIdGeneratorFactory(), + pageCache, + pageCacheTracer, + fileSystemAbstraction, + NullLogProvider.getInstance(), + cursorContextFactory, + false, + logTailMetadata + )).openAllNeoStores(); + return new LenientStoreInput( + neoStores, + readBehaviour.decorateTokenHolders(this.loadReadOnlyTokens(neoStores, true, cursorContextFactory)), + true, + cursorContextFactory, + readBehaviour + ); + } + + @Override + public long optimalAvailableConsistencyCheckerMemory( + FileSystemAbstraction fileSystemAbstraction, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache + ) { + return 0; + } + + @Override + public String name() { + return IN_MEMORY_STORAGE_ENGINE_NAME; + } + + @Override + public Set supportedFormats(boolean includeFormatsUnderDevelopment) { + return Set.of(IN_MEMORY_STORAGE_ENGINE_NAME); + } + + @Override + public boolean supportedFormat(String format, boolean includeFormatsUnderDevelopment) { + return format.equals(IN_MEMORY_STORAGE_ENGINE_NAME); + } + + @Override + public MetadataProvider transactionMetaDataStore( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + DatabaseReadOnlyChecker readOnlyChecker, + CursorContextFactory contextFactory, + LogTailLogVersionsMetadata logTailMetadata, + PageCacheTracer pageCacheTracer + ) { + return new InMemoryMetaDataProviderImpl(); + } + + @Override + public StoreVersionCheck versionCheck( + FileSystemAbstraction fileSystemAbstraction, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + LogService logService, + CursorContextFactory cursorContextFactory + ) { + return new InMemoryVersionCheck(); + } + + @Override + public List loadSchemaRules( + FileSystemAbstraction fileSystemAbstraction, + PageCache pageCache, + PageCacheTracer pageCacheTracer, + Config config, + DatabaseLayout databaseLayout, + boolean b, + Function function, + CursorContextFactory cursorContextFactory + ) { + return List.of(); + } + + @Override + public List load44SchemaRules( + FileSystemAbstraction fs, + PageCache pageCache, + PageCacheTracer pageCacheTracer, + Config config, + DatabaseLayout databaseLayout, + CursorContextFactory contextFactory, + LogTailLogVersionsMetadata logTailMetadata + ) { + return List.of(); + } + + @Override + public TokenHolders loadReadOnlyTokens( + FileSystemAbstraction fileSystemAbstraction, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + PageCacheTracer pageCacheTracer, + boolean lenient, + CursorContextFactory cursorContextFactory + ) { + StoreFactory factory = new StoreFactory( + databaseLayout, + config, + new ScanOnOpenReadOnlyIdGeneratorFactory(), + pageCache, + pageCacheTracer, + fileSystemAbstraction, + NullLogProvider.getInstance(), + cursorContextFactory, + false, + LogTailMetadata.EMPTY_LOG_TAIL + ); + try ( NeoStores stores = factory.openNeoStores( + StoreType.PROPERTY_KEY_TOKEN, StoreType.PROPERTY_KEY_TOKEN_NAME, + StoreType.LABEL_TOKEN, StoreType.LABEL_TOKEN_NAME, + StoreType.RELATIONSHIP_TYPE_TOKEN, StoreType.RELATIONSHIP_TYPE_TOKEN_NAME ) ) + { + return loadReadOnlyTokens(stores, lenient, cursorContextFactory); + } + } + + private TokenHolders loadReadOnlyTokens( + NeoStores stores, + boolean lenient, + CursorContextFactory cursorContextFactory + ) + { + try ( var cursorContext = cursorContextFactory.create("loadReadOnlyTokens"); + var storeCursors = new CachedStoreCursors( stores, cursorContext ) ) + { + stores.start( cursorContext ); + TokensLoader loader = lenient ? StoreTokens.allReadableTokens( stores ) : StoreTokens.allTokens( stores ); + TokenHolder propertyKeys = new DelegatingTokenHolder( ReadOnlyTokenCreator.READ_ONLY, TokenHolder.TYPE_PROPERTY_KEY ); + TokenHolder labels = new DelegatingTokenHolder( ReadOnlyTokenCreator.READ_ONLY, TokenHolder.TYPE_LABEL ); + TokenHolder relationshipTypes = new DelegatingTokenHolder( ReadOnlyTokenCreator.READ_ONLY, TokenHolder.TYPE_RELATIONSHIP_TYPE ); + + propertyKeys.setInitialTokens( lenient ? unique( loader.getPropertyKeyTokens( storeCursors ) ) : loader.getPropertyKeyTokens( storeCursors ) ); + labels.setInitialTokens( lenient ? unique( loader.getLabelTokens( storeCursors ) ) : loader.getLabelTokens( storeCursors ) ); + relationshipTypes.setInitialTokens( + lenient ? unique( loader.getRelationshipTypeTokens( storeCursors ) ) : loader.getRelationshipTypeTokens( storeCursors ) ); + return new TokenHolders( propertyKeys, labels, relationshipTypes ); + } + catch ( IOException e ) + { + throw new UncheckedIOException( e ); + } + } + + private static List unique( List tokens ) + { + if ( !tokens.isEmpty() ) + { + Set names = new HashSet<>( tokens.size() ); + int i = 0; + while ( i < tokens.size() ) + { + if ( names.add( tokens.get( i ).name() ) ) + { + i++; + } + else + { + // Remove the token at the given index, by replacing it with the last token in the list. + // This changes the order of elements, but can be done in constant time instead of linear time. + int lastIndex = tokens.size() - 1; + NamedToken endToken = tokens.remove( lastIndex ); + if ( i < lastIndex ) + { + tokens.set( i, endToken ); + } + } + } + } + return tokens; + } + + @Override + public CommandReaderFactory commandReaderFactory() { + return InMemoryStorageCommandReaderFactory510.INSTANCE; + } + + @Override + public void consistencyCheck( + FileSystemAbstraction fileSystem, + DatabaseLayout layout, + Config config, + PageCache pageCache, + IndexProviderMap indexProviders, + InternalLog log, + ConsistencySummaryStatistics summary, + int numberOfThreads, + long maxOffHeapCachingMemory, + OutputStream progressOutput, + boolean verbose, + ConsistencyFlags flags, + CursorContextFactory contextFactory, + PageCacheTracer pageCacheTracer, + LogTailMetadata logTailMetadata + ) { + // we can do no-op, since our "database" is _always_ consistent + } + + @Override + public ImmutableSet getStoreOpenOptions( + FileSystemAbstraction fs, + PageCache pageCache, + DatabaseLayout layout, + CursorContextFactory contextFactory + ) { + // Not sure about this, empty set is returned when the store files are in `little-endian` format + // See: `org.neo4j.kernel.impl.store.format.PageCacheOptionsSelector.select` + return Sets.immutable.empty(); + } + + @Override + public StoreId retrieveStoreId( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) throws IOException { + return StoreId.retrieveFromStore(fs, databaseLayout, pageCache, cursorContext); + } + + + @Override + public Optional versionInformation(StoreVersionIdentifier storeVersionIdentifier) { + return Optional.of(new InMemoryStoreVersion()); + } + + @Override + public void resetMetadata( + FileSystemAbstraction fileSystemAbstraction, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + CursorContextFactory cursorContextFactory, + PageCacheTracer pageCacheTracer, + StoreId storeId, + UUID externalStoreId + ) { + throw new UnsupportedOperationException(); + } + + @Override + public IncrementalBatchImporter incrementalBatchImporter( + DatabaseLayout databaseLayout, + FileSystemAbstraction fileSystem, + PageCacheTracer pageCacheTracer, + Configuration config, + LogService logService, + PrintStream progressOutput, + boolean verboseProgressOutput, + AdditionalInitialIds additionalInitialIds, + ThrowingSupplier logTailMetadataSupplier, + Config dbConfig, + Monitor monitor, + JobScheduler jobScheduler, + Collector badCollector, + LogFilesInitializer logFilesInitializer, + IndexImporterFactory indexImporterFactory, + MemoryTracker memoryTracker, + CursorContextFactory contextFactory, + IndexProvidersAccess indexProvidersAccess + ) { + throw new UnsupportedOperationException(); + } + + @Override + public LockManager createLockManager(Config config, SystemNanoClock clock) { + return LockManager.NO_LOCKS_LOCK_MANAGER; + } + + @Override + public List listStorageFiles( + FileSystemAbstraction fileSystem, DatabaseLayout databaseLayout + ) { + return Collections.emptyList(); + } + + @Override + public StorageFilesState checkStoreFileState( + FileSystemAbstraction fs, DatabaseLayout databaseLayout, PageCache pageCache + ) { + return StorageFilesState.recoveredState(); + } +} diff --git a/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/InMemoryStorageEngineImpl.java b/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/InMemoryStorageEngineImpl.java new file mode 100644 index 0000000000..484f0c1a19 --- /dev/null +++ b/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/InMemoryStorageEngineImpl.java @@ -0,0 +1,347 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._510; + +import org.neo4j.configuration.Config; +import org.neo4j.counts.CountsAccessor; +import org.neo4j.exceptions.KernelException; +import org.neo4j.gds.compat.TokenManager; +import org.neo4j.gds.config.GraphProjectConfig; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.gds.core.loading.GraphStoreCatalog; +import org.neo4j.gds.storageengine.InMemoryDatabaseCreationCatalog; +import org.neo4j.gds.storageengine.InMemoryTransactionStateVisitor; +import org.neo4j.internal.diagnostics.DiagnosticsLogger; +import org.neo4j.internal.recordstorage.InMemoryStorageReader510; +import org.neo4j.internal.schema.StorageEngineIndexingBehaviour; +import org.neo4j.io.layout.DatabaseLayout; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.tracing.DatabaseFlushEvent; +import org.neo4j.kernel.KernelVersion; +import org.neo4j.kernel.impl.store.stats.StoreEntityCounters; +import org.neo4j.kernel.lifecycle.Lifecycle; +import org.neo4j.kernel.lifecycle.LifecycleAdapter; +import org.neo4j.lock.LockGroup; +import org.neo4j.lock.LockService; +import org.neo4j.lock.LockTracer; +import org.neo4j.lock.ResourceLocker; +import org.neo4j.logging.InternalLog; +import org.neo4j.memory.MemoryTracker; +import org.neo4j.storageengine.api.CommandBatchToApply; +import org.neo4j.storageengine.api.CommandCreationContext; +import org.neo4j.storageengine.api.CommandStream; +import org.neo4j.storageengine.api.IndexUpdateListener; +import org.neo4j.storageengine.api.InternalErrorTracer; +import org.neo4j.storageengine.api.MetadataProvider; +import org.neo4j.storageengine.api.StorageCommand; +import org.neo4j.storageengine.api.StorageEngine; +import org.neo4j.storageengine.api.StorageEngineFactory; +import org.neo4j.storageengine.api.StorageLocks; +import org.neo4j.storageengine.api.StorageReader; +import org.neo4j.storageengine.api.StoreFileMetadata; +import org.neo4j.storageengine.api.StoreId; +import org.neo4j.storageengine.api.TransactionApplicationMode; +import org.neo4j.storageengine.api.cursor.StoreCursors; +import org.neo4j.storageengine.api.enrichment.Enrichment; +import org.neo4j.storageengine.api.enrichment.EnrichmentCommand; +import org.neo4j.storageengine.api.txstate.ReadableTransactionState; +import org.neo4j.storageengine.api.txstate.TxStateVisitor; +import org.neo4j.storageengine.api.txstate.validation.TransactionValidatorFactory; +import org.neo4j.time.SystemNanoClock; +import org.neo4j.token.TokenHolders; + +import java.util.Collection; +import java.util.List; +import java.util.stream.Collectors; + +import static org.neo4j.gds.utils.StringFormatting.formatWithLocale; + +public final class InMemoryStorageEngineImpl implements StorageEngine { + + public static final byte ID = 42; + private final MetadataProvider metadataProvider; + private final CypherGraphStore graphStore; + private final DatabaseLayout databaseLayout; + private final InMemoryTransactionStateVisitor txStateVisitor; + + private final CommandCreationContext commandCreationContext; + + private final TokenManager tokenManager; + private final InMemoryCountsStoreImpl countsStore; + + private static final StorageEngineIndexingBehaviour INDEXING_BEHAVIOUR = new StorageEngineIndexingBehaviour() { + @Override + public boolean useNodeIdsInRelationshipTokenIndex() { + return false; + } + + @Override + public boolean requireCoordinationLocks() { + return false; + } + + @Override + public int nodesPerPage() { + return 0; + } + + @Override + public int relationshipsPerPage() { + return 0; + } + }; + + InMemoryStorageEngineImpl( + DatabaseLayout databaseLayout, + TokenHolders tokenHolders + ) { + this.databaseLayout = databaseLayout; + this.graphStore = getGraphStoreFromCatalog(databaseLayout.getDatabaseName()); + this.txStateVisitor = new InMemoryTransactionStateVisitor(graphStore, tokenHolders); + this.commandCreationContext = new InMemoryCommandCreationContextImpl(); + this.tokenManager = new TokenManager( + tokenHolders, + InMemoryStorageEngineImpl.this.txStateVisitor, + InMemoryStorageEngineImpl.this.graphStore, + commandCreationContext + ); + InMemoryStorageEngineImpl.this.graphStore.initialize(tokenHolders); + this.countsStore = new InMemoryCountsStoreImpl(graphStore, tokenHolders); + this.metadataProvider = new InMemoryMetaDataProviderImpl(); + } + + private static CypherGraphStore getGraphStoreFromCatalog(String databaseName) { + var graphName = InMemoryDatabaseCreationCatalog.getRegisteredDbCreationGraphName(databaseName); + return (CypherGraphStore) GraphStoreCatalog.getAllGraphStores() + .filter(graphStoreWithUserNameAndConfig -> graphStoreWithUserNameAndConfig + .config() + .graphName() + .equals(graphName)) + .findFirst() + .orElseThrow(() -> new IllegalArgumentException(formatWithLocale( + "No graph with name `%s` was found in GraphStoreCatalog. Available graph names are %s", + graphName, + GraphStoreCatalog.getAllGraphStores() + .map(GraphStoreCatalog.GraphStoreWithUserNameAndConfig::config) + .map(GraphProjectConfig::graphName) + .collect(Collectors.toList()) + ))) + .graphStore(); + } + + @Override + public StoreEntityCounters storeEntityCounters() { + return new StoreEntityCounters() { + @Override + public long nodes() { + return graphStore.nodeCount(); + } + + @Override + public long relationships() { + return graphStore.relationshipCount(); + } + + @Override + public long properties() { + return graphStore.nodePropertyKeys().size() + graphStore.relationshipPropertyKeys().size(); + } + + @Override + public long relationshipTypes() { + return graphStore.relationshipTypes().size(); + } + + @Override + public long allNodesCountStore(CursorContext cursorContext) { + return graphStore.nodeCount(); + } + + @Override + public long allRelationshipsCountStore(CursorContext cursorContext) { + return graphStore.relationshipCount(); + } + }; + } + + @Override + public void preAllocateStoreFilesForCommands( + CommandBatchToApply commandBatchToApply, + TransactionApplicationMode transactionApplicationMode + ) { + } + + @Override + public StoreCursors createStorageCursors(CursorContext initialContext) { + return StoreCursors.NULL; + } + + @Override + public StorageLocks createStorageLocks(ResourceLocker locker) { + return new InMemoryStorageLocksImpl(locker); + } + + @Override + public List createCommands( + ReadableTransactionState state, + StorageReader storageReader, + CommandCreationContext creationContext, + LockTracer lockTracer, + TxStateVisitor.Decorator additionalTxStateVisitor, + CursorContext cursorContext, + StoreCursors storeCursors, + MemoryTracker memoryTracker + ) throws KernelException { + state.accept(txStateVisitor); + return List.of(); + } + + @Override + public void dumpDiagnostics(InternalLog internalLog, DiagnosticsLogger diagnosticsLogger) { + } + + @Override + public List createUpgradeCommands( + KernelVersion versionToUpgradeFrom, + KernelVersion versionToUpgradeTo + ) { + return List.of(); + } + + @Override + public EnrichmentCommand createEnrichmentCommand(KernelVersion kernelVersion, Enrichment enrichment) { + throw new UnsupportedOperationException(); + } + + @Override + public StoreId retrieveStoreId() { + return metadataProvider.getStoreId(); + } + + @Override + public StorageEngineIndexingBehaviour indexingBehaviour() { + return INDEXING_BEHAVIOUR; + } + + @Override + public StorageReader newReader() { + return new InMemoryStorageReader510(graphStore, tokenManager.tokenHolders(), countsStore); + } + + @Override + public void addIndexUpdateListener(IndexUpdateListener listener) { + + } + + @Override + public void apply(CommandBatchToApply batch, TransactionApplicationMode mode) { + } + + @Override + public void init() { + } + + @Override + public void start() { + + } + + @Override + public void stop() { + shutdown(); + } + + @Override + public void shutdown() { + InMemoryDatabaseCreationCatalog.removeDatabaseEntry(databaseLayout.getDatabaseName()); + } + + @Override + public void listStorageFiles( + Collection atomic, Collection replayable + ) { + + } + + @Override + public Lifecycle schemaAndTokensLifecycle() { + return new LifecycleAdapter() { + @Override + public void init() { + + } + }; + } + + @Override + public CountsAccessor countsAccessor() { + return countsStore; + } + + @Override + public MetadataProvider metadataProvider() { + return metadataProvider; + } + + @Override + public String name() { + return "gds in-memory storage engine"; + } + + @Override + public byte id() { + return ID; + } + + @Override + public CommandCreationContext newCommandCreationContext() { + return commandCreationContext; + } + + @Override + public TransactionValidatorFactory createTransactionValidatorFactory(StorageEngineFactory storageEngineFactory, Config config, SystemNanoClock clock) { + return TransactionValidatorFactory.EMPTY_VALIDATOR_FACTORY; + } + + @Override + public void lockRecoveryCommands( + CommandStream commands, LockService lockService, LockGroup lockGroup, TransactionApplicationMode mode + ) { + + } + + @Override + public void rollback(ReadableTransactionState txState, CursorContext cursorContext) { + // rollback is not supported but it is also called when we fail for something else + // that we do not support, such as removing node properties + // TODO: do we want to inspect the txState to infer if rollback was called explicitly or not? + } + + @Override + public void checkpoint(DatabaseFlushEvent flushEvent, CursorContext cursorContext) { + // checkpoint is not supported but it is also called when we fail for something else + // that we do not support, such as removing node properties + } + + @Override + public InternalErrorTracer internalErrorTracer() { + return InternalErrorTracer.NO_TRACER; + } +} diff --git a/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/InMemoryStorageLocksImpl.java b/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/InMemoryStorageLocksImpl.java new file mode 100644 index 0000000000..bbc08d7e80 --- /dev/null +++ b/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/InMemoryStorageLocksImpl.java @@ -0,0 +1,86 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._510; + +import org.neo4j.lock.LockTracer; +import org.neo4j.lock.ResourceLocker; +import org.neo4j.storageengine.api.StorageLocks; +import org.neo4j.storageengine.api.txstate.ReadableTransactionState; + +public class InMemoryStorageLocksImpl implements StorageLocks { + + InMemoryStorageLocksImpl(ResourceLocker locker) {} + + @Override + public void acquireExclusiveNodeLock(LockTracer lockTracer, long... ids) {} + + @Override + public void releaseExclusiveNodeLock(long... ids) {} + + @Override + public void acquireSharedNodeLock(LockTracer lockTracer, long... ids) {} + + @Override + public void releaseSharedNodeLock(long... ids) {} + + @Override + public void acquireExclusiveRelationshipLock(LockTracer lockTracer, long... ids) {} + + @Override + public void releaseExclusiveRelationshipLock(long... ids) {} + + @Override + public void acquireSharedRelationshipLock(LockTracer lockTracer, long... ids) {} + + @Override + public void releaseSharedRelationshipLock(long... ids) {} + + @Override + public void acquireRelationshipCreationLock( + LockTracer lockTracer, + long sourceNode, + long targetNode, + boolean sourceNodeAddedInTx, + boolean targetNodeAddedInTx + ) { + } + + @Override + public void acquireRelationshipDeletionLock( + LockTracer lockTracer, + long sourceNode, + long targetNode, + long relationship, + boolean relationshipAddedInTx, + boolean sourceNodeAddedInTx, + boolean targetNodeAddedInTx + ) { + } + + @Override + public void acquireNodeDeletionLock( + ReadableTransactionState readableTransactionState, + LockTracer lockTracer, + long node + ) {} + + @Override + public void acquireNodeLabelChangeLock(LockTracer lockTracer, long node, int labelId) {} +} diff --git a/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/InMemoryStoreVersion.java b/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/InMemoryStoreVersion.java new file mode 100644 index 0000000000..669b0bf498 --- /dev/null +++ b/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/InMemoryStoreVersion.java @@ -0,0 +1,70 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._510; + +import org.neo4j.configuration.Config; +import org.neo4j.storageengine.api.StoreVersion; +import org.neo4j.storageengine.api.format.Capability; +import org.neo4j.storageengine.api.format.CapabilityType; + +import java.util.Optional; + +public class InMemoryStoreVersion implements StoreVersion { + + public static final String STORE_VERSION = "gds-experimental"; + + @Override + public String getStoreVersionUserString() { + return "Unknown"; + } + + @Override + public Optional successorStoreVersion(Config config) { + return Optional.empty(); + } + + @Override + public String formatName() { + return getClass().getSimpleName(); + } + + @Override + public boolean onlyForMigration() { + return false; + } + + @Override + public boolean hasCapability(Capability capability) { + return false; + } + + @Override + public boolean hasCompatibleCapabilities( + StoreVersion otherVersion, CapabilityType type + ) { + return false; + } + + @Override + public String introductionNeo4jVersion() { + return "foo"; + } + +} diff --git a/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/InMemoryTransactionIdStoreImpl.java b/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/InMemoryTransactionIdStoreImpl.java new file mode 100644 index 0000000000..2a7a57be26 --- /dev/null +++ b/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/InMemoryTransactionIdStoreImpl.java @@ -0,0 +1,117 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._510; + +import org.neo4j.internal.recordstorage.AbstractTransactionIdStore; +import org.neo4j.io.pagecache.context.TransactionIdSnapshot; +import org.neo4j.kernel.impl.transaction.log.LogPosition; +import org.neo4j.storageengine.api.ClosedTransactionMetadata; +import org.neo4j.storageengine.api.TransactionId; +import org.neo4j.storageengine.api.TransactionIdStore; + +public class InMemoryTransactionIdStoreImpl extends AbstractTransactionIdStore { + + @Override + protected void initLastCommittedAndClosedTransactionId( + long previouslyCommittedTxId, + int checksum, + long previouslyCommittedTxCommitTimestamp, + long previouslyCommittedTxLogByteOffset, + long previouslyCommittedTxLogVersion + ) { + this.setLastCommittedAndClosedTransactionId( + previouslyCommittedTxId, + checksum, + previouslyCommittedTxCommitTimestamp, + TransactionIdStore.UNKNOWN_CONSENSUS_INDEX, + previouslyCommittedTxLogByteOffset, + previouslyCommittedTxLogVersion + ); + } + + @Override + public ClosedTransactionMetadata getLastClosedTransaction() { + long[] metaData = this.closedTransactionId.get(); + return new ClosedTransactionMetadata( + metaData[0], + new LogPosition(metaData[1], metaData[2]), + (int) metaData[3], + metaData[4], + metaData[5] + ); + } + + @Override + public TransactionIdSnapshot getClosedTransactionSnapshot() { + return new TransactionIdSnapshot(this.getLastClosedTransactionId()); + } + + @Override + protected TransactionId transactionId(long transactionId, int checksum, long commitTimestamp) { + return new TransactionId(transactionId, checksum, commitTimestamp, TransactionIdStore.UNKNOWN_CONSENSUS_INDEX); + } + + @Override + public void transactionCommitted(long transactionId, int checksum, long commitTimestamp, long consensusIndex) { + + } + + @Override + public void setLastCommittedAndClosedTransactionId( + long transactionId, + int checksum, + long commitTimestamp, + long consensusIndex, + long byteOffset, + long logVersion + ) { + + } + + @Override + public void transactionClosed( + long transactionId, + long logVersion, + long byteOffset, + int checksum, + long commitTimestamp, + long consensusIndex + ) { + this.closedTransactionId.offer( + transactionId, + new long[]{logVersion, byteOffset, checksum, commitTimestamp, consensusIndex} + ); + } + + @Override + public void resetLastClosedTransaction( + long transactionId, + long logVersion, + long byteOffset, + int checksum, + long commitTimestamp, + long consensusIndex + ) { + this.closedTransactionId.set( + transactionId, + new long[]{logVersion, byteOffset, checksum, commitTimestamp, consensusIndex} + ); + } +} diff --git a/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/InMemoryVersionCheck.java b/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/InMemoryVersionCheck.java new file mode 100644 index 0000000000..8f32317048 --- /dev/null +++ b/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/InMemoryVersionCheck.java @@ -0,0 +1,61 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._510; + +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.kernel.impl.store.format.FormatFamily; +import org.neo4j.storageengine.api.StoreVersionCheck; +import org.neo4j.storageengine.api.StoreVersionIdentifier; + +import static org.neo4j.gds.compat._510.InMemoryStoreVersion.STORE_VERSION; + +public class InMemoryVersionCheck implements StoreVersionCheck { + + private static final StoreVersionIdentifier STORE_IDENTIFIER = new StoreVersionIdentifier( + STORE_VERSION, + FormatFamily.STANDARD.name(), + 0, + 0 + ); + + @Override + public boolean isCurrentStoreVersionFullySupported(CursorContext cursorContext) { + return true; + } + + @Override + public MigrationCheckResult getAndCheckMigrationTargetVersion(String formatFamily, CursorContext cursorContext) { + return new StoreVersionCheck.MigrationCheckResult(MigrationOutcome.NO_OP, STORE_IDENTIFIER, null, null); + } + + @Override + public UpgradeCheckResult getAndCheckUpgradeTargetVersion(CursorContext cursorContext) { + return new StoreVersionCheck.UpgradeCheckResult(UpgradeOutcome.NO_OP, STORE_IDENTIFIER, null, null); + } + + @Override + public String getIntroductionVersionFromVersion(StoreVersionIdentifier storeVersionIdentifier) { + return STORE_VERSION; + } + + public StoreVersionIdentifier findLatestVersion(String s) { + return STORE_IDENTIFIER; + } +} diff --git a/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/StorageEngineProxyFactoryImpl.java b/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/StorageEngineProxyFactoryImpl.java new file mode 100644 index 0000000000..9f5c42c0f5 --- /dev/null +++ b/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/StorageEngineProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._510; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jVersion; +import org.neo4j.gds.compat.StorageEngineProxyApi; +import org.neo4j.gds.compat.StorageEngineProxyFactory; + +@ServiceProvider +public class StorageEngineProxyFactoryImpl implements StorageEngineProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return version == Neo4jVersion.V_5_10; + } + + @Override + public StorageEngineProxyApi load() { + return new StorageEngineProxyImpl(); + } + + @Override + public String description() { + return "Storage Engine 5.10"; + } +} diff --git a/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/StorageEngineProxyImpl.java b/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/StorageEngineProxyImpl.java new file mode 100644 index 0000000000..ec0aa469dd --- /dev/null +++ b/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/StorageEngineProxyImpl.java @@ -0,0 +1,156 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._510; + +import org.neo4j.common.Edition; +import org.neo4j.configuration.Config; +import org.neo4j.configuration.GraphDatabaseInternalSettings; +import org.neo4j.counts.CountsAccessor; +import org.neo4j.dbms.api.DatabaseManagementService; +import org.neo4j.gds.compat.AbstractInMemoryNodeCursor; +import org.neo4j.gds.compat.AbstractInMemoryNodePropertyCursor; +import org.neo4j.gds.compat.AbstractInMemoryRelationshipPropertyCursor; +import org.neo4j.gds.compat.AbstractInMemoryRelationshipTraversalCursor; +import org.neo4j.gds.compat.GdsDatabaseManagementServiceBuilder; +import org.neo4j.gds.compat.GraphDatabaseApiProxy; +import org.neo4j.gds.compat.StorageEngineProxyApi; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.graphdb.Direction; +import org.neo4j.graphdb.GraphDatabaseService; +import org.neo4j.internal.recordstorage.AbstractInMemoryRelationshipScanCursor; +import org.neo4j.internal.recordstorage.InMemoryStorageReader510; +import org.neo4j.io.layout.DatabaseLayout; +import org.neo4j.storageengine.api.CommandCreationContext; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.RelationshipSelection; +import org.neo4j.storageengine.api.StorageEngine; +import org.neo4j.storageengine.api.StorageEntityCursor; +import org.neo4j.storageengine.api.StoragePropertyCursor; +import org.neo4j.storageengine.api.StorageReader; +import org.neo4j.storageengine.api.StorageRelationshipTraversalCursor; +import org.neo4j.token.TokenHolders; + +import static org.neo4j.configuration.GraphDatabaseSettings.db_format; + +public class StorageEngineProxyImpl implements StorageEngineProxyApi { + + @Override + public CommandCreationContext inMemoryCommandCreationContext() { + return new InMemoryCommandCreationContextImpl(); + } + + @Override + public void initRelationshipTraversalCursorForRelType( + StorageRelationshipTraversalCursor cursor, + long sourceNodeId, + int relTypeToken + ) { + var relationshipSelection = RelationshipSelection.selection( + relTypeToken, + Direction.OUTGOING + ); + cursor.init(sourceNodeId, -1, relationshipSelection); + } + + @Override + public StorageReader inMemoryStorageReader( + CypherGraphStore graphStore, TokenHolders tokenHolders, CountsAccessor counts + ) { + return new InMemoryStorageReader510(graphStore, tokenHolders, counts); + } + + @Override + public StorageEngine createInMemoryStorageEngine(DatabaseLayout databaseLayout, TokenHolders tokenHolders) { + return new InMemoryStorageEngineImpl(databaseLayout, tokenHolders); + } + + @Override + public void createInMemoryDatabase( + DatabaseManagementService dbms, + String dbName, + Config config + ) { + config.set(db_format, InMemoryStorageEngineFactory.IN_MEMORY_STORAGE_ENGINE_NAME); + dbms.createDatabase(dbName, config); + } + + @Override + public GraphDatabaseService startAndGetInMemoryDatabase(DatabaseManagementService dbms, String dbName) { + dbms.startDatabase(dbName); + return dbms.database(dbName); + } + + @Override + public GdsDatabaseManagementServiceBuilder setSkipDefaultIndexesOnCreationSetting(GdsDatabaseManagementServiceBuilder dbmsBuilder) { + return dbmsBuilder.setConfig(GraphDatabaseInternalSettings.skip_default_indexes_on_creation, true); + } + + @Override + public AbstractInMemoryNodeCursor inMemoryNodeCursor(CypherGraphStore graphStore, TokenHolders tokenHolders) { + return new InMemoryNodeCursor(graphStore, tokenHolders); + } + + @Override + public AbstractInMemoryNodePropertyCursor inMemoryNodePropertyCursor( + CypherGraphStore graphStore, + TokenHolders tokenHolders + ) { + return new InMemoryNodePropertyCursor(graphStore, tokenHolders); + } + + @Override + public AbstractInMemoryRelationshipTraversalCursor inMemoryRelationshipTraversalCursor( + CypherGraphStore graphStore, TokenHolders tokenHolders + ) { + return new InMemoryRelationshipTraversalCursor(graphStore, tokenHolders); + } + + @Override + public AbstractInMemoryRelationshipScanCursor inMemoryRelationshipScanCursor( + CypherGraphStore graphStore, TokenHolders tokenHolders + ) { + return new InMemoryRelationshipScanCursor(graphStore, tokenHolders); + } + + @Override + public AbstractInMemoryRelationshipPropertyCursor inMemoryRelationshipPropertyCursor( + CypherGraphStore graphStore, TokenHolders tokenHolders + ) { + return new InMemoryRelationshipPropertyCursor(graphStore, tokenHolders); + } + + @Override + public void properties( + StorageEntityCursor storageCursor, StoragePropertyCursor propertyCursor, int[] propertySelection + ) { + PropertySelection selection; + if (propertySelection.length == 0) { + selection = PropertySelection.ALL_PROPERTIES; + } else { + selection = PropertySelection.selection(propertySelection); + } + storageCursor.properties(propertyCursor, selection); + } + + @Override + public Edition dbmsEdition(GraphDatabaseService databaseService) { + return GraphDatabaseApiProxy.dbmsInfo(databaseService).edition; + } +} diff --git a/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository510.java b/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository510.java new file mode 100644 index 0000000000..35b0e6c8db --- /dev/null +++ b/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository510.java @@ -0,0 +1,71 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.internal.recordstorage; + +import org.neo4j.storageengine.api.LogVersionRepository; + +import java.util.concurrent.atomic.AtomicLong; + +public class InMemoryLogVersionRepository510 implements LogVersionRepository { + + private final AtomicLong logVersion; + private final AtomicLong checkpointLogVersion; + + public InMemoryLogVersionRepository510() { + this(0, 0); + } + + private InMemoryLogVersionRepository510(long initialLogVersion, long initialCheckpointLogVersion) { + this.logVersion = new AtomicLong(); + this.checkpointLogVersion = new AtomicLong(); + this.logVersion.set(initialLogVersion); + this.checkpointLogVersion.set(initialCheckpointLogVersion); + } + + @Override + public void setCurrentLogVersion(long version) { + this.logVersion.set(version); + } + + @Override + public long incrementAndGetVersion() { + return this.logVersion.incrementAndGet(); + } + + @Override + public void setCheckpointLogVersion(long version) { + this.checkpointLogVersion.set(version); + } + + @Override + public long incrementAndGetCheckpointLogVersion() { + return this.checkpointLogVersion.incrementAndGet(); + } + + @Override + public long getCurrentLogVersion() { + return this.logVersion.get(); + } + + @Override + public long getCheckpointLogVersion() { + return this.checkpointLogVersion.get(); + } +} diff --git a/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory510.java b/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory510.java new file mode 100644 index 0000000000..710f94121e --- /dev/null +++ b/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory510.java @@ -0,0 +1,43 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.internal.recordstorage; + +import org.neo4j.kernel.KernelVersion; +import org.neo4j.storageengine.api.CommandReader; +import org.neo4j.storageengine.api.CommandReaderFactory; + +public class InMemoryStorageCommandReaderFactory510 implements CommandReaderFactory { + + public static final CommandReaderFactory INSTANCE = new InMemoryStorageCommandReaderFactory510(); + + @Override + public CommandReader get(KernelVersion kernelVersion) { + switch (kernelVersion) { + case V4_2: + return LogCommandSerializationV4_2.INSTANCE; + case V4_3_D4: + return LogCommandSerializationV4_3_D3.INSTANCE; + case V5_0: + return LogCommandSerializationV5_0.INSTANCE; + default: + throw new IllegalArgumentException("Unsupported kernel version " + kernelVersion); + } + } +} diff --git a/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageReader510.java b/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageReader510.java new file mode 100644 index 0000000000..7f5095a152 --- /dev/null +++ b/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageReader510.java @@ -0,0 +1,332 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.internal.recordstorage; + +import org.eclipse.collections.api.set.primitive.IntSet; +import org.eclipse.collections.impl.set.immutable.primitive.ImmutableIntSetFactoryImpl; +import org.neo4j.common.EntityType; +import org.neo4j.common.TokenNameLookup; +import org.neo4j.counts.CountsAccessor; +import org.neo4j.gds.compat._510.InMemoryNodeCursor; +import org.neo4j.gds.compat._510.InMemoryPropertyCursor; +import org.neo4j.gds.compat._510.InMemoryRelationshipScanCursor; +import org.neo4j.gds.compat._510.InMemoryRelationshipTraversalCursor; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.internal.schema.ConstraintDescriptor; +import org.neo4j.internal.schema.IndexDescriptor; +import org.neo4j.internal.schema.IndexType; +import org.neo4j.internal.schema.SchemaDescriptor; +import org.neo4j.internal.schema.constraints.IndexBackedConstraintDescriptor; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.memory.MemoryTracker; +import org.neo4j.storageengine.api.AllNodeScan; +import org.neo4j.storageengine.api.AllRelationshipsScan; +import org.neo4j.storageengine.api.StorageNodeCursor; +import org.neo4j.storageengine.api.StoragePropertyCursor; +import org.neo4j.storageengine.api.StorageReader; +import org.neo4j.storageengine.api.StorageRelationshipScanCursor; +import org.neo4j.storageengine.api.StorageRelationshipTraversalCursor; +import org.neo4j.storageengine.api.StorageSchemaReader; +import org.neo4j.storageengine.api.cursor.StoreCursors; +import org.neo4j.token.TokenHolders; + +import java.util.Collection; +import java.util.Collections; +import java.util.Iterator; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Function; + +public class InMemoryStorageReader510 implements StorageReader { + + protected final CypherGraphStore graphStore; + protected final TokenHolders tokenHolders; + protected final CountsAccessor counts; + private final Map, Object> dependantState; + private boolean closed; + + public InMemoryStorageReader510( + CypherGraphStore graphStore, + TokenHolders tokenHolders, + CountsAccessor counts + ) { + this.graphStore = graphStore; + + this.tokenHolders = tokenHolders; + this.counts = counts; + this.dependantState = new ConcurrentHashMap<>(); + } + + @Override + public Collection uniquenessConstraintsGetRelated( + long[] changedLabels, + long[] unchangedLabels, + int[] propertyKeyIds, + boolean propertyKeyListIsComplete, + EntityType entityType + ) { + return Collections.emptyList(); + } + + @Override + public long relationshipsGetCount(CursorContext cursorTracer) { + return graphStore.relationshipCount(); + } + + @Override + public boolean nodeExists(long id, StoreCursors storeCursors) { + var originalId = graphStore.nodes().toOriginalNodeId(id); + return graphStore.nodes().containsOriginalId(originalId); + } + + @Override + public boolean relationshipExists(long id, StoreCursors storeCursors) { + return true; + } + + @Override + public StorageNodeCursor allocateNodeCursor( + CursorContext cursorContext, StoreCursors storeCursors + ) { + return new InMemoryNodeCursor(graphStore, tokenHolders); + } + + @Override + public StoragePropertyCursor allocatePropertyCursor( + CursorContext cursorContext, StoreCursors storeCursors, MemoryTracker memoryTracker + ) { + return new InMemoryPropertyCursor(graphStore, tokenHolders); + } + + @Override + public StorageRelationshipTraversalCursor allocateRelationshipTraversalCursor( + CursorContext cursorContext, StoreCursors storeCursors + ) { + return new InMemoryRelationshipTraversalCursor(graphStore, tokenHolders); + } + + @Override + public StorageRelationshipScanCursor allocateRelationshipScanCursor( + CursorContext cursorContext, StoreCursors storeCursors + ) { + return new InMemoryRelationshipScanCursor(graphStore, tokenHolders); + } + + @Override + public IndexDescriptor indexGetForSchemaAndType( + SchemaDescriptor descriptor, IndexType type + ) { + return null; + } + + @Override + public AllRelationshipsScan allRelationshipScan() { + return new AbstractInMemoryAllRelationshipScan() { + @Override + boolean scanRange(AbstractInMemoryRelationshipScanCursor cursor, long start, long stopInclusive) { + return cursor.scanRange(start, stopInclusive); + } + + @Override + public boolean scanBatch(long sizeHint, AbstractInMemoryRelationshipScanCursor cursor) { + return super.scanBatch(sizeHint, cursor); + } + }; + } + + @Override + public Iterator indexGetForSchema(SchemaDescriptor descriptor) { + return Collections.emptyIterator(); + } + + @Override + public Iterator indexesGetForLabel(int labelId) { + return Collections.emptyIterator(); + } + + @Override + public Iterator indexesGetForRelationshipType(int relationshipType) { + return Collections.emptyIterator(); + } + + @Override + public IndexDescriptor indexGetForName(String name) { + return null; + } + + @Override + public ConstraintDescriptor constraintGetForName(String name) { + return null; + } + + @Override + public boolean indexExists(IndexDescriptor index) { + return false; + } + + @Override + public Iterator indexesGetAll() { + return Collections.emptyIterator(); + } + + @Override + public Collection valueIndexesGetRelated( + long[] tokens, int propertyKeyId, EntityType entityType + ) { + return valueIndexesGetRelated(tokens, new int[]{propertyKeyId}, entityType); + } + + @Override + public Collection valueIndexesGetRelated( + long[] tokens, int[] propertyKeyIds, EntityType entityType + ) { + return Collections.emptyList(); + } + + @Override + public Collection uniquenessConstraintsGetRelated( + long[] labels, + int propertyKeyId, + EntityType entityType + ) { + return Collections.emptyList(); + } + + @Override + public Collection uniquenessConstraintsGetRelated( + long[] tokens, + int[] propertyKeyIds, + EntityType entityType + ) { + return Collections.emptyList(); + } + + @Override + public boolean hasRelatedSchema(long[] labels, int propertyKey, EntityType entityType) { + return false; + } + + @Override + public boolean hasRelatedSchema(int label, EntityType entityType) { + return false; + } + + @Override + public Iterator constraintsGetForSchema(SchemaDescriptor descriptor) { + return Collections.emptyIterator(); + } + + @Override + public boolean constraintExists(ConstraintDescriptor descriptor) { + return false; + } + + @Override + public Iterator constraintsGetForLabel(int labelId) { + return Collections.emptyIterator(); + } + + @Override + public Iterator constraintsGetForRelationshipType(int typeId) { + return Collections.emptyIterator(); + } + + @Override + public Iterator constraintsGetAll() { + return Collections.emptyIterator(); + } + + @Override + public IntSet constraintsGetPropertyTokensForLogicalKey(int token, EntityType entityType) { + return ImmutableIntSetFactoryImpl.INSTANCE.empty(); + } + + @Override + public Long indexGetOwningUniquenessConstraintId(IndexDescriptor index) { + return null; + } + + @Override + public long countsForNode(int labelId, CursorContext cursorContext) { + return counts.nodeCount(labelId, cursorContext); + } + + @Override + public long countsForRelationship(int startLabelId, int typeId, int endLabelId, CursorContext cursorContext) { + return counts.relationshipCount(startLabelId, typeId, endLabelId, cursorContext); + } + + @Override + public long nodesGetCount(CursorContext cursorContext) { + return graphStore.nodeCount(); + } + + @Override + public int labelCount() { + return graphStore.nodes().availableNodeLabels().size(); + } + + @Override + public int propertyKeyCount() { + int nodePropertyCount = graphStore + .schema() + .nodeSchema() + .allProperties() + .size(); + int relPropertyCount = graphStore + .schema() + .relationshipSchema() + .allProperties() + .size(); + return nodePropertyCount + relPropertyCount; + } + + @Override + public int relationshipTypeCount() { + return graphStore.schema().relationshipSchema().availableTypes().size(); + } + + @Override + public T getOrCreateSchemaDependantState(Class type, Function factory) { + return type.cast(dependantState.computeIfAbsent(type, key -> factory.apply(this))); + } + + @Override + public AllNodeScan allNodeScan() { + return new InMemoryNodeScan(); + } + + @Override + public void close() { + assert !closed; + closed = true; + } + + @Override + public StorageSchemaReader schemaSnapshot() { + return this; + } + + @Override + public TokenNameLookup tokenNameLookup() { + return tokenHolders; + } + +} diff --git a/gradle/dependencies.gradle b/gradle/dependencies.gradle index b8aa431080..df419c7bdb 100644 --- a/gradle/dependencies.gradle +++ b/gradle/dependencies.gradle @@ -10,6 +10,7 @@ ext { '5.7': properties.getOrDefault('neo4jVersion57', '5.7.0'), '5.8': properties.getOrDefault('neo4jVersion58', '5.8.0'), '5.9': properties.getOrDefault('neo4jVersion59', '5.9.0'), + '5.10': properties.getOrDefault('neo4jVersion510', '5.10.0'), ] neo4jDefault = neos.'4.4' diff --git a/neo4j-adapter/src/main/java/org/neo4j/gds/compat/Neo4jVersion.java b/neo4j-adapter/src/main/java/org/neo4j/gds/compat/Neo4jVersion.java index 080384ad0f..066e56831f 100644 --- a/neo4j-adapter/src/main/java/org/neo4j/gds/compat/Neo4jVersion.java +++ b/neo4j-adapter/src/main/java/org/neo4j/gds/compat/Neo4jVersion.java @@ -37,6 +37,7 @@ public enum Neo4jVersion { V_5_7, V_5_8, V_5_9, + V_5_10, V_RC; @Override @@ -62,6 +63,8 @@ public String toString() { return "5.8"; case V_5_9: return "5.9"; + case V_5_10: + return "5.10"; case V_RC: return "rc"; default: @@ -71,7 +74,7 @@ public String toString() { public MajorMinorVersion semanticVersion() { if (this == V_RC) { - return ImmutableMajorMinorVersion.of(5, 10); + return ImmutableMajorMinorVersion.of(5, 11); } String version = toString(); @@ -153,6 +156,8 @@ static Neo4jVersion parse(String version) { } else if (minorVersion == 9) { return Neo4jVersion.V_5_9; } else if (minorVersion == 10) { + return Neo4jVersion.V_5_10; + } else if (minorVersion == 11) { return Neo4jVersion.V_RC; } } diff --git a/neo4j-adapter/src/test/java/org/neo4j/gds/compat/Neo4jVersionTest.java b/neo4j-adapter/src/test/java/org/neo4j/gds/compat/Neo4jVersionTest.java index 87a1b115a2..a7170a9868 100644 --- a/neo4j-adapter/src/test/java/org/neo4j/gds/compat/Neo4jVersionTest.java +++ b/neo4j-adapter/src/test/java/org/neo4j/gds/compat/Neo4jVersionTest.java @@ -50,7 +50,8 @@ class Neo4jVersionTest { "5.7.0, V_5_7", "5.8.0, V_5_8", "5.9.0, V_5_9", - "5.10.0, V_RC", + "5.10.0, V_5_10", + "5.11.0, V_RC", }) void testParse(String input, Neo4jVersion expected) { assertEquals(expected.name(), Neo4jVersion.parse(input).name()); diff --git a/proc/sysinfo/src/test/java/org/neo4j/gds/SysInfoProcTest.java b/proc/sysinfo/src/test/java/org/neo4j/gds/SysInfoProcTest.java index 30d1430862..85e28e0bf2 100644 --- a/proc/sysinfo/src/test/java/org/neo4j/gds/SysInfoProcTest.java +++ b/proc/sysinfo/src/test/java/org/neo4j/gds/SysInfoProcTest.java @@ -97,6 +97,11 @@ class SysInfoProcTest extends BaseProcTest { "Neo4j Settings 5.9", "Neo4j Settings 5.9 (placeholder)", + "Neo4j 5.10", + "Neo4j 5.10 (placeholder)", + "Neo4j Settings 5.10", + "Neo4j Settings 5.10 (placeholder)", + "Neo4j DEV", "Neo4j DEV (placeholder)", "Neo4j Settings DEV", @@ -229,6 +234,14 @@ void testSysInfoProc() throws IOException { "Neo4j 5.9" ); break; + case V_5_10: + expectedCompatibilities = Set.of( + "Neo4j Settings 5.10 (placeholder)", + "Neo4j Settings 5.10", + "Neo4j 5.10 (placeholder)", + "Neo4j 5.10" + ); + break; case V_RC: expectedCompatibilities = Set.of( "Neo4j Settings RC", From 1080f6a380b97062cbc8683db3e1bfda870e60b2 Mon Sep 17 00:00:00 2001 From: Veselin Nikolov Date: Wed, 12 Jul 2023 07:10:42 +0100 Subject: [PATCH 138/273] Update AuraDS version --- gradle/version.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle/version.gradle b/gradle/version.gradle index 73ac1df4ff..8725958c72 100644 --- a/gradle/version.gradle +++ b/gradle/version.gradle @@ -1,6 +1,6 @@ ext { gdsBaseVersion = '2.4.2' - gdsAuraVersion = '26' + gdsAuraVersion = '27' gdsVersion = gdsBaseVersion + (rootProject.hasProperty('aurads') ? "+${gdsAuraVersion}" : "") } From bf2e498896da5620427902b173424fe6df120e2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B6ren=20Reichardt?= Date: Wed, 12 Jul 2023 15:37:08 +0200 Subject: [PATCH 139/273] Add documentation for procedure changes --- .../ROOT/pages/graph-catalog-node-ops.adoc | 47 ++++++++++++++++++- 1 file changed, 45 insertions(+), 2 deletions(-) diff --git a/doc/modules/ROOT/pages/graph-catalog-node-ops.adoc b/doc/modules/ROOT/pages/graph-catalog-node-ops.adoc index b8cf00fca7..267f413076 100644 --- a/doc/modules/ROOT/pages/graph-catalog-node-ops.adoc +++ b/doc/modules/ROOT/pages/graph-catalog-node-ops.adoc @@ -35,7 +35,8 @@ CALL gds.graph.nodeProperty.stream( ) YIELD nodeId: Integer, - propertyValue: Integer or Float or List of Integer or List of Float + propertyValue: Integer or Float or List of Integer or List of Float, + nodeLabels: List of Strings ---- .Parameters @@ -53,6 +54,7 @@ YIELD |=== | Name | Type | Default | Description | concurrency | Integer | 4 | The number of concurrent threads. Note, this procedure is always running single-threaded. +| listNodeLabels | Boolean | false | Whether to return a list of node labels for each node. |=== .Results @@ -65,6 +67,7 @@ YIELD * Float * List of Integer * List of Float .^| The stored property value. +| nodeLabels | List of Strings | The node labels of the node. |=== ====== @@ -81,7 +84,8 @@ CALL gds.graph.nodeProperties.stream( YIELD nodeId: Integer, nodeProperty: String, - propertyValue: Integer or Float or List of Integer or List of Float + propertyValue: Integer or Float or List of Integer or List of Float, + nodeLabels: List of Strings ---- .Parameters @@ -99,6 +103,7 @@ YIELD |=== | Name | Type | Default | Description | concurrency | Integer | 4 | The number of concurrent threads. Note, this procedure is always running single-threaded. +| listNodeLabels | Boolean | false | Whether to return a list of node labels for each node. |=== .Results @@ -112,6 +117,7 @@ YIELD * Float * List of Integer * List of Float .^| The stored property value. +| nodeLabels | List of Strings | The node labels of the node. |=== ====== @@ -438,6 +444,43 @@ When streaming multiple node properties, the name of each property is included i This adds with some overhead, as each property name must be repeated for each node in the result, but is necessary in order to distinguish properties. ==== + +[[catalog-graph-stream-node-properties-with-labels-example]] + +Additionally, when streaming one or more node properties, we can also return the node labels for each individual node by setting the `listNodeLabels` configuration option. + +[role=query-example] +-- +.Stream multiple node properties with labels: +[source, cypher, role=noplay] +---- +CALL gds.graph.nodeProperties.stream( + 'socialGraph', + ['score'], + ['*'], + { listNodeLabels: true } +) +YIELD nodeId, nodeProperty, propertyValue, nodeLabels +RETURN + gds.util.asNode(nodeId).name AS name, + nodeProperty, + propertyValue, + nodeLabels +---- + +.Results +[opts="header"] +|=== +| name | nodeProperty | propertyValue | nodeLabels +| "Florentin" | "score" | 2.0 | [Person] +| "Adam" | "score" | 1.0 | [Person] +| "Veselin" | "score" | 0.0 | [Person] +| "The Hobbit" | "score" | 0.0 | [Book] +|=== +-- + + + [[catalog-graph-write-node-properties-example]] === Write From c8fb188443add9e7ca612d624246db0ee76446ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B6ren=20Reichardt?= Date: Wed, 12 Jul 2023 15:51:24 +0200 Subject: [PATCH 140/273] Fix csv serialization for array default values --- .../java/org/neo4j/gds/api/DefaultValue.java | 1 - .../org/neo4j/gds/api/DefaultValueUtil.java | 14 +++- .../io/file/csv/CsvNodeSchemaVisitor.java | 2 +- .../io/file/csv/DefaultValueIOHelper.java | 73 +++++++++++++++++ .../core/io/file/csv/NodeSchemaLoader.java | 3 +- .../io/file/csv/CsvNodeSchemaVisitorTest.java | 22 ++++++ .../io/file/csv/DefaultValueIOHelperTest.java | 78 +++++++++++++++++++ .../io/file/csv/NodeSchemaLoaderTest.java | 18 +++++ 8 files changed, 203 insertions(+), 8 deletions(-) create mode 100644 io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/DefaultValueIOHelper.java create mode 100644 io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/DefaultValueIOHelperTest.java diff --git a/graph-projection-api/src/main/java/org/neo4j/gds/api/DefaultValue.java b/graph-projection-api/src/main/java/org/neo4j/gds/api/DefaultValue.java index f954b848ea..4f358aa6ca 100644 --- a/graph-projection-api/src/main/java/org/neo4j/gds/api/DefaultValue.java +++ b/graph-projection-api/src/main/java/org/neo4j/gds/api/DefaultValue.java @@ -241,7 +241,6 @@ public long[] longArrayValue() { return defaultValue; } - @Override public String toString() { return "DefaultValue(" + defaultValue + ')'; diff --git a/graph-projection-api/src/main/java/org/neo4j/gds/api/DefaultValueUtil.java b/graph-projection-api/src/main/java/org/neo4j/gds/api/DefaultValueUtil.java index 13cb10ba70..5fa72de267 100644 --- a/graph-projection-api/src/main/java/org/neo4j/gds/api/DefaultValueUtil.java +++ b/graph-projection-api/src/main/java/org/neo4j/gds/api/DefaultValueUtil.java @@ -27,7 +27,7 @@ import static org.neo4j.gds.utils.StringFormatting.formatWithLocale; -public final class DefaultValueUtil { +final class DefaultValueUtil { private DefaultValueUtil() {} @@ -50,7 +50,9 @@ static Object transformObjectToPrimitiveArray(Object[] defaultArrayValue) { static double[] parseDoubleArrayValue(Object defaultValue, ValueType type) { double[] defaultDoubleArray; - if (defaultValue instanceof Collection) { + if (defaultValue instanceof double[]) { + defaultDoubleArray = (double[]) defaultValue; + } else if (defaultValue instanceof Collection) { defaultDoubleArray = ((Collection) defaultValue).stream() .map(Object::toString) .mapToDouble(Double::parseDouble) @@ -73,7 +75,9 @@ static double[] parseDoubleArrayValue(Object defaultValue, ValueType type) { static long[] parseLongArrayValue(Object defaultValue, ValueType type) { long[] defaultLongArray; - if (defaultValue instanceof Collection) { + if (defaultValue instanceof long[]) { + defaultLongArray = (long[]) defaultValue; + } else if (defaultValue instanceof Collection) { defaultLongArray = ((Collection) defaultValue).stream() .map(Object::toString) .mapToLong(Long::parseLong) @@ -96,7 +100,9 @@ static long[] parseLongArrayValue(Object defaultValue, ValueType type) { static float[] parseFloatArrayValue(Object defaultValue, ValueType type) { float[] defaultFloatArray; - if (defaultValue instanceof List) { + if (defaultValue instanceof float[]) { + defaultFloatArray = (float[]) defaultValue; + } else if (defaultValue instanceof List) { var df = ((List) defaultValue); defaultFloatArray = new float[df.size()]; for (int i = 0; i < df.size(); i++) { diff --git a/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/CsvNodeSchemaVisitor.java b/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/CsvNodeSchemaVisitor.java index 358b8c22d3..e464d7aaf1 100644 --- a/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/CsvNodeSchemaVisitor.java +++ b/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/CsvNodeSchemaVisitor.java @@ -55,7 +55,7 @@ protected void export() { if (key() != null) { row.add(key()); row.add(valueType().csvName()); - row.add(defaultValue().toString()); + row.add(DefaultValueIOHelper.serialize(defaultValue())); row.add(state().name()); } csvWriter.writeRow(row); diff --git a/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/DefaultValueIOHelper.java b/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/DefaultValueIOHelper.java new file mode 100644 index 0000000000..2a852682a6 --- /dev/null +++ b/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/DefaultValueIOHelper.java @@ -0,0 +1,73 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.core.io.file.csv; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.neo4j.gds.api.DefaultValue; +import org.neo4j.gds.api.nodeproperties.ValueType; + +final class DefaultValueIOHelper { + + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + private static final String DEFAULT_VALUE_PREFIX = "DefaultValue("; + + private DefaultValueIOHelper() {} + + static String serialize(DefaultValue defaultValue) { + try { + var serializedValue = OBJECT_MAPPER.writeValueAsString(defaultValue.getObject()); + return DEFAULT_VALUE_PREFIX + serializedValue + ")"; + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } + + static DefaultValue deserialize(String value, ValueType valueType, boolean isUserDefined) { + try { + if (value == null || value.isEmpty()) { + return valueType.fallbackValue(); + } + Object parseValue; + switch (valueType) { + case DOUBLE: + parseValue = OBJECT_MAPPER.readValue(value, double.class); + break; + case LONG: + parseValue = OBJECT_MAPPER.readValue(value, long.class); + break; + case LONG_ARRAY: + parseValue = OBJECT_MAPPER.readValue(value, long[].class); + break; + case FLOAT_ARRAY: + parseValue = OBJECT_MAPPER.readValue(value, float[].class); + break; + case DOUBLE_ARRAY: + parseValue = OBJECT_MAPPER.readValue(value, double[].class); + break; + default: + throw new IllegalArgumentException("Cannot deserialize type `" + valueType + "` to DefaultValue"); + } + return DefaultValue.of(parseValue, valueType, isUserDefined); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } +} diff --git a/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/NodeSchemaLoader.java b/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/NodeSchemaLoader.java index 18fbfee3df..92ea756cc8 100644 --- a/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/NodeSchemaLoader.java +++ b/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/NodeSchemaLoader.java @@ -26,7 +26,6 @@ import com.fasterxml.jackson.dataformat.csv.CsvParser; import com.fasterxml.jackson.dataformat.csv.CsvSchema; import org.neo4j.gds.NodeLabel; -import org.neo4j.gds.api.DefaultValue; import org.neo4j.gds.api.PropertyState; import org.neo4j.gds.api.nodeproperties.ValueType; import org.neo4j.gds.api.schema.MutableNodeSchema; @@ -62,7 +61,7 @@ MutableNodeSchema load() { if (schemaLine.propertyKey != null) { schemaBuilder.key(schemaLine.propertyKey); schemaBuilder.valueType(schemaLine.valueType); - schemaBuilder.defaultValue(DefaultValue.of(schemaLine.defaultValue, schemaLine.valueType, true)); + schemaBuilder.defaultValue(DefaultValueIOHelper.deserialize(schemaLine.defaultValue, schemaLine.valueType, true)); schemaBuilder.state(schemaLine.state); } schemaBuilder.endOfEntity(); diff --git a/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/CsvNodeSchemaVisitorTest.java b/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/CsvNodeSchemaVisitorTest.java index c5b98afb5f..3a21f4f373 100644 --- a/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/CsvNodeSchemaVisitorTest.java +++ b/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/CsvNodeSchemaVisitorTest.java @@ -113,6 +113,28 @@ void writesSchemaWithMixedProperties() { ); } + @Test + void shouldWriteDoubleArrayProperty() { + var nodeSchemaVisitor = new CsvNodeSchemaVisitor(tempDir); + NodeLabel labelA = NodeLabel.of("A"); + nodeSchemaVisitor.nodeLabel(labelA); + nodeSchemaVisitor.key("prop1"); + nodeSchemaVisitor.valueType(ValueType.DOUBLE_ARRAY); + nodeSchemaVisitor.defaultValue(DefaultValue.of(new double[]{42.0, 43.0})); + nodeSchemaVisitor.state(PropertyState.PERSISTENT); + nodeSchemaVisitor.endOfEntity(); + nodeSchemaVisitor.close(); + + assertCsvFiles(List.of(NODE_SCHEMA_FILE_NAME)); + assertDataContent( + NODE_SCHEMA_FILE_NAME, + List.of( + defaultHeaderColumns(), + List.of("A", "prop1", "double[]", "\"DefaultValue([42.0,43.0])\"", "PERSISTENT") + ) + ); + } + @Override protected List defaultHeaderColumns() { return NODE_SCHEMA_COLUMNS; diff --git a/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/DefaultValueIOHelperTest.java b/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/DefaultValueIOHelperTest.java new file mode 100644 index 0000000000..430d702054 --- /dev/null +++ b/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/DefaultValueIOHelperTest.java @@ -0,0 +1,78 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.core.io.file.csv; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.neo4j.gds.api.DefaultValue; +import org.neo4j.gds.api.nodeproperties.ValueType; + +import java.util.stream.Stream; + +import static org.assertj.core.api.Assertions.assertThat; + +class DefaultValueIOHelperTest { + + static Stream defaultValuesAndSerializedFormat() { + return Stream.of( + Arguments.of(DefaultValue.of(42.0, ValueType.DOUBLE, true), "DefaultValue(42.0)", ValueType.DOUBLE), + Arguments.of(DefaultValue.of(1337, ValueType.LONG, true), "DefaultValue(1337)", ValueType.LONG), + Arguments.of(DefaultValue.of(new Float[]{ 0.1f, 0.2f }, ValueType.FLOAT_ARRAY, true), "DefaultValue([0.1,0.2])", ValueType.FLOAT_ARRAY), + Arguments.of(DefaultValue.of(new Double[]{ 1.1, 2.2, 3.3 }, ValueType.DOUBLE_ARRAY, true), "DefaultValue([1.1,2.2,3.3])", ValueType.DOUBLE_ARRAY), + Arguments.of(DefaultValue.of(new Long[]{ 42L, 43L }, ValueType.LONG_ARRAY, true), "DefaultValue([42,43])", ValueType.LONG_ARRAY) + ); + } + + @ParameterizedTest + @MethodSource("defaultValuesAndSerializedFormat") + void shouldSerializeDefaultValues(DefaultValue defaultValue, String expected, ValueType __) { + assertThat(DefaultValueIOHelper.serialize(defaultValue)).isEqualTo(expected); + } + + @Test + void shouldDeserializedDefaultValues() { + assertThat(DefaultValueIOHelper + .deserialize("[42,43]", ValueType.LONG_ARRAY, true) + .longArrayValue() + ).contains(42, 43); + + assertThat(DefaultValueIOHelper + .deserialize("[42.0,13.37]", ValueType.FLOAT_ARRAY, true) + .floatArrayValue() + ).contains(42.0f, 13.37f); + + assertThat(DefaultValueIOHelper + .deserialize("[42.0,13.37]", ValueType.DOUBLE_ARRAY, true) + .doubleArrayValue() + ).contains(42.0D, 13.37D); + + assertThat(DefaultValueIOHelper + .deserialize("42.0", ValueType.DOUBLE, true) + .doubleValue() + ).isEqualTo(42.0D); + + assertThat(DefaultValueIOHelper + .deserialize("1337", ValueType.LONG, true) + .longValue() + ).isEqualTo(1337L); + } +} diff --git a/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/NodeSchemaLoaderTest.java b/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/NodeSchemaLoaderTest.java index 2367508e6c..55a8151ef4 100644 --- a/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/NodeSchemaLoaderTest.java +++ b/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/NodeSchemaLoaderTest.java @@ -144,4 +144,22 @@ void shouldLoadMixedLabels() throws IOException { .isEqualTo(new MutableNodeSchemaEntry(NodeLabel.of("B"), Map.of())); } + @Test + void shouldReadArrayDefaultValues() throws IOException { + var nodeSchemaFile = exportDir.resolve(NODE_SCHEMA_FILE_NAME).toFile(); + var lines = List.of( + String.join(", ", NODE_SCHEMA_COLUMNS), + "A, prop1, double[], \"DefaultValue([42.0,13.37])\", PERSISTENT" + ); + FileUtils.writeLines(nodeSchemaFile, lines); + + var schemaLoader = new NodeSchemaLoader(exportDir); + var nodeSchema = schemaLoader.load(); + + assertThat(nodeSchema).isNotNull(); + + assertThat(nodeSchema.availableLabels()).containsExactlyInAnyOrder(NodeLabel.of("A")); + assertThat(nodeSchema.get(NodeLabel.of("A")).properties().get("prop1").defaultValue().doubleArrayValue()) + .containsExactly(42.0, 13.37); + } } From c64a4a24e22711475d4f7c406385b6411992af70 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B6ren=20Reichardt?= Date: Thu, 13 Jul 2023 11:39:49 +0200 Subject: [PATCH 141/273] Move default value string normalization to helper Co-authored-by: Martin Junghanns --- .../io/file/csv/DefaultValueIOHelper.java | 16 ++++-- .../file/csv/GraphPropertySchemaLoader.java | 4 +- .../core/io/file/csv/JacksonConverters.java | 7 --- .../core/io/file/csv/NodeSchemaLoader.java | 1 - .../io/file/csv/RelationshipSchemaLoader.java | 4 +- .../io/file/csv/DefaultValueIOHelperTest.java | 19 ++++--- .../io/file/csv/JacksonConvertersTest.java | 49 ------------------- 7 files changed, 25 insertions(+), 75 deletions(-) delete mode 100644 io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/JacksonConvertersTest.java diff --git a/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/DefaultValueIOHelper.java b/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/DefaultValueIOHelper.java index 2a852682a6..b3fd00d2b5 100644 --- a/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/DefaultValueIOHelper.java +++ b/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/DefaultValueIOHelper.java @@ -24,27 +24,35 @@ import org.neo4j.gds.api.DefaultValue; import org.neo4j.gds.api.nodeproperties.ValueType; +import static org.neo4j.gds.utils.StringFormatting.formatWithLocale; + final class DefaultValueIOHelper { private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); - private static final String DEFAULT_VALUE_PREFIX = "DefaultValue("; + private static final String DEFAULT_VALUE_TEMPLATE = "DefaultValue(%s)"; private DefaultValueIOHelper() {} static String serialize(DefaultValue defaultValue) { try { var serializedValue = OBJECT_MAPPER.writeValueAsString(defaultValue.getObject()); - return DEFAULT_VALUE_PREFIX + serializedValue + ")"; + return formatWithLocale(DEFAULT_VALUE_TEMPLATE, serializedValue); } catch (JsonProcessingException e) { throw new RuntimeException(e); } } - static DefaultValue deserialize(String value, ValueType valueType, boolean isUserDefined) { + static DefaultValue deserialize(String serializedValue, ValueType valueType, boolean isUserDefined) { try { - if (value == null || value.isEmpty()) { + if (serializedValue == null) { return valueType.fallbackValue(); } + + var value = serializedValue.replaceAll("DefaultValue\\(|null|NaN|\\)", ""); + if (value.isEmpty()) { + return valueType.fallbackValue(); + } + Object parseValue; switch (valueType) { case DOUBLE: diff --git a/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/GraphPropertySchemaLoader.java b/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/GraphPropertySchemaLoader.java index 82a8f6ff7e..e3b661a7db 100644 --- a/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/GraphPropertySchemaLoader.java +++ b/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/GraphPropertySchemaLoader.java @@ -25,7 +25,6 @@ import com.fasterxml.jackson.dataformat.csv.CsvMapper; import com.fasterxml.jackson.dataformat.csv.CsvParser; import com.fasterxml.jackson.dataformat.csv.CsvSchema; -import org.neo4j.gds.api.DefaultValue; import org.neo4j.gds.api.PropertyState; import org.neo4j.gds.api.nodeproperties.ValueType; import org.neo4j.gds.api.schema.PropertySchema; @@ -61,7 +60,7 @@ Map load() { var schemaLine = linesIterator.next(); schemaBuilder.key(schemaLine.propertyKey); schemaBuilder.valueType(schemaLine.valueType); - schemaBuilder.defaultValue(DefaultValue.of(schemaLine.defaultValue, schemaLine.valueType, true)); + schemaBuilder.defaultValue(DefaultValueIOHelper.deserialize(schemaLine.defaultValue, schemaLine.valueType, true)); schemaBuilder.state(schemaLine.state); schemaBuilder.endOfEntity(); @@ -84,7 +83,6 @@ public static class PropertySchemaLine { ValueType valueType; @JsonProperty - @JsonDeserialize(converter = JacksonConverters.DefaultValueConverter.class) String defaultValue; @JsonProperty diff --git a/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/JacksonConverters.java b/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/JacksonConverters.java index a3865bead2..88e3a312e0 100644 --- a/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/JacksonConverters.java +++ b/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/JacksonConverters.java @@ -46,11 +46,4 @@ public ValueType convert(String value) { return ValueType.fromCsvName(value); } } - - static class DefaultValueConverter extends StdConverter { - @Override - public String convert(String value) { - return value.replaceAll("DefaultValue\\(|null|NaN|\\)", ""); - } - } } diff --git a/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/NodeSchemaLoader.java b/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/NodeSchemaLoader.java index 92ea756cc8..6ea5d97add 100644 --- a/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/NodeSchemaLoader.java +++ b/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/NodeSchemaLoader.java @@ -88,7 +88,6 @@ public static class SchemaLine { ValueType valueType; @JsonProperty - @JsonDeserialize(converter = JacksonConverters.DefaultValueConverter.class) String defaultValue; @JsonProperty diff --git a/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/RelationshipSchemaLoader.java b/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/RelationshipSchemaLoader.java index d2802af58a..89fff99de7 100644 --- a/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/RelationshipSchemaLoader.java +++ b/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/RelationshipSchemaLoader.java @@ -26,7 +26,6 @@ import com.fasterxml.jackson.dataformat.csv.CsvParser; import com.fasterxml.jackson.dataformat.csv.CsvSchema; import org.neo4j.gds.RelationshipType; -import org.neo4j.gds.api.DefaultValue; import org.neo4j.gds.api.PropertyState; import org.neo4j.gds.api.nodeproperties.ValueType; import org.neo4j.gds.api.schema.Direction; @@ -65,7 +64,7 @@ MutableRelationshipSchema load() { if (schemaLine.propertyKey != null) { schemaBuilder.key(schemaLine.propertyKey); schemaBuilder.valueType(schemaLine.valueType); - schemaBuilder.defaultValue(DefaultValue.of(schemaLine.defaultValue, schemaLine.valueType, true)); + schemaBuilder.defaultValue(DefaultValueIOHelper.deserialize(schemaLine.defaultValue, schemaLine.valueType, true)); schemaBuilder.state(schemaLine.state); schemaBuilder.aggregation(schemaLine.aggregation); } @@ -96,7 +95,6 @@ public static class SchemaLine { ValueType valueType; @JsonProperty - @JsonDeserialize(converter = JacksonConverters.DefaultValueConverter.class) String defaultValue; @JsonProperty diff --git a/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/DefaultValueIOHelperTest.java b/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/DefaultValueIOHelperTest.java index 430d702054..0a3e8bfcda 100644 --- a/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/DefaultValueIOHelperTest.java +++ b/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/DefaultValueIOHelperTest.java @@ -19,16 +19,18 @@ */ package org.neo4j.gds.core.io.file.csv; -import org.junit.jupiter.api.Test; +import org.intellij.lang.annotations.Subst; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.params.provider.ValueSource; import org.neo4j.gds.api.DefaultValue; import org.neo4j.gds.api.nodeproperties.ValueType; import java.util.stream.Stream; import static org.assertj.core.api.Assertions.assertThat; +import static org.neo4j.gds.utils.StringFormatting.formatWithLocale; class DefaultValueIOHelperTest { @@ -48,30 +50,31 @@ void shouldSerializeDefaultValues(DefaultValue defaultValue, String expected, Va assertThat(DefaultValueIOHelper.serialize(defaultValue)).isEqualTo(expected); } - @Test - void shouldDeserializedDefaultValues() { + @ParameterizedTest + @ValueSource(strings = { "DefaultValue(%s)", "%s" }) + void shouldDeserializedDefaultValues(@Subst("") String defaultValueTemplate) { assertThat(DefaultValueIOHelper - .deserialize("[42,43]", ValueType.LONG_ARRAY, true) + .deserialize(formatWithLocale(defaultValueTemplate, "[42,43]"), ValueType.LONG_ARRAY, true) .longArrayValue() ).contains(42, 43); assertThat(DefaultValueIOHelper - .deserialize("[42.0,13.37]", ValueType.FLOAT_ARRAY, true) + .deserialize(formatWithLocale(defaultValueTemplate, "[42.0,13.37]"), ValueType.FLOAT_ARRAY, true) .floatArrayValue() ).contains(42.0f, 13.37f); assertThat(DefaultValueIOHelper - .deserialize("[42.0,13.37]", ValueType.DOUBLE_ARRAY, true) + .deserialize(formatWithLocale(defaultValueTemplate, "[42.0,13.37]"), ValueType.DOUBLE_ARRAY, true) .doubleArrayValue() ).contains(42.0D, 13.37D); assertThat(DefaultValueIOHelper - .deserialize("42.0", ValueType.DOUBLE, true) + .deserialize(formatWithLocale(defaultValueTemplate, "42.0"), ValueType.DOUBLE, true) .doubleValue() ).isEqualTo(42.0D); assertThat(DefaultValueIOHelper - .deserialize("1337", ValueType.LONG, true) + .deserialize(formatWithLocale(defaultValueTemplate, "1337"), ValueType.LONG, true) .longValue() ).isEqualTo(1337L); } diff --git a/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/JacksonConvertersTest.java b/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/JacksonConvertersTest.java deleted file mode 100644 index acd0231573..0000000000 --- a/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/JacksonConvertersTest.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright (c) "Neo4j" - * Neo4j Sweden AB [http://neo4j.com] - * - * This file is part of Neo4j. - * - * Neo4j is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -package org.neo4j.gds.core.io.file.csv; - -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.Arguments; -import org.junit.jupiter.params.provider.MethodSource; -import org.neo4j.gds.api.DefaultValue; - -import java.util.stream.Stream; - -import static org.assertj.core.api.AssertionsForClassTypes.assertThat; - -class JacksonConvertersTest { - - private static Stream defaultValues() { - return Stream.of( - Arguments.of(DefaultValue.forDouble().toString(), ""), - Arguments.of(DefaultValue.forLong().toString(), Long.toString(Long.MIN_VALUE)), - Arguments.of(DefaultValue.forDoubleArray().toString(), ""), - Arguments.of(DefaultValue.forLongArray().toString(), ""), - Arguments.of(DefaultValue.forFloatArray().toString(), "") - ); - } - - @ParameterizedTest - @MethodSource("defaultValues") - void shouldDeserializeDefaultValues(String defaultValue, String expected) { - var defaultValueConverter = new JacksonConverters.DefaultValueConverter(); - assertThat(defaultValueConverter.convert(defaultValue)).isEqualTo(expected); - } -} From 2883ff84f3e907cf3b7b46f3b6f6730c2d248c44 Mon Sep 17 00:00:00 2001 From: ioannispan Date: Fri, 14 Jul 2023 14:48:16 +0200 Subject: [PATCH 142/273] Fix bug for K1Coloring and empty graphs when number of colors is asked --- .../gds/k1coloring/K1ColoringSpecificationHelper.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/proc/community/src/main/java/org/neo4j/gds/k1coloring/K1ColoringSpecificationHelper.java b/proc/community/src/main/java/org/neo4j/gds/k1coloring/K1ColoringSpecificationHelper.java index 55629c752a..f89b2ecc51 100644 --- a/proc/community/src/main/java/org/neo4j/gds/k1coloring/K1ColoringSpecificationHelper.java +++ b/proc/community/src/main/java/org/neo4j/gds/k1coloring/K1ColoringSpecificationHelper.java @@ -35,13 +35,14 @@ static AbstractResultBuilder computeResult, ProcedureReturnColumns returnColumns ) { + var algorithm=computeResult.algorithm(); if (returnColumns.contains(COLOR_COUNT_FIELD_NAME)) { - procResultBuilder.withColorCount(computeResult.algorithm().usedColors().cardinality()); + procResultBuilder.withColorCount(computeResult.isGraphEmpty() ? 0: algorithm.usedColors().cardinality()); } return procResultBuilder - .withRanIterations(computeResult.isGraphEmpty() ? 0 : computeResult.algorithm().ranIterations()) - .withDidConverge(computeResult.isGraphEmpty() ? false : computeResult.algorithm().didConverge()); + .withRanIterations(computeResult.isGraphEmpty() ? 0 : algorithm.ranIterations()) + .withDidConverge(computeResult.isGraphEmpty() ? false : algorithm.didConverge()); } } From 72e2b6d6d1c376d7ea57da976e7a373cde913b49 Mon Sep 17 00:00:00 2001 From: Veselin Nikolov Date: Mon, 17 Jul 2023 09:36:39 +0100 Subject: [PATCH 143/273] Don't validate if graph is writable if we don't write Co-authored-by: Ioannis Panagiotas --- .../SpeakerListenerLPAMutateProcTest.java | 47 +++++++++++++++++ .../SpeakerListenerLPAStreamProcTest.java | 47 +++++++++++++++++ .../SpeakerListenerLPAWriteProcTest.java | 51 +++++++++++++++++++ .../beta/pregel/PregelProcedureConfig.java | 28 ++++++++++ 4 files changed, 173 insertions(+) create mode 100644 alpha/alpha-proc/src/test/java/org/neo4j/gds/pregel/SpeakerListenerLPAMutateProcTest.java create mode 100644 alpha/alpha-proc/src/test/java/org/neo4j/gds/pregel/SpeakerListenerLPAStreamProcTest.java create mode 100644 alpha/alpha-proc/src/test/java/org/neo4j/gds/pregel/SpeakerListenerLPAWriteProcTest.java diff --git a/alpha/alpha-proc/src/test/java/org/neo4j/gds/pregel/SpeakerListenerLPAMutateProcTest.java b/alpha/alpha-proc/src/test/java/org/neo4j/gds/pregel/SpeakerListenerLPAMutateProcTest.java new file mode 100644 index 0000000000..bd0230b44e --- /dev/null +++ b/alpha/alpha-proc/src/test/java/org/neo4j/gds/pregel/SpeakerListenerLPAMutateProcTest.java @@ -0,0 +1,47 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.pregel; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.neo4j.gds.BaseProcTest; +import org.neo4j.gds.beta.generator.GraphGenerateProc; + +import static org.assertj.core.api.Assertions.assertThatNoException; + +class SpeakerListenerLPAMutateProcTest extends BaseProcTest { + + @BeforeEach + void setup() throws Exception { + registerProcedures( + SpeakerListenerLPAMutateProc.class, + GraphGenerateProc.class + ); + } + + @Test + void shouldNotFailWhenRunningOnNonWritableGraph() { + runQuery("CALL gds.beta.graph.generate('randomGraph', 5, 2, {relationshipSeed:19}) YIELD name, nodes, relationships, relationshipDistribution"); + + assertThatNoException().isThrownBy( + () -> runQuery("CALL gds.alpha.sllpa.mutate('randomGraph', {mutateProperty: 'm', maxIterations: 4, minAssociationStrength: 0.1})") + ); + } +} diff --git a/alpha/alpha-proc/src/test/java/org/neo4j/gds/pregel/SpeakerListenerLPAStreamProcTest.java b/alpha/alpha-proc/src/test/java/org/neo4j/gds/pregel/SpeakerListenerLPAStreamProcTest.java new file mode 100644 index 0000000000..b38e06f46b --- /dev/null +++ b/alpha/alpha-proc/src/test/java/org/neo4j/gds/pregel/SpeakerListenerLPAStreamProcTest.java @@ -0,0 +1,47 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.pregel; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.neo4j.gds.BaseProcTest; +import org.neo4j.gds.beta.generator.GraphGenerateProc; + +import static org.assertj.core.api.Assertions.assertThatNoException; + +class SpeakerListenerLPAStreamProcTest extends BaseProcTest { + + @BeforeEach + void setup() throws Exception { + registerProcedures( + SpeakerListenerLPAStreamProc.class, + GraphGenerateProc.class + ); + } + + @Test + void shouldNotFailWhenRunningOnNonWritableGraph() { + runQuery("CALL gds.beta.graph.generate('randomGraph', 5, 2, {relationshipSeed:19}) YIELD name, nodes, relationships, relationshipDistribution"); + + assertThatNoException().isThrownBy( + () -> runQuery("CALL gds.alpha.sllpa.stream('randomGraph', {maxIterations: 4, minAssociationStrength: 0.1})") + ); + } +} diff --git a/alpha/alpha-proc/src/test/java/org/neo4j/gds/pregel/SpeakerListenerLPAWriteProcTest.java b/alpha/alpha-proc/src/test/java/org/neo4j/gds/pregel/SpeakerListenerLPAWriteProcTest.java new file mode 100644 index 0000000000..8ade6371ef --- /dev/null +++ b/alpha/alpha-proc/src/test/java/org/neo4j/gds/pregel/SpeakerListenerLPAWriteProcTest.java @@ -0,0 +1,51 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.pregel; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.neo4j.gds.BaseProcTest; +import org.neo4j.gds.beta.generator.GraphGenerateProc; +import org.neo4j.graphdb.QueryExecutionException; + +import static org.assertj.core.api.Assertions.assertThatExceptionOfType; + +class SpeakerListenerLPAWriteProcTest extends BaseProcTest { + + @BeforeEach + void setup() throws Exception { + registerProcedures( + SpeakerListenerLPAWriteProc.class, + GraphGenerateProc.class + ); + } + + @Test + void shouldFailWhenRunningOnNonWritableGraph() { + runQuery("CALL gds.beta.graph.generate('randomGraph', 5, 2, {relationshipSeed:19}) YIELD name, nodes, relationships, relationshipDistribution"); + + assertThatExceptionOfType(QueryExecutionException.class) + .isThrownBy( + () -> runQuery("CALL gds.alpha.sllpa.write('randomGraph', {writeProperty: 'm', maxIterations: 4, minAssociationStrength: 0.1})") + ) + .withRootCauseInstanceOf(IllegalArgumentException.class) + .withMessageContaining("The provided graph does not support `write` execution mode."); + } +} diff --git a/pregel/src/main/java/org/neo4j/gds/beta/pregel/PregelProcedureConfig.java b/pregel/src/main/java/org/neo4j/gds/beta/pregel/PregelProcedureConfig.java index 6ef12ba025..2a2c568758 100644 --- a/pregel/src/main/java/org/neo4j/gds/beta/pregel/PregelProcedureConfig.java +++ b/pregel/src/main/java/org/neo4j/gds/beta/pregel/PregelProcedureConfig.java @@ -20,12 +20,17 @@ package org.neo4j.gds.beta.pregel; import org.immutables.value.Value; +import org.neo4j.gds.NodeLabel; +import org.neo4j.gds.RelationshipType; import org.neo4j.gds.annotation.Configuration; import org.neo4j.gds.annotation.ValueClass; +import org.neo4j.gds.api.GraphStore; import org.neo4j.gds.config.MutateNodePropertyConfig; import org.neo4j.gds.config.WritePropertyConfig; import org.neo4j.gds.core.CypherMapWrapper; +import java.util.Collection; + @ValueClass @Configuration @SuppressWarnings("immutables:subtype") @@ -44,6 +49,29 @@ default String mutateProperty() { return ""; } + @Override + @Configuration.GraphStoreValidationCheck + @Value.Default + default void validateGraphIsSuitableForWrite( + GraphStore graphStore, + @SuppressWarnings("unused") Collection selectedLabels, + @SuppressWarnings("unused") Collection selectedRelationshipTypes + ) { + // VN/IP: HACK! + // Since we are using the same configuration for all the modes (`stream`, `write` and `mutate`) we check if the + // graph is writable in all modes. + // We only want to raise the error if the user actually doing writes, + // which we assume is when there is a `writeProperty` present + if (writeProperty().isBlank()) { + return; + } + + if (!graphStore.capabilities().canWriteToDatabase() && !graphStore.capabilities().canWriteToRemoteDatabase()) { + throw new IllegalArgumentException("The provided graph does not support `write` execution mode."); + } + } + + static PregelProcedureConfig of(CypherMapWrapper userInput) { return new PregelProcedureConfigImpl(userInput); } From 09470ce17c33456c82ced5e2e89fdcb87ec18342 Mon Sep 17 00:00:00 2001 From: Veselin Nikolov Date: Wed, 19 Jul 2023 12:19:17 +0100 Subject: [PATCH 144/273] Add Neo4j 5.10 compatibility to OpenGDS Co-authored-by: Ioannis Panagiotas --- build.gradle | 2 ++ settings.gradle | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/build.gradle b/build.gradle index 16f260a93b..a11e882cc0 100644 --- a/build.gradle +++ b/build.gradle @@ -36,6 +36,7 @@ ext { project(':neo4j-kernel-adapter-5.7'), project(':neo4j-kernel-adapter-5.8'), project(':neo4j-kernel-adapter-5.9'), + project(':neo4j-kernel-adapter-5.10'), ], 'storage-engine-adapter': [ project(':storage-engine-adapter-4.4'), @@ -48,6 +49,7 @@ ext { project(':storage-engine-adapter-5.7'), project(':storage-engine-adapter-5.8'), project(':storage-engine-adapter-5.9'), + project(':storage-engine-adapter-5.10'), ] ] } diff --git a/settings.gradle b/settings.gradle index a73bc99bc3..dc27ab8a74 100644 --- a/settings.gradle +++ b/settings.gradle @@ -157,6 +157,9 @@ project(':neo4j-kernel-adapter-5.8').projectDir = file('compatibility/5.8/neo4j- include('neo4j-kernel-adapter-5.9') project(':neo4j-kernel-adapter-5.9').projectDir = file('compatibility/5.9/neo4j-kernel-adapter') +include('neo4j-kernel-adapter-5.10') +project(':neo4j-kernel-adapter-5.10').projectDir = file('compatibility/5.10/neo4j-kernel-adapter') + include('neo4j-kernel-adapter-api') project(':neo4j-kernel-adapter-api').projectDir = file('compatibility/api/neo4j-kernel-adapter') @@ -262,6 +265,9 @@ project(':storage-engine-adapter-5.8').projectDir = file('compatibility/5.8/stor include('storage-engine-adapter-5.9') project(':storage-engine-adapter-5.9').projectDir = file('compatibility/5.9/storage-engine-adapter') +include('storage-engine-adapter-5.10') +project(':storage-engine-adapter-5.10').projectDir = file('compatibility/5.10/storage-engine-adapter') + include('storage-engine-adapter-api') project(':storage-engine-adapter-api').projectDir = file('compatibility/api/storage-engine-adapter') From 29b74a811aac0f6345908ae7b7ffb1441d186e3c Mon Sep 17 00:00:00 2001 From: Veselin Nikolov Date: Wed, 19 Jul 2023 12:22:55 +0100 Subject: [PATCH 145/273] Update compatibility documentation Co-authored-by: Ioannis Panagiotas --- README.adoc | 5 +++-- .../ROOT/pages/installation/supported-neo4j-versions.adoc | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/README.adoc b/README.adoc index bdb16c96ee..4b35e482ba 100644 --- a/README.adoc +++ b/README.adoc @@ -26,9 +26,10 @@ When installing GDS manually, please refer to the below compatibility matrix: .Compatibility matrix (italicized version is in development) |=== |GDS version | Neo4j version | Java Version -.10+<.^|_GDS 2.4.x_ +.11+<.^|_GDS 2.4.x_ +|Neo4j 5.10.0 +.10+.^|Java 17 |Neo4j 5.9.0 -.9+.^|Java 17 |Neo4j 5.8.0 |Neo4j 5.7.0 |Neo4j 5.6.0 diff --git a/doc/modules/ROOT/pages/installation/supported-neo4j-versions.adoc b/doc/modules/ROOT/pages/installation/supported-neo4j-versions.adoc index 9def15f78f..eefdd0c98b 100644 --- a/doc/modules/ROOT/pages/installation/supported-neo4j-versions.adoc +++ b/doc/modules/ROOT/pages/installation/supported-neo4j-versions.adoc @@ -10,6 +10,7 @@ If your version of GDS or Neo4j is not listed in the matrix, you should upgrade. [opts=header] |=== | Neo4j version | Neo4j Graph Data Science +| `5.10` | `2.4.2` or later | `5.9` | `2.4`, `2.3.9` or later footnote:eol[This version series is end-of-life and will not receive further patches. Please use a later version.] | `5.8` | `2.4`, `2.3.6` or later footnote:eol[] | `5.7` | `2.4`, `2.3.3` or later footnote:eol[] From 9803b3189750beda75cdc663aaf07187b57043a6 Mon Sep 17 00:00:00 2001 From: ioannispan Date: Wed, 19 Jul 2023 16:19:27 +0200 Subject: [PATCH 146/273] Add cosine as metric for NodeSimilarity Co-Authored-By: Ioannis Panagiotas --- .../nodesim/CosineSimilarityComputer.java | 96 +++++++++++++++++++ .../nodesim/MetricSimilarityComputer.java | 6 +- .../nodesim/NodeSimilarityMetric.java | 3 +- .../nodesim/CosineSimilarityComputerTest.java | 90 +++++++++++++++++ .../nodesim/MetricSimilarityComputerTest.java | 8 ++ .../specific-configuration.adoc | 2 +- .../NodeSimilarityStreamProcCleanTest.java | 74 ++++++++++++++ 7 files changed, 276 insertions(+), 3 deletions(-) create mode 100644 algo/src/main/java/org/neo4j/gds/similarity/nodesim/CosineSimilarityComputer.java create mode 100644 algo/src/test/java/org/neo4j/gds/similarity/nodesim/CosineSimilarityComputerTest.java create mode 100644 proc/similarity/src/test/java/org/neo4j/gds/similarity/nodesim/NodeSimilarityStreamProcCleanTest.java diff --git a/algo/src/main/java/org/neo4j/gds/similarity/nodesim/CosineSimilarityComputer.java b/algo/src/main/java/org/neo4j/gds/similarity/nodesim/CosineSimilarityComputer.java new file mode 100644 index 0000000000..2279547d23 --- /dev/null +++ b/algo/src/main/java/org/neo4j/gds/similarity/nodesim/CosineSimilarityComputer.java @@ -0,0 +1,96 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.similarity.nodesim; + +import org.neo4j.gds.core.utils.Intersections; + +public class CosineSimilarityComputer implements MetricSimilarityComputer { + private final double similarityCutoff; + + public CosineSimilarityComputer(double similarityCutoff) { + this.similarityCutoff = similarityCutoff; + } + + @Override + public double computeSimilarity(long[] vector1, long[] vector2) { + var intersection = Intersections.intersection3(vector1, vector2); + var similarity = intersection / (Math.sqrt(vector1.length) * Math.sqrt(vector2.length)); + return similarity >= similarityCutoff ? similarity : Double.NaN; + } + + @Override + public double computeWeightedSimilarity(long[] vector1, long[] vector2, double[] weights1, double[] weights2) { + assert vector1.length == weights1.length; + assert vector2.length == weights2.length; + + double vector1SquaredSum = 0; + double vector2SquaredSum = 0; + double above=0; + + int offset1 = 0; + int offset2 = 0; + int length1 = weights1.length; + int length2 = weights2.length; + + while (offset1 < length1 && offset2 < length2) { + long target1 = vector1[offset1]; + long target2 = vector2[offset2]; + double w1 = weights1[offset1]; + double w2 = weights2[offset2]; + + if (target1 == target2) { + above+=w1*w2; + vector1SquaredSum+= w1*w1; + vector2SquaredSum+= w2*w2; + offset1++; + offset2++; + } else if (target1 < target2) { + vector1SquaredSum += w1*w1; + offset1++; + } else { + vector2SquaredSum += w2*w2; + offset2++; + } + } + + for (; offset1 < length1; offset1++) { + vector1SquaredSum += weights1[offset1] * weights1[offset1]; + } + for (; offset2 < length2; offset2++) { + vector2SquaredSum += weights2[offset2]* weights2[offset2]; + } + + + double similarity = above/(Math.sqrt(vector1SquaredSum) * Math.sqrt(vector2SquaredSum)); + return similarity >= similarityCutoff ? similarity : Double.NaN; + + } + + static class Builder implements MetricSimilarityComputerBuilder { + public MetricSimilarityComputer build(double similarityCutoff) { + return new CosineSimilarityComputer(similarityCutoff); + } + + @Override + public String render() { + return "COSINE"; + } + } +} diff --git a/algo/src/main/java/org/neo4j/gds/similarity/nodesim/MetricSimilarityComputer.java b/algo/src/main/java/org/neo4j/gds/similarity/nodesim/MetricSimilarityComputer.java index a1d2626385..191491f4cf 100644 --- a/algo/src/main/java/org/neo4j/gds/similarity/nodesim/MetricSimilarityComputer.java +++ b/algo/src/main/java/org/neo4j/gds/similarity/nodesim/MetricSimilarityComputer.java @@ -50,6 +50,8 @@ private static NodeSimilarityMetric valueOf(String userInput) { return NodeSimilarityMetric.JACCARD; } else if (userInputInCaps.equals("OVERLAP")) { return NodeSimilarityMetric.OVERLAP; + } else if (userInputInCaps.equals("COSINE")) { + return NodeSimilarityMetric.COSINE; } else { throw new IllegalArgumentException(userInput + " is not a valid metric. Available metrics include Jaccard and Overlap"); } @@ -58,8 +60,10 @@ private static NodeSimilarityMetric valueOf(String userInput) { private static MetricSimilarityComputerBuilder create(NodeSimilarityMetric metric) { if (metric == NodeSimilarityMetric.JACCARD) { return new JaccardSimilarityComputer.Builder(); - } else { + } else if (metric == NodeSimilarityMetric.OVERLAP) { return new OverlapSimilarityComputer.Builder(); + } else { + return new CosineSimilarityComputer.Builder(); } } diff --git a/algo/src/main/java/org/neo4j/gds/similarity/nodesim/NodeSimilarityMetric.java b/algo/src/main/java/org/neo4j/gds/similarity/nodesim/NodeSimilarityMetric.java index 19024e0d01..54de85d445 100644 --- a/algo/src/main/java/org/neo4j/gds/similarity/nodesim/NodeSimilarityMetric.java +++ b/algo/src/main/java/org/neo4j/gds/similarity/nodesim/NodeSimilarityMetric.java @@ -21,5 +21,6 @@ public enum NodeSimilarityMetric { JACCARD, - OVERLAP + OVERLAP, + COSINE, } diff --git a/algo/src/test/java/org/neo4j/gds/similarity/nodesim/CosineSimilarityComputerTest.java b/algo/src/test/java/org/neo4j/gds/similarity/nodesim/CosineSimilarityComputerTest.java new file mode 100644 index 0000000000..9da7449cd8 --- /dev/null +++ b/algo/src/test/java/org/neo4j/gds/similarity/nodesim/CosineSimilarityComputerTest.java @@ -0,0 +1,90 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.similarity.nodesim; + +import org.assertj.core.data.Offset; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.AssertionsForClassTypes.assertThat; + +class CosineSimilarityComputerTest { + + @Test + void shouldComputeUnweightedCosineSimilarity() { + var array1 = new long[]{0, 1, 2, 4, 6}; + var array2 = new long[]{0, 1, 2, 3, 5, 6, 7}; + var similarityComputer = new CosineSimilarityComputer(0); + + var cosineSimilarity = similarityComputer.computeSimilarity(array1, array2); + + assertThat(cosineSimilarity).isCloseTo(0.6761234037828131, Offset.offset(1e-5)); + } + + @Test + void shouldComputeUnweightedCosineSimilarityRespectingCutoff() { + var array1 = new long[]{0, 1, 2, 4, 6}; + var array2 = new long[]{0, 1, 2, 3, 5, 6, 7}; + var similarityComputer = new CosineSimilarityComputer(0.7); + + var cosineSimilarity = similarityComputer.computeSimilarity(array1, array2); + + assertThat(cosineSimilarity).isNaN(); + } + + @Test + void shouldComputeCosineOnArraysWithNoMissingElements() { + var similarityComputer = new CosineSimilarityComputer(0); + var array1 = new long[]{0, 1, 2}; + var weight1 = new double[]{0.5, 0.2, 0.6}; + var array2 = new long[]{0, 1, 2}; + var weight2 = new double[]{1.3, 0.9, 0.2}; + + assertThat(similarityComputer.computeWeightedSimilarity(array1, array2, weight1, weight2)).isCloseTo( + 0.73935, + Offset.offset(1e-5) + ); + } + + @Test + void shouldComputeCosineOnArraysWithNoMissingElementsRespectingCutoff() { + var similarityComputer = new CosineSimilarityComputer(1); + var array1 = new long[]{0, 1, 2}; + var weight1 = new double[]{0.5, 0.2, 0.6}; + var array2 = new long[]{0, 1, 2}; + var weight2 = new double[]{1.3, 0.9, 0.2}; + + assertThat(similarityComputer.computeWeightedSimilarity(array1, array2, weight1, weight2)).isNaN(); + } + + @Test + void shouldComputeCosineOnArraysWithMissingElements() { + var similarityComputer = new CosineSimilarityComputer(0); + var array1 = new long[]{0, 1, 2, 4, 6}; + var weight1 = new double[]{0.5, 0.2, 0.6, 0.8, 1}; + var array2 = new long[]{0, 1, 2, 3, 5, 6, 7}; + var weight2 = new double[]{1.3, 0.9, 0.2, 4.2, 1, 2, 3}; + + assertThat(similarityComputer.computeWeightedSimilarity(array1, array2, weight1, weight2)).isCloseTo( + 0.33344, + Offset.offset(1e-5) + ); + } + +} diff --git a/algo/src/test/java/org/neo4j/gds/similarity/nodesim/MetricSimilarityComputerTest.java b/algo/src/test/java/org/neo4j/gds/similarity/nodesim/MetricSimilarityComputerTest.java index f36a1ba506..f944ddc92a 100644 --- a/algo/src/test/java/org/neo4j/gds/similarity/nodesim/MetricSimilarityComputerTest.java +++ b/algo/src/test/java/org/neo4j/gds/similarity/nodesim/MetricSimilarityComputerTest.java @@ -21,6 +21,7 @@ import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; class MetricSimilarityComputerTest { @@ -31,4 +32,11 @@ void shouldThrowOnWrongMetric2() { .isInstanceOf(IllegalArgumentException.class) .hasMessageContaining("ovErLaPPPPP is not a valid metric"); } + + @Test + void shouldParseCosineSimilarityComputer() { + var cosineComputer = MetricSimilarityComputer.parse("CosiNe").build(0); + assertThat(cosineComputer) + .isExactlyInstanceOf(CosineSimilarityComputer.class); + } } diff --git a/doc/modules/ROOT/partials/algorithms/node-similarity/specific-configuration.adoc b/doc/modules/ROOT/partials/algorithms/node-similarity/specific-configuration.adoc index 27a68af42c..8984914797 100644 --- a/doc/modules/ROOT/partials/algorithms/node-similarity/specific-configuration.adoc +++ b/doc/modules/ROOT/partials/algorithms/node-similarity/specific-configuration.adoc @@ -24,5 +24,5 @@ This value cannot be negative, a value of 0 means no global limit. If unspecified, the algorithm runs unweighted. | similarityMetric | String | JACCARD | yes | The metric used to compute similarity. -Can be either `JACCARD` or `OVERLAP`. +Can be either `JACCARD`, `OVERLAP` or `COSINE`. |=== diff --git a/proc/similarity/src/test/java/org/neo4j/gds/similarity/nodesim/NodeSimilarityStreamProcCleanTest.java b/proc/similarity/src/test/java/org/neo4j/gds/similarity/nodesim/NodeSimilarityStreamProcCleanTest.java new file mode 100644 index 0000000000..776bded725 --- /dev/null +++ b/proc/similarity/src/test/java/org/neo4j/gds/similarity/nodesim/NodeSimilarityStreamProcCleanTest.java @@ -0,0 +1,74 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.similarity.nodesim; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.neo4j.gds.BaseProcTest; +import org.neo4j.gds.catalog.GraphProjectProc; +import org.neo4j.gds.core.loading.GraphStoreCatalog; +import org.neo4j.gds.extension.Neo4jGraph; + +import static org.assertj.core.api.Assertions.assertThatNoException; + +class NodeSimilarityStreamProcCleanTest extends BaseProcTest { + + @Neo4jGraph + public static final String DB_CYPHER = + "CREATE" + + " (a:Person {id: 0, name: 'Alice'})" + + ", (b:Person {id: 1, name: 'Bob'})" + + ", (c:Person {id: 2, name: 'Charlie'})" + + ", (d:Person {id: 3, name: 'Dave'})" + + ", (i1:Item {id: 10, name: 'p1'})" + + ", (i2:Item {id: 11, name: 'p2'})" + + ", (i3:Item {id: 12, name: 'p3'})" + + ", (i4:Item {id: 13, name: 'p4'})" + + ", (a)-[:LIKES]->(i1)" + + ", (a)-[:LIKES]->(i2)" + + ", (a)-[:LIKES]->(i3)" + + ", (b)-[:LIKES]->(i1)" + + ", (b)-[:LIKES]->(i2)" + + ", (c)-[:LIKES]->(i3)"; + + + @BeforeEach + void setUp() throws Exception { + registerProcedures( + NodeSimilarityStreamProc.class, + GraphProjectProc.class + ); + + runQuery("CALL gds.graph.project('g', '*', 'LIKES')"); + } + + @AfterEach + void tearDown() { + GraphStoreCatalog.removeAllLoadedGraphs(); + } + + @Test + void shouldWorkWithCosineSimilarityMetric() { + assertThatNoException().isThrownBy( + () -> runQuery("CALL gds.nodeSimilarity.stream('g', {similarityCutoff: 0.0, similarityMetric: 'cosine'})") + ); + } +} From 662ea2fd2acf807f5f1dc8230625feb8b909c7f4 Mon Sep 17 00:00:00 2001 From: Veselin Nikolov Date: Wed, 19 Jul 2023 16:25:46 +0100 Subject: [PATCH 147/273] Update GDS to the next version --- doc/modules/ROOT/pages/management-ops/utility-functions.adoc | 2 +- gradle/version.gradle | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/modules/ROOT/pages/management-ops/utility-functions.adoc b/doc/modules/ROOT/pages/management-ops/utility-functions.adoc index cb81f3e9f8..5c369412a9 100644 --- a/doc/modules/ROOT/pages/management-ops/utility-functions.adoc +++ b/doc/modules/ROOT/pages/management-ops/utility-functions.adoc @@ -27,7 +27,7 @@ RETURN gds.version() AS version [opts="header"] |=== | version -| "2.4.2" +| "2.4.3" |=== -- diff --git a/gradle/version.gradle b/gradle/version.gradle index 8725958c72..6fd9200ae0 100644 --- a/gradle/version.gradle +++ b/gradle/version.gradle @@ -1,5 +1,5 @@ ext { - gdsBaseVersion = '2.4.2' + gdsBaseVersion = '2.4.3' gdsAuraVersion = '27' gdsVersion = gdsBaseVersion + (rootProject.hasProperty('aurads') ? "+${gdsAuraVersion}" : "") From 28477e31b07da47e49c5319caad7617493212791 Mon Sep 17 00:00:00 2001 From: Veselin Nikolov Date: Wed, 19 Jul 2023 16:38:16 +0100 Subject: [PATCH 148/273] Update to Gradle 8.2.1 --- .../gradle/wrapper/gradle-wrapper.jar | Bin 62076 -> 63375 bytes .../gradle/wrapper/gradle-wrapper.properties | 3 ++- examples/pregel-bootstrap/gradlew | 5 ++++- gradle/wrapper/gradle-wrapper.jar | Bin 62076 -> 63375 bytes gradle/wrapper/gradle-wrapper.properties | 3 ++- gradlew | 5 ++++- 6 files changed, 12 insertions(+), 4 deletions(-) diff --git a/examples/pregel-bootstrap/gradle/wrapper/gradle-wrapper.jar b/examples/pregel-bootstrap/gradle/wrapper/gradle-wrapper.jar index c1962a79e29d3e0ab67b14947c167a862655af9b..033e24c4cdf41af1ab109bc7f253b2b887023340 100644 GIT binary patch delta 16170 zcmZv@1C%B~(=OPyZQHhOo71+Qo{!^7;G&o^S)+pkaqdJWHm~1r7od1qA}a4m7bN0H~O_TWh$Qcv`r+nb?b4TbS8d zxH6g9o4C29YUpd@YhrwdLs-IyGpjd3(n_D1EQ+2>M}EC_Qd^DMB&z+Y-R@$d*<|Y<~_L?8O}c#13DZ`CI-je^V*!p27iTh zVF^v_sc+#ATfG`o!(m-#)8OIgpcJaaK&dTtcz~bzH_spvFh(X~Nd=l%)i95)K-yk?O~JY-q9yJKyNwGpuUo601UzzZnZP2>f~C7ET%*JQ`7U^c%Ay= z*VXGhB(=zePs-uvej`1AV`+URCzI7opL{ct^|Lg3`JRQ#N2liRT0J3kn2{O5?+)Xh zg+2W4_vVGeL^tu5mNC*w+M@qOsA?i7Q5Y!W}0%`WElV9J|}=8*@{O1`1(!wCebWJz&EbIE09Ar_<&ldhsD}pR(~NfS=IJb>x%X z{2ulD!5`cb!w+v^IGu~jd3D$fUs>e3cW|v_Cm{8={NL)ZoxNQqikAB&nbiz7mbKz( zWjH73t*#;8Rv5%^+JhrK!zDSutNaUZF#xIcX-J?XTXJMUzc0+Q{3)Xt)KYbRR4)MYT4?1fDz4 z0NVFLz!!^q(*mC;cfO~%{B}A^V3|1aPPqpOYCO4o^)?p?Hn17_0AbdX$f;k!9sL^g z{n_Q5yM!yp{oU))sbp&r6v}Au6R`9Z#h@0oM&1n0>wAP27GtH zG#~tyCu38r+Xh)31z*ShTdXWfb`4h!sraW8_kR1VGraUOtA9}O2g{N$S+1{3q>z*< zDEs&xo6@|O7lJlzn%!gmnJL@mh6XY?H2^>+tYwAp2aD&ve*;dNlFRUUD4uJsz0s{jA0wM|`g_Bk- z2nGTI4FLio^iSgCYQ<~?w6VhgXuFy?J6pI)*tog7+L(H{+c-IDy4s67IsWSv-2ZoX zkgKk*j4q1tU51^udPJsziAoFE%s5Wgi({t%V=JasWm6hHcE*-AVByK0i}t9!4^NT& zYJ1?sHp;I5vxtJi@z=?8N5Bc2Rp96QJ7Pawo_W$pO{f?a?6fX`?dHe8J+yAg-F$LU zXmTjqP`_JciO)bHLs}L><&(2CORPpITFZ5y{Ha$rW};;c-n)RcD`TyHnL?)Fx{0?I zqQ|D4T`xLJy`A}h{D57UR@bD8{Bw{9rlPt&U?{4 zTbO4-nHnPS!as<)ecV@VpH~W*$zoPr8f09_MZBPjoU zamA5hmU=F0q4v*u)BvEyDNo)GJxs9tiPkp2uhlGLR2bUD{NSjGGCixR9?$LKAlsip zUIa{WQs#68GH3NL{(FUyk-k=lrtx{V24k>kq~uc+St1uH0Yf3s547xvD5T*@n^+VN zKO~$H#RFW+Sd*M?`&+A$L<%DwNmIW&h>4j}vyxu3PmHrGwp?hXJp!{^>$Ax2WY&9} z5fJvDKBT&~%2QWqTGf{=6Pv2U+0HUQRv9%RZLR`G^XNdKRZt`Zs z)vuUr#7C#oQ00KL7$M$(yHa*C4XZ~*t9NPMJU`fACD3v+wvLzMJipnOfRmh_kN5oD zZ;)G|-j$^OF~-yWW*p1m#1)%%tWgg_?ps;<cvxwa&b=_7Iu)xM#KIHR~gWVSQGmujR;bCgI%H#(_~8O`LAHbJ%9L?R(Dt zq%5@6HsP4(%%tF4t#7v$y&h*i|KihD+E^Q7n~`1KzELK>5I8-`H|JF2Cq9CgniYyS z_4op2_>b9Il(p8PquZ{h8Gy$%WA+8t)o_gCdb75|9NJ&}Y*D~a6)VE@eT3!qvvSPz z4-A4Vw^rS17uWVctor@Gky4eiT6nF=PVY~8jzjKM-GlQzF5I-V&Z7d^G3?o9`C9gHU5GOAMLIZIOBw|s--tIy=R#b8@3;?-9Y8jeFt`AhO z8tTwGxksHRNk>;%uqWW&Q!^M?CwVDvX-*wTji*J^X%}1`6Z(#9OsQQfUI9x&CAj=W z-tDF7TYPVS7zfx~aje8Z@J>er!E<@63gEY)W{b!AF%?j%VG;B3b;Kt6VVH0qxBLrC z*82l$taUKcm}zRM=K+>H%w7(10hX25ud7r}c#sEK;mnBsVbD;$qu_|UEarcuS7aYi zcMjgkjmj=#d&K?NX=qgouhsLh{iYTe8qtsU~kLwg4&&Q1YGyz6D@(-w< zl~tx6ulu}VfKZ@_gt2aL@E`A`ULme@K+ zek2hch6FNgHdbowNo)mBs0da-}bhPw|R1u{4 zEZ?T!7j&^lNPs1je%@Em^CPp$cX%GrCBn66>D{`Ugf%+~@)w+gX2xGJ1qCy6|1f8m zkW@0=CvkEuR0$mn*wuIvn?-qRMNjtj*c5Z_P}N^he{2=<@XK4^ zC{Zs89DIB6QjEE2PRx9Le^?_kvTpBWr~%L249F}8N&xTV?+_;?oyfV?V^T(ioIxw@ zYNZUlBAc=A{A709=R`$--jqG{jPQj-7f_Sr1$o&kapsFL3jBVIE*Z4&L}1ve?@wh=%eda^BRYm=>pJ z{p#Gotpa1aH^l+Oclp_+$Whjp_q3(G8zS<1;!#*67K0Du1}RQPo&G8mVeftaJ&a++ zYlh?j&;3LJA5Q4fDBsWauFn>VvG_9Tcrr2Yt-#+%rO0ST1GFitK8f10=rq|6lf1q? zZgVH$pWLo_(3QZ@KH}q%V;KT>r!K|?t?LSBWRUoPcv3to`%wC6ZRPF|G1tKl`(7G_xblMQANQ+j&NIeH&TK6-$u*4Uh&0t&ePU zPJkhRuh#-@_X+0}aV*Jb0Bfa+LZNqQVWJ0#=KA~Bqt%4}(36~^U)lvrj$CQX%P=?D ziHvZYaHPO6-Q>+|s~lNFW0?Bv%tzi)3M>X`;!RfF3<~0HjHc|}*l~bKATK4IXdR!B zMf+A}Up#I+)T8aogDs8)j}J)JK!%rH9&J59H~Q@Ntd^EV{~c7kTX%dQB_?kfOR-tn zA=NR@abtm5k{N9NS^G$1>>Td<278}g(`E7_k5+?RgoT&-Nqa5AjkAAn7s8#Vc=*sd zmyzfjfeIp0Fehg1gbSQ(_~qXV=y0ShN7ck^V@6t(5C%IxDmYn-~2#bGniWG#vS zWlnC*Dbfin3QX!ZI-YRxCO7uBG+d>=s@*c0sPmByGDc2mN&24$GkoH0oitsFTV0_} z4iATfIz{jBODQY1t{lpUS%Q1Hzdel~82P1N#Cura_7k&{mUoI@q?W7&Jzo61$}3G7 zl`3shFi_Vnoh`5OIKHqV;wTULz2GkZgW0zNjk3t#5aH8tz(R^=;i?c~(3-;#WM50snq>qF)cu>}tWC*wTO7r93>;1Cbif%d{o% zC1Eyo7UwX41o7QLvdU_to(vzDD`*KK^3HBZvx@j@i1Nbt-w8Z5`>?)c;rXTjdt#k# zOfJED_)awGGGg*Z0Rgo!JN?rDkpZFr6pE4%K}BPXJ>0O@93hgvCGJz?oUweJQjnVi zNQKWhxNpSd36=ip(-D4iOtMG99MY(y86GtXS~1%=jipBb#D;tZpKmMRZ_t=10TL%p z21RJ%0X=&&WUDYBbTcwsof1(CDGDD)eW`d#Y*Z87@k z^{dy_GcUp~J?qJ=i#H#EeSsp^TSr@dt$%q>c3_o1F9sr_ta1PLWYBdi1BNUNu0`v` zvgB;K@#gLmv#tD2Mf21LHU0Hq2~Ro}Upex$#h~)93nAvxcS6wkM&UVy#4RnSG6QX9 zQ;r$p=AKnBnUe=hZPH*u-Q4Ta4COuQ7TQGIqbUi4&eot$D2GHljdSdbc-MK-t1R86opRwDuUN+ zw(1^ybD7grBO>ySm29}i&+s{~7uz?*?K;N9?Yw~zd6 z*Xfoqv-*O~(QBAVpOqwZ``Qmd5qbL#d`>U7rT&?h?FN=iYu*vFfck~?6h=b48;n}$ zQrzUxWJ{eaR2!*MSX=+F*)ECE#91?SmduzuZwQ! z!ydL4;ljZ(9R_<=q z!=`&+*DUw>CsM8xVDT-;zFYUu%hn$rxPXhKztEb98>7ow#=fdMWJ!i$jJ=MIBspC; zvoJ2R96iz*(%23uM#WtAe661ynV`4t?K~eV&7!-r+tg^aw3Jiql zX^)V(pEN2WfQOL4!JgVGIoQ~a8}Gy_4l92Wst~iEI zANmgs#tUnQcv2E7>g!{jjC+X-g)LH8&8VQNoBvicmuID9WQoa^S-h?S(POL5f({Fs zWfe|-nRh@hz|Ck@iKm0C75R&`CWwUy<05TSN_IH3aMaO_Kw>0#Pv&-Dfl7b}3qfofON-WA!AB)QpF2FTnvu;s>T;lA1&Fh0 zBl$6%ODbhP1gIh2T%!8 zZ%&Q`_{;znmFQruzy3PWP@echTsS*JR65#1s^Yda=tWMNX?a%+u|@dSu2I$CfK@Jn zawQv>0i4QnlbtbIr{`+ihYt_GdJHR=O@6{5LHt~olXhcS{M}I*a8tl}U4uzgBx*jp zRji6=dfc!=jHsx4K9~%u9#`zIn~cO6$jl}Nco#8;2pDgqvpvO#S|Y1K4rie3vqVCS zI#QhtFED4h{9VA1j=@RcVQaORXzjNxK8$SAK4wPeIC%aePdZXEx8yE+0I;$3%avkwY+41*ee; z&@xvi6UvJOhfU)RKMMK5Ge)~VT{PNe>z_T^X7?!+cO%0O9;nBI39kOtN@7LUz)ZmX zVkxf)8QPZBxVNXV%s6vVeKr}hCJ=hY`pM{cihwK~6q{=~trr;R=dFS{Nx9;4Zr!`7 zG7^c|#x2=Z`)Um#l$|b#-4ZUow`yGvfCXce%qd#AG~sxuJ6eX@lQ?Gjjp4vuTv(to zGf_0z8b@Z3BzdaEB6`wXLwFwkyA*4$k{>ml#wj!^5x4DqDUFA|FW+@VD-FJyK3ynY z+{Gi9YbWOrqc_u1`$TYn+)Y1`=FhpVDRPdVzJ(>N;7R=OCBBghMVep-7atEDV6AsR zbPurLbCNf;oXDMCcEh;jgbeA|IE5ZbQ52ds%s}TJ-6?8~*qMF3@X8c=bL@w}r$Eeo zYUC@E6+viob;vjUn;z&lgCas{XLW zcxyK?xbJRX+WU9|%5bsaPbm!Tu)E}a&!br8FTR3?Cb%vZ7|$~!=Ixn55uZS#3NRZZ zs<82Gtkto2fzIEbE1T5-++IkANc74_ zARU;|ap|KEBu3}J?H?y>a845^ydr)R0F1K65>38_s0!GY|0t(o^g;aU(_1BuV33!b zi%`3stu>SZm%sRQ;lF#YPI4YIjsAv*0wm?LyvmEf2gKw__$W9yX+jR-P0o&>kaw+` zGf&tUrybKn0W_!YI0F{}d-V@ih~H2E^+PAzPlxaLf!!ly_BXZb`x{oX?}Ft-Yf}M7 zL{95Z!O*@rVV2j3Pjafo*D)wz$d3nQ2r{c~F-B4MlK60ouc3wU3}PEHhb{(moORi; zz5Hl)0M*Q# zOMmV8+5Oqz@+KiFk}x13`>Sg5)om(PI7B*n7hy<%)eZ%l1W=X?1Jtm2HUs`O#YFrj z9oFV(XD8)A{GK75(qMrd3jxUxPO`+Y7MVo#OtQX}E3fEqAVqj*?6JOOe$$5fn+5s? zx6moNC@o%1rwax68*VH@V-ANJ;x0GK{o3~V@1MKuiCN^IycAo;ZVc_;2O7q6eCH1I zoe1{_eg#}yXybiKf2$)I+FsNMa7IrsH~HZ|$A{s0LJf%{UQD;+jsdG?0>7hBQV)4Z z9Aj3a;Zp^Un5Ljqh`L5U{X*^*a6hqP--eRfh0}0|6M_IUiNtOni5Fk^t?onDM*MD^ zJegBUHkuv4>|8kN#xJYTzk`=4HR0PzpzJwG>KT()`#P3VF~fM5zGtG$RvQ|WmyaWj zqa&<4PU$5f921)o=e5(&Jm@$x-k);(lbnuD;XVQ&-lY< z+qf+FM4LeIsrObq4%f816^m|}8*00qF5^nxMS|H$dd#|s?}S(ciSghkJ(SJ=5y+twusP{MwkwIq zG2jBiouA4dgIuopX4Fp~UOni({ADA{&bB1_SYl{Q1wI*BTif%ee(N*7Z#OJCY z`He1l4dzecQ4W@TWAOkMgb_`GjENXd#_HoZ02Mr-Do>Xl9w;r*JD0R$si9tO6>US| zW|-ViVwqmhC1e{PTM51QN-HWn*EaOG$)PA8f8Q$HRNa&V^1`9Dp(-VE<`-cJRki~l zeQ) zV@HnYenHV4B4{V-j?tY(Fc2FsQ|x6Gw;Our*EHIetWC6h>UX4AD|F*5bjP5T z@3kaY0O%|F3o`0WTWlQP;ddr(jcn4KyY(k|Jxi~yT38Bltin0O;H6rTSn6Vcdf`n& z3VU99zPfSZtoV`jNq@?f5~?~6My$>J%7mhCr9$Go0cVO)?rpbQDqH4OAWGC zt!B23yF^#B>^~P@O$qgThx4S#JI`u=3Vb8kfuoSrCVyU3+I_TDPtMd zh77hUa;@t9$3OrpW1;dq;7e|B=27+?L&)R206N7fz6u?Vpo*g6vIY5v1DKt|AK$2M zJi?{ZR|-bTbSdNw@;C%KmF)oF@02bTYv#S(-3CkWy`T4^;;km9dfr10T|IR>C-<0| zdFuPGMJ!X;7kkg1rSdU~d23f8Z6O>Wa7!Q!!DKWHYFT(lU)%HbfN|7|CApdi!p6M* zZmPd41(qS*oGsEeT8dw)S%!yhgr&Tky+y^toYWPz1+9)DO8jzecE{}r$;iVGY{|@p zrp?%)e$c+T^FP36!i|qrv2(?@HIV=2NN1;L5puOPYfUZcG0NMuFx0O6`UePVOQ79wGgMj)l5<4?a<`Yl_RhY_C7U=0zKBC2$EhP^_G|S) zwv*z48K19@_pT*WUhAAZmlp){uf+E+7CcPp@0fe!wZ0R-R5-^z@HriduQz zZow5@W~ILN%8FlEM2p$(xE>5I81*!?MyluZ_h+)_1Ug0r&e(>Yv0M~3hqW5MAzFyu zT~rkx=9&{Z2Vck0$yI7kx_X*?*}kLE$UCA?X#yX}J5mqJIW0vPm&dE7bya_O96Z%~ zl$ilJ>NzFyNQyi0rMf#i6p;Rs2}#%Va%#q3X3af9vR@Gu^|I*Uw9XEY{t`plKE}Dw z8XFLZIremOfC4J$_eo{BWTsF}V-fd#;9O9P@gDn1IpW}EqCsR)gC7BFD#!|v9*h%1 z*&6syZPLg3GRsaVn+HT0jx{p1-AFJ$!XJPR;zEERi4XWy8F%Ob0bCHy{|+cVgt zxUeBR@Fg+_?_9G>{k)>Pg*RYkst}Ve&Yr9ku!oPKAT5$zr_hh$bio?MkK~VXg<}A0 z(xHUlM(j$|fxDCvX(ON*g)b7>LKCWPKjS0%J1wRdl;<;+3;S1WAQF7)9UG>EBPO4+ z+60A8s;x%l0#{t#>M3qq-pVQOPavJPiz)V?3tAxyIwpNpQ#BQ7cUn49TfXdRMw84e znq4y_=;tRzm6)Uu*a@=Cyn@(7`XL|*GokZSuV40Fdtg?L=UjQd71V&Il|4)T&J8z^ zX>1PZv)eLcn%pp%s3)`~`Cg;oBWcd_nBp_R7 z(cbpAAxWQ&^ZmRDkLbO=Jfb(k(=z$y_Dzc|sd{p_6S+9#Fbr7HEPqyXNdaJ3`3u6( zWDF@;ybOj>Le%rvVTGL7*S;P6;T6lI#?Yp@KX&- zeXq*<7IsOCb=uS5s0Mmf25>+hk)wj?se_5MedT~~WtEfn%Dxk#_W?Lj?3>GwN46fK z!IYgVw^_>#<=3oy;69J;(4rMSQ*bk#e z*O9H2VyX^(Rhj_h2~RKjRb;#jfWoVR_7xu0|7d;#jJeOlwzc=%h&6f;S#I99}wvxDNo zQFoYVq&-Mp!>+&et%Z3e-=EL?u?LUtia5D*zj}rztU#KX9V6C7;j7Q8S0 zlB*6q%yF@-Yf+q;a1)&^0$8&K{HXDYS&Ed)vJ!l6r$n9U8P`MUQZI)eK-^u6*Kdpf zzNar-y5wx;ZtRJpbYCGEd0*84PVL8&+BWu$y*{?sk&bhCehjZArP1SSX2_6(z{nE6M^R*|f6 z$ynra_U-VwV*BF1^ho4}C9XiaVprNH`hGFmgiUX%Pv*@VcTI~^;m|JEntHi&{_L&; zNnO;cWA4aJODk4op9K>jC_D0@eyJFuB2hh`Cwo{)#83w{6&Ky2xe7(Qnzks)2SH`f z9MmfjA!;HpQ_Q@C+Q5Zs>7ASx!lG`27XazRsQ1uR^eWQATS z(PqV@o6r#!swbqh-w^cNgLo54+nw2GAw@~>UnR!SfLMDZrFXJ!$OoPmtDTp_b;9`K z6tL5XDPoLt$~OS+O>IkYa^+oW@Jfg_g4g+JCAzGU4dsZ-rcx~ZL}!pigv95Pq3LG} zPEIepL$%a4dNpm5R9%Wqxwu3dl8$7pq4pjr{XIuHbFK8kLrI(}DqKPN12YQ2t3qzdnN!ez3Fd zp@($04skG7>K4pGr(&g2KJoRf`ea1&(??Wp<%O(8*U+X0RR*C;2`Ok6Xl&E2*5VdI zwm9bdWnitI-|PHYdRgj21CFGr*CO^yY1 zJkS;V*|!ymL(H~{Vz-foW=m%#Bb9256n3?)QAHTMGkd{94WY{Y;*C_3_M$LA@*1`k zcOc;KRtbu3LZZcSJ$Y@4f9q(6`;*$pPvvNuPTT!YP)11=@3hLs*qSRmT&kfVB_E~J`wO&l5No9Hxys8+F-y1{*16v=L0gph z26scBjUWa-_NHH!@XYfp&9h5bno!vSYX-@^Wni0>qJlmngFgNZ=RDuIzHu6Ja}IZ- zz~}h(TRXn514hbq<};7Yp!(msmGT0$WLE$i%+~T+S)Z&w;Z3dPlWkfIw!BJ{{~Rcq z;&sxPHBu7o@hrM#E2pGw2J~6gLR;dze8@5(Xd~jE(gF~%!U~&-tl;CBXIrbO$!#%# z7Wnm3NH%VXo`JPuS>tD|@@o51t zvF6hSTV`=L1picH03CEV53d&h8m~F=xI^xq$^KQg$S?s!Y>X4C8px}6>=*DKtGGqORX z>@+KMD)Z8^xQbawX$BD?6-3UNB<=xuVC8wB+3{ z$(6jJF;?=cj{Vw_x`S}-Rt)sM&?wC`WeCKUYuI|Su&3BBDm>S9B?@}*DAYqI@VH5J zx@#>WGMvy{SU5}Z-ds4VIzM&)$RV?;m6yYnO)4jn1+66*NN(r@8i51e)@X?XxljW& z!Mqh9S&j$#%jy30)1H zmLPP5mM-sO3a)B03I-**B$D}Mg=LNdyPsRNgzN$c%7l1~0s5sGk5LwCFlp`b1}{tY z`Ax$;Fh0h_WqU?!RsMi?(oU6P#~_3MRFz6_$2S%Y&}kOb(M&MiPm~{! zI`z;?7q`8^+qCNSK{t`or*wkUEAx){Js`RRh|P9E(`1{cvg-PRvg+x{^u&;j#m+6UDx{Mo^f1Zw);JI=wvFcnuMO()EMgA1m%4ZN)t=+tTUo{-mt26* z+YtnDP|`%#Mc4r*9=JNUppLb2m|;RLP_~8+D>BB^VX@~;nM(ASLh@oz5vUeD^CYnE z%sZ0<+!;U4eDkEZZ{0f~Z`$qI8Kw{pGxP)o=!I`)$0qyhKYNP`j1A-|^8Q z(IE~i2!?diQoAET^xIFq^XF(^gAzEOveZ#&@hY^0Wsx#jKD!&*f^7=zg?p!e4zYCx zm`g2=4;L3|Jv~$BIf>zyPp4%@okJzf`yPuSHMH7A&2cKN05YV1W^!P1%kc4LP+B=1 z_v)WD&+J|8+5u@+^?n)Tl-y?P6@xH|G0q5VL4U@?0e!W-O=L>!?VrBX+I?s$~ z+R^j|7)h>Gl(Pq9{aK<-m@9xaP!=*m9OgP;S(LE4#j`zVvSzF=uH6#r*@8;YNf6h? zM?C0=;hrzuLP9<(sJ`tcn#1=oI}cKoBNT{G4h~EsKbQ$)+upOKO24nXjex~C@DYjI z^H-KT^YiY_{qyYHG3Y~NID^UJ%(tUUUwxScD9C&CqBy=;?RY2TQ!LL8zEHK#JA-4h zjyvrS%@N-z=x&oyw-C1sVCr+(u(?A&MbAjX;!_=O(G+RJ=S%0kDY{G5j7R%f*!3Lu z4g14hdT%|ONka2%Mt^)pzcR6H!Ci>hDIGNc zI{I>=8v><;f>XvXd#l3P8Sj{536jWYa>{EhzwaYB%d0E%34 zs;&Z4pI+PJX=`lcUrsKkWLbX_E%z}twRY>ZWZ*ayyQpMM6JFI513Q{C3N3tqjZF3}4n~f@ z1^DS=&vW?GO_0n2{*g|QW&^Pcv|^Nh{_vAra`IX=Q)i-TJ>vbBs9PT;-Zf8d37A(w z!a&fT*gXFS6Cl`Ms(4TK0AUu%bg;1yNP>Qg`Kw6&A z+==jRb-{oPy?$sWM+5q(TH6-Hfq2}yOJs1A)gEt5iq_r(A0M%haJb?CJEE%{9MDb_ z?k8%7DL9hlwp;KtwOhovV+jatf2)5LG6%b3u;fgv&Cg)q9kg70Pa;_(Dp@-f085&lb{lrqjJ8XBwmAHz2ZU?>J&&Qt_utVGrOC;QXfP8-` z4(gvV_VMBckHXq0&CBQV*-Eb~g%i_xDBsc{u4VJ4V# z)zc`WeInwd{2}6{tnH<*T%#<~5YXqUVk1X0kyKV;V?B|?2qvfZWWJ%1d`v`{qzb8V z0%GqJ)!KpL8n(^YXvhTEPbM&N*Par2=zIcS*g*o-ew6NnE^4gHYxS2%ry#CtVr*@z zwt5j^SX@|L!FP+QdTwr(_G}*BfVwZnBq>D@EX6A;D}&V7K($g}Tv*OMQeQ4@(&KM| z2s5;`v-L$^DpBPqp^j)l1@*YY?SXH7bfVx?iP_RDr0jm5SQh>h;Fr&o!O%Lp_!MyQ(3)9E>d8DS=Y4e zX)UA3i+h_{j7JFweESq*VAY`P6_?Kr-?5{BV5qBo;43bLHH`A=dgd&kl&zpM)0G~- zkYP(@b$G@?HAcPDoRnK_YmTf}Ws}xe`c;l-nL+x$=@8O8&cTz-?T`>Xcq?7!eD(4w3I*^4gr*Mix$f6~Eu zL$d6&d$SyJiHzaTS(jn`-^OdoV(+^g%*5}4xiC2Aak%H8E}-9`mywb6OE#R#DUKP0 zdVGquO}fc|BHvLQwJS8k9BrC71m+*>?CBUI*L5bKEk5sD9UG+hR$T?L*a!IL8`Y<} z&x+sOGNWy`IELU&chBa@Wn5*JQwk!Xhw9c?0vrmnKecLQ>fuH_$bg-=YRIa%TxyLo zrXGl{;J`Zv|A^Xvbl*h*J0&R$R$Rl=v^#;vag}wz+Rgq4TQ~~#9XPJ=@F5%1fwVd6 zwJpeIYBSy8SmYE>Y_|F5&zWOuclzUs*!*9kb2>WvSW?oMoqvilS#gEiSRGUE;I)7W z)|E64QMUT8l=6U7@`hl*Ovr9SK?>h|yCXrQs?Za{(SF-2A^8r&;ma$yVXAv`?iY{Ruo_RpDc?$_mYe{$)!^{E%qV{M2lfi_`V{uh1LEo>ktW3KNwUB-O7WqdeNMZ^^ls8k6M-)JZs71vu_ddp;A!#g zw=wtYZZm1OVjZP72UQC)kLNf_2zE52^+~SYDd|&iCX;n0jA1Nw6}NY_8G`LN)DBhy zlWWng+oB7p6uXX_xHm4%EQ_n-YYtYEm)n7Ire#_8@fetEqAR^npHzl3SwWn01Ob3= z!A_Q3z;1)Bo}q*_D{yf z0m3N7l%x{&a?jd;^375PLG6R;IOpFh&DIHCqCl1a+`{_Se9*!4zMNmwTXL?t-{>jE z$Xie}xGj0iG^@ABlUF;!?(uq#xzp6Mx6Ul| z3hNeNoe5K6q?JwT%srU~F1bBLqFO8mC)Wd7Dz-`Q%l1u3F$h{!@}CpLAq!dM@jwH~ zzHhAgn;pmsF?>(7CxarmhWJxMrq1YZGA3Wz1@87!l!Y$CN7tfF!$-OzeglAe#;Fqa zb|lGe83*!xm~EW<$fAy1pN?N+1jh^7N;Fv(sOA#NdztDyHWHT705>9F7bCiiL`lba zuDrfhCqn3b@|o;We}3e5IwV1`^#tA^5N0csa*5^|Uaps2XI>j8J}+D#EV;>^A;+$G z{+Fs8c|#Tpo@yv3lRlyn4l|&^Jq!=;RL~3`^STI9=)eF$xiBRN8|}78od%veM~uY) z0C)8CXU0XqVAmNhW(c_;_7qO7P9Tn+s_`f9{trxKU`5_w6P2pjL)u0+J>yQ3gVFf0 zp=6XES5&pbv1@k6pqhcrgVuVtUW~TY!ys3EARHo4$Ke6b!DtC%RRM6oORchPV{wJY zZ}*hbvZAiz_e>FnKS<7#U`cJvJ>LqprgBT)h+^0Ho6q_}){b232RhdecEVytoPMp0 zb}X+S_}3#I8U0T`m*iv^+k>vWbCBpy_!MNYRb=0pTRjiRFc832V;`7x*oAZ;SCur1 z_GrOqO9Zi1Ne1W4*j)f`>&H2fMn&F+oRYW*b=kx34~c^V9_qgv*6_HFZ~iiEJits& zJgk4!dkVNb_Yt7=p~7YNNtUeMg9d6_pr;P4dJhBf@Gx$7RFGT^gE5s7moU@iGu znT^V@qS_zWer=95u@i1Gc?UB|gCk{NS3gMhr#ad8(I`@qG)aZ|UUS{}148nldRpo!`)^i0VQ@Qq^g+rJ?5f==gq7w{|_pWO}2l;^b=O{q0k^lGSE1USIAOou2v4CCA|EEaC9V5YiIo|(O)%OZ;|4x|Tf4Ktx n;|ctiLEZX40|KDl3KEuzJmfzPJO~KSzcU9N1Z4a0|3?28SkL|f delta 14892 zcmZ9z1yJQo8#Rc#yE_c-?(Q(S!{F}j7k6iHcbDPfHu&J~?p)lRft~-Y-P-*&ovJ=b zPCcEZ(n&v^a}uv1KMo-qHSCbPyRfYTA;G}#V8Fm=QcdiL0D3mg>h?Cy%x3l`Zf@Zk z3SJA+Sf4aal*3xyaB2f3RRkn*SV?+h;Z&T^;?_1w-kD)ErLoZ*yb=~;X(Oel*}4?iD#$8Yf!k8VzF5ri5)v$q$PmQzX#Mo_b>H9f*}wI2bh=zdc02i z;^4S!nnA%cfQQqR@Co07R@RcgmP`h7cPDz8z?<;!8ogf2z0PnSL>@*)EN9FgD7y@s z^W_ap{$|BPvj8b+wJA2d1I!7ej#qC9)(e&~Sw?Q#a|)ln6^VJ?vi5;Ni+ououb+G^ zbm|dvYPlMrwgWuk=$t>1Ao1yvB?XbREP9B>-xvpj0Y61>sF)?`*NhIiIs+}cAHqbA z#70YORkWhxs)3kJHE`d?Kk|%P`D&hpDy-YSd=k`&l|TIr>W@?Z zL7A=7dW%+}=x=8RUBgWhY%o=)t?9h8a`vU_2*AxQzi`Q2Y&Xrknv0Mr<8iwXf)>)3 z<**xfFVfQ9Sj^S9l~kQrqzQej1}+|6<=p28(#4VzP*g|RLouQ|xL>)e?aY5C>-_7U9h9=6~`#trpq4ttaDv%2@Bl~{dtJGpZ!6iID=J3 z37~>*=BRr#3KFW2AQdid5m84OEL(CEP>E7qhjqrN;Lp%DwroXr!VM6>`@|fHNuBr` z{t>g6<~8>PalEtbbZBC(`aFly>9EhKigz9(ES}BLoM_Q|0o6Y{>SY{Aqqc4{Zr5*X zI`0OfN6X1}#y5Q7{PX6LhG+)g-ed;_2H^Dz0Bd=reHdru2l_+HFbl$Q#)))JFfVY0 z2mR(+8#b?wl@n0{x}?#FCITWSS^Ug%A)%Hfx4n<~VD+7|HDFIv$_ejs2eU?=a*N{T zbIheH;rgJ*?Y3!+jzB+&$C0PmaqFD$%TezQvT3GYTt)iTq zKjmqowDPDslv)ivU4X%#$N@K1ECF-hDp-2mrNhn?-^)4v+I>70b9f3qV+6V*@Ditv zb?`iIy7gXnom^~L%>eu%cA5N(D5IbCW+T{4M#9HV&8H(>#QsQilZqi^42@e5YqO&F zQ{n_Ho;R!ioIe(8K6g+`BsTc^Pq`94ZV7ENxc#v* zh8_@c;!6i4@7cb=K{P<|HTI$9Ix`Hlv{(c9KJ?5ivi$Cko0J%$i}krLp%;KdU&p4i z4Z0o?`Er31_N$*JS@>}w5(i-p%jdZe%tXWI4*>I$5;@K6-V~>|_&3QZ_v-F}*>vV@ z?v=^f!M_*r9pa9@de-xk@={dBQ9U5bsC2`~lsBm>jlTqW7o4HJsRrh87~-$faUFnl zja&?aygao`O(WNP8hDL`4V}xQh?C@#qwMHi2k(g~9LtKU^w(;q4wPS@!c-<6`?Hjc z0dpgIuOY91h3z8zosxE7X~rhZ@F7z_duOVZ4j2Jw!~^n@*Rc>X4@S9gqE8nIv&ICO z6hBj9OjKkV?_smM&Sbj}nbBGYD<6<}s)JfM!ZTHpPA2#RRJ&)X?e{) zsaJ?h!r5?}%q*t+iG5!WDiRlaNNO@wUF%HX<#?EP$b`BL4+#U|b$((L+gKw-^%k+o zemdq-`Ne!PEp&>Tu>;}L@i#@uIGVw!OYF&BWThXI93thPv}67vGrbVAeTc~dFi1e( z4(1{k?mCs^4QQ+&_(a{#rT{eCZE$nAc-IacUt9?my^(i_4~kBH&Y1LT@2F^H!=e-q zkj+wipZG3pNGbPh1LSa8G3Fi!1Z%%RO#cm>xaTldF4rrw)c~ZsNNkAZi%!mJ z&dOE#v(cX2Uu+cMjFxKjdHWL02{j_*or_hD6i*MyP^80napiFY|9~zp%j4gPXb(R^SuO z15FztfoYjWtwwZasY41y?<|FinhI;cFDDhf;L9mx-&rtGtk{ioh|zetBQM%YyCxZ3X>aQex*ifMvglV(FS&z3q(GUXhLL$HS;V=k%cV` z(NT{50gFjSd8OANbvr}{XhW^)u4KXjKcnVr##Sp{*rPks)5Zr-yOdJB)9Ccp_GfZUcyN0U9hImp{JVS8Yx8f6Q|Ck7G~m?W5yAoAnzr8^t` zK~AvPGzZzue5g$|Da;?}^wSfkZz<&+xLJ6|9&lf=4s9UgqgZWtLm#<`a`8efYc$jR zk)y(I`f4D>OSsCPZDpHHmWxo4S0$}*%ufBWWS$m>!_5GQS>zU4+SFi*q|#5)$UU6c z#Y35zp4!y0lO|O>Ap1rDUm$Be8%_poL5B6W5kcpwZM7FG~axmn>+LqRc_JB{A zHgs|13VDKZ+eT3WG44un=ElhbCE9E9>P@^g8!YC(!<1M?q~$D6zrp^uD@QhJylr8C zfd$clfsy~~$|V1ua3ny-SMQ{&6AceJJ{fBiE4{)K9ECB2Dh39edA}kAj7B#V&sd*1 z&Ge>;OC6%4X3f%aUH#Jha+$RSg!C|TaZBC)ypsO=Q}4=??#}0%k;9wF$@W?b+x+v} zd&|dU$BF-mz{y5N>dX3dfnRb|`rXW3RaoFjQ6lJ>WO9U!H5w3%J$;{)LrmfulLvia z>IE(|7K5h|evc??mKYggKxU~2F4P~6fD0c5>2=4+h80^RY0?lW@6)L>i8iPxR;Y2L zyT53k7Jx8wJ1ZzWHt61CZKnIARXVZu+l16GF@y+@Ee1l;`AHjiTRDPF5qBlKZNcD-0iG71$bXvso z%9wU8XfRVVRI~)qq_+nXKJ%nPDWD-N8sP`6=!Rymtc77w2G;i8p753S8k!dptzhL%(zsZfS9Q0-QPTKe$e+eS5>+3` zqgc&^Y9jSD4Ziw2M;GVB0YB{RKcy`ZgVN1(rGHGN<7__l%tR9-CtH$*_EaRVcd+7- zq~mpJneYG{$Ykt3;OkvZN}ELN1D1{7c__h@&rerZ=Q_&F-j9##MeVF$XV*Q?x*pe) zNJwgtGv|!G8}q9g=`a$qd{;MXBljc5Ggz5)Ha45eE9(6GWZa(9r|aW4y7V`41pGSN z+S*!MT41ts_yv|>GTWELn%gt03V&6Um37$p6?y>dI7BUmG@7ew+zhqd$QpZWgkGHC z7&tm4lKaK_Z{!@3LB^NH8rP`!Eq=vsqfzK}4yifDa{ZkWq}*u8nGW2=zl^CSH3Zq^ zZq5vz{d4o3-CXQRj|W%5i}A76^DOD89bqI|F5lpi?jZa78y!bVjCUt5wlq_@c=6|h z1Y!UK5gp$!ww8#AxG7vPiyIIkLM$nMz^VzRz>8siW%N?$*w^`Py5Zxnl5Dvrh}<+vFZv>ZLEKZM61 znA=^jf_H6OdpUq?II^raf|U3x8OOcE)sX;9GJh!Pbl0bNDr}8{^G`*6ud7v?hpfj` z@`2@WaP{kraJM_|a2CxM_HY&}TM@S4@2geyne(CmMXFr5VR$X{)_{kZ(LQ)vxkjI( z0`>3ga3t>&+CLB7m_t0sc%w9Ueua$2ozr5<+Wwv*l25*z8+B|EGOT+V?w55?U^NHG zZZY@*exrfWu@Yii6z@c3^*081sXpmKx!rFIn@QU5JG-P<+O2XHn+SzL-e#g3a#*jX zA-MEV3bT?`i*C0{qoMqX>_X}{55{MERLMan;f!Q=WPeK~+YVaHVx&<@ZYK+7gf|Ro zSj)0+E8>knKQTriVvovC*+!9k^TY>~=k2LaLe7wL1lq{=O}F!5@D%w-kdAm7vF6I# ztU4fDInuKQ^ns!yXh02hMtclcy=r^k>HO0Mv>E)B5cozpokC2;ztMjkGKw1iSY3R! zyd}b2`8nVl@5{K#Glx0uMiAJP5{Bsgre?>R*r;dcO%~E>8A-yC&SHo1Jhl&LsbrLK zm{=;pLM15opj~&<9n)R)#TJ#Dfdgt80PvpGq2)GZ@yB2ELOD03@a$JT0x7brT~( zAnYt*w8|r>_G6GF+aBl@EiH1B4E1w1gU0GD=*7lPV#jmKa^qySDD%0+jdu68!kHV)wu* zR6Hl-u7WhPx~aEPw_+yIu4Yd({{qvix|hTG$+=T|%j91(Qn0s?S$+bbJt5ecZnOE& zeN#CQ7`jmYBqErj8=3`ay~Rnl&9xA0DYIJq#TrEvE|P;C{P2kvR`9ZR=h-Tp1G>Wr zbD3vTa#2z|Be>c6g}NH*BH?vEk_k#t{|%_34w#d{W!h-2VT_g%G;8UOzG=+KZ3sz!eQ~ygG=)) zT%Q=Evo8}L*zv#VBmTU?#}^z{aDEbyYP{IQ7wk3IeK781b7sj#=2aD%-BE`>T+f+( z7RoNpy+qkOtiYW`Vkuh-jz@9{56rM7510{%%s9v4hIyU<#H*zNhstr;Bi^i3W}Q@W z_@ZB;oa`4XFH*wv5gBOVpWwv&rw#Wx%Xy#dzwVI_=k|0ub}w^AC9>G+Z`;C70`!qs z5V46cf!aei^f0+EDBUhGMDe8=maT|fh+!Pu6>YK+AC^NR#WH3QKW0mR%r(qODR|Al zaD6f_d@|W}^6LozmS6o$#hV_twsJn$58i?5y&@qr+YOOL51Dh3F#QG7XCbmp)o(7N zzmTq}q^VvZ=3= z@!L11xFzPe*9n}Fvm?L}zIy!5K>>xpk*sf>oq7*wO#Ntx8nmq9f&fGSFa6%2Zvt_S zOU>abG@r6(XZ4$EIm{8IdSVOCf~MIS#@ABWdcqZucU5F^*vD=vqFBl@UYox*F&T2?sE_)xkp3FI&R!yngE?oVegg-Dzp zd*Mm7WYf`qE)6MMpIz0c4i4P#`4a`o)=pOv=EqOD|BMGT$z*^`i9^K^V_h3lQ(xB9 zy(9tZ4$L|f@Z~}_11xufY=g~Rh(k)!=b7Q(u9L0`Wx$(rTX}7wA2=q2x@$!6!fVTZQBG?g>`Xy$nKNu-=yKs( zHygJ-npfA8B>GB}f$Rdk$MO4WW-x>}`cP#J3s!XWbL%S7!Pyz6Z^v4l#$TupA~66b zI)J&BZ`gBqu|7quLQV*y^oA{)NyNpu>+H5C}aRx7EQVnp{ z>8+Pm9_4cT;D7k?RCK)*=tgW{s!x`A*yeVsEkGlAq{E*9jLPf2YTb;vCewwCF_;!?~_F zj#y&cdU^jL2UCO(gkM5O(z0tH03ea6YX1I$GBs{O_YkImG*gjabqd1W{)C2+G!}EzMTwUoOezvH| zmI(3@ll&>VK#pt){tAp0ngH*msdJfCLo$T6Yi9y#Yrf|SYme=lZr~&!>2vm9*p)FN zJbnQ4*8z+k;+9`fXAcJKmYBK7m+k7rdv40#>VJ`~sF{v=kau#N2 zMp{qNK||@X8HyW2t*))ItW+;M#nwi?x{R(Wy}VSI|r79A-N{?=nPMZu*9baTTuQUH5DMjq?K&GXOOJ`PG3SY)+^Px zY5C=H`qRe^QP%ssvTmNlRfncZewGfN-$Nl>W!vVo638r!nlK;xy8QFRQvaQm_*dOC zQT*QFeF~mB-aT&05RqRI{B7ipTYKoaL0Y7ZSP0H?#~*9eYdoea=)ERY`sd9enjIUlGcW5Zlz$g@9=&rYg6zpL6%NdGuNe8Gd)#SceU? z4;}utA=4nk{DNmPL+8wNYS5%#rE^^Rv#)mC{CG(jG{^n(IRk<`;!#`UzgKJ?S1#b> zZ>h-y@N3%7CLs);0YS{sliIipTBdSaX-RmAjRPPeR)Z3^6Ipke(1@i0Ay$F$G# zT!I#60qDdPsMhf>cmCGzkit@dOkVA{fy(aW4}s|ZO0Zg_QzhW$Ddg4S@w)N?$!VVC zz5t1vXOpvtver4c%fi^ba8=`BYo083>S0y8rvczIISNbJw^MfS^P>lcH!RR~ML{8Z zPvZDPTi+Wr{XDEYSAgtFQ0iX;u@x64!UoEq!O!jI;#?i93&=)X-9F6dv@? z19vPwE$Ab}Q^KfBe`kzxC(~nakuH#aAwUPLJ_2Mhi9r6x3k|WM?~ib)o-a0o)Qjdk zB^yu(gJXj7z8(Dapz9C})xN;PMJOP#7Zn-%R?RnWI|vZN%BKu{K&Dx#5-sk4K&%Z? z3g1=(IfQQ~XSqeKM$3}Q&?<%xW1Kh7yRbGK4oQ%cM8@gnm^=Lvx0A+t>*vML0Jtzi zy_2f2#z~AOmL#JmR=)%^6Qx(nxi zQ-6jmd?Z_ZN8|Mgvn+~wQ?=JFnJxEAi_jpjlP&uN^F~KRg<7FKKV$BT>o1}Ey97eV zQ(C@YBKSf0@84Th9}prj`wO}YVd>=hl$7;cy!aK`azMsW?(_|(O8a3?mf}nH z3yLH>f`QJ7=#Y3m9$oY|78@E#0f00~47qn@b@_an z(;cKui-(z}*W5^|N3n4)6%UbOn40r}W2dAx#sa!ue%S(4HC?H-tz$>|_F_-vP{|Vk zV-|Vp^(=CAhOPlNwwF&vTD9^r{UdRr4Sfappztne-z{P7LhaiQ$R1mZ!nRezaIq>B zqVfsU@@z1MY@I07apAC0#48=~}&cWqTPT5bE`GNbS%`Z*cQUYku zPN}rkg5{gn8e>Zd_B-mNLAw>--*1*zrfHwCpBvovOuZBoWs)`#n;7k^B~vbQPSksX zZ=`&mEc969(0qFXFOdogw=nGp%p#~eHNi#wb|fArU*P}d$AIJ+XPC$*HoRg>_+Vh? zTwq{i|E9)pfXp>J$bc15+m3llUbGa1c1o(1bm$a=l*h)j%}q#L-HeA`PO_0rie>XN z^7E!Uog3FnNi1#~?lhHe=%$PShU+TZz}-E&Vh0-qjyY7oV*vWtqEgjHtYf z&R)rcO7l?{D7|sau1cCoFTwqL3Jea1+#Fxw_$E+OYk;GMvVfWRq)$AbaR!o-?z{0n zqxwdVct@lv0{$eI8m=XV326#86nQWtTCgdbEo}y(s&q2Il5W|GuawhgF z%Ji*EX70)PA`B>&**su(cYthaT}(esCqL)|rc855MSqY;J3jJ7+L+c&{F=NpDi3{? z^BYs&-&W{!BjqEW5TwrUQL&Laf>UB{ASj|cYU;zI`2h%@;SyJ$V3_4Yu6b59tE-Uo z+K~wtUICgLlThWUp1U%;{U}LH2Ne{mqby8L4|3MHg?&f?BW+Mx18 z_IuqP#vyk-i0aCKHvCi=m(3E)#bAX?QbuPZ)-118iSkti^dJh5Nzim59G5EAIdlJb zY*m`6JAirkmu-@-HLT@zDcWVRkUL#KCbN3>B{Y`^*ejBd0!b}zXnsk<0kWQ)&AV2a zl$KL^>yeWCg^H6Y;y2!|nID|rIx|` zq#Ak}>5JzddM76ISG7dtu6_tc3{B-45akfcc(1IQ!D=2AI&GF=IE$SDS0;KoH4|pZ z-*F6=}ZX zP6B-3OXG{vDxgF3`Zn)AYj&fx7j#vweLGQVyv+W_>i`KE9K*7njhB>IZ>QXO0^kx{ zV%a?fkOVTg87TRG`LYG*cgTSK+O>E?LGr}Uz2ftgk_!2z2If8B$>W1bYpvrJ)r&}v zVzGKu8gFW5h<_Je%EaWR6;1t{2SI?3BN9-i9rqgW7ECN{1jV-YWN>8N@(#*vRUEEs z_CIp}wMNgG_VoU12?;GXnV^>6RTO>~hSH;z-wGl_l2mHP5Yz+N{uggx-)LRZYaZv# zo1WHp4|iq`6?=U~iSB6gr*>|QznFUUC}o{)Mdz2X90t$>&o?d5{LhtBNE}qB#}NPy z*{W5Gq}aE-wOS&Kz@LR_PysU3$c4L+z+p8vKV2(nz1d<11cY4_K7|9IuKS@wU59e) ze78&T$xe1i8JLtFeffouxJynw$xjV&M+tHD9aORVVg=$-6B20~Cj7oGus_gn`Viap z)BJboiUVY?sZ|;CZF5X>h30C0D-GbtCWUZ%J%w&Z?^op!FP)h$Ls6V%B%@JekO8?} z^=y8RlqXP;S0=nVz&j8p^Nq+m0FC4pjrEh&L1F}n%&Oc?Ut4~g`7O<%n^~ZAN^JeL z1;K`*A`&gX6}%ch`46Snl;>HyKD1zQPK+Lkn%#tn?YShg(axEUrjF>3r$qq2mGyH{ zgPLNi$x>XG%$Mq(8^0ye0^hqd0P(Q(nzCe>nnid8J!)~zlA##qbVPH%+IK&&nyz%N z8e?Uj0cBpA0nEX5Tj5pMsz1bJy?glNXFZ>Oy~}OyT!wkc{9j{72)sJYBGWQoJ=^uT zfv`e29xPVysxGuKKZIOgm`#8;GnNVrHly^D0SeyYz7I`4a^JIF6aa<&nEP-t@GvSC zeJL`DR5+;j9Lz%X(x=a#eDPUe$OpDkxnyU7v@kyqDoq3;%5fcT9WYSY_et}{@slyo zoA__|C&I9DAp^+i!Rw|MXYHI+=e#eU;k4iZP)ISNBl|`R*QIgzk^xZulD_Z`1u12B z!W2RCm4WT>Plb#fQ}}d8H>YN?Y?rp#?+`*G4oEiK3AuDK?Ym>fPJ0L|=jA1gCxkXX zk~wT7Cf}>{Y=;&-6AK;kN}kxIN5194o`zVl*}SW!nv*q(9A#8gGd^O3eR2;4;KM&- zlihXQ6p)f3e4#}Jqybt78Km+Q7*W(^FI$Avw?830Yzv$6wj&bx8$EG)O8ogQ>)4;% z2!}C8Z@FLh>eSOLV}89D()PQqWc*4Fi;bwZ8uJ00UJ18Va$fAw?j7EU@pY%xmXfJZ z-*=FysHrYlxO9ujZDFRfppwe>{U@Yxg;E&!RQ5$a{88cmvIdZR(S+Y+!|uz3g=Fb> zgPzP`z93MWr+BL3&%*l1S1Xf-tPb`Q6Dd$OLv~WGeQJ_OBk&yc=uyHnepLicpa!=B zO+yecFEQk)sF1r}OND+f z_dl$LF@jH>w69IA0i0VDelSLec6+kgNDFE6x1X)mR-*-3T*689khQfgVDmog{^DJve6UL2 zpfOM8K1XHARbU6)dj|++GHrZ7u5GY<#snaz{vA-^eADde6mfEOf^mdG{Q$??z0&H7 z>0^A&bc#XnHNcMy62wo-NYEoi%Ze6`_Me`VldMrKuU$C3a|tXoK^ST=JzQIr?5=MI zRfoDio}6ZzbhefigF*-0^N3{YfZ5vRH-cC<7V>X$%NRLMkb3#mn>wkaYYqe7#kJra zJOJ3^88~|`0d_|moIAg4rK#_>E?mRA#_?mp1b=c*UHG`vV>30d**CDcJ5KY3Qn!$D^yrsscj?Ipds93(`n$^ooqcrMHbC}4R^e~s* z@oN(QQoH7L?Us<@fA<;5AuAsHN;m%VvjVWl7im3Xvc45R`D_`)+v=h;Q0E&N)huiR44j%A9>2%J}tu^aE0C(5GJfwlc7CUD&YSH z7og~Gb}dX085-HWxBJWK0p-HG0t>_EZht}|{2Xf9Z@B#>w%Uqh+E;te2iveDe;V*$ zlk&YnP&kyvS?JZ93vDB6P!=<<->x!xrnsd$q16@f(UnlpR0zewfivoad0RBYRY0&b zw0_{;SJ3G&z6w&B&f|ti82U{&A&Lig+=%V4}>fRsih>I9rCuC~c8#CLutITP?(|K!XI#F^&^Q!n$&r<`H5kgFIH)fL4j^lqC% zDGfR6vE!rJregSe;df&_J&+{%iWc~mBgo*mJ9b1{i%%Xc;%c4e?OV_<;$SPMPBhIj z9w%}hr!w(v>4jJSp}&aM%uX}1=Vf%!3gGj<8KM<@*f=R|0@AB7Zh>5z3Eth0X6V7hwjBSz*NeBs(mee4F;T#Wh^5{VBx(@>%50I0zG0< z?Ge8|>d9J53NBU6VQmrdsN539WKQv!lImkfwTJHRQQDJ5Fm7S$M2JT5NPZ2NxI&zs zz*Bpf@WJN0ZqZ2I`i#SM#VuhLecRH(5W}(aE|@lioo}*a-51G;R_>4cPf{Sx@DmyW zZg7S!&OddG3S6p6C4MT)G7-Q~eL)l}Vn*C%9RuX`iiM7~UMMN10vW#u*N5+v z`Evxr9+O7SVr1tqe0tSo1Q8Gv94+D- zgdlPskSuN>0xSo7wRqx$)7)kiXBT=(fb(KL36qRPG&o3SfpKH8nhBuK;SNz!=5_?6 zIIm_RO^eNeqR4wR99DxL+RTqAUO7Toe&FADR{k{uM3_!~&B{3gVMVY2|`3xZnLaGl<1%Q3Z?Hrn7U$R!j3_EeY zh@o7%phu}7pj;P>T#ij8&uffc$p&odBoLdA~JY!NX3VK1=>$E-Ts;5ku zZp6iCT`jln?22p}!Do05z|{8K^1^NNo*Hv^VwqX*5nUeKBDV4sC}(wiWC~Y#+_RM? zuetB9Ydz^p!4MA0rFFg$l0uh3&c%Y{B-A|3`ODJ469JpA?1LVh;oj9PtiR)y?!(}i>(!_)`nF|-6$ z=H)stA;(hDEeJTa80sT}5pO^^;1t$$DKPG3_zOib470JDYWm3yH_g9W8>;5cHXpHf zoiM=^m%95W6O1$;UHl7c-cX(b}i%B@^N z(48q?hEh9s_zHZTiK#`byC0sf%dIlYi%88e<3v>Zp&9_{e>M(=+&2@$X(x+KIu3r( zL4)T~2oMF;g8K29qxwP^-NdMb|JAjHmMy5V1CYA=A#sgl=LSjd{z>RK=8#-D0ir1+ zqmaz9LC|BaV(G7B;5g>ETphw>bf}WYAyB$WLd>HQ!m>%wKJnQ+0iq*%l~ED{~uvln@+CJ20R#8EjAb!?f*%+ zQ+L*I0Y1i9N7!FVO*v~wsm9z?XmFjTKP|k-V^q=5j^He~w1M!P#yQH|spjTD;PkYs zb=|O*9qOqZ(^G5RB96X2c~QAMYD`_v^?UF2dwI)s0LR6&BaFh=>TAMt?@rgw^JVIn z&w~pX!>toOOY-eJno)Tn0!xNVLkJlPZPE<_VB4oGPCNX@7QaE&8P}+$5C;}}vL773 zL7f#B);9WH__I4-B=TkV?}rbh`VQVej<-L@b$7Ux6Y`#epm1M7TjUK2$(@zKdwc8eqGw!Ul?mCN02fgw_ z1sxrjMi+_dg-{jciw)MsB?$u+X+?)E0BiSMbxovt=oZHDwd@me1&r^z00X+vPxEO$rzdR_YR9ymou&{zu)K*!1TTRG9EJbU-s*MS=o_hC%b+vx%ubY~WHvf~kvu^k( z5pmgY2w27`=qy|49b6uyb7#+OJnQHsOt(0BjVOgw7~8a(Se~jJWZER><~%m{0M;5o zc6#qr?vfMz1t`DV8uFQE*&q<@*=6K_9fs0c*K~>rpyeR$fzF7o$>#L6a$T5)Ev43t zG=)!cA%nhN1c`IC*7WVAx}!}uuJgEBlZK4OW^o0;3eyISSh1N>zW?cF&azuQEW}fo zSb~#)2xg93dj0}q05G{CmynJXFj{CK+fLRwiJr7{`PBbO1xw|GQ|nHrK^>!}LB?{R zZeCnwR{}9l)XeTqW@cLwklzf4uRHEyn8Ua(CjAZA5prqYkalZ>UyyvO>-yF1=(j|< zWnIB|gRwvN^-aOt&^t(R4S$QT>*^yZ#UL^(j>VzGX1%l^{d{?qd8)|+pfE&NsC!`U zP?CtGHsDM~-7K6Z3V$!{e>0~>w|Hr z{igU10dQ2imGX}!2pl{96kq11c{C-Kmu=^llHW~cQ=@5mnE#j`t(2RnwUK$~(a>Y4 zESJ~mq1+tN@W=mQV)LVH+C9IlY(ER6Jr_@c-2+l*>+iJ1Q@!N^_~(Vi`JQ=~q_1fD zL+)s}FgR-8GNo&b%vG#m()Ugg?Ui`q@qrCczxDc%7!lF@K(wN=2eDBW(^L2% z`B5|}?3|R!2v=0Zvq_M~;KGvgIkqp?Oo{*XN<6g;PH?wten{#-W9 z_rNmg^|2;7o{))iC!W*!4!BmsBbye}a}YO# zcX;ps;ANN!1ZbY1~hv1vdNMKW4PuVRTmoAo2vMh?jDvQ6SwCzL6R=1Fh;lLRni zs4|%^F2D`JQwD3*-i*q(TV9}bt1%$EKMRPL5fQ`9PFJmRp22%Fga2?QLjE=65@vRL zU>%pr9eHCc=mK$X`X`D#zMPIT*2Y^HRb7V_5T8!R=>CMm=T~Ry^b6=!1oT4pp=A$` z&6}d0KBf-&HMQ2YxYnh3!Q}B&JiXmylVr6Y`KwW;-Lm5#o43pIl~XI%Kg>R6mz;<^ zmAJxQ3^JgB3~>X5`Y1m+n0EMvvfr7#-;0o8#&xvJg%!t@Iiz>-ho5MuCCo*rsP@kw zpgrL;)Cp@k4t;#kdIWe&w0EYCH{u4)W(KQZI+CSMZLk$rT>)2`9YS9sU;g`vlg2uO zl>Ol-Nk2?i%8Zb&r6*P};1x6X`%i^Gv%KL9)>hOI`u|k24S4iaxBXVs0{XMJYHH39iKO+wUILxLBh*iwb~6HP zr-J@!ayCPucsqKI`V0+_1SPgC-2tpu z20?po6xi5Ery?X5|1|Q@5Tf@m%DwmCehnz%HKbl&khnib{k#VcnGMy6MLCJzSB{mSru-M7YIf>C&TK{asy8rb%F zI0J2{ddgkg_P%$+U07>uEGhXiF>IfuY*B?>PFp<)8O#cFMIu9gxRzhM_L}3WRT{(! zvT|tI;t12!ldM-%E8S>_&bSt*Tav&3U>3F(GdoBbt{YJLcz(+}1Y;VCwPqn}(iVHf z53|_BuBEQ;iZwYadD~U5D^_qs=rnYt?Nd6s5K`OA@DnPsV>+8ZJEPbe4*AOef=KN@ zBm%x3kRkp5OocQz^sxW8sW27%1Sj>?1r6z+7vaC9G#Jh)buJJ)mB^JS74`%zRpOQa z95ogEmOeG=mKDOx^WQ;|)F2<&)SX*2qW>&VP+(xI|I7@513LtG>3`6<67&CD5z+tri~66YM#}#Y z6(QF8{)=7u$PE!b_#a#uLrxjR`|p0xJP|MOB diff --git a/examples/pregel-bootstrap/gradle/wrapper/gradle-wrapper.properties b/examples/pregel-bootstrap/gradle/wrapper/gradle-wrapper.properties index a21c6ebe28..c747538fb3 100644 --- a/examples/pregel-bootstrap/gradle/wrapper/gradle-wrapper.properties +++ b/examples/pregel-bootstrap/gradle/wrapper/gradle-wrapper.properties @@ -1,6 +1,7 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-8.1-all.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.2.1-all.zip networkTimeout=10000 +validateDistributionUrl=true zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/examples/pregel-bootstrap/gradlew b/examples/pregel-bootstrap/gradlew index aeb74cbb43..fcb6fca147 100755 --- a/examples/pregel-bootstrap/gradlew +++ b/examples/pregel-bootstrap/gradlew @@ -130,10 +130,13 @@ location of your Java installation." fi else JAVACMD=java - which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + if ! command -v java >/dev/null 2>&1 + then + die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. Please set the JAVA_HOME variable in your environment to match the location of your Java installation." + fi fi # Increase the maximum file descriptors if we can. diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar index c1962a79e29d3e0ab67b14947c167a862655af9b..033e24c4cdf41af1ab109bc7f253b2b887023340 100644 GIT binary patch delta 16170 zcmZv@1C%B~(=OPyZQHhOo71+Qo{!^7;G&o^S)+pkaqdJWHm~1r7od1qA}a4m7bN0H~O_TWh$Qcv`r+nb?b4TbS8d zxH6g9o4C29YUpd@YhrwdLs-IyGpjd3(n_D1EQ+2>M}EC_Qd^DMB&z+Y-R@$d*<|Y<~_L?8O}c#13DZ`CI-je^V*!p27iTh zVF^v_sc+#ATfG`o!(m-#)8OIgpcJaaK&dTtcz~bzH_spvFh(X~Nd=l%)i95)K-yk?O~JY-q9yJKyNwGpuUo601UzzZnZP2>f~C7ET%*JQ`7U^c%Ay= z*VXGhB(=zePs-uvej`1AV`+URCzI7opL{ct^|Lg3`JRQ#N2liRT0J3kn2{O5?+)Xh zg+2W4_vVGeL^tu5mNC*w+M@qOsA?i7Q5Y!W}0%`WElV9J|}=8*@{O1`1(!wCebWJz&EbIE09Ar_<&ldhsD}pR(~NfS=IJb>x%X z{2ulD!5`cb!w+v^IGu~jd3D$fUs>e3cW|v_Cm{8={NL)ZoxNQqikAB&nbiz7mbKz( zWjH73t*#;8Rv5%^+JhrK!zDSutNaUZF#xIcX-J?XTXJMUzc0+Q{3)Xt)KYbRR4)MYT4?1fDz4 z0NVFLz!!^q(*mC;cfO~%{B}A^V3|1aPPqpOYCO4o^)?p?Hn17_0AbdX$f;k!9sL^g z{n_Q5yM!yp{oU))sbp&r6v}Au6R`9Z#h@0oM&1n0>wAP27GtH zG#~tyCu38r+Xh)31z*ShTdXWfb`4h!sraW8_kR1VGraUOtA9}O2g{N$S+1{3q>z*< zDEs&xo6@|O7lJlzn%!gmnJL@mh6XY?H2^>+tYwAp2aD&ve*;dNlFRUUD4uJsz0s{jA0wM|`g_Bk- z2nGTI4FLio^iSgCYQ<~?w6VhgXuFy?J6pI)*tog7+L(H{+c-IDy4s67IsWSv-2ZoX zkgKk*j4q1tU51^udPJsziAoFE%s5Wgi({t%V=JasWm6hHcE*-AVByK0i}t9!4^NT& zYJ1?sHp;I5vxtJi@z=?8N5Bc2Rp96QJ7Pawo_W$pO{f?a?6fX`?dHe8J+yAg-F$LU zXmTjqP`_JciO)bHLs}L><&(2CORPpITFZ5y{Ha$rW};;c-n)RcD`TyHnL?)Fx{0?I zqQ|D4T`xLJy`A}h{D57UR@bD8{Bw{9rlPt&U?{4 zTbO4-nHnPS!as<)ecV@VpH~W*$zoPr8f09_MZBPjoU zamA5hmU=F0q4v*u)BvEyDNo)GJxs9tiPkp2uhlGLR2bUD{NSjGGCixR9?$LKAlsip zUIa{WQs#68GH3NL{(FUyk-k=lrtx{V24k>kq~uc+St1uH0Yf3s547xvD5T*@n^+VN zKO~$H#RFW+Sd*M?`&+A$L<%DwNmIW&h>4j}vyxu3PmHrGwp?hXJp!{^>$Ax2WY&9} z5fJvDKBT&~%2QWqTGf{=6Pv2U+0HUQRv9%RZLR`G^XNdKRZt`Zs z)vuUr#7C#oQ00KL7$M$(yHa*C4XZ~*t9NPMJU`fACD3v+wvLzMJipnOfRmh_kN5oD zZ;)G|-j$^OF~-yWW*p1m#1)%%tWgg_?ps;<cvxwa&b=_7Iu)xM#KIHR~gWVSQGmujR;bCgI%H#(_~8O`LAHbJ%9L?R(Dt zq%5@6HsP4(%%tF4t#7v$y&h*i|KihD+E^Q7n~`1KzELK>5I8-`H|JF2Cq9CgniYyS z_4op2_>b9Il(p8PquZ{h8Gy$%WA+8t)o_gCdb75|9NJ&}Y*D~a6)VE@eT3!qvvSPz z4-A4Vw^rS17uWVctor@Gky4eiT6nF=PVY~8jzjKM-GlQzF5I-V&Z7d^G3?o9`C9gHU5GOAMLIZIOBw|s--tIy=R#b8@3;?-9Y8jeFt`AhO z8tTwGxksHRNk>;%uqWW&Q!^M?CwVDvX-*wTji*J^X%}1`6Z(#9OsQQfUI9x&CAj=W z-tDF7TYPVS7zfx~aje8Z@J>er!E<@63gEY)W{b!AF%?j%VG;B3b;Kt6VVH0qxBLrC z*82l$taUKcm}zRM=K+>H%w7(10hX25ud7r}c#sEK;mnBsVbD;$qu_|UEarcuS7aYi zcMjgkjmj=#d&K?NX=qgouhsLh{iYTe8qtsU~kLwg4&&Q1YGyz6D@(-w< zl~tx6ulu}VfKZ@_gt2aL@E`A`ULme@K+ zek2hch6FNgHdbowNo)mBs0da-}bhPw|R1u{4 zEZ?T!7j&^lNPs1je%@Em^CPp$cX%GrCBn66>D{`Ugf%+~@)w+gX2xGJ1qCy6|1f8m zkW@0=CvkEuR0$mn*wuIvn?-qRMNjtj*c5Z_P}N^he{2=<@XK4^ zC{Zs89DIB6QjEE2PRx9Le^?_kvTpBWr~%L249F}8N&xTV?+_;?oyfV?V^T(ioIxw@ zYNZUlBAc=A{A709=R`$--jqG{jPQj-7f_Sr1$o&kapsFL3jBVIE*Z4&L}1ve?@wh=%eda^BRYm=>pJ z{p#Gotpa1aH^l+Oclp_+$Whjp_q3(G8zS<1;!#*67K0Du1}RQPo&G8mVeftaJ&a++ zYlh?j&;3LJA5Q4fDBsWauFn>VvG_9Tcrr2Yt-#+%rO0ST1GFitK8f10=rq|6lf1q? zZgVH$pWLo_(3QZ@KH}q%V;KT>r!K|?t?LSBWRUoPcv3to`%wC6ZRPF|G1tKl`(7G_xblMQANQ+j&NIeH&TK6-$u*4Uh&0t&ePU zPJkhRuh#-@_X+0}aV*Jb0Bfa+LZNqQVWJ0#=KA~Bqt%4}(36~^U)lvrj$CQX%P=?D ziHvZYaHPO6-Q>+|s~lNFW0?Bv%tzi)3M>X`;!RfF3<~0HjHc|}*l~bKATK4IXdR!B zMf+A}Up#I+)T8aogDs8)j}J)JK!%rH9&J59H~Q@Ntd^EV{~c7kTX%dQB_?kfOR-tn zA=NR@abtm5k{N9NS^G$1>>Td<278}g(`E7_k5+?RgoT&-Nqa5AjkAAn7s8#Vc=*sd zmyzfjfeIp0Fehg1gbSQ(_~qXV=y0ShN7ck^V@6t(5C%IxDmYn-~2#bGniWG#vS zWlnC*Dbfin3QX!ZI-YRxCO7uBG+d>=s@*c0sPmByGDc2mN&24$GkoH0oitsFTV0_} z4iATfIz{jBODQY1t{lpUS%Q1Hzdel~82P1N#Cura_7k&{mUoI@q?W7&Jzo61$}3G7 zl`3shFi_Vnoh`5OIKHqV;wTULz2GkZgW0zNjk3t#5aH8tz(R^=;i?c~(3-;#WM50snq>qF)cu>}tWC*wTO7r93>;1Cbif%d{o% zC1Eyo7UwX41o7QLvdU_to(vzDD`*KK^3HBZvx@j@i1Nbt-w8Z5`>?)c;rXTjdt#k# zOfJED_)awGGGg*Z0Rgo!JN?rDkpZFr6pE4%K}BPXJ>0O@93hgvCGJz?oUweJQjnVi zNQKWhxNpSd36=ip(-D4iOtMG99MY(y86GtXS~1%=jipBb#D;tZpKmMRZ_t=10TL%p z21RJ%0X=&&WUDYBbTcwsof1(CDGDD)eW`d#Y*Z87@k z^{dy_GcUp~J?qJ=i#H#EeSsp^TSr@dt$%q>c3_o1F9sr_ta1PLWYBdi1BNUNu0`v` zvgB;K@#gLmv#tD2Mf21LHU0Hq2~Ro}Upex$#h~)93nAvxcS6wkM&UVy#4RnSG6QX9 zQ;r$p=AKnBnUe=hZPH*u-Q4Ta4COuQ7TQGIqbUi4&eot$D2GHljdSdbc-MK-t1R86opRwDuUN+ zw(1^ybD7grBO>ySm29}i&+s{~7uz?*?K;N9?Yw~zd6 z*Xfoqv-*O~(QBAVpOqwZ``Qmd5qbL#d`>U7rT&?h?FN=iYu*vFfck~?6h=b48;n}$ zQrzUxWJ{eaR2!*MSX=+F*)ECE#91?SmduzuZwQ! z!ydL4;ljZ(9R_<=q z!=`&+*DUw>CsM8xVDT-;zFYUu%hn$rxPXhKztEb98>7ow#=fdMWJ!i$jJ=MIBspC; zvoJ2R96iz*(%23uM#WtAe661ynV`4t?K~eV&7!-r+tg^aw3Jiql zX^)V(pEN2WfQOL4!JgVGIoQ~a8}Gy_4l92Wst~iEI zANmgs#tUnQcv2E7>g!{jjC+X-g)LH8&8VQNoBvicmuID9WQoa^S-h?S(POL5f({Fs zWfe|-nRh@hz|Ck@iKm0C75R&`CWwUy<05TSN_IH3aMaO_Kw>0#Pv&-Dfl7b}3qfofON-WA!AB)QpF2FTnvu;s>T;lA1&Fh0 zBl$6%ODbhP1gIh2T%!8 zZ%&Q`_{;znmFQruzy3PWP@echTsS*JR65#1s^Yda=tWMNX?a%+u|@dSu2I$CfK@Jn zawQv>0i4QnlbtbIr{`+ihYt_GdJHR=O@6{5LHt~olXhcS{M}I*a8tl}U4uzgBx*jp zRji6=dfc!=jHsx4K9~%u9#`zIn~cO6$jl}Nco#8;2pDgqvpvO#S|Y1K4rie3vqVCS zI#QhtFED4h{9VA1j=@RcVQaORXzjNxK8$SAK4wPeIC%aePdZXEx8yE+0I;$3%avkwY+41*ee; z&@xvi6UvJOhfU)RKMMK5Ge)~VT{PNe>z_T^X7?!+cO%0O9;nBI39kOtN@7LUz)ZmX zVkxf)8QPZBxVNXV%s6vVeKr}hCJ=hY`pM{cihwK~6q{=~trr;R=dFS{Nx9;4Zr!`7 zG7^c|#x2=Z`)Um#l$|b#-4ZUow`yGvfCXce%qd#AG~sxuJ6eX@lQ?Gjjp4vuTv(to zGf_0z8b@Z3BzdaEB6`wXLwFwkyA*4$k{>ml#wj!^5x4DqDUFA|FW+@VD-FJyK3ynY z+{Gi9YbWOrqc_u1`$TYn+)Y1`=FhpVDRPdVzJ(>N;7R=OCBBghMVep-7atEDV6AsR zbPurLbCNf;oXDMCcEh;jgbeA|IE5ZbQ52ds%s}TJ-6?8~*qMF3@X8c=bL@w}r$Eeo zYUC@E6+viob;vjUn;z&lgCas{XLW zcxyK?xbJRX+WU9|%5bsaPbm!Tu)E}a&!br8FTR3?Cb%vZ7|$~!=Ixn55uZS#3NRZZ zs<82Gtkto2fzIEbE1T5-++IkANc74_ zARU;|ap|KEBu3}J?H?y>a845^ydr)R0F1K65>38_s0!GY|0t(o^g;aU(_1BuV33!b zi%`3stu>SZm%sRQ;lF#YPI4YIjsAv*0wm?LyvmEf2gKw__$W9yX+jR-P0o&>kaw+` zGf&tUrybKn0W_!YI0F{}d-V@ih~H2E^+PAzPlxaLf!!ly_BXZb`x{oX?}Ft-Yf}M7 zL{95Z!O*@rVV2j3Pjafo*D)wz$d3nQ2r{c~F-B4MlK60ouc3wU3}PEHhb{(moORi; zz5Hl)0M*Q# zOMmV8+5Oqz@+KiFk}x13`>Sg5)om(PI7B*n7hy<%)eZ%l1W=X?1Jtm2HUs`O#YFrj z9oFV(XD8)A{GK75(qMrd3jxUxPO`+Y7MVo#OtQX}E3fEqAVqj*?6JOOe$$5fn+5s? zx6moNC@o%1rwax68*VH@V-ANJ;x0GK{o3~V@1MKuiCN^IycAo;ZVc_;2O7q6eCH1I zoe1{_eg#}yXybiKf2$)I+FsNMa7IrsH~HZ|$A{s0LJf%{UQD;+jsdG?0>7hBQV)4Z z9Aj3a;Zp^Un5Ljqh`L5U{X*^*a6hqP--eRfh0}0|6M_IUiNtOni5Fk^t?onDM*MD^ zJegBUHkuv4>|8kN#xJYTzk`=4HR0PzpzJwG>KT()`#P3VF~fM5zGtG$RvQ|WmyaWj zqa&<4PU$5f921)o=e5(&Jm@$x-k);(lbnuD;XVQ&-lY< z+qf+FM4LeIsrObq4%f816^m|}8*00qF5^nxMS|H$dd#|s?}S(ciSghkJ(SJ=5y+twusP{MwkwIq zG2jBiouA4dgIuopX4Fp~UOni({ADA{&bB1_SYl{Q1wI*BTif%ee(N*7Z#OJCY z`He1l4dzecQ4W@TWAOkMgb_`GjENXd#_HoZ02Mr-Do>Xl9w;r*JD0R$si9tO6>US| zW|-ViVwqmhC1e{PTM51QN-HWn*EaOG$)PA8f8Q$HRNa&V^1`9Dp(-VE<`-cJRki~l zeQ) zV@HnYenHV4B4{V-j?tY(Fc2FsQ|x6Gw;Our*EHIetWC6h>UX4AD|F*5bjP5T z@3kaY0O%|F3o`0WTWlQP;ddr(jcn4KyY(k|Jxi~yT38Bltin0O;H6rTSn6Vcdf`n& z3VU99zPfSZtoV`jNq@?f5~?~6My$>J%7mhCr9$Go0cVO)?rpbQDqH4OAWGC zt!B23yF^#B>^~P@O$qgThx4S#JI`u=3Vb8kfuoSrCVyU3+I_TDPtMd zh77hUa;@t9$3OrpW1;dq;7e|B=27+?L&)R206N7fz6u?Vpo*g6vIY5v1DKt|AK$2M zJi?{ZR|-bTbSdNw@;C%KmF)oF@02bTYv#S(-3CkWy`T4^;;km9dfr10T|IR>C-<0| zdFuPGMJ!X;7kkg1rSdU~d23f8Z6O>Wa7!Q!!DKWHYFT(lU)%HbfN|7|CApdi!p6M* zZmPd41(qS*oGsEeT8dw)S%!yhgr&Tky+y^toYWPz1+9)DO8jzecE{}r$;iVGY{|@p zrp?%)e$c+T^FP36!i|qrv2(?@HIV=2NN1;L5puOPYfUZcG0NMuFx0O6`UePVOQ79wGgMj)l5<4?a<`Yl_RhY_C7U=0zKBC2$EhP^_G|S) zwv*z48K19@_pT*WUhAAZmlp){uf+E+7CcPp@0fe!wZ0R-R5-^z@HriduQz zZow5@W~ILN%8FlEM2p$(xE>5I81*!?MyluZ_h+)_1Ug0r&e(>Yv0M~3hqW5MAzFyu zT~rkx=9&{Z2Vck0$yI7kx_X*?*}kLE$UCA?X#yX}J5mqJIW0vPm&dE7bya_O96Z%~ zl$ilJ>NzFyNQyi0rMf#i6p;Rs2}#%Va%#q3X3af9vR@Gu^|I*Uw9XEY{t`plKE}Dw z8XFLZIremOfC4J$_eo{BWTsF}V-fd#;9O9P@gDn1IpW}EqCsR)gC7BFD#!|v9*h%1 z*&6syZPLg3GRsaVn+HT0jx{p1-AFJ$!XJPR;zEERi4XWy8F%Ob0bCHy{|+cVgt zxUeBR@Fg+_?_9G>{k)>Pg*RYkst}Ve&Yr9ku!oPKAT5$zr_hh$bio?MkK~VXg<}A0 z(xHUlM(j$|fxDCvX(ON*g)b7>LKCWPKjS0%J1wRdl;<;+3;S1WAQF7)9UG>EBPO4+ z+60A8s;x%l0#{t#>M3qq-pVQOPavJPiz)V?3tAxyIwpNpQ#BQ7cUn49TfXdRMw84e znq4y_=;tRzm6)Uu*a@=Cyn@(7`XL|*GokZSuV40Fdtg?L=UjQd71V&Il|4)T&J8z^ zX>1PZv)eLcn%pp%s3)`~`Cg;oBWcd_nBp_R7 z(cbpAAxWQ&^ZmRDkLbO=Jfb(k(=z$y_Dzc|sd{p_6S+9#Fbr7HEPqyXNdaJ3`3u6( zWDF@;ybOj>Le%rvVTGL7*S;P6;T6lI#?Yp@KX&- zeXq*<7IsOCb=uS5s0Mmf25>+hk)wj?se_5MedT~~WtEfn%Dxk#_W?Lj?3>GwN46fK z!IYgVw^_>#<=3oy;69J;(4rMSQ*bk#e z*O9H2VyX^(Rhj_h2~RKjRb;#jfWoVR_7xu0|7d;#jJeOlwzc=%h&6f;S#I99}wvxDNo zQFoYVq&-Mp!>+&et%Z3e-=EL?u?LUtia5D*zj}rztU#KX9V6C7;j7Q8S0 zlB*6q%yF@-Yf+q;a1)&^0$8&K{HXDYS&Ed)vJ!l6r$n9U8P`MUQZI)eK-^u6*Kdpf zzNar-y5wx;ZtRJpbYCGEd0*84PVL8&+BWu$y*{?sk&bhCehjZArP1SSX2_6(z{nE6M^R*|f6 z$ynra_U-VwV*BF1^ho4}C9XiaVprNH`hGFmgiUX%Pv*@VcTI~^;m|JEntHi&{_L&; zNnO;cWA4aJODk4op9K>jC_D0@eyJFuB2hh`Cwo{)#83w{6&Ky2xe7(Qnzks)2SH`f z9MmfjA!;HpQ_Q@C+Q5Zs>7ASx!lG`27XazRsQ1uR^eWQATS z(PqV@o6r#!swbqh-w^cNgLo54+nw2GAw@~>UnR!SfLMDZrFXJ!$OoPmtDTp_b;9`K z6tL5XDPoLt$~OS+O>IkYa^+oW@Jfg_g4g+JCAzGU4dsZ-rcx~ZL}!pigv95Pq3LG} zPEIepL$%a4dNpm5R9%Wqxwu3dl8$7pq4pjr{XIuHbFK8kLrI(}DqKPN12YQ2t3qzdnN!ez3Fd zp@($04skG7>K4pGr(&g2KJoRf`ea1&(??Wp<%O(8*U+X0RR*C;2`Ok6Xl&E2*5VdI zwm9bdWnitI-|PHYdRgj21CFGr*CO^yY1 zJkS;V*|!ymL(H~{Vz-foW=m%#Bb9256n3?)QAHTMGkd{94WY{Y;*C_3_M$LA@*1`k zcOc;KRtbu3LZZcSJ$Y@4f9q(6`;*$pPvvNuPTT!YP)11=@3hLs*qSRmT&kfVB_E~J`wO&l5No9Hxys8+F-y1{*16v=L0gph z26scBjUWa-_NHH!@XYfp&9h5bno!vSYX-@^Wni0>qJlmngFgNZ=RDuIzHu6Ja}IZ- zz~}h(TRXn514hbq<};7Yp!(msmGT0$WLE$i%+~T+S)Z&w;Z3dPlWkfIw!BJ{{~Rcq z;&sxPHBu7o@hrM#E2pGw2J~6gLR;dze8@5(Xd~jE(gF~%!U~&-tl;CBXIrbO$!#%# z7Wnm3NH%VXo`JPuS>tD|@@o51t zvF6hSTV`=L1picH03CEV53d&h8m~F=xI^xq$^KQg$S?s!Y>X4C8px}6>=*DKtGGqORX z>@+KMD)Z8^xQbawX$BD?6-3UNB<=xuVC8wB+3{ z$(6jJF;?=cj{Vw_x`S}-Rt)sM&?wC`WeCKUYuI|Su&3BBDm>S9B?@}*DAYqI@VH5J zx@#>WGMvy{SU5}Z-ds4VIzM&)$RV?;m6yYnO)4jn1+66*NN(r@8i51e)@X?XxljW& z!Mqh9S&j$#%jy30)1H zmLPP5mM-sO3a)B03I-**B$D}Mg=LNdyPsRNgzN$c%7l1~0s5sGk5LwCFlp`b1}{tY z`Ax$;Fh0h_WqU?!RsMi?(oU6P#~_3MRFz6_$2S%Y&}kOb(M&MiPm~{! zI`z;?7q`8^+qCNSK{t`or*wkUEAx){Js`RRh|P9E(`1{cvg-PRvg+x{^u&;j#m+6UDx{Mo^f1Zw);JI=wvFcnuMO()EMgA1m%4ZN)t=+tTUo{-mt26* z+YtnDP|`%#Mc4r*9=JNUppLb2m|;RLP_~8+D>BB^VX@~;nM(ASLh@oz5vUeD^CYnE z%sZ0<+!;U4eDkEZZ{0f~Z`$qI8Kw{pGxP)o=!I`)$0qyhKYNP`j1A-|^8Q z(IE~i2!?diQoAET^xIFq^XF(^gAzEOveZ#&@hY^0Wsx#jKD!&*f^7=zg?p!e4zYCx zm`g2=4;L3|Jv~$BIf>zyPp4%@okJzf`yPuSHMH7A&2cKN05YV1W^!P1%kc4LP+B=1 z_v)WD&+J|8+5u@+^?n)Tl-y?P6@xH|G0q5VL4U@?0e!W-O=L>!?VrBX+I?s$~ z+R^j|7)h>Gl(Pq9{aK<-m@9xaP!=*m9OgP;S(LE4#j`zVvSzF=uH6#r*@8;YNf6h? zM?C0=;hrzuLP9<(sJ`tcn#1=oI}cKoBNT{G4h~EsKbQ$)+upOKO24nXjex~C@DYjI z^H-KT^YiY_{qyYHG3Y~NID^UJ%(tUUUwxScD9C&CqBy=;?RY2TQ!LL8zEHK#JA-4h zjyvrS%@N-z=x&oyw-C1sVCr+(u(?A&MbAjX;!_=O(G+RJ=S%0kDY{G5j7R%f*!3Lu z4g14hdT%|ONka2%Mt^)pzcR6H!Ci>hDIGNc zI{I>=8v><;f>XvXd#l3P8Sj{536jWYa>{EhzwaYB%d0E%34 zs;&Z4pI+PJX=`lcUrsKkWLbX_E%z}twRY>ZWZ*ayyQpMM6JFI513Q{C3N3tqjZF3}4n~f@ z1^DS=&vW?GO_0n2{*g|QW&^Pcv|^Nh{_vAra`IX=Q)i-TJ>vbBs9PT;-Zf8d37A(w z!a&fT*gXFS6Cl`Ms(4TK0AUu%bg;1yNP>Qg`Kw6&A z+==jRb-{oPy?$sWM+5q(TH6-Hfq2}yOJs1A)gEt5iq_r(A0M%haJb?CJEE%{9MDb_ z?k8%7DL9hlwp;KtwOhovV+jatf2)5LG6%b3u;fgv&Cg)q9kg70Pa;_(Dp@-f085&lb{lrqjJ8XBwmAHz2ZU?>J&&Qt_utVGrOC;QXfP8-` z4(gvV_VMBckHXq0&CBQV*-Eb~g%i_xDBsc{u4VJ4V# z)zc`WeInwd{2}6{tnH<*T%#<~5YXqUVk1X0kyKV;V?B|?2qvfZWWJ%1d`v`{qzb8V z0%GqJ)!KpL8n(^YXvhTEPbM&N*Par2=zIcS*g*o-ew6NnE^4gHYxS2%ry#CtVr*@z zwt5j^SX@|L!FP+QdTwr(_G}*BfVwZnBq>D@EX6A;D}&V7K($g}Tv*OMQeQ4@(&KM| z2s5;`v-L$^DpBPqp^j)l1@*YY?SXH7bfVx?iP_RDr0jm5SQh>h;Fr&o!O%Lp_!MyQ(3)9E>d8DS=Y4e zX)UA3i+h_{j7JFweESq*VAY`P6_?Kr-?5{BV5qBo;43bLHH`A=dgd&kl&zpM)0G~- zkYP(@b$G@?HAcPDoRnK_YmTf}Ws}xe`c;l-nL+x$=@8O8&cTz-?T`>Xcq?7!eD(4w3I*^4gr*Mix$f6~Eu zL$d6&d$SyJiHzaTS(jn`-^OdoV(+^g%*5}4xiC2Aak%H8E}-9`mywb6OE#R#DUKP0 zdVGquO}fc|BHvLQwJS8k9BrC71m+*>?CBUI*L5bKEk5sD9UG+hR$T?L*a!IL8`Y<} z&x+sOGNWy`IELU&chBa@Wn5*JQwk!Xhw9c?0vrmnKecLQ>fuH_$bg-=YRIa%TxyLo zrXGl{;J`Zv|A^Xvbl*h*J0&R$R$Rl=v^#;vag}wz+Rgq4TQ~~#9XPJ=@F5%1fwVd6 zwJpeIYBSy8SmYE>Y_|F5&zWOuclzUs*!*9kb2>WvSW?oMoqvilS#gEiSRGUE;I)7W z)|E64QMUT8l=6U7@`hl*Ovr9SK?>h|yCXrQs?Za{(SF-2A^8r&;ma$yVXAv`?iY{Ruo_RpDc?$_mYe{$)!^{E%qV{M2lfi_`V{uh1LEo>ktW3KNwUB-O7WqdeNMZ^^ls8k6M-)JZs71vu_ddp;A!#g zw=wtYZZm1OVjZP72UQC)kLNf_2zE52^+~SYDd|&iCX;n0jA1Nw6}NY_8G`LN)DBhy zlWWng+oB7p6uXX_xHm4%EQ_n-YYtYEm)n7Ire#_8@fetEqAR^npHzl3SwWn01Ob3= z!A_Q3z;1)Bo}q*_D{yf z0m3N7l%x{&a?jd;^375PLG6R;IOpFh&DIHCqCl1a+`{_Se9*!4zMNmwTXL?t-{>jE z$Xie}xGj0iG^@ABlUF;!?(uq#xzp6Mx6Ul| z3hNeNoe5K6q?JwT%srU~F1bBLqFO8mC)Wd7Dz-`Q%l1u3F$h{!@}CpLAq!dM@jwH~ zzHhAgn;pmsF?>(7CxarmhWJxMrq1YZGA3Wz1@87!l!Y$CN7tfF!$-OzeglAe#;Fqa zb|lGe83*!xm~EW<$fAy1pN?N+1jh^7N;Fv(sOA#NdztDyHWHT705>9F7bCiiL`lba zuDrfhCqn3b@|o;We}3e5IwV1`^#tA^5N0csa*5^|Uaps2XI>j8J}+D#EV;>^A;+$G z{+Fs8c|#Tpo@yv3lRlyn4l|&^Jq!=;RL~3`^STI9=)eF$xiBRN8|}78od%veM~uY) z0C)8CXU0XqVAmNhW(c_;_7qO7P9Tn+s_`f9{trxKU`5_w6P2pjL)u0+J>yQ3gVFf0 zp=6XES5&pbv1@k6pqhcrgVuVtUW~TY!ys3EARHo4$Ke6b!DtC%RRM6oORchPV{wJY zZ}*hbvZAiz_e>FnKS<7#U`cJvJ>LqprgBT)h+^0Ho6q_}){b232RhdecEVytoPMp0 zb}X+S_}3#I8U0T`m*iv^+k>vWbCBpy_!MNYRb=0pTRjiRFc832V;`7x*oAZ;SCur1 z_GrOqO9Zi1Ne1W4*j)f`>&H2fMn&F+oRYW*b=kx34~c^V9_qgv*6_HFZ~iiEJits& zJgk4!dkVNb_Yt7=p~7YNNtUeMg9d6_pr;P4dJhBf@Gx$7RFGT^gE5s7moU@iGu znT^V@qS_zWer=95u@i1Gc?UB|gCk{NS3gMhr#ad8(I`@qG)aZ|UUS{}148nldRpo!`)^i0VQ@Qq^g+rJ?5f==gq7w{|_pWO}2l;^b=O{q0k^lGSE1USIAOou2v4CCA|EEaC9V5YiIo|(O)%OZ;|4x|Tf4Ktx n;|ctiLEZX40|KDl3KEuzJmfzPJO~KSzcU9N1Z4a0|3?28SkL|f delta 14892 zcmZ9z1yJQo8#Rc#yE_c-?(Q(S!{F}j7k6iHcbDPfHu&J~?p)lRft~-Y-P-*&ovJ=b zPCcEZ(n&v^a}uv1KMo-qHSCbPyRfYTA;G}#V8Fm=QcdiL0D3mg>h?Cy%x3l`Zf@Zk z3SJA+Sf4aal*3xyaB2f3RRkn*SV?+h;Z&T^;?_1w-kD)ErLoZ*yb=~;X(Oel*}4?iD#$8Yf!k8VzF5ri5)v$q$PmQzX#Mo_b>H9f*}wI2bh=zdc02i z;^4S!nnA%cfQQqR@Co07R@RcgmP`h7cPDz8z?<;!8ogf2z0PnSL>@*)EN9FgD7y@s z^W_ap{$|BPvj8b+wJA2d1I!7ej#qC9)(e&~Sw?Q#a|)ln6^VJ?vi5;Ni+ououb+G^ zbm|dvYPlMrwgWuk=$t>1Ao1yvB?XbREP9B>-xvpj0Y61>sF)?`*NhIiIs+}cAHqbA z#70YORkWhxs)3kJHE`d?Kk|%P`D&hpDy-YSd=k`&l|TIr>W@?Z zL7A=7dW%+}=x=8RUBgWhY%o=)t?9h8a`vU_2*AxQzi`Q2Y&Xrknv0Mr<8iwXf)>)3 z<**xfFVfQ9Sj^S9l~kQrqzQej1}+|6<=p28(#4VzP*g|RLouQ|xL>)e?aY5C>-_7U9h9=6~`#trpq4ttaDv%2@Bl~{dtJGpZ!6iID=J3 z37~>*=BRr#3KFW2AQdid5m84OEL(CEP>E7qhjqrN;Lp%DwroXr!VM6>`@|fHNuBr` z{t>g6<~8>PalEtbbZBC(`aFly>9EhKigz9(ES}BLoM_Q|0o6Y{>SY{Aqqc4{Zr5*X zI`0OfN6X1}#y5Q7{PX6LhG+)g-ed;_2H^Dz0Bd=reHdru2l_+HFbl$Q#)))JFfVY0 z2mR(+8#b?wl@n0{x}?#FCITWSS^Ug%A)%Hfx4n<~VD+7|HDFIv$_ejs2eU?=a*N{T zbIheH;rgJ*?Y3!+jzB+&$C0PmaqFD$%TezQvT3GYTt)iTq zKjmqowDPDslv)ivU4X%#$N@K1ECF-hDp-2mrNhn?-^)4v+I>70b9f3qV+6V*@Ditv zb?`iIy7gXnom^~L%>eu%cA5N(D5IbCW+T{4M#9HV&8H(>#QsQilZqi^42@e5YqO&F zQ{n_Ho;R!ioIe(8K6g+`BsTc^Pq`94ZV7ENxc#v* zh8_@c;!6i4@7cb=K{P<|HTI$9Ix`Hlv{(c9KJ?5ivi$Cko0J%$i}krLp%;KdU&p4i z4Z0o?`Er31_N$*JS@>}w5(i-p%jdZe%tXWI4*>I$5;@K6-V~>|_&3QZ_v-F}*>vV@ z?v=^f!M_*r9pa9@de-xk@={dBQ9U5bsC2`~lsBm>jlTqW7o4HJsRrh87~-$faUFnl zja&?aygao`O(WNP8hDL`4V}xQh?C@#qwMHi2k(g~9LtKU^w(;q4wPS@!c-<6`?Hjc z0dpgIuOY91h3z8zosxE7X~rhZ@F7z_duOVZ4j2Jw!~^n@*Rc>X4@S9gqE8nIv&ICO z6hBj9OjKkV?_smM&Sbj}nbBGYD<6<}s)JfM!ZTHpPA2#RRJ&)X?e{) zsaJ?h!r5?}%q*t+iG5!WDiRlaNNO@wUF%HX<#?EP$b`BL4+#U|b$((L+gKw-^%k+o zemdq-`Ne!PEp&>Tu>;}L@i#@uIGVw!OYF&BWThXI93thPv}67vGrbVAeTc~dFi1e( z4(1{k?mCs^4QQ+&_(a{#rT{eCZE$nAc-IacUt9?my^(i_4~kBH&Y1LT@2F^H!=e-q zkj+wipZG3pNGbPh1LSa8G3Fi!1Z%%RO#cm>xaTldF4rrw)c~ZsNNkAZi%!mJ z&dOE#v(cX2Uu+cMjFxKjdHWL02{j_*or_hD6i*MyP^80napiFY|9~zp%j4gPXb(R^SuO z15FztfoYjWtwwZasY41y?<|FinhI;cFDDhf;L9mx-&rtGtk{ioh|zetBQM%YyCxZ3X>aQex*ifMvglV(FS&z3q(GUXhLL$HS;V=k%cV` z(NT{50gFjSd8OANbvr}{XhW^)u4KXjKcnVr##Sp{*rPks)5Zr-yOdJB)9Ccp_GfZUcyN0U9hImp{JVS8Yx8f6Q|Ck7G~m?W5yAoAnzr8^t` zK~AvPGzZzue5g$|Da;?}^wSfkZz<&+xLJ6|9&lf=4s9UgqgZWtLm#<`a`8efYc$jR zk)y(I`f4D>OSsCPZDpHHmWxo4S0$}*%ufBWWS$m>!_5GQS>zU4+SFi*q|#5)$UU6c z#Y35zp4!y0lO|O>Ap1rDUm$Be8%_poL5B6W5kcpwZM7FG~axmn>+LqRc_JB{A zHgs|13VDKZ+eT3WG44un=ElhbCE9E9>P@^g8!YC(!<1M?q~$D6zrp^uD@QhJylr8C zfd$clfsy~~$|V1ua3ny-SMQ{&6AceJJ{fBiE4{)K9ECB2Dh39edA}kAj7B#V&sd*1 z&Ge>;OC6%4X3f%aUH#Jha+$RSg!C|TaZBC)ypsO=Q}4=??#}0%k;9wF$@W?b+x+v} zd&|dU$BF-mz{y5N>dX3dfnRb|`rXW3RaoFjQ6lJ>WO9U!H5w3%J$;{)LrmfulLvia z>IE(|7K5h|evc??mKYggKxU~2F4P~6fD0c5>2=4+h80^RY0?lW@6)L>i8iPxR;Y2L zyT53k7Jx8wJ1ZzWHt61CZKnIARXVZu+l16GF@y+@Ee1l;`AHjiTRDPF5qBlKZNcD-0iG71$bXvso z%9wU8XfRVVRI~)qq_+nXKJ%nPDWD-N8sP`6=!Rymtc77w2G;i8p753S8k!dptzhL%(zsZfS9Q0-QPTKe$e+eS5>+3` zqgc&^Y9jSD4Ziw2M;GVB0YB{RKcy`ZgVN1(rGHGN<7__l%tR9-CtH$*_EaRVcd+7- zq~mpJneYG{$Ykt3;OkvZN}ELN1D1{7c__h@&rerZ=Q_&F-j9##MeVF$XV*Q?x*pe) zNJwgtGv|!G8}q9g=`a$qd{;MXBljc5Ggz5)Ha45eE9(6GWZa(9r|aW4y7V`41pGSN z+S*!MT41ts_yv|>GTWELn%gt03V&6Um37$p6?y>dI7BUmG@7ew+zhqd$QpZWgkGHC z7&tm4lKaK_Z{!@3LB^NH8rP`!Eq=vsqfzK}4yifDa{ZkWq}*u8nGW2=zl^CSH3Zq^ zZq5vz{d4o3-CXQRj|W%5i}A76^DOD89bqI|F5lpi?jZa78y!bVjCUt5wlq_@c=6|h z1Y!UK5gp$!ww8#AxG7vPiyIIkLM$nMz^VzRz>8siW%N?$*w^`Py5Zxnl5Dvrh}<+vFZv>ZLEKZM61 znA=^jf_H6OdpUq?II^raf|U3x8OOcE)sX;9GJh!Pbl0bNDr}8{^G`*6ud7v?hpfj` z@`2@WaP{kraJM_|a2CxM_HY&}TM@S4@2geyne(CmMXFr5VR$X{)_{kZ(LQ)vxkjI( z0`>3ga3t>&+CLB7m_t0sc%w9Ueua$2ozr5<+Wwv*l25*z8+B|EGOT+V?w55?U^NHG zZZY@*exrfWu@Yii6z@c3^*081sXpmKx!rFIn@QU5JG-P<+O2XHn+SzL-e#g3a#*jX zA-MEV3bT?`i*C0{qoMqX>_X}{55{MERLMan;f!Q=WPeK~+YVaHVx&<@ZYK+7gf|Ro zSj)0+E8>knKQTriVvovC*+!9k^TY>~=k2LaLe7wL1lq{=O}F!5@D%w-kdAm7vF6I# ztU4fDInuKQ^ns!yXh02hMtclcy=r^k>HO0Mv>E)B5cozpokC2;ztMjkGKw1iSY3R! zyd}b2`8nVl@5{K#Glx0uMiAJP5{Bsgre?>R*r;dcO%~E>8A-yC&SHo1Jhl&LsbrLK zm{=;pLM15opj~&<9n)R)#TJ#Dfdgt80PvpGq2)GZ@yB2ELOD03@a$JT0x7brT~( zAnYt*w8|r>_G6GF+aBl@EiH1B4E1w1gU0GD=*7lPV#jmKa^qySDD%0+jdu68!kHV)wu* zR6Hl-u7WhPx~aEPw_+yIu4Yd({{qvix|hTG$+=T|%j91(Qn0s?S$+bbJt5ecZnOE& zeN#CQ7`jmYBqErj8=3`ay~Rnl&9xA0DYIJq#TrEvE|P;C{P2kvR`9ZR=h-Tp1G>Wr zbD3vTa#2z|Be>c6g}NH*BH?vEk_k#t{|%_34w#d{W!h-2VT_g%G;8UOzG=+KZ3sz!eQ~ygG=)) zT%Q=Evo8}L*zv#VBmTU?#}^z{aDEbyYP{IQ7wk3IeK781b7sj#=2aD%-BE`>T+f+( z7RoNpy+qkOtiYW`Vkuh-jz@9{56rM7510{%%s9v4hIyU<#H*zNhstr;Bi^i3W}Q@W z_@ZB;oa`4XFH*wv5gBOVpWwv&rw#Wx%Xy#dzwVI_=k|0ub}w^AC9>G+Z`;C70`!qs z5V46cf!aei^f0+EDBUhGMDe8=maT|fh+!Pu6>YK+AC^NR#WH3QKW0mR%r(qODR|Al zaD6f_d@|W}^6LozmS6o$#hV_twsJn$58i?5y&@qr+YOOL51Dh3F#QG7XCbmp)o(7N zzmTq}q^VvZ=3= z@!L11xFzPe*9n}Fvm?L}zIy!5K>>xpk*sf>oq7*wO#Ntx8nmq9f&fGSFa6%2Zvt_S zOU>abG@r6(XZ4$EIm{8IdSVOCf~MIS#@ABWdcqZucU5F^*vD=vqFBl@UYox*F&T2?sE_)xkp3FI&R!yngE?oVegg-Dzp zd*Mm7WYf`qE)6MMpIz0c4i4P#`4a`o)=pOv=EqOD|BMGT$z*^`i9^K^V_h3lQ(xB9 zy(9tZ4$L|f@Z~}_11xufY=g~Rh(k)!=b7Q(u9L0`Wx$(rTX}7wA2=q2x@$!6!fVTZQBG?g>`Xy$nKNu-=yKs( zHygJ-npfA8B>GB}f$Rdk$MO4WW-x>}`cP#J3s!XWbL%S7!Pyz6Z^v4l#$TupA~66b zI)J&BZ`gBqu|7quLQV*y^oA{)NyNpu>+H5C}aRx7EQVnp{ z>8+Pm9_4cT;D7k?RCK)*=tgW{s!x`A*yeVsEkGlAq{E*9jLPf2YTb;vCewwCF_;!?~_F zj#y&cdU^jL2UCO(gkM5O(z0tH03ea6YX1I$GBs{O_YkImG*gjabqd1W{)C2+G!}EzMTwUoOezvH| zmI(3@ll&>VK#pt){tAp0ngH*msdJfCLo$T6Yi9y#Yrf|SYme=lZr~&!>2vm9*p)FN zJbnQ4*8z+k;+9`fXAcJKmYBK7m+k7rdv40#>VJ`~sF{v=kau#N2 zMp{qNK||@X8HyW2t*))ItW+;M#nwi?x{R(Wy}VSI|r79A-N{?=nPMZu*9baTTuQUH5DMjq?K&GXOOJ`PG3SY)+^Px zY5C=H`qRe^QP%ssvTmNlRfncZewGfN-$Nl>W!vVo638r!nlK;xy8QFRQvaQm_*dOC zQT*QFeF~mB-aT&05RqRI{B7ipTYKoaL0Y7ZSP0H?#~*9eYdoea=)ERY`sd9enjIUlGcW5Zlz$g@9=&rYg6zpL6%NdGuNe8Gd)#SceU? z4;}utA=4nk{DNmPL+8wNYS5%#rE^^Rv#)mC{CG(jG{^n(IRk<`;!#`UzgKJ?S1#b> zZ>h-y@N3%7CLs);0YS{sliIipTBdSaX-RmAjRPPeR)Z3^6Ipke(1@i0Ay$F$G# zT!I#60qDdPsMhf>cmCGzkit@dOkVA{fy(aW4}s|ZO0Zg_QzhW$Ddg4S@w)N?$!VVC zz5t1vXOpvtver4c%fi^ba8=`BYo083>S0y8rvczIISNbJw^MfS^P>lcH!RR~ML{8Z zPvZDPTi+Wr{XDEYSAgtFQ0iX;u@x64!UoEq!O!jI;#?i93&=)X-9F6dv@? z19vPwE$Ab}Q^KfBe`kzxC(~nakuH#aAwUPLJ_2Mhi9r6x3k|WM?~ib)o-a0o)Qjdk zB^yu(gJXj7z8(Dapz9C})xN;PMJOP#7Zn-%R?RnWI|vZN%BKu{K&Dx#5-sk4K&%Z? z3g1=(IfQQ~XSqeKM$3}Q&?<%xW1Kh7yRbGK4oQ%cM8@gnm^=Lvx0A+t>*vML0Jtzi zy_2f2#z~AOmL#JmR=)%^6Qx(nxi zQ-6jmd?Z_ZN8|Mgvn+~wQ?=JFnJxEAi_jpjlP&uN^F~KRg<7FKKV$BT>o1}Ey97eV zQ(C@YBKSf0@84Th9}prj`wO}YVd>=hl$7;cy!aK`azMsW?(_|(O8a3?mf}nH z3yLH>f`QJ7=#Y3m9$oY|78@E#0f00~47qn@b@_an z(;cKui-(z}*W5^|N3n4)6%UbOn40r}W2dAx#sa!ue%S(4HC?H-tz$>|_F_-vP{|Vk zV-|Vp^(=CAhOPlNwwF&vTD9^r{UdRr4Sfappztne-z{P7LhaiQ$R1mZ!nRezaIq>B zqVfsU@@z1MY@I07apAC0#48=~}&cWqTPT5bE`GNbS%`Z*cQUYku zPN}rkg5{gn8e>Zd_B-mNLAw>--*1*zrfHwCpBvovOuZBoWs)`#n;7k^B~vbQPSksX zZ=`&mEc969(0qFXFOdogw=nGp%p#~eHNi#wb|fArU*P}d$AIJ+XPC$*HoRg>_+Vh? zTwq{i|E9)pfXp>J$bc15+m3llUbGa1c1o(1bm$a=l*h)j%}q#L-HeA`PO_0rie>XN z^7E!Uog3FnNi1#~?lhHe=%$PShU+TZz}-E&Vh0-qjyY7oV*vWtqEgjHtYf z&R)rcO7l?{D7|sau1cCoFTwqL3Jea1+#Fxw_$E+OYk;GMvVfWRq)$AbaR!o-?z{0n zqxwdVct@lv0{$eI8m=XV326#86nQWtTCgdbEo}y(s&q2Il5W|GuawhgF z%Ji*EX70)PA`B>&**su(cYthaT}(esCqL)|rc855MSqY;J3jJ7+L+c&{F=NpDi3{? z^BYs&-&W{!BjqEW5TwrUQL&Laf>UB{ASj|cYU;zI`2h%@;SyJ$V3_4Yu6b59tE-Uo z+K~wtUICgLlThWUp1U%;{U}LH2Ne{mqby8L4|3MHg?&f?BW+Mx18 z_IuqP#vyk-i0aCKHvCi=m(3E)#bAX?QbuPZ)-118iSkti^dJh5Nzim59G5EAIdlJb zY*m`6JAirkmu-@-HLT@zDcWVRkUL#KCbN3>B{Y`^*ejBd0!b}zXnsk<0kWQ)&AV2a zl$KL^>yeWCg^H6Y;y2!|nID|rIx|` zq#Ak}>5JzddM76ISG7dtu6_tc3{B-45akfcc(1IQ!D=2AI&GF=IE$SDS0;KoH4|pZ z-*F6=}ZX zP6B-3OXG{vDxgF3`Zn)AYj&fx7j#vweLGQVyv+W_>i`KE9K*7njhB>IZ>QXO0^kx{ zV%a?fkOVTg87TRG`LYG*cgTSK+O>E?LGr}Uz2ftgk_!2z2If8B$>W1bYpvrJ)r&}v zVzGKu8gFW5h<_Je%EaWR6;1t{2SI?3BN9-i9rqgW7ECN{1jV-YWN>8N@(#*vRUEEs z_CIp}wMNgG_VoU12?;GXnV^>6RTO>~hSH;z-wGl_l2mHP5Yz+N{uggx-)LRZYaZv# zo1WHp4|iq`6?=U~iSB6gr*>|QznFUUC}o{)Mdz2X90t$>&o?d5{LhtBNE}qB#}NPy z*{W5Gq}aE-wOS&Kz@LR_PysU3$c4L+z+p8vKV2(nz1d<11cY4_K7|9IuKS@wU59e) ze78&T$xe1i8JLtFeffouxJynw$xjV&M+tHD9aORVVg=$-6B20~Cj7oGus_gn`Viap z)BJboiUVY?sZ|;CZF5X>h30C0D-GbtCWUZ%J%w&Z?^op!FP)h$Ls6V%B%@JekO8?} z^=y8RlqXP;S0=nVz&j8p^Nq+m0FC4pjrEh&L1F}n%&Oc?Ut4~g`7O<%n^~ZAN^JeL z1;K`*A`&gX6}%ch`46Snl;>HyKD1zQPK+Lkn%#tn?YShg(axEUrjF>3r$qq2mGyH{ zgPLNi$x>XG%$Mq(8^0ye0^hqd0P(Q(nzCe>nnid8J!)~zlA##qbVPH%+IK&&nyz%N z8e?Uj0cBpA0nEX5Tj5pMsz1bJy?glNXFZ>Oy~}OyT!wkc{9j{72)sJYBGWQoJ=^uT zfv`e29xPVysxGuKKZIOgm`#8;GnNVrHly^D0SeyYz7I`4a^JIF6aa<&nEP-t@GvSC zeJL`DR5+;j9Lz%X(x=a#eDPUe$OpDkxnyU7v@kyqDoq3;%5fcT9WYSY_et}{@slyo zoA__|C&I9DAp^+i!Rw|MXYHI+=e#eU;k4iZP)ISNBl|`R*QIgzk^xZulD_Z`1u12B z!W2RCm4WT>Plb#fQ}}d8H>YN?Y?rp#?+`*G4oEiK3AuDK?Ym>fPJ0L|=jA1gCxkXX zk~wT7Cf}>{Y=;&-6AK;kN}kxIN5194o`zVl*}SW!nv*q(9A#8gGd^O3eR2;4;KM&- zlihXQ6p)f3e4#}Jqybt78Km+Q7*W(^FI$Avw?830Yzv$6wj&bx8$EG)O8ogQ>)4;% z2!}C8Z@FLh>eSOLV}89D()PQqWc*4Fi;bwZ8uJ00UJ18Va$fAw?j7EU@pY%xmXfJZ z-*=FysHrYlxO9ujZDFRfppwe>{U@Yxg;E&!RQ5$a{88cmvIdZR(S+Y+!|uz3g=Fb> zgPzP`z93MWr+BL3&%*l1S1Xf-tPb`Q6Dd$OLv~WGeQJ_OBk&yc=uyHnepLicpa!=B zO+yecFEQk)sF1r}OND+f z_dl$LF@jH>w69IA0i0VDelSLec6+kgNDFE6x1X)mR-*-3T*689khQfgVDmog{^DJve6UL2 zpfOM8K1XHARbU6)dj|++GHrZ7u5GY<#snaz{vA-^eADde6mfEOf^mdG{Q$??z0&H7 z>0^A&bc#XnHNcMy62wo-NYEoi%Ze6`_Me`VldMrKuU$C3a|tXoK^ST=JzQIr?5=MI zRfoDio}6ZzbhefigF*-0^N3{YfZ5vRH-cC<7V>X$%NRLMkb3#mn>wkaYYqe7#kJra zJOJ3^88~|`0d_|moIAg4rK#_>E?mRA#_?mp1b=c*UHG`vV>30d**CDcJ5KY3Qn!$D^yrsscj?Ipds93(`n$^ooqcrMHbC}4R^e~s* z@oN(QQoH7L?Us<@fA<;5AuAsHN;m%VvjVWl7im3Xvc45R`D_`)+v=h;Q0E&N)huiR44j%A9>2%J}tu^aE0C(5GJfwlc7CUD&YSH z7og~Gb}dX085-HWxBJWK0p-HG0t>_EZht}|{2Xf9Z@B#>w%Uqh+E;te2iveDe;V*$ zlk&YnP&kyvS?JZ93vDB6P!=<<->x!xrnsd$q16@f(UnlpR0zewfivoad0RBYRY0&b zw0_{;SJ3G&z6w&B&f|ti82U{&A&Lig+=%V4}>fRsih>I9rCuC~c8#CLutITP?(|K!XI#F^&^Q!n$&r<`H5kgFIH)fL4j^lqC% zDGfR6vE!rJregSe;df&_J&+{%iWc~mBgo*mJ9b1{i%%Xc;%c4e?OV_<;$SPMPBhIj z9w%}hr!w(v>4jJSp}&aM%uX}1=Vf%!3gGj<8KM<@*f=R|0@AB7Zh>5z3Eth0X6V7hwjBSz*NeBs(mee4F;T#Wh^5{VBx(@>%50I0zG0< z?Ge8|>d9J53NBU6VQmrdsN539WKQv!lImkfwTJHRQQDJ5Fm7S$M2JT5NPZ2NxI&zs zz*Bpf@WJN0ZqZ2I`i#SM#VuhLecRH(5W}(aE|@lioo}*a-51G;R_>4cPf{Sx@DmyW zZg7S!&OddG3S6p6C4MT)G7-Q~eL)l}Vn*C%9RuX`iiM7~UMMN10vW#u*N5+v z`Evxr9+O7SVr1tqe0tSo1Q8Gv94+D- zgdlPskSuN>0xSo7wRqx$)7)kiXBT=(fb(KL36qRPG&o3SfpKH8nhBuK;SNz!=5_?6 zIIm_RO^eNeqR4wR99DxL+RTqAUO7Toe&FADR{k{uM3_!~&B{3gVMVY2|`3xZnLaGl<1%Q3Z?Hrn7U$R!j3_EeY zh@o7%phu}7pj;P>T#ij8&uffc$p&odBoLdA~JY!NX3VK1=>$E-Ts;5ku zZp6iCT`jln?22p}!Do05z|{8K^1^NNo*Hv^VwqX*5nUeKBDV4sC}(wiWC~Y#+_RM? zuetB9Ydz^p!4MA0rFFg$l0uh3&c%Y{B-A|3`ODJ469JpA?1LVh;oj9PtiR)y?!(}i>(!_)`nF|-6$ z=H)stA;(hDEeJTa80sT}5pO^^;1t$$DKPG3_zOib470JDYWm3yH_g9W8>;5cHXpHf zoiM=^m%95W6O1$;UHl7c-cX(b}i%B@^N z(48q?hEh9s_zHZTiK#`byC0sf%dIlYi%88e<3v>Zp&9_{e>M(=+&2@$X(x+KIu3r( zL4)T~2oMF;g8K29qxwP^-NdMb|JAjHmMy5V1CYA=A#sgl=LSjd{z>RK=8#-D0ir1+ zqmaz9LC|BaV(G7B;5g>ETphw>bf}WYAyB$WLd>HQ!m>%wKJnQ+0iq*%l~ED{~uvln@+CJ20R#8EjAb!?f*%+ zQ+L*I0Y1i9N7!FVO*v~wsm9z?XmFjTKP|k-V^q=5j^He~w1M!P#yQH|spjTD;PkYs zb=|O*9qOqZ(^G5RB96X2c~QAMYD`_v^?UF2dwI)s0LR6&BaFh=>TAMt?@rgw^JVIn z&w~pX!>toOOY-eJno)Tn0!xNVLkJlPZPE<_VB4oGPCNX@7QaE&8P}+$5C;}}vL773 zL7f#B);9WH__I4-B=TkV?}rbh`VQVej<-L@b$7Ux6Y`#epm1M7TjUK2$(@zKdwc8eqGw!Ul?mCN02fgw_ z1sxrjMi+_dg-{jciw)MsB?$u+X+?)E0BiSMbxovt=oZHDwd@me1&r^z00X+vPxEO$rzdR_YR9ymou&{zu)K*!1TTRG9EJbU-s*MS=o_hC%b+vx%ubY~WHvf~kvu^k( z5pmgY2w27`=qy|49b6uyb7#+OJnQHsOt(0BjVOgw7~8a(Se~jJWZER><~%m{0M;5o zc6#qr?vfMz1t`DV8uFQE*&q<@*=6K_9fs0c*K~>rpyeR$fzF7o$>#L6a$T5)Ev43t zG=)!cA%nhN1c`IC*7WVAx}!}uuJgEBlZK4OW^o0;3eyISSh1N>zW?cF&azuQEW}fo zSb~#)2xg93dj0}q05G{CmynJXFj{CK+fLRwiJr7{`PBbO1xw|GQ|nHrK^>!}LB?{R zZeCnwR{}9l)XeTqW@cLwklzf4uRHEyn8Ua(CjAZA5prqYkalZ>UyyvO>-yF1=(j|< zWnIB|gRwvN^-aOt&^t(R4S$QT>*^yZ#UL^(j>VzGX1%l^{d{?qd8)|+pfE&NsC!`U zP?CtGHsDM~-7K6Z3V$!{e>0~>w|Hr z{igU10dQ2imGX}!2pl{96kq11c{C-Kmu=^llHW~cQ=@5mnE#j`t(2RnwUK$~(a>Y4 zESJ~mq1+tN@W=mQV)LVH+C9IlY(ER6Jr_@c-2+l*>+iJ1Q@!N^_~(Vi`JQ=~q_1fD zL+)s}FgR-8GNo&b%vG#m()Ugg?Ui`q@qrCczxDc%7!lF@K(wN=2eDBW(^L2% z`B5|}?3|R!2v=0Zvq_M~;KGvgIkqp?Oo{*XN<6g;PH?wten{#-W9 z_rNmg^|2;7o{))iC!W*!4!BmsBbye}a}YO# zcX;ps;ANN!1ZbY1~hv1vdNMKW4PuVRTmoAo2vMh?jDvQ6SwCzL6R=1Fh;lLRni zs4|%^F2D`JQwD3*-i*q(TV9}bt1%$EKMRPL5fQ`9PFJmRp22%Fga2?QLjE=65@vRL zU>%pr9eHCc=mK$X`X`D#zMPIT*2Y^HRb7V_5T8!R=>CMm=T~Ry^b6=!1oT4pp=A$` z&6}d0KBf-&HMQ2YxYnh3!Q}B&JiXmylVr6Y`KwW;-Lm5#o43pIl~XI%Kg>R6mz;<^ zmAJxQ3^JgB3~>X5`Y1m+n0EMvvfr7#-;0o8#&xvJg%!t@Iiz>-ho5MuCCo*rsP@kw zpgrL;)Cp@k4t;#kdIWe&w0EYCH{u4)W(KQZI+CSMZLk$rT>)2`9YS9sU;g`vlg2uO zl>Ol-Nk2?i%8Zb&r6*P};1x6X`%i^Gv%KL9)>hOI`u|k24S4iaxBXVs0{XMJYHH39iKO+wUILxLBh*iwb~6HP zr-J@!ayCPucsqKI`V0+_1SPgC-2tpu z20?po6xi5Ery?X5|1|Q@5Tf@m%DwmCehnz%HKbl&khnib{k#VcnGMy6MLCJzSB{mSru-M7YIf>C&TK{asy8rb%F zI0J2{ddgkg_P%$+U07>uEGhXiF>IfuY*B?>PFp<)8O#cFMIu9gxRzhM_L}3WRT{(! zvT|tI;t12!ldM-%E8S>_&bSt*Tav&3U>3F(GdoBbt{YJLcz(+}1Y;VCwPqn}(iVHf z53|_BuBEQ;iZwYadD~U5D^_qs=rnYt?Nd6s5K`OA@DnPsV>+8ZJEPbe4*AOef=KN@ zBm%x3kRkp5OocQz^sxW8sW27%1Sj>?1r6z+7vaC9G#Jh)buJJ)mB^JS74`%zRpOQa z95ogEmOeG=mKDOx^WQ;|)F2<&)SX*2qW>&VP+(xI|I7@513LtG>3`6<67&CD5z+tri~66YM#}#Y z6(QF8{)=7u$PE!b_#a#uLrxjR`|p0xJP|MOB diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index a21c6ebe28..c747538fb3 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,6 +1,7 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-8.1-all.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.2.1-all.zip networkTimeout=10000 +validateDistributionUrl=true zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/gradlew b/gradlew index aeb74cbb43..fcb6fca147 100755 --- a/gradlew +++ b/gradlew @@ -130,10 +130,13 @@ location of your Java installation." fi else JAVACMD=java - which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + if ! command -v java >/dev/null 2>&1 + then + die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. Please set the JAVA_HOME variable in your environment to match the location of your Java installation." + fi fi # Increase the maximum file descriptors if we can. From 000122cae7a1b170a719d98e25ba0267472cbdaa Mon Sep 17 00:00:00 2001 From: Paul Horn Date: Wed, 19 Jul 2023 14:47:21 +0200 Subject: [PATCH 149/273] Add migration guide for Alpha Aggregation to new Cypher projection --- doc/modules/ROOT/content-nav.adoc | 1 + .../migration-alpha-cp-to-cpv2/index.adoc | 185 ++++++++++++++++++ 2 files changed, 186 insertions(+) create mode 100644 doc/modules/ROOT/pages/migration-alpha-cp-to-cpv2/index.adoc diff --git a/doc/modules/ROOT/content-nav.adoc b/doc/modules/ROOT/content-nav.adoc index 61dd42e6c5..75e849c4e4 100644 --- a/doc/modules/ROOT/content-nav.adoc +++ b/doc/modules/ROOT/content-nav.adoc @@ -169,3 +169,4 @@ *** xref:migration-gds-1-to-gds-2/migration-algorithms.adoc[] *** xref:migration-gds-1-to-gds-2/migration-ml.adoc[] ** xref:migration-lcp-to-cpv2/index.adoc[] +** xref:migration-alpha-cp-to-cpv2/index.adoc[] diff --git a/doc/modules/ROOT/pages/migration-alpha-cp-to-cpv2/index.adoc b/doc/modules/ROOT/pages/migration-alpha-cp-to-cpv2/index.adoc new file mode 100644 index 0000000000..92e74e8c0b --- /dev/null +++ b/doc/modules/ROOT/pages/migration-alpha-cp-to-cpv2/index.adoc @@ -0,0 +1,185 @@ +[appendix] +[[appendix-d]] += Migration from Alpha Cypher Aggregation to new Cypher projection +:description: If you have been using `gds.alpha.graph.project` Cypher aggregation, you can find the info you will need to migrate to using the new Cypher projection. + + +== Who should read this guide + +This guide is intended for users who have been using the Alpha Cypher Aggregation https://neo4j.com/docs/graph-data-science/2.3/management-ops/projections/graph-project-cypher-aggregation/[`gds.alpha.graph.project`]. +Cypher projections are now done using the `gds.graph.project` aggregation function. +We assume that most of the mentioned operations and concepts can be understood with little explanation. +Thus we are intentionally brief in the examples and comparisons. +Please see xref:management-ops/graph-creation/graph-project-cypher-projection.adoc[the documentation for the Cypher projection] for more details. + +== API Changes + +The new Cypher projection is a replacement for the Alpha Cypher Aggregation. +Like the Alpha Cypher Aggregation, the new Cypher projection is an aggregation function that is called as part of a Cypher query. + +The following changes have been made to the API: + +* The new Cypher projection is called using `gds.graph.project` instead of `gds.alpha.graph.project`. +* The new Cypher projection defines a single map parameter for defining projection related information such as labels or properties. +** There is still a separate map parameter for defining the graph configuration. +* The `properties` key of the relationship configuration map has been renamed to `relationshipProperties`. +* Additional validation to reduce mis-use of the API: +** Validate that each `sourceNode*` entry has a corresponding `targetNode*` entry and vice-versa. +** Identify end help with migration to this new API if any of the points above have not been followed. + + +.Structural changes between the Alpha aggregation/new projections: +[opts=header,cols="1a,1a"] +|=== +| Legacy | New +| +[source, cypher, role=noplay] +---- +$query +RETURN gds.alpha.graph.project( + $graphName, + sourceNode, + targetNode, + $nodeConfig, + $relationshipConfig, + $configuration +) +---- +| +[source, cypher, role=noplay] +---- +$relationshipQuery +RETURN gds.graph.project( + $graphName, + sourceNode, + targetNode, + $dataConfig, + $configuration +) +---- +|=== + +== Examples + +The following examples not include the full Cypher queries before the aggreation function is called, nor do they include and YIELD of return fields. +There no changes related to those two aspects. + +.Side-by-side comparison +[opts=header,cols="1a,1a"] +|=== +| Alpha | New +2+| : Projection without any configuration +| +[source, cypher, role=noplay] +---- +... +RETURN gds.alpha.graph.project('g', source, target) +---- +| +[source, cypher, role=noplay] +---- +... +RETURN gds.graph.project('g', source, target) +---- +2+| : Multi-graph projection +| +[source, cypher, role=noplay] +---- +... +RETURN gds.alpha.graph.project( + 'g', + source, + target, + { + sourceNodeLabels: labels(source), + targetNodeLabels: labels(target), + }, { + relationshipType: type(r) + } +) +---- +| +[source, cypher, role=noplay] +---- +... +RETURN gds.graph.project( + 'g', + source, + target, + { + sourceNodeLabels: labels(source), + targetNodeLabels: labels(target), + relationshipType: type(rel) + } +) +---- +2+| : Graph projection with properties +| +[source, cypher, role=noplay] +---- +... +RETURN gds.alpha.graph.project( + 'g', + source, + target, + { + sourceNodeLabels: labels(source), + targetNodeLabels: labels(target), + sourceNodeProperties: source { .age }, + targetNodeProperties: target { .age }, + }, { + relationshipType: type(rel), + properties: rel { .numberOfPages } + } +) +---- +| +[source, cypher, role=noplay] +---- +... +RETURN gds.graph.project( + 'g', + source, + target, + { + sourceNodeLabels: labels(source), + targetNodeLabels: labels(target), + sourceNodeProperties: source { .age }, + targetNodeProperties: target { .age }, + relationshipType: type(rel), + relationshipProperties: rel { .numberOfPages } + } +) +---- +2+| : Graph projection with one-sided properties +| +[source, cypher, role=noplay] +---- +... +RETURN gds.alpha.graph.project( + 'g', + source, + target, + { + sourceNodeLabels: labels(source), + sourceNodeProperties: source { .age }, + } +) +---- +| +[source, cypher, role=noplay] +---- +... +RETURN gds.graph.project( + 'g', + source, + target, + { + sourceNodeLabels: labels(source), + targetNodeLabels: NULL, + sourceNodeProperties: source { .age }, + targetNodeProperties: NULL, + } +) +---- +|=== From dacccbf74f201438f6accc761e11ecfd5e00b744 Mon Sep 17 00:00:00 2001 From: Paul Horn Date: Thu, 20 Jul 2023 10:43:21 +0200 Subject: [PATCH 150/273] Fix cross link --- doc/modules/ROOT/pages/migration-alpha-cp-to-cpv2/index.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/modules/ROOT/pages/migration-alpha-cp-to-cpv2/index.adoc b/doc/modules/ROOT/pages/migration-alpha-cp-to-cpv2/index.adoc index 92e74e8c0b..9f053ef870 100644 --- a/doc/modules/ROOT/pages/migration-alpha-cp-to-cpv2/index.adoc +++ b/doc/modules/ROOT/pages/migration-alpha-cp-to-cpv2/index.adoc @@ -10,7 +10,7 @@ This guide is intended for users who have been using the Alpha Cypher Aggregatio Cypher projections are now done using the `gds.graph.project` aggregation function. We assume that most of the mentioned operations and concepts can be understood with little explanation. Thus we are intentionally brief in the examples and comparisons. -Please see xref:management-ops/graph-creation/graph-project-cypher-projection.adoc[the documentation for the Cypher projection] for more details. +Please see xref:management-ops/projections/graph-project-cypher-projection.adoc[the documentation for the Cypher projection] for more details. == API Changes From 25f29cafb87904a4d0117e14dd6f73e30518f2e8 Mon Sep 17 00:00:00 2001 From: Paul Horn Date: Thu, 20 Jul 2023 10:43:29 +0200 Subject: [PATCH 151/273] Add missing verb MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Sören Reichardt --- doc/modules/ROOT/pages/migration-alpha-cp-to-cpv2/index.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/modules/ROOT/pages/migration-alpha-cp-to-cpv2/index.adoc b/doc/modules/ROOT/pages/migration-alpha-cp-to-cpv2/index.adoc index 9f053ef870..cf8df31d8e 100644 --- a/doc/modules/ROOT/pages/migration-alpha-cp-to-cpv2/index.adoc +++ b/doc/modules/ROOT/pages/migration-alpha-cp-to-cpv2/index.adoc @@ -62,7 +62,7 @@ RETURN gds.graph.project( == Examples The following examples not include the full Cypher queries before the aggreation function is called, nor do they include and YIELD of return fields. -There no changes related to those two aspects. +There are no changes related to those two aspects. .Side-by-side comparison [opts=header,cols="1a,1a"] From 5877eba67c731274551ca935a8e15ab28d4c0f20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Thu, 20 Jul 2023 15:23:04 +0200 Subject: [PATCH 152/273] Fix redirect --- .../projections/graph-project-cypher-projection.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/modules/ROOT/pages/management-ops/projections/graph-project-cypher-projection.adoc b/doc/modules/ROOT/pages/management-ops/projections/graph-project-cypher-projection.adoc index 9ae8a056f8..03b2ef762b 100644 --- a/doc/modules/ROOT/pages/management-ops/projections/graph-project-cypher-projection.adoc +++ b/doc/modules/ROOT/pages/management-ops/projections/graph-project-cypher-projection.adoc @@ -1,7 +1,7 @@ [[catalog-graph-project-cypher-projection]] = Projecting graphs using Cypher :description: This section details projecting GDS graphs using `Cypher` projections. -:page-aliases: managments-ops/projections/graph-project-cypher-aggregation.adoc +:page-aliases: management-ops/projections/graph-project-cypher-aggregation Using Cypher projection is a more flexible and expressive approach with diminished focus on performance compared to the xref:management-ops/projections/graph-project.adoc[native projections]. From 416cee2dbee930aa6b55fe2833a0d73c2602574a Mon Sep 17 00:00:00 2001 From: Nicola Vitucci Date: Thu, 20 Jul 2023 14:32:01 +0200 Subject: [PATCH 153/273] Fix reference to paper --- doc/modules/ROOT/pages/algorithms/kmeans.adoc | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/doc/modules/ROOT/pages/algorithms/kmeans.adoc b/doc/modules/ROOT/pages/algorithms/kmeans.adoc index 400ee19410..468b33e572 100644 --- a/doc/modules/ROOT/pages/algorithms/kmeans.adoc +++ b/doc/modules/ROOT/pages/algorithms/kmeans.adoc @@ -37,20 +37,19 @@ For more information on this algorithm, see: [[algorithms-kmeans-sampling]] == Initial Centroid Sampling -The algorithm starts by picking `k` centroids by randomly sampling from the set of available nodes. +The algorithm starts by picking `k` centroids by randomly sampling from the set of available nodes. There are two different sampling strategies. Uniform:: -With uniform sampling, each node has the same probability to be picked as one of the `k` initial centroids. +With uniform sampling, each node has the same probability to be picked as one of the `k` initial centroids. This is the default sampler for K-Means denoted with the `uniform` parameter. K-Means++:: -This sampling strategy adapts the well-known -K-means\++ http://ilpubs.stanford.edu:8090/778/1/2006-13.pdf[initialization algorithm] for K-Means. -The sampling begins by choosing the first centroid uniformly at random. -Then, the remaining `k-1` centroids are picked one-by-one based on weighted random sampling. +This sampling strategy adapts the well-known K-means\++ initialization algorithmfootnote:[Arthur, David and Sergei Vassilvitskii. "k-means{plus}{plus}: The Advantages of Careful Seeding." _ACM-SIAM Symposium on Discrete Algorithms_, 2007.] for K-Means. +The sampling begins by choosing the first centroid uniformly at random. +Then, the remaining `k-1` centroids are picked one-by-one based on weighted random sampling. That is, the probability a node is chosen as the next centroid is proportional to its minimum distance from the already picked centroids. Nodes with larger distance hence have higher chance to be picked as a centroid. This sampling strategy tries to spread the initial clusters more evenly so as to obtain a better final clustering. This option can be enabled by choosing `kmeans++` as the initial sampler in the configuration. @@ -105,7 +104,7 @@ include::partial$/algorithms/kmeans/specific-configuration.adoc[] | nodeId | Integer | Node ID. | communityId | Integer | The community ID. | distanceFromCentroid | Float | Distance of the node from the centroid of its community. -| silhouette | Float | Silhouette score of the node. +| silhouette | Float | Silhouette score of the node. |=== ====== From 81f89ba3340f696f27b27bef337780671d1743ba Mon Sep 17 00:00:00 2001 From: Nicola Vitucci Date: Thu, 20 Jul 2023 14:34:46 +0200 Subject: [PATCH 154/273] Update citation --- doc/modules/ROOT/pages/algorithms/kmeans.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/modules/ROOT/pages/algorithms/kmeans.adoc b/doc/modules/ROOT/pages/algorithms/kmeans.adoc index 468b33e572..b7f276cd17 100644 --- a/doc/modules/ROOT/pages/algorithms/kmeans.adoc +++ b/doc/modules/ROOT/pages/algorithms/kmeans.adoc @@ -47,7 +47,7 @@ This is the default sampler for K-Means denoted with the `uniform` parameter. K-Means++:: -This sampling strategy adapts the well-known K-means\++ initialization algorithmfootnote:[Arthur, David and Sergei Vassilvitskii. "k-means{plus}{plus}: The Advantages of Careful Seeding." _ACM-SIAM Symposium on Discrete Algorithms_, 2007.] for K-Means. +This sampling strategy adapts the well-known K-means\++ initialization algorithmfootnote:[Arthur, David and Sergei Vassilvitskii. "k-means{plus}{plus}: The Advantages of Careful Seeding." _ACM-SIAM Symposium on Discrete Algorithms_ (2007).] for K-Means. The sampling begins by choosing the first centroid uniformly at random. Then, the remaining `k-1` centroids are picked one-by-one based on weighted random sampling. That is, the probability a node is chosen as the next centroid is proportional to its minimum distance from the already picked centroids. From 46da99d609e5f8c36cf2779081f01495eba423e6 Mon Sep 17 00:00:00 2001 From: yuval Date: Tue, 18 Jul 2023 10:32:20 +0200 Subject: [PATCH 155/273] Using label mapping for export The mapping was written to a new file and not added to the existing node schema. The advantage is easier support for old exports (without mapping), because the import code maintains compatibility. --- .../neo4j/gds/core/io/GraphStoreExporter.java | 10 ++- .../neo4j/gds/core/io/GraphStoreInput.java | 5 ++ .../neo4j/gds/core/io/NodeLabelMapping.java | 47 ++++++++++++ .../java/org/neo4j/gds/core/io/NodeStore.java | 24 +++++-- .../io/file/GraphStoreToFileExporter.java | 21 ++++++ .../gds/core/io/schema/SimpleVisitor.java | 27 +++++++ .../file/csv/CsvNodeLabelMappingVisitor.java | 72 +++++++++++++++++++ .../io/file/csv/GraphStoreToCsvExporter.java | 1 + 8 files changed, 202 insertions(+), 5 deletions(-) create mode 100644 io/core/src/main/java/org/neo4j/gds/core/io/NodeLabelMapping.java create mode 100644 io/core/src/main/java/org/neo4j/gds/core/io/schema/SimpleVisitor.java create mode 100644 io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/CsvNodeLabelMappingVisitor.java diff --git a/io/core/src/main/java/org/neo4j/gds/core/io/GraphStoreExporter.java b/io/core/src/main/java/org/neo4j/gds/core/io/GraphStoreExporter.java index 7d199c705a..8ed9039957 100644 --- a/io/core/src/main/java/org/neo4j/gds/core/io/GraphStoreExporter.java +++ b/io/core/src/main/java/org/neo4j/gds/core/io/GraphStoreExporter.java @@ -20,6 +20,7 @@ package org.neo4j.gds.core.io; import org.neo4j.common.Validator; +import org.neo4j.gds.NodeLabel; import org.neo4j.gds.annotation.ValueClass; import org.neo4j.gds.api.GraphStore; import org.neo4j.gds.api.IdMap; @@ -99,7 +100,14 @@ protected GraphStoreExporter( public ExportedProperties run() { var metaDataStore = MetaDataStore.of(graphStore); - var nodeStore = NodeStore.of(graphStore, neoNodeProperties); + // todo: map only on config + var nodeStore = NodeStore.of( + graphStore, + neoNodeProperties, + true + ? Optional.of(new NodeLabelMapping(graphStore.nodeLabels())) + : Optional.empty() + ); var relationshipStore = RelationshipStore.of(graphStore, config.defaultRelationshipType()); var graphProperties = graphStore .graphPropertyKeys() diff --git a/io/core/src/main/java/org/neo4j/gds/core/io/GraphStoreInput.java b/io/core/src/main/java/org/neo4j/gds/core/io/GraphStoreInput.java index 7a74e29dbc..d3e2b5e581 100644 --- a/io/core/src/main/java/org/neo4j/gds/core/io/GraphStoreInput.java +++ b/io/core/src/main/java/org/neo4j/gds/core/io/GraphStoreInput.java @@ -48,6 +48,7 @@ import java.util.Iterator; import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.Queue; import java.util.Set; import java.util.Spliterator; @@ -181,6 +182,10 @@ public InputIterable relationships(Collector badCollector) { return () -> new RelationshipImporter(relationshipStore, batchSize, idMode.get(), idMapFunction); } + public Optional labelMapping() { + return nodeStore.labelMapping(); + } + @Override public IdType idType() { return idMode.idType; diff --git a/io/core/src/main/java/org/neo4j/gds/core/io/NodeLabelMapping.java b/io/core/src/main/java/org/neo4j/gds/core/io/NodeLabelMapping.java new file mode 100644 index 0000000000..563ccea684 --- /dev/null +++ b/io/core/src/main/java/org/neo4j/gds/core/io/NodeLabelMapping.java @@ -0,0 +1,47 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.core.io; + +import org.neo4j.gds.NodeLabel; + +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +public class NodeLabelMapping { + private final HashMap map; + + NodeLabelMapping(Collection labels) { + var indexGenerator = IntStream.range(0, labels.size()).iterator(); + map = new HashMap<>(labels.stream() + .collect(Collectors.toMap(label -> label, label -> Integer.toString(indexGenerator.next())))); + } + + public String get(NodeLabel label) { + return map.get(label); + } + + public Set> entrySet() { + return map.entrySet(); + } +} diff --git a/io/core/src/main/java/org/neo4j/gds/core/io/NodeStore.java b/io/core/src/main/java/org/neo4j/gds/core/io/NodeStore.java index bf75afe7c0..867baa366a 100644 --- a/io/core/src/main/java/org/neo4j/gds/core/io/NodeStore.java +++ b/io/core/src/main/java/org/neo4j/gds/core/io/NodeStore.java @@ -27,7 +27,9 @@ import java.util.HashMap; import java.util.Map; +import java.util.Optional; import java.util.Set; +import java.util.function.Function; import java.util.function.LongFunction; public class NodeStore { @@ -47,13 +49,17 @@ public class NodeStore { private final boolean hasLabels; + private final Optional nodeLabelMapping; + private final Function labelNameFunction; + private NodeStore( long nodeCount, HugeIntArray labelCounts, IdMap idMap, boolean hasLabels, Map> nodeProperties, - Map> additionalProperties + Map> additionalProperties, + Optional nodeLabelMapping ) { this.nodeCount = nodeCount; this.labelCounts = labelCounts; @@ -62,6 +68,10 @@ private NodeStore( this.hasLabels = hasLabels; this.availableNodeLabels = idMap.availableNodeLabels(); this.additionalProperties = additionalProperties; + this.nodeLabelMapping = nodeLabelMapping; + this.labelNameFunction = nodeLabelMapping.isPresent() + ? nodeLabelMapping.get()::get + : (NodeLabel nodeLabel) -> nodeLabel.name(); } boolean hasLabels() { @@ -76,6 +86,10 @@ int labelCount() { return !hasLabels() ? 0 : idMap.availableNodeLabels().size(); } + Optional labelMapping() { + return nodeLabelMapping; + } + int propertyCount() { if (nodeProperties == null) { return 0; @@ -94,7 +108,7 @@ String[] labels(long nodeId) { int i = 0; for (var nodeLabel : availableNodeLabels) { if (idMap.hasLabel(nodeId, nodeLabel)) { - labels[i++] = nodeLabel.name; + labels[i++] = labelNameFunction.apply(nodeLabel); } } @@ -103,7 +117,8 @@ String[] labels(long nodeId) { static NodeStore of( GraphStore graphStore, - Map> additionalProperties + Map> additionalProperties, + Optional nodeLabelMapping ) { HugeIntArray labelCounts = null; @@ -137,7 +152,8 @@ static NodeStore of( nodeLabels, hasNodeLabels, nodeProperties.isEmpty() ? null : nodeProperties, - additionalProperties + additionalProperties, + nodeLabelMapping ); } } diff --git a/io/core/src/main/java/org/neo4j/gds/core/io/file/GraphStoreToFileExporter.java b/io/core/src/main/java/org/neo4j/gds/core/io/file/GraphStoreToFileExporter.java index e3096445ed..b1cc9b1358 100644 --- a/io/core/src/main/java/org/neo4j/gds/core/io/file/GraphStoreToFileExporter.java +++ b/io/core/src/main/java/org/neo4j/gds/core/io/file/GraphStoreToFileExporter.java @@ -19,6 +19,7 @@ */ package org.neo4j.gds.core.io.file; +import org.neo4j.gds.NodeLabel; import org.neo4j.gds.api.GraphStore; import org.neo4j.gds.compat.CompatInput; import org.neo4j.gds.core.concurrency.ParallelUtil; @@ -29,6 +30,7 @@ import org.neo4j.gds.core.io.schema.ElementSchemaVisitor; import org.neo4j.gds.core.io.schema.NodeSchemaVisitor; import org.neo4j.gds.core.io.schema.RelationshipSchemaVisitor; +import org.neo4j.gds.core.io.schema.SimpleVisitor; import org.neo4j.gds.core.loading.Capabilities; import org.neo4j.gds.core.utils.progress.TaskRegistryFactory; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; @@ -41,6 +43,7 @@ import java.io.IOException; import java.io.UncheckedIOException; import java.util.ArrayList; +import java.util.Map; import java.util.Optional; import java.util.function.Supplier; @@ -53,6 +56,8 @@ public class GraphStoreToFileExporter extends GraphStoreExporter> userInfoVisitorSupplier; private final Supplier> graphInfoVisitorSupplier; private final Supplier nodeSchemaVisitorSupplier; + private final Supplier>> labelMappingVisitorSupplierVisitorSupplier; + private final Supplier relationshipSchemaVisitorSupplier; private final Supplier graphPropertySchemaVisitorSupplier; private final Supplier> graphCapabilitiesWriterSupplier; @@ -68,6 +73,7 @@ public GraphStoreToFileExporter( Supplier> userInfoVisitorSupplier, Supplier> graphInfoVisitorSupplier, Supplier nodeSchemaVisitorSupplier, + Supplier>> labelMappingVisitorSupplierVisitorSupplier, Supplier relationshipSchemaVisitorSupplier, Supplier graphPropertySchemaVisitorSupplier, Supplier> graphCapabilitiesWriterSupplier, @@ -85,6 +91,7 @@ public GraphStoreToFileExporter( this.userInfoVisitorSupplier = userInfoVisitorSupplier; this.graphInfoVisitorSupplier = graphInfoVisitorSupplier; this.nodeSchemaVisitorSupplier = nodeSchemaVisitorSupplier; + this.labelMappingVisitorSupplierVisitorSupplier = labelMappingVisitorSupplierVisitorSupplier; this.relationshipSchemaVisitorSupplier = relationshipSchemaVisitorSupplier; this.graphPropertySchemaVisitorSupplier = graphPropertySchemaVisitorSupplier; this.graphCapabilitiesWriterSupplier = graphCapabilitiesWriterSupplier; @@ -103,6 +110,9 @@ protected void export(GraphStoreInput graphStoreInput) { exportGraphPropertySchema(graphStoreInput); exportGraphCapabilities(graphStoreInput); } + // todo: if configuration then: + exportNodeLabelMapping(graphStoreInput); + var progressTracker = createProgressTracker(graphStoreInput); try { @@ -246,6 +256,17 @@ private void exportNodeSchema(GraphStoreInput graphStoreInput) { } } + private void exportNodeLabelMapping(GraphStoreInput graphStoreInput) { + var labelMapping = graphStoreInput.labelMapping(); + if (labelMapping.isPresent()) { + try (var labelMappingVisitor = labelMappingVisitorSupplierVisitorSupplier.get()) { + labelMapping.get().entrySet().forEach(entry -> labelMappingVisitor.export(entry)); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + } + private void exportRelationshipSchema(GraphStoreInput graphStoreInput) { var relationshipSchema = graphStoreInput.metaDataStore().relationshipSchema(); try (var relationshipSchemaVisitor = relationshipSchemaVisitorSupplier.get()) { diff --git a/io/core/src/main/java/org/neo4j/gds/core/io/schema/SimpleVisitor.java b/io/core/src/main/java/org/neo4j/gds/core/io/schema/SimpleVisitor.java new file mode 100644 index 0000000000..4eaaab61ed --- /dev/null +++ b/io/core/src/main/java/org/neo4j/gds/core/io/schema/SimpleVisitor.java @@ -0,0 +1,27 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.core.io.schema; + +import java.io.Closeable; +import java.io.IOException; + +public interface SimpleVisitor extends Closeable { + void export(T value); +} diff --git a/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/CsvNodeLabelMappingVisitor.java b/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/CsvNodeLabelMappingVisitor.java new file mode 100644 index 0000000000..a556c375f7 --- /dev/null +++ b/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/CsvNodeLabelMappingVisitor.java @@ -0,0 +1,72 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.core.io.file.csv; + +import de.siegmar.fastcsv.writer.CsvWriter; +import org.neo4j.gds.NodeLabel; +import org.neo4j.gds.core.io.schema.SimpleVisitor; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Map; +import java.util.Set; + +public class CsvNodeLabelMappingVisitor implements SimpleVisitor> { + + private static final String LABEL_MAPPING = "index"; + private static final String LABEL_COLUMN_NAME = "label"; + private static final String LABEL_MAPPING_FILE_NAME = "label-mappings.csv"; + private final CsvWriter csvWriter; + + CsvNodeLabelMappingVisitor(Path fileLocation) { + try { + this.csvWriter = CsvWriter.builder().build(fileLocation.resolve(LABEL_MAPPING_FILE_NAME), StandardCharsets.UTF_8); + writeHeader(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Override + public void export(Map.Entry nodeLabelMapping) { + var row = new ArrayList(); + row.add(nodeLabelMapping.getValue()); + row.add(nodeLabelMapping.getKey().name()); + csvWriter.writeRow(row); + } + + private void writeHeader() { + csvWriter.writeRow( + LABEL_MAPPING, + LABEL_COLUMN_NAME + ); + } + + @Override + public void close() throws IOException { + try { + csvWriter.close(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } +} diff --git a/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/GraphStoreToCsvExporter.java b/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/GraphStoreToCsvExporter.java index e606f114b9..477e85dd70 100644 --- a/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/GraphStoreToCsvExporter.java +++ b/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/GraphStoreToCsvExporter.java @@ -76,6 +76,7 @@ public static GraphStoreToFileExporter create( () -> new UserInfoVisitor(exportPath), () -> new CsvGraphInfoVisitor(exportPath), () -> new CsvNodeSchemaVisitor(exportPath), + () -> new CsvNodeLabelMappingVisitor(exportPath), () -> new CsvRelationshipSchemaVisitor(exportPath), () -> new CsvGraphPropertySchemaVisitor(exportPath), () -> new CsvGraphCapabilitiesWriter(exportPath), From 9c31187cacfd150e30015ad366df344fced521e6 Mon Sep 17 00:00:00 2001 From: yuval Date: Tue, 18 Jul 2023 14:16:10 +0200 Subject: [PATCH 156/273] Added label mapping conf to CSV export --- .../java/org/neo4j/gds/core/io/GraphStoreExporter.java | 9 ++++++--- .../gds/core/io/db/GraphStoreToDatabaseExporter.java | 2 +- .../neo4j/gds/core/io/file/GraphStoreToFileExporter.java | 3 +-- .../gds/core/io/file/GraphStoreToFileExporterConfig.java | 7 +++++++ 4 files changed, 15 insertions(+), 6 deletions(-) diff --git a/io/core/src/main/java/org/neo4j/gds/core/io/GraphStoreExporter.java b/io/core/src/main/java/org/neo4j/gds/core/io/GraphStoreExporter.java index 8ed9039957..6b6cc4ba6b 100644 --- a/io/core/src/main/java/org/neo4j/gds/core/io/GraphStoreExporter.java +++ b/io/core/src/main/java/org/neo4j/gds/core/io/GraphStoreExporter.java @@ -41,6 +41,8 @@ public abstract class GraphStoreExporter> neoNodeProperties; + private final boolean useLabelMapping; + public enum IdMappingType implements IdMapFunction { MAPPED { @Override @@ -85,13 +87,15 @@ interface IdMapFunction { protected GraphStoreExporter( GraphStore graphStore, CONFIG config, - Optional neoNodeProperties + Optional neoNodeProperties, + boolean useLabelMapping ) { this.graphStore = graphStore; this.config = config; this.neoNodeProperties = neoNodeProperties .map(NeoNodeProperties::neoNodeProperties) .orElse(Map.of()); + this.useLabelMapping = useLabelMapping; } protected abstract void export(GraphStoreInput graphStoreInput); @@ -100,11 +104,10 @@ protected GraphStoreExporter( public ExportedProperties run() { var metaDataStore = MetaDataStore.of(graphStore); - // todo: map only on config var nodeStore = NodeStore.of( graphStore, neoNodeProperties, - true + useLabelMapping ? Optional.of(new NodeLabelMapping(graphStore.nodeLabels())) : Optional.empty() ); diff --git a/io/core/src/main/java/org/neo4j/gds/core/io/db/GraphStoreToDatabaseExporter.java b/io/core/src/main/java/org/neo4j/gds/core/io/db/GraphStoreToDatabaseExporter.java index a4b3e28128..88ad125533 100644 --- a/io/core/src/main/java/org/neo4j/gds/core/io/db/GraphStoreToDatabaseExporter.java +++ b/io/core/src/main/java/org/neo4j/gds/core/io/db/GraphStoreToDatabaseExporter.java @@ -64,7 +64,7 @@ private GraphStoreToDatabaseExporter( Log log, ProgressTracker progressTracker ) { - super(graphStore, config, neoNodeProperties); + super(graphStore, config, neoNodeProperties, false); var executionMonitor = ProgressTrackerExecutionMonitor.of( progressTracker, ClockService.clock(), diff --git a/io/core/src/main/java/org/neo4j/gds/core/io/file/GraphStoreToFileExporter.java b/io/core/src/main/java/org/neo4j/gds/core/io/file/GraphStoreToFileExporter.java index b1cc9b1358..ee4c801f0b 100644 --- a/io/core/src/main/java/org/neo4j/gds/core/io/file/GraphStoreToFileExporter.java +++ b/io/core/src/main/java/org/neo4j/gds/core/io/file/GraphStoreToFileExporter.java @@ -84,7 +84,7 @@ public GraphStoreToFileExporter( Log log, String rootTaskName ) { - super(graphStore, config, neoNodeProperties); + super(graphStore, config, neoNodeProperties, config.useLabelMapping()); this.nodeVisitorSupplier = nodeVisitorSupplier; this.relationshipVisitorSupplier = relationshipVisitorSupplier; this.graphPropertyVisitorSupplier = graphPropertyVisitorSupplier; @@ -110,7 +110,6 @@ protected void export(GraphStoreInput graphStoreInput) { exportGraphPropertySchema(graphStoreInput); exportGraphCapabilities(graphStoreInput); } - // todo: if configuration then: exportNodeLabelMapping(graphStoreInput); var progressTracker = createProgressTracker(graphStoreInput); diff --git a/io/core/src/main/java/org/neo4j/gds/core/io/file/GraphStoreToFileExporterConfig.java b/io/core/src/main/java/org/neo4j/gds/core/io/file/GraphStoreToFileExporterConfig.java index 5fbc0e944f..d5ae4d843e 100644 --- a/io/core/src/main/java/org/neo4j/gds/core/io/file/GraphStoreToFileExporterConfig.java +++ b/io/core/src/main/java/org/neo4j/gds/core/io/file/GraphStoreToFileExporterConfig.java @@ -42,6 +42,13 @@ default String username() { String exportName(); + @Value.Default + default boolean useLabelMapping() { + // the default should generally be true, however neo4j admin, backwards compatability and maybe some feedback + // about this change from field engineers should be taken into account first. + return false; + } + static GraphStoreToFileExporterConfig of(String username, CypherMapWrapper config) { return new GraphStoreToFileExporterConfigImpl(username, config); } From b586887965ee77988cc322b7e2066f880d02ddaa Mon Sep 17 00:00:00 2001 From: yuval Date: Thu, 20 Jul 2023 22:35:18 +0200 Subject: [PATCH 157/273] Implemented label mapping for CSV import --- .../gds/core/io/NodeLabelInverseMapping.java | 47 ++++++++++++ .../org/neo4j/gds/core/io/file/FileInput.java | 5 ++ .../io/file/FileToGraphStoreImporter.java | 8 ++ .../gds/core/io/file/csv/CsvFileInput.java | 22 +++++- .../core/io/file/csv/CsvImportFileUtil.java | 15 +++- .../file/csv/CsvNodeLabelMappingVisitor.java | 2 +- .../io/file/csv/NodeLabelMappingLoader.java | 76 +++++++++++++++++++ .../io/file/csv/CsvImportFileUtilTest.java | 2 +- 8 files changed, 168 insertions(+), 9 deletions(-) create mode 100644 io/core/src/main/java/org/neo4j/gds/core/io/NodeLabelInverseMapping.java create mode 100644 io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/NodeLabelMappingLoader.java diff --git a/io/core/src/main/java/org/neo4j/gds/core/io/NodeLabelInverseMapping.java b/io/core/src/main/java/org/neo4j/gds/core/io/NodeLabelInverseMapping.java new file mode 100644 index 0000000000..3257fccc20 --- /dev/null +++ b/io/core/src/main/java/org/neo4j/gds/core/io/NodeLabelInverseMapping.java @@ -0,0 +1,47 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.core.io; + +import org.neo4j.gds.NodeLabel; + +import java.util.HashMap; +import java.util.Map; +import java.util.Set; + +public class NodeLabelInverseMapping { + // index -> label name + private final HashMap map; + + public NodeLabelInverseMapping() { + map = new HashMap<>(); + } + + public void add(String index, String labelName) { + map.put(index, labelName); + } + + public String get(String index) { + return map.get(index); + } + + public Set> entrySet() { + return map.entrySet(); + } +} diff --git a/io/core/src/main/java/org/neo4j/gds/core/io/file/FileInput.java b/io/core/src/main/java/org/neo4j/gds/core/io/file/FileInput.java index 5122e3a5e5..694da523f1 100644 --- a/io/core/src/main/java/org/neo4j/gds/core/io/file/FileInput.java +++ b/io/core/src/main/java/org/neo4j/gds/core/io/file/FileInput.java @@ -23,16 +23,21 @@ import org.neo4j.gds.api.schema.MutableRelationshipSchema; import org.neo4j.gds.api.schema.PropertySchema; import org.neo4j.gds.compat.CompatInput; +import org.neo4j.gds.core.io.NodeLabelInverseMapping; import org.neo4j.gds.core.loading.Capabilities; import org.neo4j.internal.batchimport.InputIterable; +import org.neo4j.internal.batchimport.input.Collector; import java.util.Map; +import java.util.Optional; +import java.util.function.Function; public interface FileInput extends CompatInput { InputIterable graphProperties(); String userName(); GraphInfo graphInfo(); MutableNodeSchema nodeSchema(); + Optional labelMapping(); MutableRelationshipSchema relationshipSchema(); Map graphPropertySchema(); Capabilities capabilities(); diff --git a/io/core/src/main/java/org/neo4j/gds/core/io/file/FileToGraphStoreImporter.java b/io/core/src/main/java/org/neo4j/gds/core/io/file/FileToGraphStoreImporter.java index f40955fbca..961d36b1d6 100644 --- a/io/core/src/main/java/org/neo4j/gds/core/io/file/FileToGraphStoreImporter.java +++ b/io/core/src/main/java/org/neo4j/gds/core/io/file/FileToGraphStoreImporter.java @@ -150,6 +150,14 @@ private Nodes importNodes(FileInput fileInput) { progressTracker.beginSubTask(); MutableNodeSchema nodeSchema = fileInput.nodeSchema(); graphSchemaBuilder.nodeSchema(nodeSchema); + nodeSchema.entries().stream().forEach(entry -> log.info("Imported node label schema: %s", entry.identifier())); + var labelMapping = fileInput.labelMapping(); + if (labelMapping.isPresent()) { + labelMapping.get().entrySet().forEach(entry -> log.info("Label mapping: %s -> %s", entry.getKey(), entry.getValue())); + } + else { + log.info("Label mapping file was not found, continuing import without label mapping"); + } NodesBuilder nodesBuilder = GraphFactory.initNodesBuilder(nodeSchema) .maxOriginalId(fileInput.graphInfo().maxOriginalId()) diff --git a/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/CsvFileInput.java b/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/CsvFileInput.java index 4a4a776d3f..d70784bd41 100644 --- a/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/CsvFileInput.java +++ b/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/CsvFileInput.java @@ -31,6 +31,7 @@ import org.neo4j.gds.api.schema.RelationshipPropertySchema; import org.neo4j.gds.compat.CompatPropertySizeCalculator; import org.neo4j.gds.core.io.GraphStoreInput; +import org.neo4j.gds.core.io.NodeLabelInverseMapping; import org.neo4j.gds.core.io.file.FileHeader; import org.neo4j.gds.core.io.file.FileInput; import org.neo4j.gds.core.io.file.GraphInfo; @@ -54,6 +55,7 @@ import java.nio.file.Path; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.stream.Collectors; final class CsvFileInput implements FileInput { @@ -80,6 +82,7 @@ final class CsvFileInput implements FileInput { private final String userName; private final GraphInfo graphInfo; private final MutableNodeSchema nodeSchema; + private final Optional labelMapping; private final MutableRelationshipSchema relationshipSchema; private final Map graphPropertySchema; private final Capabilities capabilities; @@ -89,6 +92,7 @@ final class CsvFileInput implements FileInput { this.userName = new UserInfoLoader(importPath).load(); this.graphInfo = new GraphInfoLoader(importPath, CSV_MAPPER).load(); this.nodeSchema = new NodeSchemaLoader(importPath).load(); + this.labelMapping = new NodeLabelMappingLoader(importPath).load(); this.relationshipSchema = new RelationshipSchemaLoader(importPath).load(); this.graphPropertySchema = new GraphPropertySchemaLoader(importPath).load(); this.capabilities = new GraphCapabilitiesLoader(importPath, CSV_MAPPER).load(); @@ -97,10 +101,15 @@ final class CsvFileInput implements FileInput { @Override public InputIterable nodes(Collector badCollector) { Map> pathMapping = CsvImportFileUtil.nodeHeaderToFileMapping(importPath); - Map> headerToDataFilesMapping = pathMapping.entrySet().stream().collect(Collectors.toMap( - entry -> CsvImportFileUtil.parseNodeHeader(entry.getKey()), - Map.Entry::getValue - )); + Map> headerToDataFilesMapping = pathMapping.entrySet() + .stream() + .collect(Collectors.toMap( + entry -> CsvImportFileUtil.parseNodeHeader( + entry.getKey(), + labelMapping.isPresent() ? labelMapping.get()::get : x -> x + ), + Map.Entry::getValue + )); return () -> new NodeImporter(headerToDataFilesMapping, nodeSchema); } @@ -157,6 +166,11 @@ public MutableNodeSchema nodeSchema() { return nodeSchema; } + @Override + public Optional labelMapping() { + return labelMapping; + } + @Override public MutableRelationshipSchema relationshipSchema() { return relationshipSchema; diff --git a/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/CsvImportFileUtil.java b/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/CsvImportFileUtil.java index 60505e61dc..47bc43eb9d 100644 --- a/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/CsvImportFileUtil.java +++ b/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/CsvImportFileUtil.java @@ -30,6 +30,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.HashMap; import java.util.List; @@ -44,13 +45,18 @@ public final class CsvImportFileUtil { private CsvImportFileUtil() {} - public static NodeFileHeader parseNodeHeader(Path headerFile) { + public static NodeFileHeader parseNodeHeader(Path headerFile, Function labelMapping) { try (MappingIterator iterator = HEADER_FILE_READER.readValues(headerFile.toFile())) { var headerLine = iterator.next(); if (headerLine == null) { throw new UncheckedIOException(new IOException("Header line was null")); } - return NodeFileHeader.of(headerLine, inferNodeLabels(headerFile)); + return NodeFileHeader.of( + headerLine, + Arrays.stream(inferNodeLabels(headerFile)) + .map(label -> labelMapping.apply(label)) + .toArray(String[]::new) + ); } catch (IOException e) { throw new UncheckedIOException(e); } @@ -107,7 +113,10 @@ static List getGraphPropertyHeaderFiles(Path csvDirectory) { return getFilesByRegex(csvDirectory, graphPropertyFilesPattern); } - private static Map> headerToFileMapping(Path csvDirectory, Function> headerPaths) { + private static Map> headerToFileMapping( + Path csvDirectory, + Function> headerPaths + ) { Map> headerToDataFileMapping = new HashMap<>(); for (Path headerFile : headerPaths.apply(csvDirectory)) { String dataFilePattern = headerFile.getFileName().toString().replace("_header", "(_\\d+)"); diff --git a/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/CsvNodeLabelMappingVisitor.java b/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/CsvNodeLabelMappingVisitor.java index a556c375f7..7e4490cf63 100644 --- a/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/CsvNodeLabelMappingVisitor.java +++ b/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/CsvNodeLabelMappingVisitor.java @@ -34,7 +34,7 @@ public class CsvNodeLabelMappingVisitor implements SimpleVisitor. + */ +package org.neo4j.gds.core.io.file.csv; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.ObjectReader; +import com.fasterxml.jackson.dataformat.csv.CsvMapper; +import com.fasterxml.jackson.dataformat.csv.CsvParser; +import com.fasterxml.jackson.dataformat.csv.CsvSchema; +import org.neo4j.gds.core.io.NodeLabelInverseMapping; + +import java.io.BufferedReader; +import java.io.FileReader; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.util.Optional; + +public class NodeLabelMappingLoader { + + private final ObjectReader objectReader; + private final Path labelMappingPath; + private final NodeLabelInverseMapping mapping; + + NodeLabelMappingLoader(Path csvDirectory) { + this.mapping = new NodeLabelInverseMapping(); + this.labelMappingPath = csvDirectory.resolve(CsvNodeLabelMappingVisitor.LABEL_MAPPING_FILE_NAME); + CsvMapper csvMapper = new CsvMapper(); + csvMapper.enable(CsvParser.Feature.TRIM_SPACES); + CsvSchema schema = CsvSchema.emptySchema().withHeader(); + this.objectReader = csvMapper.readerFor(MappingLine.class).with(schema); + } + + Optional load() { + var file = labelMappingPath.toFile(); + if (! file.isFile()) { + return Optional.empty(); + } + + try(var reader = new BufferedReader(new FileReader(file, StandardCharsets.UTF_8))) { + var linesIterator = objectReader.readValues(reader); + while(linesIterator.hasNext()) { + var mappingLine = linesIterator.next(); + mapping.add(mappingLine.index, mappingLine.label); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + return Optional.of(mapping); + } + + public static class MappingLine { + @JsonProperty + String index; + + @JsonProperty + String label; + } +} diff --git a/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/CsvImportFileUtilTest.java b/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/CsvImportFileUtilTest.java index 9649c149e6..994d4c2704 100644 --- a/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/CsvImportFileUtilTest.java +++ b/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/CsvImportFileUtilTest.java @@ -164,7 +164,7 @@ void shouldParseNodeHeaderFile() throws IOException { var headerPath = tempDir.resolve("nodes_Person_King_header.csv"); FileUtils.writeLines(headerPath.toFile(), List.of(":ID,foo:long,bar:double")); - var parsedHeader = CsvImportFileUtil.parseNodeHeader(headerPath); + var parsedHeader = CsvImportFileUtil.parseNodeHeader(headerPath, x -> x); assertThat(parsedHeader.nodeLabels()).containsExactlyInAnyOrder("Person", "King"); assertThat(parsedHeader.propertyMappings()).containsExactlyInAnyOrder( From d7a59c1e63e4cde8c257b75d60f2e0d3d47d728f Mon Sep 17 00:00:00 2001 From: yuval Date: Fri, 21 Jul 2023 16:29:20 +0200 Subject: [PATCH 158/273] Added label mapping conf to backup proc and Aura --- .../neo4j/gds/core/io/file/GraphStoreToFileExporterConfig.java | 1 + 1 file changed, 1 insertion(+) diff --git a/io/core/src/main/java/org/neo4j/gds/core/io/file/GraphStoreToFileExporterConfig.java b/io/core/src/main/java/org/neo4j/gds/core/io/file/GraphStoreToFileExporterConfig.java index d5ae4d843e..a364f58295 100644 --- a/io/core/src/main/java/org/neo4j/gds/core/io/file/GraphStoreToFileExporterConfig.java +++ b/io/core/src/main/java/org/neo4j/gds/core/io/file/GraphStoreToFileExporterConfig.java @@ -46,6 +46,7 @@ default String username() { default boolean useLabelMapping() { // the default should generally be true, however neo4j admin, backwards compatability and maybe some feedback // about this change from field engineers should be taken into account first. + // the same is true for BackupConfig return false; } From 4e299d1541e5168d8ce034f79f79a7139f44477d Mon Sep 17 00:00:00 2001 From: yuvalr1neo <117662497+yuvalr1neo@users.noreply.github.com> Date: Mon, 24 Jul 2023 14:22:19 +0200 Subject: [PATCH 159/273] style fix MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Sören Reichardt --- .../org/neo4j/gds/core/io/file/csv/NodeLabelMappingLoader.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/NodeLabelMappingLoader.java b/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/NodeLabelMappingLoader.java index db36ee8ac0..531e5f3527 100644 --- a/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/NodeLabelMappingLoader.java +++ b/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/NodeLabelMappingLoader.java @@ -50,7 +50,7 @@ public class NodeLabelMappingLoader { Optional load() { var file = labelMappingPath.toFile(); - if (! file.isFile()) { + if (!file.isFile()) { return Optional.empty(); } From 953387d310ef71758431a0d2437c4c63fa03b400 Mon Sep 17 00:00:00 2001 From: yuval Date: Tue, 25 Jul 2023 08:23:28 +0200 Subject: [PATCH 160/273] Rename variable fix MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Sören Reichardt --- .../org/neo4j/gds/core/io/file/GraphStoreToFileExporter.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/io/core/src/main/java/org/neo4j/gds/core/io/file/GraphStoreToFileExporter.java b/io/core/src/main/java/org/neo4j/gds/core/io/file/GraphStoreToFileExporter.java index ee4c801f0b..ef0ba1f794 100644 --- a/io/core/src/main/java/org/neo4j/gds/core/io/file/GraphStoreToFileExporter.java +++ b/io/core/src/main/java/org/neo4j/gds/core/io/file/GraphStoreToFileExporter.java @@ -73,7 +73,7 @@ public GraphStoreToFileExporter( Supplier> userInfoVisitorSupplier, Supplier> graphInfoVisitorSupplier, Supplier nodeSchemaVisitorSupplier, - Supplier>> labelMappingVisitorSupplierVisitorSupplier, + Supplier>> labelMappingVisitorSupplier, Supplier relationshipSchemaVisitorSupplier, Supplier graphPropertySchemaVisitorSupplier, Supplier> graphCapabilitiesWriterSupplier, @@ -91,7 +91,7 @@ public GraphStoreToFileExporter( this.userInfoVisitorSupplier = userInfoVisitorSupplier; this.graphInfoVisitorSupplier = graphInfoVisitorSupplier; this.nodeSchemaVisitorSupplier = nodeSchemaVisitorSupplier; - this.labelMappingVisitorSupplierVisitorSupplier = labelMappingVisitorSupplierVisitorSupplier; + this.labelMappingVisitorSupplierVisitorSupplier = labelMappingVisitorSupplier; this.relationshipSchemaVisitorSupplier = relationshipSchemaVisitorSupplier; this.graphPropertySchemaVisitorSupplier = graphPropertySchemaVisitorSupplier; this.graphCapabilitiesWriterSupplier = graphCapabilitiesWriterSupplier; From 2b909344cf3509d18956892269c79e2f84e2abbc Mon Sep 17 00:00:00 2001 From: yuval Date: Tue, 25 Jul 2023 08:28:10 +0200 Subject: [PATCH 161/273] Using static reference instead of a new object MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Sören Reichardt --- .../main/java/org/neo4j/gds/core/io/file/csv/CsvFileInput.java | 3 ++- .../org/neo4j/gds/core/io/file/csv/CsvImportFileUtilTest.java | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/CsvFileInput.java b/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/CsvFileInput.java index d70784bd41..55b953288b 100644 --- a/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/CsvFileInput.java +++ b/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/CsvFileInput.java @@ -25,6 +25,7 @@ import com.fasterxml.jackson.dataformat.csv.CsvParser; import com.fasterxml.jackson.dataformat.csv.CsvSchema; import org.apache.commons.lang3.tuple.Pair; +import org.eclipse.collections.impl.block.factory.Functions; import org.neo4j.gds.api.schema.MutableNodeSchema; import org.neo4j.gds.api.schema.MutableRelationshipSchema; import org.neo4j.gds.api.schema.PropertySchema; @@ -106,7 +107,7 @@ public InputIterable nodes(Collector badCollector) { .collect(Collectors.toMap( entry -> CsvImportFileUtil.parseNodeHeader( entry.getKey(), - labelMapping.isPresent() ? labelMapping.get()::get : x -> x + labelMapping.isPresent() ? labelMapping.get()::get : Functions.identity() ), Map.Entry::getValue )); diff --git a/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/CsvImportFileUtilTest.java b/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/CsvImportFileUtilTest.java index 994d4c2704..7d78a5bf78 100644 --- a/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/CsvImportFileUtilTest.java +++ b/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/CsvImportFileUtilTest.java @@ -20,6 +20,7 @@ package org.neo4j.gds.core.io.file.csv; import org.apache.commons.io.FileUtils; +import org.eclipse.collections.impl.block.factory.Functions; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; import org.junit.jupiter.params.ParameterizedTest; @@ -164,7 +165,7 @@ void shouldParseNodeHeaderFile() throws IOException { var headerPath = tempDir.resolve("nodes_Person_King_header.csv"); FileUtils.writeLines(headerPath.toFile(), List.of(":ID,foo:long,bar:double")); - var parsedHeader = CsvImportFileUtil.parseNodeHeader(headerPath, x -> x); + var parsedHeader = CsvImportFileUtil.parseNodeHeader(headerPath, Functions.identity()); assertThat(parsedHeader.nodeLabels()).containsExactlyInAnyOrder("Person", "King"); assertThat(parsedHeader.propertyMappings()).containsExactlyInAnyOrder( From 33d874e8de2c7481c6e81c5ba0d13d2d353cee90 Mon Sep 17 00:00:00 2001 From: yuval Date: Tue, 25 Jul 2023 08:34:50 +0200 Subject: [PATCH 162/273] using MutableInt for more readable code MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Sören Reichardt --- .../main/java/org/neo4j/gds/core/io/NodeLabelMapping.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/io/core/src/main/java/org/neo4j/gds/core/io/NodeLabelMapping.java b/io/core/src/main/java/org/neo4j/gds/core/io/NodeLabelMapping.java index 563ccea684..eb35293481 100644 --- a/io/core/src/main/java/org/neo4j/gds/core/io/NodeLabelMapping.java +++ b/io/core/src/main/java/org/neo4j/gds/core/io/NodeLabelMapping.java @@ -19,6 +19,7 @@ */ package org.neo4j.gds.core.io; +import org.apache.commons.lang3.mutable.MutableInt; import org.neo4j.gds.NodeLabel; import java.util.Collection; @@ -26,15 +27,14 @@ import java.util.Map; import java.util.Set; import java.util.stream.Collectors; -import java.util.stream.IntStream; public class NodeLabelMapping { private final HashMap map; NodeLabelMapping(Collection labels) { - var indexGenerator = IntStream.range(0, labels.size()).iterator(); + MutableInt index = new MutableInt(); map = new HashMap<>(labels.stream() - .collect(Collectors.toMap(label -> label, label -> Integer.toString(indexGenerator.next())))); + .collect(Collectors.toMap(label -> label, label -> Integer.toString(index.getAndIncrement())))); } public String get(NodeLabel label) { From e9c216ee018318c5decc7a25fc27f20794d2a838 Mon Sep 17 00:00:00 2001 From: yuval Date: Tue, 25 Jul 2023 08:40:45 +0200 Subject: [PATCH 163/273] Removed unneeded class, using map instead MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Sören Reichardt --- .../gds/core/io/NodeLabelInverseMapping.java | 47 ------------------- .../org/neo4j/gds/core/io/file/FileInput.java | 5 +- .../gds/core/io/file/csv/CsvFileInput.java | 6 +-- .../io/file/csv/NodeLabelMappingLoader.java | 10 ++-- 4 files changed, 10 insertions(+), 58 deletions(-) delete mode 100644 io/core/src/main/java/org/neo4j/gds/core/io/NodeLabelInverseMapping.java diff --git a/io/core/src/main/java/org/neo4j/gds/core/io/NodeLabelInverseMapping.java b/io/core/src/main/java/org/neo4j/gds/core/io/NodeLabelInverseMapping.java deleted file mode 100644 index 3257fccc20..0000000000 --- a/io/core/src/main/java/org/neo4j/gds/core/io/NodeLabelInverseMapping.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright (c) "Neo4j" - * Neo4j Sweden AB [http://neo4j.com] - * - * This file is part of Neo4j. - * - * Neo4j is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -package org.neo4j.gds.core.io; - -import org.neo4j.gds.NodeLabel; - -import java.util.HashMap; -import java.util.Map; -import java.util.Set; - -public class NodeLabelInverseMapping { - // index -> label name - private final HashMap map; - - public NodeLabelInverseMapping() { - map = new HashMap<>(); - } - - public void add(String index, String labelName) { - map.put(index, labelName); - } - - public String get(String index) { - return map.get(index); - } - - public Set> entrySet() { - return map.entrySet(); - } -} diff --git a/io/core/src/main/java/org/neo4j/gds/core/io/file/FileInput.java b/io/core/src/main/java/org/neo4j/gds/core/io/file/FileInput.java index 694da523f1..fca0a6f28c 100644 --- a/io/core/src/main/java/org/neo4j/gds/core/io/file/FileInput.java +++ b/io/core/src/main/java/org/neo4j/gds/core/io/file/FileInput.java @@ -23,11 +23,10 @@ import org.neo4j.gds.api.schema.MutableRelationshipSchema; import org.neo4j.gds.api.schema.PropertySchema; import org.neo4j.gds.compat.CompatInput; -import org.neo4j.gds.core.io.NodeLabelInverseMapping; import org.neo4j.gds.core.loading.Capabilities; import org.neo4j.internal.batchimport.InputIterable; -import org.neo4j.internal.batchimport.input.Collector; +import java.util.HashMap; import java.util.Map; import java.util.Optional; import java.util.function.Function; @@ -37,7 +36,7 @@ public interface FileInput extends CompatInput { String userName(); GraphInfo graphInfo(); MutableNodeSchema nodeSchema(); - Optional labelMapping(); + Optional> labelMapping(); MutableRelationshipSchema relationshipSchema(); Map graphPropertySchema(); Capabilities capabilities(); diff --git a/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/CsvFileInput.java b/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/CsvFileInput.java index 55b953288b..b4ccab3764 100644 --- a/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/CsvFileInput.java +++ b/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/CsvFileInput.java @@ -32,7 +32,6 @@ import org.neo4j.gds.api.schema.RelationshipPropertySchema; import org.neo4j.gds.compat.CompatPropertySizeCalculator; import org.neo4j.gds.core.io.GraphStoreInput; -import org.neo4j.gds.core.io.NodeLabelInverseMapping; import org.neo4j.gds.core.io.file.FileHeader; import org.neo4j.gds.core.io.file.FileInput; import org.neo4j.gds.core.io.file.GraphInfo; @@ -54,6 +53,7 @@ import java.io.IOException; import java.nio.file.Path; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; @@ -83,7 +83,7 @@ final class CsvFileInput implements FileInput { private final String userName; private final GraphInfo graphInfo; private final MutableNodeSchema nodeSchema; - private final Optional labelMapping; + private final Optional> labelMapping; private final MutableRelationshipSchema relationshipSchema; private final Map graphPropertySchema; private final Capabilities capabilities; @@ -168,7 +168,7 @@ public MutableNodeSchema nodeSchema() { } @Override - public Optional labelMapping() { + public Optional> labelMapping() { return labelMapping; } diff --git a/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/NodeLabelMappingLoader.java b/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/NodeLabelMappingLoader.java index 531e5f3527..da7a8f5576 100644 --- a/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/NodeLabelMappingLoader.java +++ b/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/NodeLabelMappingLoader.java @@ -24,23 +24,23 @@ import com.fasterxml.jackson.dataformat.csv.CsvMapper; import com.fasterxml.jackson.dataformat.csv.CsvParser; import com.fasterxml.jackson.dataformat.csv.CsvSchema; -import org.neo4j.gds.core.io.NodeLabelInverseMapping; import java.io.BufferedReader; import java.io.FileReader; import java.io.IOException; import java.nio.charset.StandardCharsets; import java.nio.file.Path; +import java.util.HashMap; import java.util.Optional; public class NodeLabelMappingLoader { private final ObjectReader objectReader; private final Path labelMappingPath; - private final NodeLabelInverseMapping mapping; + private final HashMap mapping; NodeLabelMappingLoader(Path csvDirectory) { - this.mapping = new NodeLabelInverseMapping(); + this.mapping = new HashMap(); this.labelMappingPath = csvDirectory.resolve(CsvNodeLabelMappingVisitor.LABEL_MAPPING_FILE_NAME); CsvMapper csvMapper = new CsvMapper(); csvMapper.enable(CsvParser.Feature.TRIM_SPACES); @@ -48,7 +48,7 @@ public class NodeLabelMappingLoader { this.objectReader = csvMapper.readerFor(MappingLine.class).with(schema); } - Optional load() { + Optional> load() { var file = labelMappingPath.toFile(); if (!file.isFile()) { return Optional.empty(); @@ -58,7 +58,7 @@ Optional load() { var linesIterator = objectReader.readValues(reader); while(linesIterator.hasNext()) { var mappingLine = linesIterator.next(); - mapping.add(mappingLine.index, mappingLine.label); + mapping.put(mappingLine.index, mappingLine.label); } } catch (IOException e) { throw new RuntimeException(e); From c383b1c40839f5321140e4690f88e8b855b5a182 Mon Sep 17 00:00:00 2001 From: yuval Date: Tue, 25 Jul 2023 08:45:48 +0200 Subject: [PATCH 164/273] Removed unused imports --- .../src/main/java/org/neo4j/gds/core/io/GraphStoreExporter.java | 1 - io/core/src/main/java/org/neo4j/gds/core/io/file/FileInput.java | 1 - .../main/java/org/neo4j/gds/core/io/schema/SimpleVisitor.java | 1 - .../neo4j/gds/core/io/file/csv/CsvNodeLabelMappingVisitor.java | 1 - 4 files changed, 4 deletions(-) diff --git a/io/core/src/main/java/org/neo4j/gds/core/io/GraphStoreExporter.java b/io/core/src/main/java/org/neo4j/gds/core/io/GraphStoreExporter.java index 6b6cc4ba6b..7cc3afc14c 100644 --- a/io/core/src/main/java/org/neo4j/gds/core/io/GraphStoreExporter.java +++ b/io/core/src/main/java/org/neo4j/gds/core/io/GraphStoreExporter.java @@ -20,7 +20,6 @@ package org.neo4j.gds.core.io; import org.neo4j.common.Validator; -import org.neo4j.gds.NodeLabel; import org.neo4j.gds.annotation.ValueClass; import org.neo4j.gds.api.GraphStore; import org.neo4j.gds.api.IdMap; diff --git a/io/core/src/main/java/org/neo4j/gds/core/io/file/FileInput.java b/io/core/src/main/java/org/neo4j/gds/core/io/file/FileInput.java index fca0a6f28c..78efdb0563 100644 --- a/io/core/src/main/java/org/neo4j/gds/core/io/file/FileInput.java +++ b/io/core/src/main/java/org/neo4j/gds/core/io/file/FileInput.java @@ -29,7 +29,6 @@ import java.util.HashMap; import java.util.Map; import java.util.Optional; -import java.util.function.Function; public interface FileInput extends CompatInput { InputIterable graphProperties(); diff --git a/io/core/src/main/java/org/neo4j/gds/core/io/schema/SimpleVisitor.java b/io/core/src/main/java/org/neo4j/gds/core/io/schema/SimpleVisitor.java index 4eaaab61ed..8c4a56608f 100644 --- a/io/core/src/main/java/org/neo4j/gds/core/io/schema/SimpleVisitor.java +++ b/io/core/src/main/java/org/neo4j/gds/core/io/schema/SimpleVisitor.java @@ -20,7 +20,6 @@ package org.neo4j.gds.core.io.schema; import java.io.Closeable; -import java.io.IOException; public interface SimpleVisitor extends Closeable { void export(T value); diff --git a/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/CsvNodeLabelMappingVisitor.java b/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/CsvNodeLabelMappingVisitor.java index 7e4490cf63..18303b7789 100644 --- a/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/CsvNodeLabelMappingVisitor.java +++ b/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/CsvNodeLabelMappingVisitor.java @@ -28,7 +28,6 @@ import java.nio.file.Path; import java.util.ArrayList; import java.util.Map; -import java.util.Set; public class CsvNodeLabelMappingVisitor implements SimpleVisitor> { From fa1c2c9fe77f6022019173f90777a73e35f15b40 Mon Sep 17 00:00:00 2001 From: yuval Date: Tue, 25 Jul 2023 11:14:04 +0200 Subject: [PATCH 165/273] Changed backup config to use label mapping by default --- .../gds/core/io/file/GraphStoreToFileExporterConfig.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/io/core/src/main/java/org/neo4j/gds/core/io/file/GraphStoreToFileExporterConfig.java b/io/core/src/main/java/org/neo4j/gds/core/io/file/GraphStoreToFileExporterConfig.java index a364f58295..db58b6c69a 100644 --- a/io/core/src/main/java/org/neo4j/gds/core/io/file/GraphStoreToFileExporterConfig.java +++ b/io/core/src/main/java/org/neo4j/gds/core/io/file/GraphStoreToFileExporterConfig.java @@ -44,9 +44,7 @@ default String username() { @Value.Default default boolean useLabelMapping() { - // the default should generally be true, however neo4j admin, backwards compatability and maybe some feedback - // about this change from field engineers should be taken into account first. - // the same is true for BackupConfig + // the reason for default false (unlike BackupConfig) is that Neo4j Admin import is not aware of label mapping return false; } From 6483494d75360c36675520da52ee775af79c55f9 Mon Sep 17 00:00:00 2001 From: Veselin Nikolov Date: Tue, 25 Jul 2023 12:46:30 +0100 Subject: [PATCH 166/273] Update Aura version --- gradle/version.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle/version.gradle b/gradle/version.gradle index 6fd9200ae0..3bd37271dc 100644 --- a/gradle/version.gradle +++ b/gradle/version.gradle @@ -1,6 +1,6 @@ ext { gdsBaseVersion = '2.4.3' - gdsAuraVersion = '27' + gdsAuraVersion = '28' gdsVersion = gdsBaseVersion + (rootProject.hasProperty('aurads') ? "+${gdsAuraVersion}" : "") } From cc1787f1547b10311b3440a157b961fc5d5f1cb2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Tue, 25 Jul 2023 15:17:08 +0200 Subject: [PATCH 167/273] Mark alpha version of RWR as deprecated --- .../src/main/java/org/neo4j/gds/catalog/GraphSampleProc.java | 1 + 1 file changed, 1 insertion(+) diff --git a/proc/catalog/src/main/java/org/neo4j/gds/catalog/GraphSampleProc.java b/proc/catalog/src/main/java/org/neo4j/gds/catalog/GraphSampleProc.java index 251a0c8310..6ed93cb418 100644 --- a/proc/catalog/src/main/java/org/neo4j/gds/catalog/GraphSampleProc.java +++ b/proc/catalog/src/main/java/org/neo4j/gds/catalog/GraphSampleProc.java @@ -47,6 +47,7 @@ public class GraphSampleProc extends CatalogProc { private static final String CNARW_DESCRIPTION = "Constructs a random subgraph based on common neighbour aware random walks"; @Internal + @Deprecated(forRemoval = true) @Procedure(name = "gds.alpha.graph.sample.rwr", mode = READ, deprecatedBy = "gds.graph.sample.rwr") @Description(RWR_DESCRIPTION) public Stream sampleRandomWalkWithRestartsAlpha( From 9596e76e508d1a8ce8ee32def622b40978bd499f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Tue, 25 Jul 2023 15:21:36 +0200 Subject: [PATCH 168/273] Mark alpha version of scaleProperties as deprecated and internal --- .../ROOT/pages/operations-reference/algorithm-references.adoc | 3 --- .../test/java/org/neo4j/gds/OpenGdsProcedureSmokeTest.java | 4 +--- .../java/org/neo4j/gds/scaling/ScalePropertiesMutateProc.java | 3 +++ .../java/org/neo4j/gds/scaling/ScalePropertiesStreamProc.java | 3 +++ 4 files changed, 7 insertions(+), 6 deletions(-) diff --git a/doc/modules/ROOT/pages/operations-reference/algorithm-references.adoc b/doc/modules/ROOT/pages/operations-reference/algorithm-references.adoc index b7653fa121..20d3e93c5e 100644 --- a/doc/modules/ROOT/pages/operations-reference/algorithm-references.adoc +++ b/doc/modules/ROOT/pages/operations-reference/algorithm-references.adoc @@ -385,7 +385,4 @@ .2+<.^| xref:algorithms/alpha/modularity.adoc[Modularity Metric] | `gds.alpha.modularity.stats` | `gds.alpha.modularity.stream` -.2+<.^|xref:algorithms/scale-properties.adoc[Scale Properties] -| `gds.alpha.scaleProperties.mutate` (deprecated) -| `gds.alpha.scaleProperties.stream` (deprecated) |=== diff --git a/open-packaging/src/test/java/org/neo4j/gds/OpenGdsProcedureSmokeTest.java b/open-packaging/src/test/java/org/neo4j/gds/OpenGdsProcedureSmokeTest.java index bcbe9df3dc..639e49c65f 100644 --- a/open-packaging/src/test/java/org/neo4j/gds/OpenGdsProcedureSmokeTest.java +++ b/open-packaging/src/test/java/org/neo4j/gds/OpenGdsProcedureSmokeTest.java @@ -182,10 +182,8 @@ class OpenGdsProcedureSmokeTest extends BaseProcTest { "gds.alpha.scc.write", "gds.alpha.scc.stream", - "gds.alpha.scaleProperties.mutate", "gds.scaleProperties.mutate", "gds.scaleProperties.mutate.estimate", - "gds.alpha.scaleProperties.stream", "gds.scaleProperties.stream", "gds.scaleProperties.stream.estimate", "gds.scaleProperties.stats", @@ -562,7 +560,7 @@ void countShouldMatch() { ); // If you find yourself updating this count, please also update the count in SmokeTest.kt - int expectedCount = 407; + int expectedCount = 405; assertEquals( expectedCount, returnedRows, diff --git a/proc/misc/src/main/java/org/neo4j/gds/scaling/ScalePropertiesMutateProc.java b/proc/misc/src/main/java/org/neo4j/gds/scaling/ScalePropertiesMutateProc.java index b733d74008..6ceae741a7 100644 --- a/proc/misc/src/main/java/org/neo4j/gds/scaling/ScalePropertiesMutateProc.java +++ b/proc/misc/src/main/java/org/neo4j/gds/scaling/ScalePropertiesMutateProc.java @@ -26,6 +26,7 @@ import org.neo4j.gds.results.MemoryEstimateResult; import org.neo4j.gds.results.StandardMutateResult; import org.neo4j.procedure.Description; +import org.neo4j.procedure.Internal; import org.neo4j.procedure.Name; import org.neo4j.procedure.Procedure; @@ -65,6 +66,8 @@ public Stream estimate( ).computeEstimate(graphName, configuration); } + @Internal + @Deprecated(forRemoval = true) @Procedure(value = "gds.alpha.scaleProperties.mutate", deprecatedBy = "gds.scaleProperties.mutate") @Description(SCALE_PROPERTIES_DESCRIPTION) public Stream alphaMutate( diff --git a/proc/misc/src/main/java/org/neo4j/gds/scaling/ScalePropertiesStreamProc.java b/proc/misc/src/main/java/org/neo4j/gds/scaling/ScalePropertiesStreamProc.java index 5c69ff038a..71bf380cd5 100644 --- a/proc/misc/src/main/java/org/neo4j/gds/scaling/ScalePropertiesStreamProc.java +++ b/proc/misc/src/main/java/org/neo4j/gds/scaling/ScalePropertiesStreamProc.java @@ -24,6 +24,7 @@ import org.neo4j.gds.executor.ProcedureExecutor; import org.neo4j.gds.results.MemoryEstimateResult; import org.neo4j.procedure.Description; +import org.neo4j.procedure.Internal; import org.neo4j.procedure.Name; import org.neo4j.procedure.Procedure; @@ -65,6 +66,8 @@ public Stream estimate( ).computeEstimate(graphName, configuration); } + @Internal + @Deprecated(forRemoval = true) @Procedure(value = "gds.alpha.scaleProperties.stream", deprecatedBy = "gds.scaleProperties.stream") @Description(SCALE_PROPERTIES_DESCRIPTION) public Stream alphaStream( From 02d061397fcf394f0165431726739297793df375 Mon Sep 17 00:00:00 2001 From: Paul Horn Date: Thu, 27 Jul 2023 11:53:47 +0200 Subject: [PATCH 169/273] Extract error catching from various `QuerySubscriber`s --- .../gds/core/loading/NodeSubscriber.java | 24 +--------- .../core/loading/RelationshipSubscriber.java | 22 +-------- .../loading/ResultCountingSubscriber.java | 23 +-------- .../utils/ErrorCachingQuerySubscriber.java | 48 +++++++++++++++++++ 4 files changed, 54 insertions(+), 63 deletions(-) create mode 100644 core/src/main/java/org/neo4j/gds/core/utils/ErrorCachingQuerySubscriber.java diff --git a/core/src/main/java/org/neo4j/gds/core/loading/NodeSubscriber.java b/core/src/main/java/org/neo4j/gds/core/loading/NodeSubscriber.java index 99e31f19d0..67970c753b 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/NodeSubscriber.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/NodeSubscriber.java @@ -21,10 +21,9 @@ import org.neo4j.gds.core.loading.construction.NodeLabelTokens; import org.neo4j.gds.core.loading.construction.NodesBuilder; +import org.neo4j.gds.core.utils.ErrorCachingQuerySubscriber; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.graphdb.QueryStatistics; -import org.neo4j.kernel.impl.query.QueryExecutionKernelException; -import org.neo4j.kernel.impl.query.QuerySubscriber; import org.neo4j.values.AnyValue; import org.neo4j.values.SequenceValue; import org.neo4j.values.storable.NumberValue; @@ -34,12 +33,11 @@ import java.util.HashMap; import java.util.Map; -import java.util.Optional; import java.util.Set; import static org.neo4j.gds.utils.StringFormatting.formatWithLocale; -class NodeSubscriber implements QuerySubscriber { +class NodeSubscriber extends ErrorCachingQuerySubscriber { private static final String ID_COLUMN = "id"; private static final int UNINITIALIZED = -1; @@ -62,8 +60,6 @@ class NodeSubscriber implements QuerySubscriber { private int labelOffset = UNINITIALIZED; private String[] fieldNames; - private Optional error = Optional.empty(); - public NodeSubscriber( ProgressTracker progressTracker ) { @@ -83,11 +79,6 @@ void initialize(String[] fieldNames, NodesBuilder nodesBuilder) { } } - Optional error() - { - return error; - } - long rows() { return rows; } @@ -150,17 +141,6 @@ public void onRecordCompleted() { progressTracker.logProgress(); } - @Override - public void onError(Throwable throwable) { - if (throwable instanceof RuntimeException) { - this.error = Optional.of((RuntimeException) throwable); - } else if (throwable instanceof QueryExecutionKernelException) { - this.error = Optional.of(((QueryExecutionKernelException) throwable).asUserException()); - } else { - this.error = Optional.of(new RuntimeException(throwable)); - } - } - @Override public void onResultCompleted(QueryStatistics statistics) { } diff --git a/core/src/main/java/org/neo4j/gds/core/loading/RelationshipSubscriber.java b/core/src/main/java/org/neo4j/gds/core/loading/RelationshipSubscriber.java index 996c9cea89..883bb69e4b 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/RelationshipSubscriber.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/RelationshipSubscriber.java @@ -23,22 +23,20 @@ import org.neo4j.gds.RelationshipType; import org.neo4j.gds.api.IdMap; import org.neo4j.gds.core.loading.construction.RelationshipsBuilder; +import org.neo4j.gds.core.utils.ErrorCachingQuerySubscriber; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.graphdb.QueryStatistics; -import org.neo4j.kernel.impl.query.QueryExecutionKernelException; -import org.neo4j.kernel.impl.query.QuerySubscriber; import org.neo4j.values.AnyValue; import org.neo4j.values.storable.NumberValue; import org.neo4j.values.storable.TextValue; -import java.util.Optional; import java.util.Set; import static org.neo4j.gds.RelationshipType.ALL_RELATIONSHIPS; import static org.neo4j.gds.core.loading.LoadingExceptions.validateSourceNodeIsLoaded; import static org.neo4j.gds.core.loading.LoadingExceptions.validateTargetNodeIsLoaded; -class RelationshipSubscriber implements QuerySubscriber { +class RelationshipSubscriber extends ErrorCachingQuerySubscriber { private static final String SOURCE_COLUMN = "source"; private static final String TARGET_COLUMN = "target"; @@ -67,8 +65,6 @@ class RelationshipSubscriber implements QuerySubscriber { private RelationshipsBuilder allRelationshipsBuilder; - private Optional error = Optional.empty(); - RelationshipSubscriber( IdMap idMap, CypherRelationshipLoader.Context loaderContext, @@ -109,10 +105,6 @@ void initialize(String[] fieldNames, ObjectDoubleMap propertyDefaultValu this.propertyValueBuffer = new double[propertyCount]; } - Optional error() { - return error; - } - public long rows() { return rows; } @@ -189,16 +181,6 @@ public void onRecordCompleted() { progressTracker.logProgress(); } - @Override - public void onError(Throwable throwable) { - if (throwable instanceof RuntimeException) { - this.error = Optional.of((RuntimeException) throwable); - } else if (throwable instanceof QueryExecutionKernelException) { - this.error = Optional.of(((QueryExecutionKernelException) throwable).asUserException()); - } else { - this.error = Optional.of(new RuntimeException(throwable)); - } - } @Override public void onResultCompleted(QueryStatistics statistics) { diff --git a/core/src/main/java/org/neo4j/gds/core/loading/ResultCountingSubscriber.java b/core/src/main/java/org/neo4j/gds/core/loading/ResultCountingSubscriber.java index 518fd5ef98..12b2feb59a 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/ResultCountingSubscriber.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/ResultCountingSubscriber.java @@ -19,20 +19,12 @@ */ package org.neo4j.gds.core.loading; +import org.neo4j.gds.core.utils.ErrorCachingQuerySubscriber; import org.neo4j.graphdb.QueryStatistics; -import org.neo4j.kernel.impl.query.QueryExecutionKernelException; -import org.neo4j.kernel.impl.query.QuerySubscriber; import org.neo4j.values.AnyValue; -import java.util.Optional; - -class ResultCountingSubscriber implements QuerySubscriber { +class ResultCountingSubscriber extends ErrorCachingQuerySubscriber { private long rows = 0; - private Optional error = Optional.empty(); - - Optional error() { - return error; - } public long rows() { return rows; @@ -59,17 +51,6 @@ public void onRecordCompleted() { } - @Override - public void onError(Throwable throwable) { - if (throwable instanceof RuntimeException) { - this.error = Optional.of((RuntimeException) throwable); - } else if (throwable instanceof QueryExecutionKernelException) { - this.error = Optional.of(((QueryExecutionKernelException) throwable).asUserException()); - } else { - this.error = Optional.of(new RuntimeException(throwable)); - } - } - @Override public void onResultCompleted(QueryStatistics statistics) { diff --git a/core/src/main/java/org/neo4j/gds/core/utils/ErrorCachingQuerySubscriber.java b/core/src/main/java/org/neo4j/gds/core/utils/ErrorCachingQuerySubscriber.java new file mode 100644 index 0000000000..8df7c5b9cf --- /dev/null +++ b/core/src/main/java/org/neo4j/gds/core/utils/ErrorCachingQuerySubscriber.java @@ -0,0 +1,48 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.core.utils; + +import org.jetbrains.annotations.Nullable; +import org.neo4j.graphdb.QueryStatistics; +import org.neo4j.kernel.impl.query.QueryExecutionKernelException; +import org.neo4j.kernel.impl.query.QuerySubscriber; +import org.neo4j.values.AnyValue; + +import java.util.Optional; + +public abstract class ErrorCachingQuerySubscriber implements QuerySubscriber { + + @Nullable + private RuntimeException error; + + public Optional error() { + return Optional.ofNullable(error); + } + + public void onError(Throwable throwable) { + if (throwable instanceof RuntimeException) { + this.error = (RuntimeException) throwable; + } else if (throwable instanceof QueryExecutionKernelException) { + this.error = ((QueryExecutionKernelException) throwable).asUserException(); + } else { + this.error = new RuntimeException(throwable); + } + } +} From 734af5d63c6e6e60d7c4c551e00d70e774cda326 Mon Sep 17 00:00:00 2001 From: Paul Horn Date: Thu, 27 Jul 2023 11:54:34 +0200 Subject: [PATCH 170/273] Ensure that query execution errors are thrown if the runtime doesn't already do that --- .../utils/ErrorCachingQuerySubscriber.java | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/core/src/main/java/org/neo4j/gds/core/utils/ErrorCachingQuerySubscriber.java b/core/src/main/java/org/neo4j/gds/core/utils/ErrorCachingQuerySubscriber.java index 8df7c5b9cf..d8076f8680 100644 --- a/core/src/main/java/org/neo4j/gds/core/utils/ErrorCachingQuerySubscriber.java +++ b/core/src/main/java/org/neo4j/gds/core/utils/ErrorCachingQuerySubscriber.java @@ -45,4 +45,26 @@ public void onError(Throwable throwable) { this.error = new RuntimeException(throwable); } } + + public static final class DoNothingSubscriber extends ErrorCachingQuerySubscriber { + @Override + public void onResult(int numberOfFields) { + } + + @Override + public void onRecord() { + } + + @Override + public void onField(int offset, AnyValue value) { + } + + @Override + public void onRecordCompleted() { + } + + @Override + public void onResultCompleted(QueryStatistics statistics) { + } + } } From 4df4d21cb3afb9d52e83f262704b0f29e9cde41e Mon Sep 17 00:00:00 2001 From: yuval Date: Wed, 26 Jul 2023 11:54:48 +0200 Subject: [PATCH 171/273] made tests a bit more readable --- ...svToGraphStoreImporterIntegrationTest.java | 40 +++++++------------ 1 file changed, 15 insertions(+), 25 deletions(-) diff --git a/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/CsvToGraphStoreImporterIntegrationTest.java b/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/CsvToGraphStoreImporterIntegrationTest.java index 4c56e49c8c..004ee3b728 100644 --- a/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/CsvToGraphStoreImporterIntegrationTest.java +++ b/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/CsvToGraphStoreImporterIntegrationTest.java @@ -24,7 +24,6 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.EnumSource; import org.junit.jupiter.params.provider.ValueSource; -import org.neo4j.gds.api.Graph; import org.neo4j.gds.api.GraphStore; import org.neo4j.gds.api.properties.graph.DoubleArrayGraphPropertyValues; import org.neo4j.gds.api.properties.graph.LongGraphPropertyValues; @@ -36,9 +35,6 @@ import org.neo4j.gds.core.loading.HighLimitIdMapBuilder; import org.neo4j.gds.core.loading.ImmutableStaticCapabilities; import org.neo4j.gds.core.utils.progress.EmptyTaskRegistryFactory; -import org.neo4j.gds.extension.GdlExtension; -import org.neo4j.gds.extension.GdlGraph; -import org.neo4j.gds.extension.Inject; import org.neo4j.gds.gdl.GdlFactory; import java.nio.file.Path; @@ -49,11 +45,9 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.neo4j.gds.TestSupport.assertGraphEquals; -@GdlExtension class CsvToGraphStoreImporterIntegrationTest { - @GdlGraph - private static final String GDL = + private static final String GRAPH_WITH_PROPERTIES = "CREATE" + // This triggers jackson wrapping the values in quotes " (a:A:B { averylongpropertynamegreaterthantwentyfour: 0, prop2: 42, prop3: [0.30000001192092896D, 0.20000000298023224D]})" + @@ -67,18 +61,13 @@ class CsvToGraphStoreImporterIntegrationTest { ", (c)-[:REL2 { prop3: 4, prop4: 46 }]->(d)" + ", (d)-[:REL2 { prop3: 5, prop4: 47 }]->(a)"; - @Inject - GraphStore graphStore; - - @Inject - Graph graph; - @TempDir Path graphLocation; @ParameterizedTest @ValueSource(ints = {1, 4}) void shouldImportProperties(int concurrency) { + var graphStore = GdlFactory.of(GRAPH_WITH_PROPERTIES).build(); GraphStoreToCsvExporter.create(graphStore, exportConfig(concurrency), graphLocation).run(); @@ -87,29 +76,30 @@ void shouldImportProperties(int concurrency) { var importedGraphStore = userGraphStore.graphStore(); var importedGraph = importedGraphStore.getUnion(); - assertGraphEquals(graph, importedGraph); + assertGraphEquals(graphStore.getUnion(), importedGraph); } @ParameterizedTest @ValueSource(ints = {1, 4}) void shouldImportGraphStoreWithGraphProperties(int concurrency) { - addLongGraphProperty(); - addDoubleArrayGraphProperty(); - addLongNamedGraphProperty(); + var graphStore = GdlFactory.of(GRAPH_WITH_PROPERTIES).build(); + + addLongGraphProperty(graphStore); + addDoubleArrayGraphProperty(graphStore); + addLongNamedGraphProperty(graphStore); GraphStoreToCsvExporter.create(graphStore, exportConfig(concurrency), graphLocation).run(); var importer = new CsvToGraphStoreImporter(concurrency, graphLocation, Neo4jProxy.testLog(), EmptyTaskRegistryFactory.INSTANCE); - var userGraphStore = importer.run(); - var graphStore = userGraphStore.graphStore(); + var userGraphStore = importer.run().graphStore(); - assertThat(graphStore.graphPropertyKeys()).containsExactlyInAnyOrder( + assertThat(userGraphStore.graphPropertyKeys()).containsExactlyInAnyOrder( "longProp", "doubleArrayProp", "thisisaverylongnameintentionallytotriggerquoting" ); var expectedLongValues = LongStream.range(0, 10_000).toArray(); - assertThat(graphStore.graphProperty("longProp").values().longValues().toArray()) + assertThat(userGraphStore.graphProperty("longProp").values().longValues().toArray()) .containsExactlyInAnyOrder(expectedLongValues); var expectedDoubleArrayProperties = LongStream @@ -117,7 +107,7 @@ void shouldImportGraphStoreWithGraphProperties(int concurrency) { .mapToObj(i -> new double[]{(double) i, 42.0}) .collect(Collectors.toList()) .toArray(new double[0][0]); - assertThat(graphStore.graphProperty("doubleArrayProp").values().doubleArrayValues().collect(Collectors.toList())) + assertThat(userGraphStore.graphProperty("doubleArrayProp").values().doubleArrayValues().collect(Collectors.toList())) .containsExactlyInAnyOrder(expectedDoubleArrayProperties); } @@ -182,7 +172,7 @@ private GraphStoreToFileExporterConfig exportConfig(int concurrency) { .build(); } - private void addDoubleArrayGraphProperty() { + private void addDoubleArrayGraphProperty(GraphStore graphStore) { graphStore.addGraphProperty("doubleArrayProp", new DoubleArrayGraphPropertyValues() { @Override public Stream doubleArrayValues() { @@ -196,7 +186,7 @@ public long valueCount() { }); } - private void addLongGraphProperty() { + private void addLongGraphProperty(GraphStore graphStore) { graphStore.addGraphProperty("longProp", new LongGraphPropertyValues() { @Override public LongStream longValues() { @@ -210,7 +200,7 @@ public long valueCount() { }); } - private void addLongNamedGraphProperty() { + private void addLongNamedGraphProperty(GraphStore graphStore) { graphStore.addGraphProperty("thisisaverylongnameintentionallytotriggerquoting", new LongGraphPropertyValues() { @Override public LongStream longValues() { From a4a003f2a0110be807c2aace7338948d17b4f9d0 Mon Sep 17 00:00:00 2001 From: yuval Date: Wed, 26 Jul 2023 12:13:08 +0200 Subject: [PATCH 172/273] added more properties tests --- ...svToGraphStoreImporterIntegrationTest.java | 62 ++++++++++++++++--- 1 file changed, 52 insertions(+), 10 deletions(-) diff --git a/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/CsvToGraphStoreImporterIntegrationTest.java b/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/CsvToGraphStoreImporterIntegrationTest.java index 004ee3b728..05d05df1e7 100644 --- a/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/CsvToGraphStoreImporterIntegrationTest.java +++ b/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/CsvToGraphStoreImporterIntegrationTest.java @@ -22,7 +22,9 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.EnumSource; +import org.junit.jupiter.params.provider.MethodSource; import org.junit.jupiter.params.provider.ValueSource; import org.neo4j.gds.api.GraphStore; import org.neo4j.gds.api.properties.graph.DoubleArrayGraphPropertyValues; @@ -61,15 +63,39 @@ class CsvToGraphStoreImporterIntegrationTest { ", (c)-[:REL2 { prop3: 4, prop4: 46 }]->(d)" + ", (d)-[:REL2 { prop3: 5, prop4: 47 }]->(a)"; + private static final String GRAPH_WITH_UNDERSCORE_LABELS = + "CREATE" + + // This triggers jackson wrapping the values in quotes + " (a:A_B { averylongpropertynamegreaterthantwentyfour: 0, prop2: 42, prop3: [0.30000001192092896D, 0.20000000298023224D]})" + + ", (b:A:B { averylongpropertynamegreaterthantwentyfour: 1, prop2: 43})" + + ", (c:A:C { averylongpropertynamegreaterthantwentyfour: 2, prop2: 44, prop3: [-0.04D] })" + + ", (d:B { averylongpropertynamegreaterthantwentyfour: 3 })" + + ", (a)-[:REL1 { prop1: 0, prop2: 42 }]->(a)" + + ", (a)-[:REL1 { prop1: 1, prop2: 43 }]->(b)" + + ", (b)-[:REL1 { prop1: 2, prop2: 44 }]->(a)" + + ", (b)-[:REL2 { prop3: 3, prop4: 45 }]->(c)" + + ", (c)-[:REL2 { prop3: 4, prop4: 46 }]->(d)" + + ", (d)-[:REL2 { prop3: 5, prop4: 47 }]->(a)"; + + @TempDir Path graphLocation; + static Stream concurrencyLabelMappingArgs() { + return Stream.of( + Arguments.of(1, false), + Arguments.of(4, false), + Arguments.of(1, true), + Arguments.of(4, true) + ); + } + @ParameterizedTest - @ValueSource(ints = {1, 4}) - void shouldImportProperties(int concurrency) { + @MethodSource("concurrencyLabelMappingArgs") + void shouldImportProperties(int concurrency, boolean useLabelMapping) { var graphStore = GdlFactory.of(GRAPH_WITH_PROPERTIES).build(); - GraphStoreToCsvExporter.create(graphStore, exportConfig(concurrency), graphLocation).run(); + GraphStoreToCsvExporter.create(graphStore, exportConfig(concurrency, useLabelMapping), graphLocation).run(); var importer = new CsvToGraphStoreImporter(concurrency, graphLocation, Neo4jProxy.testLog(), EmptyTaskRegistryFactory.INSTANCE); var userGraphStore = importer.run(); @@ -80,15 +106,15 @@ void shouldImportProperties(int concurrency) { } @ParameterizedTest - @ValueSource(ints = {1, 4}) - void shouldImportGraphStoreWithGraphProperties(int concurrency) { + @MethodSource("concurrencyLabelMappingArgs") + void shouldImportGraphStoreWithGraphProperties(int concurrency, boolean useLabelMapping) { var graphStore = GdlFactory.of(GRAPH_WITH_PROPERTIES).build(); addLongGraphProperty(graphStore); addDoubleArrayGraphProperty(graphStore); addLongNamedGraphProperty(graphStore); - GraphStoreToCsvExporter.create(graphStore, exportConfig(concurrency), graphLocation).run(); + GraphStoreToCsvExporter.create(graphStore, exportConfig(concurrency, useLabelMapping), graphLocation).run(); var importer = new CsvToGraphStoreImporter(concurrency, graphLocation, Neo4jProxy.testLog(), EmptyTaskRegistryFactory.INSTANCE); var userGraphStore = importer.run().graphStore(); @@ -111,11 +137,26 @@ void shouldImportGraphStoreWithGraphProperties(int concurrency) { .containsExactlyInAnyOrder(expectedDoubleArrayProperties); } + @ParameterizedTest + @ValueSource(ints = {1, 4}) + void shouldImportGraphWithPropertiesAndUnderscoreLabels(int concurrency) { + var graphStore = GdlFactory.of(GRAPH_WITH_UNDERSCORE_LABELS).build(); + + GraphStoreToCsvExporter.create(graphStore, exportConfig(concurrency, true /* will not work without label mapping */), graphLocation).run(); + + var importer = new CsvToGraphStoreImporter(concurrency, graphLocation, Neo4jProxy.testLog(), EmptyTaskRegistryFactory.INSTANCE); + var userGraphStore = importer.run(); + + var importedGraphStore = userGraphStore.graphStore(); + var importedGraph = importedGraphStore.getUnion(); + assertGraphEquals(graphStore.getUnion(), importedGraph); + } + @Test void shouldImportGraphWithNoLabels() { var graphStore = GdlFactory.of("()-[]->()").build(); - GraphStoreToCsvExporter.create(graphStore, exportConfig(4), graphLocation).run(); + GraphStoreToCsvExporter.create(graphStore, exportConfig(4, false), graphLocation).run(); var importer = new CsvToGraphStoreImporter(4, graphLocation, Neo4jProxy.testLog(), EmptyTaskRegistryFactory.INSTANCE); var userGraphStore = importer.run(); @@ -134,7 +175,7 @@ void shouldImportCapabilities(WriteMode writeMode) { .build() .build(); - GraphStoreToCsvExporter.create(graphStoreWithCapabilities, exportConfig(1), graphLocation).run(); + GraphStoreToCsvExporter.create(graphStoreWithCapabilities, exportConfig(1, false), graphLocation).run(); var importer = new CsvToGraphStoreImporter(1, graphLocation, Neo4jProxy.testLog(), EmptyTaskRegistryFactory.INSTANCE); var userGraphStore = importer.run(); @@ -155,7 +196,7 @@ void shouldConsiderIdMapBuilderType(String idMapBuilderType) { .build() .build(); - GraphStoreToCsvExporter.create(graphStore, exportConfig(1), graphLocation).run(); + GraphStoreToCsvExporter.create(graphStore, exportConfig(1, false), graphLocation).run(); var importer = new CsvToGraphStoreImporter(1, graphLocation, Neo4jProxy.testLog(), EmptyTaskRegistryFactory.INSTANCE); var userGraphStore = importer.run(); @@ -163,12 +204,13 @@ void shouldConsiderIdMapBuilderType(String idMapBuilderType) { assertThat(userGraphStore.graphStore().nodes().typeId()).startsWith(idMapBuilderType); } - private GraphStoreToFileExporterConfig exportConfig(int concurrency) { + private GraphStoreToFileExporterConfig exportConfig(int concurrency, boolean useLabelMapping) { return GraphStoreToFileExporterConfigImpl.builder() .exportName("my-export") .writeConcurrency(concurrency) .username("") .includeMetaData(true) + .useLabelMapping(useLabelMapping) .build(); } From 324370c7709afb3edaa456f466db8ecc4f553d92 Mon Sep 17 00:00:00 2001 From: yuval Date: Wed, 26 Jul 2023 23:35:17 +0200 Subject: [PATCH 173/273] Rename fix MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Sören Reichardt --- .../neo4j/gds/core/io/file/GraphStoreToFileExporter.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/io/core/src/main/java/org/neo4j/gds/core/io/file/GraphStoreToFileExporter.java b/io/core/src/main/java/org/neo4j/gds/core/io/file/GraphStoreToFileExporter.java index ef0ba1f794..71d5c0b151 100644 --- a/io/core/src/main/java/org/neo4j/gds/core/io/file/GraphStoreToFileExporter.java +++ b/io/core/src/main/java/org/neo4j/gds/core/io/file/GraphStoreToFileExporter.java @@ -56,7 +56,7 @@ public class GraphStoreToFileExporter extends GraphStoreExporter> userInfoVisitorSupplier; private final Supplier> graphInfoVisitorSupplier; private final Supplier nodeSchemaVisitorSupplier; - private final Supplier>> labelMappingVisitorSupplierVisitorSupplier; + private final Supplier>> labelMappingVisitorSupplier; private final Supplier relationshipSchemaVisitorSupplier; private final Supplier graphPropertySchemaVisitorSupplier; @@ -91,7 +91,7 @@ public GraphStoreToFileExporter( this.userInfoVisitorSupplier = userInfoVisitorSupplier; this.graphInfoVisitorSupplier = graphInfoVisitorSupplier; this.nodeSchemaVisitorSupplier = nodeSchemaVisitorSupplier; - this.labelMappingVisitorSupplierVisitorSupplier = labelMappingVisitorSupplier; + this.labelMappingVisitorSupplier = labelMappingVisitorSupplier; this.relationshipSchemaVisitorSupplier = relationshipSchemaVisitorSupplier; this.graphPropertySchemaVisitorSupplier = graphPropertySchemaVisitorSupplier; this.graphCapabilitiesWriterSupplier = graphCapabilitiesWriterSupplier; @@ -258,7 +258,7 @@ private void exportNodeSchema(GraphStoreInput graphStoreInput) { private void exportNodeLabelMapping(GraphStoreInput graphStoreInput) { var labelMapping = graphStoreInput.labelMapping(); if (labelMapping.isPresent()) { - try (var labelMappingVisitor = labelMappingVisitorSupplierVisitorSupplier.get()) { + try (var labelMappingVisitor = labelMappingVisitorSupplier.get()) { labelMapping.get().entrySet().forEach(entry -> labelMappingVisitor.export(entry)); } catch (IOException e) { throw new UncheckedIOException(e); From 72ac7c3704900f4edaf918ed96c36a6681b9dd52 Mon Sep 17 00:00:00 2001 From: yuval Date: Fri, 28 Jul 2023 10:46:10 +0200 Subject: [PATCH 174/273] fixed node label mapping with properties --- .../neo4j/gds/core/io/GraphStoreExporter.java | 10 +++---- .../neo4j/gds/core/io/GraphStoreInput.java | 12 ++++---- .../neo4j/gds/core/io/NodeLabelMapping.java | 10 ++++++- .../java/org/neo4j/gds/core/io/NodeStore.java | 15 +++++++++- .../io/db/GraphStoreToDatabaseExporter.java | 2 +- .../io/file/GraphStoreToFileExporter.java | 4 ++- .../neo4j/gds/core/io/file/NodeVisitor.java | 6 ++-- .../gds/core/io/file/csv/CsvNodeVisitor.java | 28 +++++++++++++++++-- .../io/file/csv/GraphStoreToCsvExporter.java | 11 ++++++-- 9 files changed, 75 insertions(+), 23 deletions(-) diff --git a/io/core/src/main/java/org/neo4j/gds/core/io/GraphStoreExporter.java b/io/core/src/main/java/org/neo4j/gds/core/io/GraphStoreExporter.java index 7cc3afc14c..11477e04f2 100644 --- a/io/core/src/main/java/org/neo4j/gds/core/io/GraphStoreExporter.java +++ b/io/core/src/main/java/org/neo4j/gds/core/io/GraphStoreExporter.java @@ -40,7 +40,7 @@ public abstract class GraphStoreExporter> neoNodeProperties; - private final boolean useLabelMapping; + private final Optional nodeLabelMapping; public enum IdMappingType implements IdMapFunction { MAPPED { @@ -87,14 +87,14 @@ protected GraphStoreExporter( GraphStore graphStore, CONFIG config, Optional neoNodeProperties, - boolean useLabelMapping + Optional nodeLabelMapping ) { this.graphStore = graphStore; this.config = config; this.neoNodeProperties = neoNodeProperties .map(NeoNodeProperties::neoNodeProperties) .orElse(Map.of()); - this.useLabelMapping = useLabelMapping; + this.nodeLabelMapping = nodeLabelMapping; } protected abstract void export(GraphStoreInput graphStoreInput); @@ -106,9 +106,7 @@ public ExportedProperties run() { var nodeStore = NodeStore.of( graphStore, neoNodeProperties, - useLabelMapping - ? Optional.of(new NodeLabelMapping(graphStore.nodeLabels())) - : Optional.empty() + nodeLabelMapping ); var relationshipStore = RelationshipStore.of(graphStore, config.defaultRelationshipType()); var graphProperties = graphStore diff --git a/io/core/src/main/java/org/neo4j/gds/core/io/GraphStoreInput.java b/io/core/src/main/java/org/neo4j/gds/core/io/GraphStoreInput.java index d3e2b5e581..4cb59a1e60 100644 --- a/io/core/src/main/java/org/neo4j/gds/core/io/GraphStoreInput.java +++ b/io/core/src/main/java/org/neo4j/gds/core/io/GraphStoreInput.java @@ -182,10 +182,6 @@ public InputIterable relationships(Collector badCollector) { return () -> new RelationshipImporter(relationshipStore, batchSize, idMode.get(), idMapFunction); } - public Optional labelMapping() { - return nodeStore.labelMapping(); - } - @Override public IdType idType() { return idMode.idType; @@ -224,6 +220,10 @@ public InputIterable graphProperties() { return () -> new GraphPropertyIterator(graphProperties.iterator(), concurrency); } + public Optional labelMapping() { + return nodeStore.labelMapping(); + } + static class GraphPropertyIterator implements InputIterator { private final Iterator graphPropertyIterator; @@ -473,7 +473,7 @@ public boolean next(InputEntityVisitor visitor) throws IOException { if (hasProperties) { for (var label : labels) { - nodeStore.nodeProperties + nodeStore.labelToNodeProperties() .getOrDefault(label, Map.of()) .forEach((propertyKey, properties) -> exportProperty( visitor, @@ -483,7 +483,7 @@ public boolean next(InputEntityVisitor visitor) throws IOException { } } } else if (hasProperties) { // no label information, but node properties - nodeStore.nodeProperties.forEach((label, nodeProperties) -> nodeProperties.forEach((propertyKey, properties) -> exportProperty( + nodeStore.labelToNodeProperties().forEach((label, nodeProperties) -> nodeProperties.forEach((propertyKey, properties) -> exportProperty( visitor, propertyKey, properties::getObject diff --git a/io/core/src/main/java/org/neo4j/gds/core/io/NodeLabelMapping.java b/io/core/src/main/java/org/neo4j/gds/core/io/NodeLabelMapping.java index eb35293481..6712b7da48 100644 --- a/io/core/src/main/java/org/neo4j/gds/core/io/NodeLabelMapping.java +++ b/io/core/src/main/java/org/neo4j/gds/core/io/NodeLabelMapping.java @@ -30,17 +30,25 @@ public class NodeLabelMapping { private final HashMap map; + private final HashMap inverseMap; - NodeLabelMapping(Collection labels) { + public NodeLabelMapping(Collection labels) { MutableInt index = new MutableInt(); map = new HashMap<>(labels.stream() .collect(Collectors.toMap(label -> label, label -> Integer.toString(index.getAndIncrement())))); + index.setValue(0); + inverseMap = new HashMap<>(labels.stream() + .collect(Collectors.toMap(label -> Integer.toString(index.getAndIncrement()), label -> label))); } public String get(NodeLabel label) { return map.get(label); } + public NodeLabel get(String labelName) { + return inverseMap.get(labelName); + } + public Set> entrySet() { return map.entrySet(); } diff --git a/io/core/src/main/java/org/neo4j/gds/core/io/NodeStore.java b/io/core/src/main/java/org/neo4j/gds/core/io/NodeStore.java index 867baa366a..ccdfc0b3ba 100644 --- a/io/core/src/main/java/org/neo4j/gds/core/io/NodeStore.java +++ b/io/core/src/main/java/org/neo4j/gds/core/io/NodeStore.java @@ -31,6 +31,7 @@ import java.util.Set; import java.util.function.Function; import java.util.function.LongFunction; +import java.util.stream.Collectors; public class NodeStore { @@ -42,7 +43,8 @@ public class NodeStore { final IdMap idMap; - final Map> nodeProperties; + private final Map> nodeProperties; + final Map> additionalProperties; private final Set availableNodeLabels; @@ -82,6 +84,17 @@ boolean hasProperties() { return nodeProperties != null; } + Map> labelToNodeProperties () { + return nodeLabelMapping.isPresent() + ? nodeProperties.entrySet() + .stream() + .collect(Collectors.toMap( + entry -> nodeLabelMapping.get().get(NodeLabel.of(entry.getKey())), + entry -> entry.getValue() + )) + : nodeProperties; + } + int labelCount() { return !hasLabels() ? 0 : idMap.availableNodeLabels().size(); } diff --git a/io/core/src/main/java/org/neo4j/gds/core/io/db/GraphStoreToDatabaseExporter.java b/io/core/src/main/java/org/neo4j/gds/core/io/db/GraphStoreToDatabaseExporter.java index 88ad125533..9d62cb0968 100644 --- a/io/core/src/main/java/org/neo4j/gds/core/io/db/GraphStoreToDatabaseExporter.java +++ b/io/core/src/main/java/org/neo4j/gds/core/io/db/GraphStoreToDatabaseExporter.java @@ -64,7 +64,7 @@ private GraphStoreToDatabaseExporter( Log log, ProgressTracker progressTracker ) { - super(graphStore, config, neoNodeProperties, false); + super(graphStore, config, neoNodeProperties, Optional.empty()); var executionMonitor = ProgressTrackerExecutionMonitor.of( progressTracker, ClockService.clock(), diff --git a/io/core/src/main/java/org/neo4j/gds/core/io/file/GraphStoreToFileExporter.java b/io/core/src/main/java/org/neo4j/gds/core/io/file/GraphStoreToFileExporter.java index 71d5c0b151..c86360ae2f 100644 --- a/io/core/src/main/java/org/neo4j/gds/core/io/file/GraphStoreToFileExporter.java +++ b/io/core/src/main/java/org/neo4j/gds/core/io/file/GraphStoreToFileExporter.java @@ -27,6 +27,7 @@ import org.neo4j.gds.core.io.GraphStoreExporter; import org.neo4j.gds.core.io.GraphStoreInput; import org.neo4j.gds.core.io.NeoNodeProperties; +import org.neo4j.gds.core.io.NodeLabelMapping; import org.neo4j.gds.core.io.schema.ElementSchemaVisitor; import org.neo4j.gds.core.io.schema.NodeSchemaVisitor; import org.neo4j.gds.core.io.schema.RelationshipSchemaVisitor; @@ -70,6 +71,7 @@ public GraphStoreToFileExporter( GraphStore graphStore, GraphStoreToFileExporterConfig config, Optional neoNodeProperties, + Optional nodeLabelMapping, Supplier> userInfoVisitorSupplier, Supplier> graphInfoVisitorSupplier, Supplier nodeSchemaVisitorSupplier, @@ -84,7 +86,7 @@ public GraphStoreToFileExporter( Log log, String rootTaskName ) { - super(graphStore, config, neoNodeProperties, config.useLabelMapping()); + super(graphStore, config, neoNodeProperties, nodeLabelMapping); this.nodeVisitorSupplier = nodeVisitorSupplier; this.relationshipVisitorSupplier = relationshipVisitorSupplier; this.graphPropertyVisitorSupplier = graphPropertyVisitorSupplier; diff --git a/io/core/src/main/java/org/neo4j/gds/core/io/file/NodeVisitor.java b/io/core/src/main/java/org/neo4j/gds/core/io/file/NodeVisitor.java index 5cd311e16e..2358255b49 100644 --- a/io/core/src/main/java/org/neo4j/gds/core/io/file/NodeVisitor.java +++ b/io/core/src/main/java/org/neo4j/gds/core/io/file/NodeVisitor.java @@ -35,11 +35,11 @@ public abstract class NodeVisitor extends ElementVisitor { private static final List EMPTY_LABELS = Collections.emptyList(); - private static final Set EMPTY_LABELS_LABEL = Set.of(NodeLabel.ALL_NODES); + protected static final Set EMPTY_LABELS_LABEL = Set.of(NodeLabel.ALL_NODES); - private final NodeSchema nodeSchema; + protected final NodeSchema nodeSchema; private long currentId; - private List currentLabels; + protected List currentLabels; private String labelIdentifier; protected NodeVisitor(NodeSchema nodeSchema) { diff --git a/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/CsvNodeVisitor.java b/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/CsvNodeVisitor.java index 009d380f08..505dc7e71b 100644 --- a/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/CsvNodeVisitor.java +++ b/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/CsvNodeVisitor.java @@ -21,18 +21,24 @@ import com.fasterxml.jackson.dataformat.csv.CsvSchema; import org.jetbrains.annotations.TestOnly; +import org.neo4j.gds.NodeLabel; import org.neo4j.gds.api.schema.NodeSchema; import org.neo4j.gds.api.schema.PropertySchema; +import org.neo4j.gds.core.io.NodeLabelMapping; import org.neo4j.gds.core.io.file.NodeVisitor; import java.io.IOException; import java.nio.file.Path; +import java.util.ArrayList; import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; +import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.function.UnaryOperator; +import java.util.stream.Collectors; import static org.neo4j.gds.utils.StringFormatting.formatWithLocale; @@ -45,22 +51,26 @@ public class CsvNodeVisitor extends NodeVisitor { private final Map csvAppenders; private final Set headerFiles; + private final Optional nodeLabelMapping; + CsvNodeVisitor( Path fileLocation, NodeSchema nodeSchema, Set headerFiles, - int visitorId + int visitorId, + Optional nodeLabelMapping ) { super(nodeSchema); this.fileLocation = fileLocation; this.headerFiles = headerFiles; this.visitorId = visitorId; + this.nodeLabelMapping = nodeLabelMapping; this.csvAppenders = new HashMap<>(); } @TestOnly public CsvNodeVisitor(Path fileLocation, NodeSchema nodeSchema) { - this(fileLocation, nodeSchema, new HashSet<>(), 0); + this(fileLocation, nodeSchema, new HashSet<>(), 0, Optional.empty()); } @Override @@ -160,4 +170,18 @@ private JacksonFileAppender fileAppender(Path filePath, UnaryOperator getPropertySchema() { + var nodeLabelList = currentLabels.isEmpty() + ? EMPTY_LABELS_LABEL + : currentLabels.stream() + .map(nodeLabelMapping.isPresent() + ? nodeLabelMapping.get()::get + : NodeLabel::of + ) + .collect(Collectors.toSet()); + var propertySchemaForLabels = nodeSchema.filter(nodeLabelList); + return new ArrayList<>(propertySchemaForLabels.unionProperties().values()); + } } diff --git a/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/GraphStoreToCsvExporter.java b/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/GraphStoreToCsvExporter.java index 477e85dd70..6e94b7f086 100644 --- a/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/GraphStoreToCsvExporter.java +++ b/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/GraphStoreToCsvExporter.java @@ -24,6 +24,7 @@ import org.neo4j.gds.api.nodeproperties.ValueType; import org.neo4j.gds.api.schema.MutableNodeSchema; import org.neo4j.gds.core.io.NeoNodeProperties; +import org.neo4j.gds.core.io.NodeLabelMapping; import org.neo4j.gds.core.io.file.GraphStoreToFileExporter; import org.neo4j.gds.core.io.file.GraphStoreToFileExporterConfig; import org.neo4j.gds.core.utils.progress.TaskRegistryFactory; @@ -69,11 +70,16 @@ public static GraphStoreToFileExporter create( .forEach(label -> neoNodeSchema.getOrCreateLabel(label).addProperty(key, ValueType.STRING)) )); + Optional nodeLabelMapping = config.useLabelMapping() + ? Optional.of(new NodeLabelMapping(graphStore.nodeLabels())) + : Optional.empty(); + return new GraphStoreToFileExporter( graphStore, config, neoNodeProperties, - () -> new UserInfoVisitor(exportPath), + nodeLabelMapping, + () -> new UserInfoVisitor(exportPath), () -> new CsvGraphInfoVisitor(exportPath), () -> new CsvNodeSchemaVisitor(exportPath), () -> new CsvNodeLabelMappingVisitor(exportPath), @@ -84,7 +90,8 @@ public static GraphStoreToFileExporter create( exportPath, nodeSchema.union(neoNodeSchema), headerFiles, - index + index, + nodeLabelMapping ), (index) -> new CsvRelationshipVisitor(exportPath, relationshipSchema, headerFiles, index), (index) -> new CsvGraphPropertyVisitor( From f064927ea04c5e8dc38ec96bf01a7daeef3e2ab9 Mon Sep 17 00:00:00 2001 From: yuval Date: Fri, 28 Jul 2023 14:43:28 +0200 Subject: [PATCH 175/273] Fix style MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Sören Reichardt --- io/core/src/main/java/org/neo4j/gds/core/io/NodeStore.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/io/core/src/main/java/org/neo4j/gds/core/io/NodeStore.java b/io/core/src/main/java/org/neo4j/gds/core/io/NodeStore.java index ccdfc0b3ba..97957d71cd 100644 --- a/io/core/src/main/java/org/neo4j/gds/core/io/NodeStore.java +++ b/io/core/src/main/java/org/neo4j/gds/core/io/NodeStore.java @@ -84,7 +84,7 @@ boolean hasProperties() { return nodeProperties != null; } - Map> labelToNodeProperties () { + Map> labelToNodeProperties() { return nodeLabelMapping.isPresent() ? nodeProperties.entrySet() .stream() From 7b90a8719f326bee00f96414bcab2ce568323f84 Mon Sep 17 00:00:00 2001 From: yuval Date: Mon, 31 Jul 2023 17:37:23 +0200 Subject: [PATCH 176/273] Bumped Aura version to +29 --- gradle/version.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle/version.gradle b/gradle/version.gradle index 3bd37271dc..a4c6760613 100644 --- a/gradle/version.gradle +++ b/gradle/version.gradle @@ -1,6 +1,6 @@ ext { gdsBaseVersion = '2.4.3' - gdsAuraVersion = '28' + gdsAuraVersion = '29' gdsVersion = gdsBaseVersion + (rootProject.hasProperty('aurads') ? "+${gdsAuraVersion}" : "") } From 8f810484126d290259c24308a9fbfd33a45e6c57 Mon Sep 17 00:00:00 2001 From: Veselin Nikolov Date: Tue, 1 Aug 2023 08:00:57 +0100 Subject: [PATCH 177/273] Update checkstyle tool version --- gradle/dependencies.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle/dependencies.gradle b/gradle/dependencies.gradle index df419c7bdb..f80dbfb555 100644 --- a/gradle/dependencies.gradle +++ b/gradle/dependencies.gradle @@ -55,7 +55,7 @@ ext { 'auto-common' : '1.2.1', 'auto-service' : '1.0.1', 'bouncycastle-prov' : '1.72', - 'checkStyle' : '10.3.4', + 'checkStyle' : '10.12.1', 'commons-io' : '2.11.0', 'commons-lang3' : '3.12.0', 'commons-math3' : '3.6.1', From 9e26657ae9a220753ce09263d845d8bba3fba633 Mon Sep 17 00:00:00 2001 From: Veselin Nikolov Date: Tue, 1 Aug 2023 09:07:28 +0100 Subject: [PATCH 178/273] Use Google auto-service in only as `compileOnly` --- config-generator/build.gradle | 2 +- procedure-collector/build.gradle | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/config-generator/build.gradle b/config-generator/build.gradle index 1faebbcd9c..c86f22675c 100644 --- a/config-generator/build.gradle +++ b/config-generator/build.gradle @@ -10,7 +10,7 @@ dependencies { annotationProcessor group: 'org.immutables', name: 'builder', version: ver.'immutables' annotationProcessor group: 'org.immutables', name: 'value', version: ver.'immutables' - implementation group: 'com.google.auto.service', name: 'auto-service', version: ver.'auto-service' + compileOnly group: 'com.google.auto.service', name: 'auto-service', version: ver.'auto-service' implementation project(':annotations') implementation group: 'org.immutables', name: 'value-annotations', version: ver.'immutables' diff --git a/procedure-collector/build.gradle b/procedure-collector/build.gradle index 6e3d84126c..b65aa3860d 100644 --- a/procedure-collector/build.gradle +++ b/procedure-collector/build.gradle @@ -10,7 +10,7 @@ dependencies { annotationProcessor group: 'org.immutables', name: 'builder', version: ver.'immutables' annotationProcessor group: 'org.immutables', name: 'value', version: ver.'immutables' - implementation group: 'com.google.auto.service', name: 'auto-service', version: ver.'auto-service' + compileOnly group: 'com.google.auto.service', name: 'auto-service', version: ver.'auto-service' implementation project(':annotations') implementation project(':executor') From 3801e8bf8ccc968d0a1b57f2884976a93a8403dd Mon Sep 17 00:00:00 2001 From: yuval Date: Tue, 1 Aug 2023 10:04:45 +0200 Subject: [PATCH 179/273] Node label mapping docs --- .../pages/management-ops/backup-restore.adoc | 1 + .../graph-catalog/graph-export-csv.adoc | 26 +++++++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/doc/modules/ROOT/pages/management-ops/backup-restore.adoc b/doc/modules/ROOT/pages/management-ops/backup-restore.adoc index 56afdf9793..140b4cc7d5 100644 --- a/doc/modules/ROOT/pages/management-ops/backup-restore.adoc +++ b/doc/modules/ROOT/pages/management-ops/backup-restore.adoc @@ -47,6 +47,7 @@ YIELD | Name | Type | Default | Description | concurrency | Integer | 4 | The number of concurrent threads used for performing the backup. | includeGraphs | Boolean | true | Flag to decide whether only models or also graphs should be backed up. +| useLabelMapping | Boolean | true | Whether to use node label mapping when exporting the graph. See xref:management-ops/graph-export/graph-export-csv.adoc#node_label_mapping[exporting graphs to csv] for details. |=== .Results diff --git a/doc/modules/ROOT/partials/management-ops/graph-catalog/graph-export-csv.adoc b/doc/modules/ROOT/partials/management-ops/graph-catalog/graph-export-csv.adoc index 858700857b..5870e4cb94 100644 --- a/doc/modules/ROOT/partials/management-ops/graph-catalog/graph-export-csv.adoc +++ b/doc/modules/ROOT/partials/management-ops/graph-catalog/graph-export-csv.adoc @@ -51,6 +51,7 @@ YIELD | writeConcurrency | Boolean | 4 | yes | The number of concurrent threads used for writing the database. | defaultRelationshipType | String | +__ALL__+ | yes | Relationship type used for `*` relationship projections. | additionalNodeProperties | String, List or Map | {} | yes | Allows for exporting additional node properties from the original graph backing the projected graph. +| useLabelMapping | Boolean | false | yes | Whether to use node label mapping when exporting the graph |=== @@ -132,6 +133,8 @@ YIELD The format of the exported CSV files is based on the format that is supported by the Neo4j Admin https://neo4j.com/docs/operations-manual/current/tools/neo4j-admin/neo4j-admin-import/[import] command. +GDS does not add a column for node labels and relationship types in the data. In order to import them using Neo4j Admin, the labels and types should be set using the `--nodes` and `--relationship` parameters. + === Nodes @@ -160,6 +163,29 @@ nodes_A_B_1.csv nodes_A_B_2.csv ---- +==== Nodes label mapping [[node_label_mapping]] +When the configuration parameter `useLabelMapping` is set to true, the names of the labels will be mapped to integers during the export. This mapping will be written to a new file named `label-mappings.csv`. This parameter is required when label names contain underscores or special characters that are forbidden in file names by the OS. + +Using the example above, if label mapping is enabled, the content of `label-mappings.csv` may be: +---- +index,label +0,A +1,B +---- + +In this case, these files will be created for the nodes: + +---- +nodes_0_header.csv +nodes_0_0.csv +nodes_1_header.csv +nodes_1_0.csv +nodes_1_2.csv +nodes_0_1_header.csv +nodes_0_1_0.csv +nodes_0_1_1.csv +nodes_0_1_2.csv +---- === Relationships From d5da93513e07a5027e083d21d30d550b25b0a4d5 Mon Sep 17 00:00:00 2001 From: yuval Date: Tue, 1 Aug 2023 10:09:24 +0200 Subject: [PATCH 180/273] updated comment --- .../neo4j/gds/core/io/file/GraphStoreToFileExporterConfig.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/io/core/src/main/java/org/neo4j/gds/core/io/file/GraphStoreToFileExporterConfig.java b/io/core/src/main/java/org/neo4j/gds/core/io/file/GraphStoreToFileExporterConfig.java index db58b6c69a..c1c4ead72d 100644 --- a/io/core/src/main/java/org/neo4j/gds/core/io/file/GraphStoreToFileExporterConfig.java +++ b/io/core/src/main/java/org/neo4j/gds/core/io/file/GraphStoreToFileExporterConfig.java @@ -44,7 +44,7 @@ default String username() { @Value.Default default boolean useLabelMapping() { - // the reason for default false (unlike BackupConfig) is that Neo4j Admin import is not aware of label mapping + // the reason for default false (unlike BackupConfig) is to prevent a breaking change in CSV export return false; } From 25721192ea999149c7c862435351e45cca8eae1d Mon Sep 17 00:00:00 2001 From: yuval Date: Tue, 1 Aug 2023 11:19:52 +0200 Subject: [PATCH 181/273] Changed phrasing Co-authored-by: Martin Junghanns --- doc/modules/ROOT/pages/management-ops/backup-restore.adoc | 8 ++++---- .../management-ops/graph-catalog/graph-export-csv.adoc | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/modules/ROOT/pages/management-ops/backup-restore.adoc b/doc/modules/ROOT/pages/management-ops/backup-restore.adoc index 140b4cc7d5..c57290a97b 100644 --- a/doc/modules/ROOT/pages/management-ops/backup-restore.adoc +++ b/doc/modules/ROOT/pages/management-ops/backup-restore.adoc @@ -44,10 +44,10 @@ YIELD .Configuration [opts="header",cols="1,1,1,4"] |=== -| Name | Type | Default | Description -| concurrency | Integer | 4 | The number of concurrent threads used for performing the backup. -| includeGraphs | Boolean | true | Flag to decide whether only models or also graphs should be backed up. -| useLabelMapping | Boolean | true | Whether to use node label mapping when exporting the graph. See xref:management-ops/graph-export/graph-export-csv.adoc#node_label_mapping[exporting graphs to csv] for details. +| Name | Type | Default | Description +| concurrency | Integer | 4 | The number of concurrent threads used for performing the backup. +| includeGraphs | Boolean | true | Flag to decide whether only models or also graphs should be backed up. +| useLabelMapping | Boolean | true | Flag to decide whether to use node label mapping when exporting the graph. See xref:management-ops/graph-export/graph-export-csv.adoc#node_label_mapping[exporting graphs to csv] for details. |=== .Results diff --git a/doc/modules/ROOT/partials/management-ops/graph-catalog/graph-export-csv.adoc b/doc/modules/ROOT/partials/management-ops/graph-catalog/graph-export-csv.adoc index 5870e4cb94..7dc15e6f56 100644 --- a/doc/modules/ROOT/partials/management-ops/graph-catalog/graph-export-csv.adoc +++ b/doc/modules/ROOT/partials/management-ops/graph-catalog/graph-export-csv.adoc @@ -51,7 +51,7 @@ YIELD | writeConcurrency | Boolean | 4 | yes | The number of concurrent threads used for writing the database. | defaultRelationshipType | String | +__ALL__+ | yes | Relationship type used for `*` relationship projections. | additionalNodeProperties | String, List or Map | {} | yes | Allows for exporting additional node properties from the original graph backing the projected graph. -| useLabelMapping | Boolean | false | yes | Whether to use node label mapping when exporting the graph +| useLabelMapping | Boolean | false | yes | Flag to decide whether to use node label mapping when exporting the graph |=== From d9f3b5aee79c02f8c0e166025ed980e22ca60464 Mon Sep 17 00:00:00 2001 From: yuval Date: Tue, 1 Aug 2023 11:21:20 +0200 Subject: [PATCH 182/273] Fixed parameter type Co-authored-by: Martin Junghanns --- .../management-ops/graph-catalog/graph-export-csv.adoc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/modules/ROOT/partials/management-ops/graph-catalog/graph-export-csv.adoc b/doc/modules/ROOT/partials/management-ops/graph-catalog/graph-export-csv.adoc index 7dc15e6f56..4386e55145 100644 --- a/doc/modules/ROOT/partials/management-ops/graph-catalog/graph-export-csv.adoc +++ b/doc/modules/ROOT/partials/management-ops/graph-catalog/graph-export-csv.adoc @@ -48,7 +48,7 @@ YIELD |=== | Name | Type | Default | Optional | Description | exportName | String | none | No | The name of the directory where the graph is exported to. The absolute path of the exported CSV files depends on the configuration parameter `gds.export.location` in the `neo4j.conf`. -| writeConcurrency | Boolean | 4 | yes | The number of concurrent threads used for writing the database. +| writeConcurrency | Integer | 4 | yes | The number of concurrent threads used for writing the database. | defaultRelationshipType | String | +__ALL__+ | yes | Relationship type used for `*` relationship projections. | additionalNodeProperties | String, List or Map | {} | yes | Allows for exporting additional node properties from the original graph backing the projected graph. | useLabelMapping | Boolean | false | yes | Flag to decide whether to use node label mapping when exporting the graph @@ -108,7 +108,7 @@ YIELD | Name | Type | Default | Optional | Description | exportName | String | none | no | Name of the folder the exported CSV files are saved at. | samplingFactor | Double | 0.001 | yes | The fraction of nodes and relationships to sample for the estimation. -| writeConcurrency | Boolean | 4 | yes | The number of concurrent threads used for writing the database. +| writeConcurrency | Integer | 4 | yes | The number of concurrent threads used for writing the database. | defaultRelationshipType | String | +__ALL__+ | yes | Relationship type used for `*` relationship projections. |=== From f31ad2881167666eb28ad5d242a8f58e7d4533e6 Mon Sep 17 00:00:00 2001 From: yuval Date: Tue, 1 Aug 2023 11:47:20 +0200 Subject: [PATCH 183/273] Style and links Co-authored-by: Martin Junghanns --- .../management-ops/graph-catalog/graph-export-csv.adoc | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/doc/modules/ROOT/partials/management-ops/graph-catalog/graph-export-csv.adoc b/doc/modules/ROOT/partials/management-ops/graph-catalog/graph-export-csv.adoc index 4386e55145..a0432ed6ca 100644 --- a/doc/modules/ROOT/partials/management-ops/graph-catalog/graph-export-csv.adoc +++ b/doc/modules/ROOT/partials/management-ops/graph-catalog/graph-export-csv.adoc @@ -133,8 +133,11 @@ YIELD The format of the exported CSV files is based on the format that is supported by the Neo4j Admin https://neo4j.com/docs/operations-manual/current/tools/neo4j-admin/neo4j-admin-import/[import] command. -GDS does not add a column for node labels and relationship types in the data. In order to import them using Neo4j Admin, the labels and types should be set using the `--nodes` and `--relationship` parameters. +GDS does not add a column for node labels and relationship types in the data. +In order to import them using Neo4j Admin, the labels and types should be set using the `--nodes` and `--relationship` parameters. +More details here: using the same https://neo4j.com/docs/operations-manual/current/tutorial/neo4j-admin-import/#_using_the_same_label_for_every_node[label for every node], +https://neo4j.com/docs/operations-manual/current/tutorial/neo4j-admin-import/#_using_the_same_relationship_type_for_every_relationship[type for every relationship] === Nodes @@ -164,7 +167,10 @@ nodes_A_B_2.csv ---- ==== Nodes label mapping [[node_label_mapping]] -When the configuration parameter `useLabelMapping` is set to true, the names of the labels will be mapped to integers during the export. This mapping will be written to a new file named `label-mappings.csv`. This parameter is required when label names contain underscores or special characters that are forbidden in file names by the OS. + +When the configuration parameter `useLabelMapping` is set to true, the names of the labels will be mapped to integers during the export. +This mapping will be written to a new file named `label-mappings.csv`. +This parameter is required when label names contain underscores or special characters that are forbidden in file names by the OS. Using the example above, if label mapping is enabled, the content of `label-mappings.csv` may be: ---- From 188b5bfa6cb42048908c3f38d9a1d71482bcfe55 Mon Sep 17 00:00:00 2001 From: Veselin Nikolov Date: Tue, 25 Jul 2023 12:53:40 +0100 Subject: [PATCH 184/273] Post-release actions for GDS 2.4.3 --- README.adoc | 16 ++++++++-------- .../pages/management-ops/utility-functions.adoc | 2 +- examples/pregel-bootstrap/build.gradle | 2 +- gradle/version.gradle | 2 +- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/README.adoc b/README.adoc index 4b35e482ba..c07f431c18 100644 --- a/README.adoc +++ b/README.adoc @@ -95,7 +95,7 @@ For the most basic set of features, like graph loading and the graph representat org.neo4j.gds core - 2.4.1 + 2.4.3 ---- @@ -107,21 +107,21 @@ The algorithms are located in the `algo-common`, `algo` and `alpha-algo` modules org.neo4j.gds algo-common - 2.4.1 + 2.4.3 org.neo4j.gds algo - 2.4.1 + 2.4.3 org.neo4j.gds alpha-algo - 2.4.1 + 2.4.3 ---- @@ -133,28 +133,28 @@ The procedures are located in the `proc-common`, `proc` and `alpha-proc` modules org.neo4j.gds proc-common - 2.4.1 + 2.4.3 org.neo4j.gds proc - 2.4.1 + 2.4.3 org.neo4j.gds alpha-proc - 2.4.1 + 2.4.3 org.neo4j.gds open-write-services - 2.4.1 + 2.4.3 ---- diff --git a/doc/modules/ROOT/pages/management-ops/utility-functions.adoc b/doc/modules/ROOT/pages/management-ops/utility-functions.adoc index 5c369412a9..cd84872878 100644 --- a/doc/modules/ROOT/pages/management-ops/utility-functions.adoc +++ b/doc/modules/ROOT/pages/management-ops/utility-functions.adoc @@ -27,7 +27,7 @@ RETURN gds.version() AS version [opts="header"] |=== | version -| "2.4.3" +| "2.4.4" |=== -- diff --git a/examples/pregel-bootstrap/build.gradle b/examples/pregel-bootstrap/build.gradle index 5ca090ea53..ff170deb4d 100644 --- a/examples/pregel-bootstrap/build.gradle +++ b/examples/pregel-bootstrap/build.gradle @@ -7,7 +7,7 @@ plugins { ext { // Make sure these are the same as your installation of GDS and Neo4j - gdsVersion = '2.4.2' + gdsVersion = '2.4.4' neo4jVersion = '5.8.0' // Necessary to generate value classes for Pregel configs diff --git a/gradle/version.gradle b/gradle/version.gradle index a4c6760613..3c3b5e41ce 100644 --- a/gradle/version.gradle +++ b/gradle/version.gradle @@ -1,5 +1,5 @@ ext { - gdsBaseVersion = '2.4.3' + gdsBaseVersion = '2.4.4' gdsAuraVersion = '29' gdsVersion = gdsBaseVersion + (rootProject.hasProperty('aurads') ? "+${gdsAuraVersion}" : "") From f583d62ddecf21e0bc56dd16e86918b6cc7b7e77 Mon Sep 17 00:00:00 2001 From: Veselin Nikolov Date: Fri, 28 Jul 2023 08:58:38 +0100 Subject: [PATCH 185/273] Add test for Pregel bootstrap write --- examples/pregel-bootstrap/build.gradle | 4 +- .../gds/example/ExamplePregelComputation.java | 2 +- ...ExamplePregelComputationWriteProcTest.java | 68 +++++++++++++++++++ 3 files changed, 72 insertions(+), 2 deletions(-) create mode 100644 examples/pregel-bootstrap/src/test/java/gds/example/ExamplePregelComputationWriteProcTest.java diff --git a/examples/pregel-bootstrap/build.gradle b/examples/pregel-bootstrap/build.gradle index ff170deb4d..5d45ce4568 100644 --- a/examples/pregel-bootstrap/build.gradle +++ b/examples/pregel-bootstrap/build.gradle @@ -8,7 +8,7 @@ plugins { ext { // Make sure these are the same as your installation of GDS and Neo4j gdsVersion = '2.4.4' - neo4jVersion = '5.8.0' + neo4jVersion = '5.10.0' // Necessary to generate value classes for Pregel configs immutablesVersion = '2.8.1' @@ -28,6 +28,7 @@ dependencies { // We depend on the proc artifact of the GDS library compileOnly "org.neo4j.gds:proc:$gdsVersion" compileOnly "org.neo4j.gds:core:$gdsVersion" + compileOnly "org.neo4j.gds:core-write:$gdsVersion" compileOnly "org.neo4j.gds:algo-common:$gdsVersion" compileOnly "org.neo4j.gds:executor:$gdsVersion" compileOnly "org.neo4j.gds:memory-usage:$gdsVersion" @@ -40,6 +41,7 @@ dependencies { // We also need to depend on Neo4j itself for our tests. // Usually, Neo4j is available when we are running as a plugin. compileOnly "org.neo4j:neo4j:$neo4jVersion" + compileOnly "org.neo4j:neo4j-procedure-api:$neo4jVersion" // We need an annotation processor for our K1 configuration annotationProcessor "org.immutables:value:$immutablesVersion" diff --git a/examples/pregel-bootstrap/src/main/java/gds/example/ExamplePregelComputation.java b/examples/pregel-bootstrap/src/main/java/gds/example/ExamplePregelComputation.java index b06510b75b..4778e4ec8f 100644 --- a/examples/pregel-bootstrap/src/main/java/gds/example/ExamplePregelComputation.java +++ b/examples/pregel-bootstrap/src/main/java/gds/example/ExamplePregelComputation.java @@ -32,7 +32,7 @@ import org.neo4j.gds.beta.pregel.context.InitContext; import org.neo4j.gds.core.CypherMapWrapper; -@PregelProcedure(name = "pregel.example", modes = {GDSMode.STREAM}, description = "My first Pregel example") +@PregelProcedure(name = "pregel.example", modes = {GDSMode.STREAM, GDSMode.WRITE}, description = "My first Pregel example") public class ExamplePregelComputation implements PregelComputation { public static final String KEY = "key"; diff --git a/examples/pregel-bootstrap/src/test/java/gds/example/ExamplePregelComputationWriteProcTest.java b/examples/pregel-bootstrap/src/test/java/gds/example/ExamplePregelComputationWriteProcTest.java new file mode 100644 index 0000000000..600e7cfd8c --- /dev/null +++ b/examples/pregel-bootstrap/src/test/java/gds/example/ExamplePregelComputationWriteProcTest.java @@ -0,0 +1,68 @@ +/* + * Copyright (c) 2017-2020 "Neo4j," + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package gds.example; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.neo4j.gds.BaseProcTest; +import org.neo4j.gds.catalog.GraphProjectProc; +import org.neo4j.gds.extension.Neo4jGraph; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.InstanceOfAssertFactories.LONG; + +class ExamplePregelComputationWriteProcTest extends BaseProcTest { + + @Neo4jGraph + private static final String MY_TEST_GRAPH = + "CREATE" + + " (alice)" + + ", (bob)" + + ", (eve)" + + ", (alice)-[:LIKES]->(bob)" + + ", (bob)-[:LIKES]->(alice)" + + ", (eve)-[:DISLIKES]->(alice)" + + ", (eve)-[:DISLIKES]->(bob)"; + + @BeforeEach + void setup() throws Exception { + registerProcedures(ExamplePregelComputationWriteProc.class, GraphProjectProc.class); + } + + @Test + void write() { + runQueryWithRowConsumer("MATCH (n) WHERE n.pregelkey IS NOT NULL RETURN count(n) AS count ", row -> { + assertThat(row.getNumber("count")) + .asInstanceOf(LONG) + .as("There should be no nodes having the `pregelkey` property") + .isEqualTo(0); + }); + + runQuery("CALL gds.graph.project('graph', '*', '*')"); + runQuery("CALL pregel.example.write('graph', { maxIterations: 10, writeProperty: 'pregel' })"); + + runQueryWithRowConsumer("MATCH (n) WHERE n.pregelkey IS NOT NULL RETURN count(n) AS count ", row -> { + assertThat(row.getNumber("count")) + .asInstanceOf(LONG) + .as("All nodes should have the `pregelkey` property") + .isEqualTo(3); + }); + } +} From 753687721c17d97c8cd0b0551789c0a33b228d0f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B6ren=20Reichardt?= Date: Thu, 27 Jul 2023 12:06:37 +0200 Subject: [PATCH 186/273] Fix compat issue when registering procedures --- .../gds/projection/CypherAggregationTest.java | 4 +- .../gds/compat/GraphDatabaseApiProxy.java | 65 ++++++++++++++++++- 2 files changed, 63 insertions(+), 6 deletions(-) diff --git a/cypher-aggregation/src/test/java/org/neo4j/gds/projection/CypherAggregationTest.java b/cypher-aggregation/src/test/java/org/neo4j/gds/projection/CypherAggregationTest.java index 246cb56738..02e0e2d7ae 100644 --- a/cypher-aggregation/src/test/java/org/neo4j/gds/projection/CypherAggregationTest.java +++ b/cypher-aggregation/src/test/java/org/neo4j/gds/projection/CypherAggregationTest.java @@ -48,7 +48,6 @@ import org.neo4j.graphdb.Direction; import org.neo4j.graphdb.RelationshipType; import org.neo4j.graphdb.Result; -import org.neo4j.kernel.api.procedure.GlobalProcedures; import java.util.List; import java.util.Map; @@ -86,8 +85,7 @@ class CypherAggregationTest extends BaseProcTest { @BeforeEach void setup() throws Exception { - var procedures = GraphDatabaseApiProxy.resolveDependency(db, GlobalProcedures.class); - procedures.register(Neo4jProxy.callableUserAggregationFunction(CypherAggregation.newInstance()), true); + GraphDatabaseApiProxy.register(db, Neo4jProxy.callableUserAggregationFunction(CypherAggregation.newInstance())); registerProcedures(GraphDropProc.class, GraphListProc.class, WccStreamProc.class); } diff --git a/neo4j-adapter/src/main/java/org/neo4j/gds/compat/GraphDatabaseApiProxy.java b/neo4j-adapter/src/main/java/org/neo4j/gds/compat/GraphDatabaseApiProxy.java index 33451b2d65..0eca1c93c9 100644 --- a/neo4j-adapter/src/main/java/org/neo4j/gds/compat/GraphDatabaseApiProxy.java +++ b/neo4j-adapter/src/main/java/org/neo4j/gds/compat/GraphDatabaseApiProxy.java @@ -40,6 +40,8 @@ import org.neo4j.kernel.internal.GraphDatabaseAPI; import org.neo4j.storageengine.api.StorageEngineFactory; +import java.lang.reflect.InvocationTargetException; +import java.util.Arrays; import java.util.Map; import java.util.function.Consumer; import java.util.function.Function; @@ -82,7 +84,7 @@ public static void registerProcedures( ) throws KernelException { GlobalProcedures procedures = resolveDependency(db, GlobalProcedures.class); for (Class clazz : procedureClasses) { - procedures.registerProcedure(clazz, overrideCurrentImplementation); + registerProcedures(procedures, clazz, overrideCurrentImplementation); } } @@ -105,7 +107,7 @@ public static void registerAggregationFunctions(GraphDatabaseService db, Class procedureClass, + boolean overrideCurrentImplementation + ) { + var globalProceduresClass = globalProcedures.getClass(); + var registerProcedureMethod = Arrays.stream(globalProceduresClass.getDeclaredMethods()) + .filter(method -> method.getName().equals("registerProcedure")) + .findFirst() + .orElseThrow(() -> new RuntimeException("Could not find registerProcedure method")); + + try { + switch (registerProcedureMethod.getParameterCount()) { + case 1: + registerProcedureMethod.invoke(globalProcedures, procedureClass); + break; + case 2: { + registerProcedureMethod.invoke(globalProcedures, procedureClass, overrideCurrentImplementation); + break; + } + default: + throw new RuntimeException("Unexpected number of parameters for registerProcedure method"); + } + } catch (IllegalAccessException | InvocationTargetException e) { + throw new RuntimeException(e); + } + } + + private static void registerAggregationFunction(GlobalProcedures globalProcedures, CallableUserAggregationFunction function) { + var globalProceduresClass = globalProcedures.getClass(); + var registerProcedureMethod = Arrays.stream(globalProceduresClass.getDeclaredMethods()) + .filter(method -> method.getName().equals("register")) + .filter(method -> method.getParameterCount() == 1 && method.getParameterTypes()[0].equals(CallableUserAggregationFunction.class)) + .findFirst() + .orElseThrow(() -> new RuntimeException("Could not find registerAggregationFunction method")); + + try { + switch (registerProcedureMethod.getParameterCount()) { + case 1: + registerProcedureMethod.invoke(globalProcedures, function); + break; + case 2: { + registerProcedureMethod.invoke(globalProcedures, function, true); + break; + } + default: + throw new RuntimeException("Unexpected number of parameters for register method"); + } + } catch (IllegalAccessException | InvocationTargetException e) { + throw new RuntimeException(e); + } + } } From e7c42a87c78daece7481d3b3fa496ab8262bfdf8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B6ren=20Reichardt?= Date: Fri, 28 Jul 2023 09:42:50 +0200 Subject: [PATCH 187/273] Fix storage engine compat issue CountsAccessor -> CountsStore --- .../gds/compat/_44/StorageEngineProxyImpl.java | 16 ---------------- .../gds/compat/_51/StorageEngineProxyImpl.java | 16 ---------------- .../gds/compat/_510/StorageEngineProxyImpl.java | 16 ---------------- .../gds/compat/_52/StorageEngineProxyImpl.java | 16 ---------------- .../gds/compat/_53/StorageEngineProxyImpl.java | 16 ---------------- .../gds/compat/_54/StorageEngineProxyImpl.java | 16 ---------------- .../gds/compat/_55/StorageEngineProxyImpl.java | 16 ---------------- .../gds/compat/_56/StorageEngineProxyImpl.java | 16 ---------------- .../gds/compat/_57/StorageEngineProxyImpl.java | 16 ---------------- .../gds/compat/_58/StorageEngineProxyImpl.java | 16 ---------------- .../gds/compat/_59/StorageEngineProxyImpl.java | 16 ---------------- .../neo4j/gds/compat/StorageEngineProxyApi.java | 11 ----------- .../org/neo4j/gds/compat/StorageEngineProxy.java | 13 ------------- 13 files changed, 200 deletions(-) diff --git a/compatibility/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/StorageEngineProxyImpl.java b/compatibility/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/StorageEngineProxyImpl.java index e31769817d..a82ceac106 100644 --- a/compatibility/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/StorageEngineProxyImpl.java +++ b/compatibility/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/StorageEngineProxyImpl.java @@ -22,7 +22,6 @@ import org.neo4j.common.Edition; import org.neo4j.configuration.Config; import org.neo4j.configuration.GraphDatabaseInternalSettings; -import org.neo4j.counts.CountsAccessor; import org.neo4j.dbms.api.DatabaseManagementService; import org.neo4j.gds.compat.AbstractInMemoryNodeCursor; import org.neo4j.gds.compat.AbstractInMemoryNodePropertyCursor; @@ -35,15 +34,12 @@ import org.neo4j.graphdb.Direction; import org.neo4j.graphdb.GraphDatabaseService; import org.neo4j.internal.recordstorage.AbstractInMemoryRelationshipScanCursor; -import org.neo4j.internal.recordstorage.InMemoryStorageReader44; import org.neo4j.io.layout.DatabaseLayout; -import org.neo4j.storageengine.api.CommandCreationContext; import org.neo4j.storageengine.api.PropertySelection; import org.neo4j.storageengine.api.RelationshipSelection; import org.neo4j.storageengine.api.StorageEngine; import org.neo4j.storageengine.api.StorageEntityCursor; import org.neo4j.storageengine.api.StoragePropertyCursor; -import org.neo4j.storageengine.api.StorageReader; import org.neo4j.storageengine.api.StorageRelationshipTraversalCursor; import org.neo4j.token.TokenHolders; @@ -51,11 +47,6 @@ public class StorageEngineProxyImpl implements StorageEngineProxyApi { - @Override - public CommandCreationContext inMemoryCommandCreationContext() { - return new InMemoryCommandCreationContextImpl(); - } - @Override public void initRelationshipTraversalCursorForRelType( StorageRelationshipTraversalCursor cursor, @@ -69,13 +60,6 @@ public void initRelationshipTraversalCursorForRelType( cursor.init(sourceNodeId, -1, relationshipSelection); } - @Override - public StorageReader inMemoryStorageReader( - CypherGraphStore graphStore, TokenHolders tokenHolders, CountsAccessor counts - ) { - return new InMemoryStorageReader44(graphStore, tokenHolders, counts); - } - @Override public StorageEngine createInMemoryStorageEngine(DatabaseLayout databaseLayout, TokenHolders tokenHolders) { return new InMemoryStorageEngineImpl(databaseLayout, tokenHolders); diff --git a/compatibility/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/StorageEngineProxyImpl.java b/compatibility/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/StorageEngineProxyImpl.java index 532768751c..df302f95b1 100644 --- a/compatibility/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/StorageEngineProxyImpl.java +++ b/compatibility/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/StorageEngineProxyImpl.java @@ -22,7 +22,6 @@ import org.neo4j.common.Edition; import org.neo4j.configuration.Config; import org.neo4j.configuration.GraphDatabaseInternalSettings; -import org.neo4j.counts.CountsAccessor; import org.neo4j.dbms.api.DatabaseManagementService; import org.neo4j.gds.compat.AbstractInMemoryNodeCursor; import org.neo4j.gds.compat.AbstractInMemoryNodePropertyCursor; @@ -35,15 +34,12 @@ import org.neo4j.graphdb.Direction; import org.neo4j.graphdb.GraphDatabaseService; import org.neo4j.internal.recordstorage.AbstractInMemoryRelationshipScanCursor; -import org.neo4j.internal.recordstorage.InMemoryStorageReader51; import org.neo4j.io.layout.DatabaseLayout; -import org.neo4j.storageengine.api.CommandCreationContext; import org.neo4j.storageengine.api.PropertySelection; import org.neo4j.storageengine.api.RelationshipSelection; import org.neo4j.storageengine.api.StorageEngine; import org.neo4j.storageengine.api.StorageEntityCursor; import org.neo4j.storageengine.api.StoragePropertyCursor; -import org.neo4j.storageengine.api.StorageReader; import org.neo4j.storageengine.api.StorageRelationshipTraversalCursor; import org.neo4j.token.TokenHolders; @@ -51,11 +47,6 @@ public class StorageEngineProxyImpl implements StorageEngineProxyApi { - @Override - public CommandCreationContext inMemoryCommandCreationContext() { - return new InMemoryCommandCreationContextImpl(); - } - @Override public void initRelationshipTraversalCursorForRelType( StorageRelationshipTraversalCursor cursor, @@ -69,13 +60,6 @@ public void initRelationshipTraversalCursorForRelType( cursor.init(sourceNodeId, -1, relationshipSelection); } - @Override - public StorageReader inMemoryStorageReader( - CypherGraphStore graphStore, TokenHolders tokenHolders, CountsAccessor counts - ) { - return new InMemoryStorageReader51(graphStore, tokenHolders, counts); - } - @Override public StorageEngine createInMemoryStorageEngine(DatabaseLayout databaseLayout, TokenHolders tokenHolders) { return new InMemoryStorageEngineImpl(databaseLayout, tokenHolders); diff --git a/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/StorageEngineProxyImpl.java b/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/StorageEngineProxyImpl.java index ec0aa469dd..0e4e3e89cf 100644 --- a/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/StorageEngineProxyImpl.java +++ b/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/StorageEngineProxyImpl.java @@ -22,7 +22,6 @@ import org.neo4j.common.Edition; import org.neo4j.configuration.Config; import org.neo4j.configuration.GraphDatabaseInternalSettings; -import org.neo4j.counts.CountsAccessor; import org.neo4j.dbms.api.DatabaseManagementService; import org.neo4j.gds.compat.AbstractInMemoryNodeCursor; import org.neo4j.gds.compat.AbstractInMemoryNodePropertyCursor; @@ -35,15 +34,12 @@ import org.neo4j.graphdb.Direction; import org.neo4j.graphdb.GraphDatabaseService; import org.neo4j.internal.recordstorage.AbstractInMemoryRelationshipScanCursor; -import org.neo4j.internal.recordstorage.InMemoryStorageReader510; import org.neo4j.io.layout.DatabaseLayout; -import org.neo4j.storageengine.api.CommandCreationContext; import org.neo4j.storageengine.api.PropertySelection; import org.neo4j.storageengine.api.RelationshipSelection; import org.neo4j.storageengine.api.StorageEngine; import org.neo4j.storageengine.api.StorageEntityCursor; import org.neo4j.storageengine.api.StoragePropertyCursor; -import org.neo4j.storageengine.api.StorageReader; import org.neo4j.storageengine.api.StorageRelationshipTraversalCursor; import org.neo4j.token.TokenHolders; @@ -51,11 +47,6 @@ public class StorageEngineProxyImpl implements StorageEngineProxyApi { - @Override - public CommandCreationContext inMemoryCommandCreationContext() { - return new InMemoryCommandCreationContextImpl(); - } - @Override public void initRelationshipTraversalCursorForRelType( StorageRelationshipTraversalCursor cursor, @@ -69,13 +60,6 @@ public void initRelationshipTraversalCursorForRelType( cursor.init(sourceNodeId, -1, relationshipSelection); } - @Override - public StorageReader inMemoryStorageReader( - CypherGraphStore graphStore, TokenHolders tokenHolders, CountsAccessor counts - ) { - return new InMemoryStorageReader510(graphStore, tokenHolders, counts); - } - @Override public StorageEngine createInMemoryStorageEngine(DatabaseLayout databaseLayout, TokenHolders tokenHolders) { return new InMemoryStorageEngineImpl(databaseLayout, tokenHolders); diff --git a/compatibility/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/StorageEngineProxyImpl.java b/compatibility/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/StorageEngineProxyImpl.java index 96a86ce2d3..df4a47e627 100644 --- a/compatibility/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/StorageEngineProxyImpl.java +++ b/compatibility/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/StorageEngineProxyImpl.java @@ -22,7 +22,6 @@ import org.neo4j.common.Edition; import org.neo4j.configuration.Config; import org.neo4j.configuration.GraphDatabaseInternalSettings; -import org.neo4j.counts.CountsAccessor; import org.neo4j.dbms.api.DatabaseManagementService; import org.neo4j.gds.compat.AbstractInMemoryNodeCursor; import org.neo4j.gds.compat.AbstractInMemoryNodePropertyCursor; @@ -35,15 +34,12 @@ import org.neo4j.graphdb.Direction; import org.neo4j.graphdb.GraphDatabaseService; import org.neo4j.internal.recordstorage.AbstractInMemoryRelationshipScanCursor; -import org.neo4j.internal.recordstorage.InMemoryStorageReader52; import org.neo4j.io.layout.DatabaseLayout; -import org.neo4j.storageengine.api.CommandCreationContext; import org.neo4j.storageengine.api.PropertySelection; import org.neo4j.storageengine.api.RelationshipSelection; import org.neo4j.storageengine.api.StorageEngine; import org.neo4j.storageengine.api.StorageEntityCursor; import org.neo4j.storageengine.api.StoragePropertyCursor; -import org.neo4j.storageengine.api.StorageReader; import org.neo4j.storageengine.api.StorageRelationshipTraversalCursor; import org.neo4j.token.TokenHolders; @@ -51,11 +47,6 @@ public class StorageEngineProxyImpl implements StorageEngineProxyApi { - @Override - public CommandCreationContext inMemoryCommandCreationContext() { - return new InMemoryCommandCreationContextImpl(); - } - @Override public void initRelationshipTraversalCursorForRelType( StorageRelationshipTraversalCursor cursor, @@ -69,13 +60,6 @@ public void initRelationshipTraversalCursorForRelType( cursor.init(sourceNodeId, -1, relationshipSelection); } - @Override - public StorageReader inMemoryStorageReader( - CypherGraphStore graphStore, TokenHolders tokenHolders, CountsAccessor counts - ) { - return new InMemoryStorageReader52(graphStore, tokenHolders, counts); - } - @Override public StorageEngine createInMemoryStorageEngine(DatabaseLayout databaseLayout, TokenHolders tokenHolders) { return new InMemoryStorageEngineImpl(databaseLayout, tokenHolders); diff --git a/compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/StorageEngineProxyImpl.java b/compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/StorageEngineProxyImpl.java index 08f7830ad6..fd1fd03c19 100644 --- a/compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/StorageEngineProxyImpl.java +++ b/compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/StorageEngineProxyImpl.java @@ -22,7 +22,6 @@ import org.neo4j.common.Edition; import org.neo4j.configuration.Config; import org.neo4j.configuration.GraphDatabaseInternalSettings; -import org.neo4j.counts.CountsAccessor; import org.neo4j.dbms.api.DatabaseManagementService; import org.neo4j.gds.compat.AbstractInMemoryNodeCursor; import org.neo4j.gds.compat.AbstractInMemoryNodePropertyCursor; @@ -35,15 +34,12 @@ import org.neo4j.graphdb.Direction; import org.neo4j.graphdb.GraphDatabaseService; import org.neo4j.internal.recordstorage.AbstractInMemoryRelationshipScanCursor; -import org.neo4j.internal.recordstorage.InMemoryStorageReader53; import org.neo4j.io.layout.DatabaseLayout; -import org.neo4j.storageengine.api.CommandCreationContext; import org.neo4j.storageengine.api.PropertySelection; import org.neo4j.storageengine.api.RelationshipSelection; import org.neo4j.storageengine.api.StorageEngine; import org.neo4j.storageengine.api.StorageEntityCursor; import org.neo4j.storageengine.api.StoragePropertyCursor; -import org.neo4j.storageengine.api.StorageReader; import org.neo4j.storageengine.api.StorageRelationshipTraversalCursor; import org.neo4j.token.TokenHolders; @@ -51,11 +47,6 @@ public class StorageEngineProxyImpl implements StorageEngineProxyApi { - @Override - public CommandCreationContext inMemoryCommandCreationContext() { - return new InMemoryCommandCreationContextImpl(); - } - @Override public void initRelationshipTraversalCursorForRelType( StorageRelationshipTraversalCursor cursor, @@ -69,13 +60,6 @@ public void initRelationshipTraversalCursorForRelType( cursor.init(sourceNodeId, -1, relationshipSelection); } - @Override - public StorageReader inMemoryStorageReader( - CypherGraphStore graphStore, TokenHolders tokenHolders, CountsAccessor counts - ) { - return new InMemoryStorageReader53(graphStore, tokenHolders, counts); - } - @Override public StorageEngine createInMemoryStorageEngine(DatabaseLayout databaseLayout, TokenHolders tokenHolders) { return new InMemoryStorageEngineImpl(databaseLayout, tokenHolders); diff --git a/compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/StorageEngineProxyImpl.java b/compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/StorageEngineProxyImpl.java index f99cb35537..19c401d088 100644 --- a/compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/StorageEngineProxyImpl.java +++ b/compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/StorageEngineProxyImpl.java @@ -22,7 +22,6 @@ import org.neo4j.common.Edition; import org.neo4j.configuration.Config; import org.neo4j.configuration.GraphDatabaseInternalSettings; -import org.neo4j.counts.CountsAccessor; import org.neo4j.dbms.api.DatabaseManagementService; import org.neo4j.gds.compat.AbstractInMemoryNodeCursor; import org.neo4j.gds.compat.AbstractInMemoryNodePropertyCursor; @@ -35,15 +34,12 @@ import org.neo4j.graphdb.Direction; import org.neo4j.graphdb.GraphDatabaseService; import org.neo4j.internal.recordstorage.AbstractInMemoryRelationshipScanCursor; -import org.neo4j.internal.recordstorage.InMemoryStorageReader54; import org.neo4j.io.layout.DatabaseLayout; -import org.neo4j.storageengine.api.CommandCreationContext; import org.neo4j.storageengine.api.PropertySelection; import org.neo4j.storageengine.api.RelationshipSelection; import org.neo4j.storageengine.api.StorageEngine; import org.neo4j.storageengine.api.StorageEntityCursor; import org.neo4j.storageengine.api.StoragePropertyCursor; -import org.neo4j.storageengine.api.StorageReader; import org.neo4j.storageengine.api.StorageRelationshipTraversalCursor; import org.neo4j.token.TokenHolders; @@ -51,11 +47,6 @@ public class StorageEngineProxyImpl implements StorageEngineProxyApi { - @Override - public CommandCreationContext inMemoryCommandCreationContext() { - return new InMemoryCommandCreationContextImpl(); - } - @Override public void initRelationshipTraversalCursorForRelType( StorageRelationshipTraversalCursor cursor, @@ -69,13 +60,6 @@ public void initRelationshipTraversalCursorForRelType( cursor.init(sourceNodeId, -1, relationshipSelection); } - @Override - public StorageReader inMemoryStorageReader( - CypherGraphStore graphStore, TokenHolders tokenHolders, CountsAccessor counts - ) { - return new InMemoryStorageReader54(graphStore, tokenHolders, counts); - } - @Override public StorageEngine createInMemoryStorageEngine(DatabaseLayout databaseLayout, TokenHolders tokenHolders) { return new InMemoryStorageEngineImpl(databaseLayout, tokenHolders); diff --git a/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/StorageEngineProxyImpl.java b/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/StorageEngineProxyImpl.java index 7124a0829d..ae6f1c8df9 100644 --- a/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/StorageEngineProxyImpl.java +++ b/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/StorageEngineProxyImpl.java @@ -22,7 +22,6 @@ import org.neo4j.common.Edition; import org.neo4j.configuration.Config; import org.neo4j.configuration.GraphDatabaseInternalSettings; -import org.neo4j.counts.CountsAccessor; import org.neo4j.dbms.api.DatabaseManagementService; import org.neo4j.gds.compat.AbstractInMemoryNodeCursor; import org.neo4j.gds.compat.AbstractInMemoryNodePropertyCursor; @@ -35,15 +34,12 @@ import org.neo4j.graphdb.Direction; import org.neo4j.graphdb.GraphDatabaseService; import org.neo4j.internal.recordstorage.AbstractInMemoryRelationshipScanCursor; -import org.neo4j.internal.recordstorage.InMemoryStorageReader55; import org.neo4j.io.layout.DatabaseLayout; -import org.neo4j.storageengine.api.CommandCreationContext; import org.neo4j.storageengine.api.PropertySelection; import org.neo4j.storageengine.api.RelationshipSelection; import org.neo4j.storageengine.api.StorageEngine; import org.neo4j.storageengine.api.StorageEntityCursor; import org.neo4j.storageengine.api.StoragePropertyCursor; -import org.neo4j.storageengine.api.StorageReader; import org.neo4j.storageengine.api.StorageRelationshipTraversalCursor; import org.neo4j.token.TokenHolders; @@ -51,11 +47,6 @@ public class StorageEngineProxyImpl implements StorageEngineProxyApi { - @Override - public CommandCreationContext inMemoryCommandCreationContext() { - return new InMemoryCommandCreationContextImpl(); - } - @Override public void initRelationshipTraversalCursorForRelType( StorageRelationshipTraversalCursor cursor, @@ -69,13 +60,6 @@ public void initRelationshipTraversalCursorForRelType( cursor.init(sourceNodeId, -1, relationshipSelection); } - @Override - public StorageReader inMemoryStorageReader( - CypherGraphStore graphStore, TokenHolders tokenHolders, CountsAccessor counts - ) { - return new InMemoryStorageReader55(graphStore, tokenHolders, counts); - } - @Override public StorageEngine createInMemoryStorageEngine(DatabaseLayout databaseLayout, TokenHolders tokenHolders) { return new InMemoryStorageEngineImpl(databaseLayout, tokenHolders); diff --git a/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/StorageEngineProxyImpl.java b/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/StorageEngineProxyImpl.java index b5a1b5a4ff..f0c961de55 100644 --- a/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/StorageEngineProxyImpl.java +++ b/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/StorageEngineProxyImpl.java @@ -22,7 +22,6 @@ import org.neo4j.common.Edition; import org.neo4j.configuration.Config; import org.neo4j.configuration.GraphDatabaseInternalSettings; -import org.neo4j.counts.CountsAccessor; import org.neo4j.dbms.api.DatabaseManagementService; import org.neo4j.gds.compat.AbstractInMemoryNodeCursor; import org.neo4j.gds.compat.AbstractInMemoryNodePropertyCursor; @@ -35,15 +34,12 @@ import org.neo4j.graphdb.Direction; import org.neo4j.graphdb.GraphDatabaseService; import org.neo4j.internal.recordstorage.AbstractInMemoryRelationshipScanCursor; -import org.neo4j.internal.recordstorage.InMemoryStorageReader56; import org.neo4j.io.layout.DatabaseLayout; -import org.neo4j.storageengine.api.CommandCreationContext; import org.neo4j.storageengine.api.PropertySelection; import org.neo4j.storageengine.api.RelationshipSelection; import org.neo4j.storageengine.api.StorageEngine; import org.neo4j.storageengine.api.StorageEntityCursor; import org.neo4j.storageengine.api.StoragePropertyCursor; -import org.neo4j.storageengine.api.StorageReader; import org.neo4j.storageengine.api.StorageRelationshipTraversalCursor; import org.neo4j.token.TokenHolders; @@ -51,11 +47,6 @@ public class StorageEngineProxyImpl implements StorageEngineProxyApi { - @Override - public CommandCreationContext inMemoryCommandCreationContext() { - return new InMemoryCommandCreationContextImpl(); - } - @Override public void initRelationshipTraversalCursorForRelType( StorageRelationshipTraversalCursor cursor, @@ -69,13 +60,6 @@ public void initRelationshipTraversalCursorForRelType( cursor.init(sourceNodeId, -1, relationshipSelection); } - @Override - public StorageReader inMemoryStorageReader( - CypherGraphStore graphStore, TokenHolders tokenHolders, CountsAccessor counts - ) { - return new InMemoryStorageReader56(graphStore, tokenHolders, counts); - } - @Override public StorageEngine createInMemoryStorageEngine(DatabaseLayout databaseLayout, TokenHolders tokenHolders) { return new InMemoryStorageEngineImpl(databaseLayout, tokenHolders); diff --git a/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/StorageEngineProxyImpl.java b/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/StorageEngineProxyImpl.java index 8a0eadab4b..c845bc1451 100644 --- a/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/StorageEngineProxyImpl.java +++ b/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/StorageEngineProxyImpl.java @@ -22,7 +22,6 @@ import org.neo4j.common.Edition; import org.neo4j.configuration.Config; import org.neo4j.configuration.GraphDatabaseInternalSettings; -import org.neo4j.counts.CountsAccessor; import org.neo4j.dbms.api.DatabaseManagementService; import org.neo4j.gds.compat.AbstractInMemoryNodeCursor; import org.neo4j.gds.compat.AbstractInMemoryNodePropertyCursor; @@ -35,15 +34,12 @@ import org.neo4j.graphdb.Direction; import org.neo4j.graphdb.GraphDatabaseService; import org.neo4j.internal.recordstorage.AbstractInMemoryRelationshipScanCursor; -import org.neo4j.internal.recordstorage.InMemoryStorageReader57; import org.neo4j.io.layout.DatabaseLayout; -import org.neo4j.storageengine.api.CommandCreationContext; import org.neo4j.storageengine.api.PropertySelection; import org.neo4j.storageengine.api.RelationshipSelection; import org.neo4j.storageengine.api.StorageEngine; import org.neo4j.storageengine.api.StorageEntityCursor; import org.neo4j.storageengine.api.StoragePropertyCursor; -import org.neo4j.storageengine.api.StorageReader; import org.neo4j.storageengine.api.StorageRelationshipTraversalCursor; import org.neo4j.token.TokenHolders; @@ -51,11 +47,6 @@ public class StorageEngineProxyImpl implements StorageEngineProxyApi { - @Override - public CommandCreationContext inMemoryCommandCreationContext() { - return new InMemoryCommandCreationContextImpl(); - } - @Override public void initRelationshipTraversalCursorForRelType( StorageRelationshipTraversalCursor cursor, @@ -69,13 +60,6 @@ public void initRelationshipTraversalCursorForRelType( cursor.init(sourceNodeId, -1, relationshipSelection); } - @Override - public StorageReader inMemoryStorageReader( - CypherGraphStore graphStore, TokenHolders tokenHolders, CountsAccessor counts - ) { - return new InMemoryStorageReader57(graphStore, tokenHolders, counts); - } - @Override public StorageEngine createInMemoryStorageEngine(DatabaseLayout databaseLayout, TokenHolders tokenHolders) { return new InMemoryStorageEngineImpl(databaseLayout, tokenHolders); diff --git a/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/StorageEngineProxyImpl.java b/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/StorageEngineProxyImpl.java index c47d09fd39..fd8e08e5a3 100644 --- a/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/StorageEngineProxyImpl.java +++ b/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/StorageEngineProxyImpl.java @@ -22,7 +22,6 @@ import org.neo4j.common.Edition; import org.neo4j.configuration.Config; import org.neo4j.configuration.GraphDatabaseInternalSettings; -import org.neo4j.counts.CountsAccessor; import org.neo4j.dbms.api.DatabaseManagementService; import org.neo4j.gds.compat.AbstractInMemoryNodeCursor; import org.neo4j.gds.compat.AbstractInMemoryNodePropertyCursor; @@ -35,15 +34,12 @@ import org.neo4j.graphdb.Direction; import org.neo4j.graphdb.GraphDatabaseService; import org.neo4j.internal.recordstorage.AbstractInMemoryRelationshipScanCursor; -import org.neo4j.internal.recordstorage.InMemoryStorageReader58; import org.neo4j.io.layout.DatabaseLayout; -import org.neo4j.storageengine.api.CommandCreationContext; import org.neo4j.storageengine.api.PropertySelection; import org.neo4j.storageengine.api.RelationshipSelection; import org.neo4j.storageengine.api.StorageEngine; import org.neo4j.storageengine.api.StorageEntityCursor; import org.neo4j.storageengine.api.StoragePropertyCursor; -import org.neo4j.storageengine.api.StorageReader; import org.neo4j.storageengine.api.StorageRelationshipTraversalCursor; import org.neo4j.token.TokenHolders; @@ -51,11 +47,6 @@ public class StorageEngineProxyImpl implements StorageEngineProxyApi { - @Override - public CommandCreationContext inMemoryCommandCreationContext() { - return new InMemoryCommandCreationContextImpl(); - } - @Override public void initRelationshipTraversalCursorForRelType( StorageRelationshipTraversalCursor cursor, @@ -69,13 +60,6 @@ public void initRelationshipTraversalCursorForRelType( cursor.init(sourceNodeId, -1, relationshipSelection); } - @Override - public StorageReader inMemoryStorageReader( - CypherGraphStore graphStore, TokenHolders tokenHolders, CountsAccessor counts - ) { - return new InMemoryStorageReader58(graphStore, tokenHolders, counts); - } - @Override public StorageEngine createInMemoryStorageEngine(DatabaseLayout databaseLayout, TokenHolders tokenHolders) { return new InMemoryStorageEngineImpl(databaseLayout, tokenHolders); diff --git a/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/StorageEngineProxyImpl.java b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/StorageEngineProxyImpl.java index 8cda8e608e..81318ea397 100644 --- a/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/StorageEngineProxyImpl.java +++ b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/StorageEngineProxyImpl.java @@ -22,7 +22,6 @@ import org.neo4j.common.Edition; import org.neo4j.configuration.Config; import org.neo4j.configuration.GraphDatabaseInternalSettings; -import org.neo4j.counts.CountsAccessor; import org.neo4j.dbms.api.DatabaseManagementService; import org.neo4j.gds.compat.AbstractInMemoryNodeCursor; import org.neo4j.gds.compat.AbstractInMemoryNodePropertyCursor; @@ -35,15 +34,12 @@ import org.neo4j.graphdb.Direction; import org.neo4j.graphdb.GraphDatabaseService; import org.neo4j.internal.recordstorage.AbstractInMemoryRelationshipScanCursor; -import org.neo4j.internal.recordstorage.InMemoryStorageReader59; import org.neo4j.io.layout.DatabaseLayout; -import org.neo4j.storageengine.api.CommandCreationContext; import org.neo4j.storageengine.api.PropertySelection; import org.neo4j.storageengine.api.RelationshipSelection; import org.neo4j.storageengine.api.StorageEngine; import org.neo4j.storageengine.api.StorageEntityCursor; import org.neo4j.storageengine.api.StoragePropertyCursor; -import org.neo4j.storageengine.api.StorageReader; import org.neo4j.storageengine.api.StorageRelationshipTraversalCursor; import org.neo4j.token.TokenHolders; @@ -51,11 +47,6 @@ public class StorageEngineProxyImpl implements StorageEngineProxyApi { - @Override - public CommandCreationContext inMemoryCommandCreationContext() { - return new InMemoryCommandCreationContextImpl(); - } - @Override public void initRelationshipTraversalCursorForRelType( StorageRelationshipTraversalCursor cursor, @@ -69,13 +60,6 @@ public void initRelationshipTraversalCursorForRelType( cursor.init(sourceNodeId, -1, relationshipSelection); } - @Override - public StorageReader inMemoryStorageReader( - CypherGraphStore graphStore, TokenHolders tokenHolders, CountsAccessor counts - ) { - return new InMemoryStorageReader59(graphStore, tokenHolders, counts); - } - @Override public StorageEngine createInMemoryStorageEngine(DatabaseLayout databaseLayout, TokenHolders tokenHolders) { return new InMemoryStorageEngineImpl(databaseLayout, tokenHolders); diff --git a/compatibility/api/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/StorageEngineProxyApi.java b/compatibility/api/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/StorageEngineProxyApi.java index b502f0e615..c1ddd16bf8 100644 --- a/compatibility/api/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/StorageEngineProxyApi.java +++ b/compatibility/api/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/StorageEngineProxyApi.java @@ -21,17 +21,14 @@ import org.neo4j.common.Edition; import org.neo4j.configuration.Config; -import org.neo4j.counts.CountsAccessor; import org.neo4j.dbms.api.DatabaseManagementService; import org.neo4j.gds.core.cypher.CypherGraphStore; import org.neo4j.graphdb.GraphDatabaseService; import org.neo4j.internal.recordstorage.AbstractInMemoryRelationshipScanCursor; import org.neo4j.io.layout.DatabaseLayout; -import org.neo4j.storageengine.api.CommandCreationContext; import org.neo4j.storageengine.api.StorageEngine; import org.neo4j.storageengine.api.StorageEntityCursor; import org.neo4j.storageengine.api.StoragePropertyCursor; -import org.neo4j.storageengine.api.StorageReader; import org.neo4j.storageengine.api.StorageRelationshipTraversalCursor; import org.neo4j.token.TokenHolders; @@ -51,20 +48,12 @@ static void requireNeo4jVersion(Neo4jVersion version, Class self) { } } - CommandCreationContext inMemoryCommandCreationContext(); - void initRelationshipTraversalCursorForRelType( StorageRelationshipTraversalCursor cursor, long sourceNodeId, int relTypeToken ); - StorageReader inMemoryStorageReader( - CypherGraphStore graphStore, - TokenHolders tokenHolders, - CountsAccessor counts - ); - void createInMemoryDatabase( DatabaseManagementService dbms, String dbName, diff --git a/compatibility/common/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/StorageEngineProxy.java b/compatibility/common/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/StorageEngineProxy.java index 4851dd1c07..12964be986 100644 --- a/compatibility/common/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/StorageEngineProxy.java +++ b/compatibility/common/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/StorageEngineProxy.java @@ -21,18 +21,15 @@ import org.neo4j.common.Edition; import org.neo4j.configuration.Config; -import org.neo4j.counts.CountsAccessor; import org.neo4j.dbms.api.DatabaseManagementService; import org.neo4j.gds.core.cypher.CypherGraphStore; import org.neo4j.gds.storageengine.InMemoryDatabaseCreationCatalog; import org.neo4j.graphdb.GraphDatabaseService; import org.neo4j.internal.recordstorage.AbstractInMemoryRelationshipScanCursor; import org.neo4j.io.layout.DatabaseLayout; -import org.neo4j.storageengine.api.CommandCreationContext; import org.neo4j.storageengine.api.StorageEngine; import org.neo4j.storageengine.api.StorageEntityCursor; import org.neo4j.storageengine.api.StoragePropertyCursor; -import org.neo4j.storageengine.api.StorageReader; import org.neo4j.storageengine.api.StorageRelationshipTraversalCursor; import org.neo4j.token.TokenHolders; @@ -45,10 +42,6 @@ public final class StorageEngineProxy { private StorageEngineProxy() {} - public static CommandCreationContext inMemoryCommandCreationContext() { - return IMPL.inMemoryCommandCreationContext(); - } - public static void initRelationshipTraversalCursorForRelType( StorageRelationshipTraversalCursor cursor, long sourceNodeId, @@ -57,12 +50,6 @@ public static void initRelationshipTraversalCursorForRelType( IMPL.initRelationshipTraversalCursorForRelType(cursor, sourceNodeId, relTypeToken); } - public static StorageReader inMemoryStorageReader( - CypherGraphStore graphStore, TokenHolders tokenHolders, CountsAccessor counts - ) { - return IMPL.inMemoryStorageReader(graphStore, tokenHolders, counts); - } - public static StorageEngine createInMemoryStorageEngine( DatabaseLayout databaseLayout, TokenHolders tokenHolders From b69ddd84063eb102f477a22dcda60237ec983ca3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B6ren=20Reichardt?= Date: Fri, 28 Jul 2023 09:43:09 +0200 Subject: [PATCH 188/273] Fix global procedures copat issue --- .../neo4j/gds/compat/_44/Neo4jProxyImpl.java | 32 ++++++++++++ .../neo4j/gds/compat/_51/Neo4jProxyImpl.java | 32 ++++++++++++ .../neo4j/gds/compat/_510/Neo4jProxyImpl.java | 32 ++++++++++++ .../neo4j/gds/compat/_52/Neo4jProxyImpl.java | 32 ++++++++++++ .../neo4j/gds/compat/_53/Neo4jProxyImpl.java | 32 ++++++++++++ .../neo4j/gds/compat/_54/Neo4jProxyImpl.java | 32 ++++++++++++ .../neo4j/gds/compat/_55/Neo4jProxyImpl.java | 32 ++++++++++++ .../neo4j/gds/compat/_56/Neo4jProxyImpl.java | 32 ++++++++++++ .../neo4j/gds/compat/_57/Neo4jProxyImpl.java | 32 ++++++++++++ .../neo4j/gds/compat/_58/Neo4jProxyImpl.java | 32 ++++++++++++ .../neo4j/gds/compat/_59/Neo4jProxyImpl.java | 32 ++++++++++++ .../gds/compat/GlobalProcedureRegistry.java | 32 ++++++++++++ .../org/neo4j/gds/compat/Neo4jProxyApi.java | 7 +++ .../java/org/neo4j/gds/compat/Neo4jProxy.java | 25 ++++++++++ .../projection/AlphaCypherAggregation.java | 5 +- .../gds/projection/CypherAggregation.java | 5 +- .../gds/projection/CypherAggregationTest.java | 2 +- .../gds/compat/GraphDatabaseApiProxy.java | 29 +++++++++-- .../gds/internal/CustomProceduresUtil.java | 49 ------------------- 19 files changed, 445 insertions(+), 61 deletions(-) create mode 100644 compatibility/api/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/GlobalProcedureRegistry.java delete mode 100644 proc/common/src/main/java/com/neo4j/gds/internal/CustomProceduresUtil.java diff --git a/compatibility/4.4/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_44/Neo4jProxyImpl.java b/compatibility/4.4/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_44/Neo4jProxyImpl.java index 84e4af6d80..b8a3076c66 100644 --- a/compatibility/4.4/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_44/Neo4jProxyImpl.java +++ b/compatibility/4.4/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_44/Neo4jProxyImpl.java @@ -41,6 +41,7 @@ import org.neo4j.gds.compat.GdsDatabaseLayout; import org.neo4j.gds.compat.GdsDatabaseManagementServiceBuilder; import org.neo4j.gds.compat.GdsGraphDatabaseAPI; +import org.neo4j.gds.compat.GlobalProcedureRegistry; import org.neo4j.gds.compat.GraphDatabaseApiProxy; import org.neo4j.gds.compat.InputEntityIdVisitor; import org.neo4j.gds.compat.Neo4jProxyApi; @@ -84,6 +85,7 @@ import org.neo4j.internal.kernel.api.RelationshipScanCursor; import org.neo4j.internal.kernel.api.Scan; import org.neo4j.internal.kernel.api.connectioninfo.ClientConnectionInfo; +import org.neo4j.internal.kernel.api.exceptions.ProcedureException; import org.neo4j.internal.kernel.api.procs.FieldSignature; import org.neo4j.internal.kernel.api.procs.Neo4jTypes; import org.neo4j.internal.kernel.api.procs.ProcedureSignature; @@ -107,6 +109,8 @@ import org.neo4j.kernel.api.KernelTransactionHandle; import org.neo4j.kernel.api.procedure.CallableProcedure; import org.neo4j.kernel.api.procedure.CallableUserAggregationFunction; +import org.neo4j.kernel.api.procedure.Context; +import org.neo4j.kernel.api.procedure.GlobalProcedures; import org.neo4j.kernel.database.NormalizedDatabaseName; import org.neo4j.kernel.impl.coreapi.InternalTransaction; import org.neo4j.kernel.impl.index.schema.IndexImporterFactoryImpl; @@ -137,8 +141,10 @@ import java.util.Arrays; import java.util.List; import java.util.Optional; +import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; +import java.util.stream.Stream; import static org.neo4j.gds.compat.InternalReadOps.countByIdGenerator; @@ -804,4 +810,30 @@ public boolean isCompositeDatabase(GraphDatabaseService databaseService) { var databaseManager = GraphDatabaseApiProxy.resolveDependency(databaseService, FabricDatabaseManager.class); return databaseManager.isFabricDatabase(GraphDatabaseApiProxy.databaseId(databaseService).name()); } + + @Override + public T lookupComponentProvider(Context ctx, Class component, boolean safe) throws ProcedureException { + var globalProcedures = GraphDatabaseApiProxy.resolveDependency(ctx.dependencyResolver(), GlobalProcedures.class); + return globalProcedures.lookupComponentProvider(component, safe).apply(ctx); + } + + @Override + public GlobalProcedureRegistry globalProcedureRegistry(GlobalProcedures globalProcedures) { + return new GlobalProcedureRegistry() { + @Override + public Set getAllProcedures() { + return globalProcedures.getAllProcedures(); + } + + @Override + public Stream getAllNonAggregatingFunctions() { + return globalProcedures.getAllNonAggregatingFunctions(); + } + + @Override + public Stream getAllAggregatingFunctions() { + return globalProcedures.getAllAggregatingFunctions(); + } + }; + } } diff --git a/compatibility/5.1/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_51/Neo4jProxyImpl.java b/compatibility/5.1/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_51/Neo4jProxyImpl.java index fa9f9292d5..b323990b78 100644 --- a/compatibility/5.1/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_51/Neo4jProxyImpl.java +++ b/compatibility/5.1/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_51/Neo4jProxyImpl.java @@ -44,6 +44,7 @@ import org.neo4j.gds.compat.GdsDatabaseLayout; import org.neo4j.gds.compat.GdsDatabaseManagementServiceBuilder; import org.neo4j.gds.compat.GdsGraphDatabaseAPI; +import org.neo4j.gds.compat.GlobalProcedureRegistry; import org.neo4j.gds.compat.GraphDatabaseApiProxy; import org.neo4j.gds.compat.InputEntityIdVisitor; import org.neo4j.gds.compat.Neo4jProxyApi; @@ -87,6 +88,7 @@ import org.neo4j.internal.kernel.api.Scan; import org.neo4j.internal.kernel.api.TokenPredicate; import org.neo4j.internal.kernel.api.connectioninfo.ClientConnectionInfo; +import org.neo4j.internal.kernel.api.exceptions.ProcedureException; import org.neo4j.internal.kernel.api.procs.FieldSignature; import org.neo4j.internal.kernel.api.procs.Neo4jTypes; import org.neo4j.internal.kernel.api.procs.ProcedureSignature; @@ -113,6 +115,8 @@ import org.neo4j.kernel.api.KernelTransactionHandle; import org.neo4j.kernel.api.procedure.CallableProcedure; import org.neo4j.kernel.api.procedure.CallableUserAggregationFunction; +import org.neo4j.kernel.api.procedure.Context; +import org.neo4j.kernel.api.procedure.GlobalProcedures; import org.neo4j.kernel.database.NormalizedDatabaseName; import org.neo4j.kernel.impl.coreapi.InternalTransaction; import org.neo4j.kernel.impl.index.schema.IndexImporterFactoryImpl; @@ -147,8 +151,10 @@ import java.util.List; import java.util.Locale; import java.util.Optional; +import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; +import java.util.stream.Stream; import static java.lang.String.format; import static org.neo4j.gds.compat.InternalReadOps.countByIdGenerator; @@ -921,4 +927,30 @@ public boolean isCompositeDatabase(GraphDatabaseService databaseService) { var databaseManager = GraphDatabaseApiProxy.resolveDependency(databaseService, FabricDatabaseManager.class); return databaseManager.isFabricDatabase(GraphDatabaseApiProxy.databaseId(databaseService)); } + + @Override + public T lookupComponentProvider(Context ctx, Class component, boolean safe) throws ProcedureException { + var globalProcedures = GraphDatabaseApiProxy.resolveDependency(ctx.dependencyResolver(), GlobalProcedures.class); + return globalProcedures.lookupComponentProvider(component, safe).apply(ctx); + } + + @Override + public GlobalProcedureRegistry globalProcedureRegistry(GlobalProcedures globalProcedures) { + return new GlobalProcedureRegistry() { + @Override + public Set getAllProcedures() { + return globalProcedures.getAllProcedures(); + } + + @Override + public Stream getAllNonAggregatingFunctions() { + return globalProcedures.getAllNonAggregatingFunctions(); + } + + @Override + public Stream getAllAggregatingFunctions() { + return globalProcedures.getAllAggregatingFunctions(); + } + }; + } } diff --git a/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/Neo4jProxyImpl.java b/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/Neo4jProxyImpl.java index 48822655c0..3c53d58a61 100644 --- a/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/Neo4jProxyImpl.java +++ b/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/Neo4jProxyImpl.java @@ -44,6 +44,7 @@ import org.neo4j.gds.compat.GdsDatabaseLayout; import org.neo4j.gds.compat.GdsDatabaseManagementServiceBuilder; import org.neo4j.gds.compat.GdsGraphDatabaseAPI; +import org.neo4j.gds.compat.GlobalProcedureRegistry; import org.neo4j.gds.compat.GraphDatabaseApiProxy; import org.neo4j.gds.compat.InputEntityIdVisitor; import org.neo4j.gds.compat.Neo4jProxyApi; @@ -87,6 +88,7 @@ import org.neo4j.internal.kernel.api.Scan; import org.neo4j.internal.kernel.api.TokenPredicate; import org.neo4j.internal.kernel.api.connectioninfo.ClientConnectionInfo; +import org.neo4j.internal.kernel.api.exceptions.ProcedureException; import org.neo4j.internal.kernel.api.procs.FieldSignature; import org.neo4j.internal.kernel.api.procs.Neo4jTypes; import org.neo4j.internal.kernel.api.procs.ProcedureSignature; @@ -113,6 +115,8 @@ import org.neo4j.kernel.api.KernelTransactionHandle; import org.neo4j.kernel.api.procedure.CallableProcedure; import org.neo4j.kernel.api.procedure.CallableUserAggregationFunction; +import org.neo4j.kernel.api.procedure.Context; +import org.neo4j.kernel.api.procedure.GlobalProcedures; import org.neo4j.kernel.database.NormalizedDatabaseName; import org.neo4j.kernel.impl.coreapi.InternalTransaction; import org.neo4j.kernel.impl.index.schema.IndexImporterFactoryImpl; @@ -148,8 +152,10 @@ import java.util.List; import java.util.Locale; import java.util.Optional; +import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; +import java.util.stream.Stream; import static java.lang.String.format; import static org.neo4j.gds.compat.InternalReadOps.countByIdGenerator; @@ -916,4 +922,30 @@ public boolean isCompositeDatabase(GraphDatabaseService databaseService) { var databaseManager = GraphDatabaseApiProxy.resolveDependency(databaseService, FabricDatabaseManager.class); return databaseManager.isFabricDatabase(GraphDatabaseApiProxy.databaseId(databaseService)); } + + @Override + public T lookupComponentProvider(Context ctx, Class component, boolean safe) throws ProcedureException { + var globalProcedures = GraphDatabaseApiProxy.resolveDependency(ctx.dependencyResolver(), GlobalProcedures.class); + return globalProcedures.lookupComponentProvider(component, safe).apply(ctx); + } + + @Override + public GlobalProcedureRegistry globalProcedureRegistry(GlobalProcedures globalProcedures) { + return new GlobalProcedureRegistry() { + @Override + public Set getAllProcedures() { + return globalProcedures.getAllProcedures(); + } + + @Override + public Stream getAllNonAggregatingFunctions() { + return globalProcedures.getAllNonAggregatingFunctions(); + } + + @Override + public Stream getAllAggregatingFunctions() { + return globalProcedures.getAllAggregatingFunctions(); + } + }; + } } diff --git a/compatibility/5.2/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_52/Neo4jProxyImpl.java b/compatibility/5.2/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_52/Neo4jProxyImpl.java index 222a7b596d..2b7c286b17 100644 --- a/compatibility/5.2/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_52/Neo4jProxyImpl.java +++ b/compatibility/5.2/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_52/Neo4jProxyImpl.java @@ -44,6 +44,7 @@ import org.neo4j.gds.compat.GdsDatabaseLayout; import org.neo4j.gds.compat.GdsDatabaseManagementServiceBuilder; import org.neo4j.gds.compat.GdsGraphDatabaseAPI; +import org.neo4j.gds.compat.GlobalProcedureRegistry; import org.neo4j.gds.compat.GraphDatabaseApiProxy; import org.neo4j.gds.compat.InputEntityIdVisitor; import org.neo4j.gds.compat.Neo4jProxyApi; @@ -87,6 +88,7 @@ import org.neo4j.internal.kernel.api.Scan; import org.neo4j.internal.kernel.api.TokenPredicate; import org.neo4j.internal.kernel.api.connectioninfo.ClientConnectionInfo; +import org.neo4j.internal.kernel.api.exceptions.ProcedureException; import org.neo4j.internal.kernel.api.procs.FieldSignature; import org.neo4j.internal.kernel.api.procs.Neo4jTypes; import org.neo4j.internal.kernel.api.procs.ProcedureSignature; @@ -113,6 +115,8 @@ import org.neo4j.kernel.api.KernelTransactionHandle; import org.neo4j.kernel.api.procedure.CallableProcedure; import org.neo4j.kernel.api.procedure.CallableUserAggregationFunction; +import org.neo4j.kernel.api.procedure.Context; +import org.neo4j.kernel.api.procedure.GlobalProcedures; import org.neo4j.kernel.database.NormalizedDatabaseName; import org.neo4j.kernel.impl.coreapi.InternalTransaction; import org.neo4j.kernel.impl.index.schema.IndexImporterFactoryImpl; @@ -147,8 +151,10 @@ import java.util.List; import java.util.Locale; import java.util.Optional; +import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; +import java.util.stream.Stream; import static java.lang.String.format; import static org.neo4j.gds.compat.InternalReadOps.countByIdGenerator; @@ -919,4 +925,30 @@ public boolean isCompositeDatabase(GraphDatabaseService databaseService) { var databaseManager = GraphDatabaseApiProxy.resolveDependency(databaseService, FabricDatabaseManager.class); return databaseManager.isFabricDatabase(GraphDatabaseApiProxy.databaseId(databaseService)); } + + @Override + public T lookupComponentProvider(Context ctx, Class component, boolean safe) throws ProcedureException { + var globalProcedures = GraphDatabaseApiProxy.resolveDependency(ctx.dependencyResolver(), GlobalProcedures.class); + return globalProcedures.lookupComponentProvider(component, safe).apply(ctx); + } + + @Override + public GlobalProcedureRegistry globalProcedureRegistry(GlobalProcedures globalProcedures) { + return new GlobalProcedureRegistry() { + @Override + public Set getAllProcedures() { + return globalProcedures.getAllProcedures(); + } + + @Override + public Stream getAllNonAggregatingFunctions() { + return globalProcedures.getAllNonAggregatingFunctions(); + } + + @Override + public Stream getAllAggregatingFunctions() { + return globalProcedures.getAllAggregatingFunctions(); + } + }; + } } diff --git a/compatibility/5.3/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_53/Neo4jProxyImpl.java b/compatibility/5.3/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_53/Neo4jProxyImpl.java index e326af19cc..13bdeedd6f 100644 --- a/compatibility/5.3/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_53/Neo4jProxyImpl.java +++ b/compatibility/5.3/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_53/Neo4jProxyImpl.java @@ -44,6 +44,7 @@ import org.neo4j.gds.compat.GdsDatabaseLayout; import org.neo4j.gds.compat.GdsDatabaseManagementServiceBuilder; import org.neo4j.gds.compat.GdsGraphDatabaseAPI; +import org.neo4j.gds.compat.GlobalProcedureRegistry; import org.neo4j.gds.compat.GraphDatabaseApiProxy; import org.neo4j.gds.compat.InputEntityIdVisitor; import org.neo4j.gds.compat.Neo4jProxyApi; @@ -87,6 +88,7 @@ import org.neo4j.internal.kernel.api.Scan; import org.neo4j.internal.kernel.api.TokenPredicate; import org.neo4j.internal.kernel.api.connectioninfo.ClientConnectionInfo; +import org.neo4j.internal.kernel.api.exceptions.ProcedureException; import org.neo4j.internal.kernel.api.procs.FieldSignature; import org.neo4j.internal.kernel.api.procs.Neo4jTypes; import org.neo4j.internal.kernel.api.procs.ProcedureSignature; @@ -113,6 +115,8 @@ import org.neo4j.kernel.api.KernelTransactionHandle; import org.neo4j.kernel.api.procedure.CallableProcedure; import org.neo4j.kernel.api.procedure.CallableUserAggregationFunction; +import org.neo4j.kernel.api.procedure.Context; +import org.neo4j.kernel.api.procedure.GlobalProcedures; import org.neo4j.kernel.database.NormalizedDatabaseName; import org.neo4j.kernel.impl.coreapi.InternalTransaction; import org.neo4j.kernel.impl.index.schema.IndexImporterFactoryImpl; @@ -147,8 +151,10 @@ import java.util.List; import java.util.Locale; import java.util.Optional; +import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; +import java.util.stream.Stream; import static java.lang.String.format; import static org.neo4j.gds.compat.InternalReadOps.countByIdGenerator; @@ -920,4 +926,30 @@ public boolean isCompositeDatabase(GraphDatabaseService databaseService) { var databaseManager = GraphDatabaseApiProxy.resolveDependency(databaseService, FabricDatabaseManager.class); return databaseManager.isFabricDatabase(GraphDatabaseApiProxy.databaseId(databaseService)); } + + @Override + public T lookupComponentProvider(Context ctx, Class component, boolean safe) throws ProcedureException { + var globalProcedures = GraphDatabaseApiProxy.resolveDependency(ctx.dependencyResolver(), GlobalProcedures.class); + return globalProcedures.lookupComponentProvider(component, safe).apply(ctx); + } + + @Override + public GlobalProcedureRegistry globalProcedureRegistry(GlobalProcedures globalProcedures) { + return new GlobalProcedureRegistry() { + @Override + public Set getAllProcedures() { + return globalProcedures.getAllProcedures(); + } + + @Override + public Stream getAllNonAggregatingFunctions() { + return globalProcedures.getAllNonAggregatingFunctions(); + } + + @Override + public Stream getAllAggregatingFunctions() { + return globalProcedures.getAllAggregatingFunctions(); + } + }; + } } diff --git a/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/Neo4jProxyImpl.java b/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/Neo4jProxyImpl.java index 680bf0acec..421cbabe2b 100644 --- a/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/Neo4jProxyImpl.java +++ b/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/Neo4jProxyImpl.java @@ -44,6 +44,7 @@ import org.neo4j.gds.compat.GdsDatabaseLayout; import org.neo4j.gds.compat.GdsDatabaseManagementServiceBuilder; import org.neo4j.gds.compat.GdsGraphDatabaseAPI; +import org.neo4j.gds.compat.GlobalProcedureRegistry; import org.neo4j.gds.compat.GraphDatabaseApiProxy; import org.neo4j.gds.compat.InputEntityIdVisitor; import org.neo4j.gds.compat.Neo4jProxyApi; @@ -87,6 +88,7 @@ import org.neo4j.internal.kernel.api.Scan; import org.neo4j.internal.kernel.api.TokenPredicate; import org.neo4j.internal.kernel.api.connectioninfo.ClientConnectionInfo; +import org.neo4j.internal.kernel.api.exceptions.ProcedureException; import org.neo4j.internal.kernel.api.procs.FieldSignature; import org.neo4j.internal.kernel.api.procs.Neo4jTypes; import org.neo4j.internal.kernel.api.procs.ProcedureSignature; @@ -113,6 +115,8 @@ import org.neo4j.kernel.api.KernelTransactionHandle; import org.neo4j.kernel.api.procedure.CallableProcedure; import org.neo4j.kernel.api.procedure.CallableUserAggregationFunction; +import org.neo4j.kernel.api.procedure.Context; +import org.neo4j.kernel.api.procedure.GlobalProcedures; import org.neo4j.kernel.database.NormalizedDatabaseName; import org.neo4j.kernel.impl.coreapi.InternalTransaction; import org.neo4j.kernel.impl.index.schema.IndexImporterFactoryImpl; @@ -147,8 +151,10 @@ import java.util.List; import java.util.Locale; import java.util.Optional; +import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; +import java.util.stream.Stream; import static java.lang.String.format; import static org.neo4j.gds.compat.InternalReadOps.countByIdGenerator; @@ -919,4 +925,30 @@ public boolean isCompositeDatabase(GraphDatabaseService databaseService) { var databaseManager = GraphDatabaseApiProxy.resolveDependency(databaseService, FabricDatabaseManager.class); return databaseManager.isFabricDatabase(GraphDatabaseApiProxy.databaseId(databaseService)); } + + @Override + public T lookupComponentProvider(Context ctx, Class component, boolean safe) throws ProcedureException { + var globalProcedures = GraphDatabaseApiProxy.resolveDependency(ctx.dependencyResolver(), GlobalProcedures.class); + return globalProcedures.lookupComponentProvider(component, safe).apply(ctx); + } + + @Override + public GlobalProcedureRegistry globalProcedureRegistry(GlobalProcedures globalProcedures) { + return new GlobalProcedureRegistry() { + @Override + public Set getAllProcedures() { + return globalProcedures.getAllProcedures(); + } + + @Override + public Stream getAllNonAggregatingFunctions() { + return globalProcedures.getAllNonAggregatingFunctions(); + } + + @Override + public Stream getAllAggregatingFunctions() { + return globalProcedures.getAllAggregatingFunctions(); + } + }; + } } diff --git a/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/Neo4jProxyImpl.java b/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/Neo4jProxyImpl.java index 64e14d50a5..b3e9f68823 100644 --- a/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/Neo4jProxyImpl.java +++ b/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/Neo4jProxyImpl.java @@ -44,6 +44,7 @@ import org.neo4j.gds.compat.GdsDatabaseLayout; import org.neo4j.gds.compat.GdsDatabaseManagementServiceBuilder; import org.neo4j.gds.compat.GdsGraphDatabaseAPI; +import org.neo4j.gds.compat.GlobalProcedureRegistry; import org.neo4j.gds.compat.GraphDatabaseApiProxy; import org.neo4j.gds.compat.InputEntityIdVisitor; import org.neo4j.gds.compat.Neo4jProxyApi; @@ -87,6 +88,7 @@ import org.neo4j.internal.kernel.api.Scan; import org.neo4j.internal.kernel.api.TokenPredicate; import org.neo4j.internal.kernel.api.connectioninfo.ClientConnectionInfo; +import org.neo4j.internal.kernel.api.exceptions.ProcedureException; import org.neo4j.internal.kernel.api.procs.FieldSignature; import org.neo4j.internal.kernel.api.procs.Neo4jTypes; import org.neo4j.internal.kernel.api.procs.ProcedureSignature; @@ -113,6 +115,8 @@ import org.neo4j.kernel.api.KernelTransactionHandle; import org.neo4j.kernel.api.procedure.CallableProcedure; import org.neo4j.kernel.api.procedure.CallableUserAggregationFunction; +import org.neo4j.kernel.api.procedure.Context; +import org.neo4j.kernel.api.procedure.GlobalProcedures; import org.neo4j.kernel.database.NormalizedDatabaseName; import org.neo4j.kernel.impl.coreapi.InternalTransaction; import org.neo4j.kernel.impl.index.schema.IndexImporterFactoryImpl; @@ -148,8 +152,10 @@ import java.util.List; import java.util.Locale; import java.util.Optional; +import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; +import java.util.stream.Stream; import static java.lang.String.format; import static org.neo4j.gds.compat.InternalReadOps.countByIdGenerator; @@ -920,4 +926,30 @@ public boolean isCompositeDatabase(GraphDatabaseService databaseService) { var databaseManager = GraphDatabaseApiProxy.resolveDependency(databaseService, FabricDatabaseManager.class); return databaseManager.isFabricDatabase(GraphDatabaseApiProxy.databaseId(databaseService)); } + + @Override + public T lookupComponentProvider(Context ctx, Class component, boolean safe) throws ProcedureException { + var globalProcedures = GraphDatabaseApiProxy.resolveDependency(ctx.dependencyResolver(), GlobalProcedures.class); + return globalProcedures.lookupComponentProvider(component, safe).apply(ctx); + } + + @Override + public GlobalProcedureRegistry globalProcedureRegistry(GlobalProcedures globalProcedures) { + return new GlobalProcedureRegistry() { + @Override + public Set getAllProcedures() { + return globalProcedures.getAllProcedures(); + } + + @Override + public Stream getAllNonAggregatingFunctions() { + return globalProcedures.getAllNonAggregatingFunctions(); + } + + @Override + public Stream getAllAggregatingFunctions() { + return globalProcedures.getAllAggregatingFunctions(); + } + }; + } } diff --git a/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/Neo4jProxyImpl.java b/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/Neo4jProxyImpl.java index 420eeb3474..9074bc65de 100644 --- a/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/Neo4jProxyImpl.java +++ b/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/Neo4jProxyImpl.java @@ -44,6 +44,7 @@ import org.neo4j.gds.compat.GdsDatabaseLayout; import org.neo4j.gds.compat.GdsDatabaseManagementServiceBuilder; import org.neo4j.gds.compat.GdsGraphDatabaseAPI; +import org.neo4j.gds.compat.GlobalProcedureRegistry; import org.neo4j.gds.compat.GraphDatabaseApiProxy; import org.neo4j.gds.compat.InputEntityIdVisitor; import org.neo4j.gds.compat.Neo4jProxyApi; @@ -87,6 +88,7 @@ import org.neo4j.internal.kernel.api.Scan; import org.neo4j.internal.kernel.api.TokenPredicate; import org.neo4j.internal.kernel.api.connectioninfo.ClientConnectionInfo; +import org.neo4j.internal.kernel.api.exceptions.ProcedureException; import org.neo4j.internal.kernel.api.procs.FieldSignature; import org.neo4j.internal.kernel.api.procs.Neo4jTypes; import org.neo4j.internal.kernel.api.procs.ProcedureSignature; @@ -113,6 +115,8 @@ import org.neo4j.kernel.api.KernelTransactionHandle; import org.neo4j.kernel.api.procedure.CallableProcedure; import org.neo4j.kernel.api.procedure.CallableUserAggregationFunction; +import org.neo4j.kernel.api.procedure.Context; +import org.neo4j.kernel.api.procedure.GlobalProcedures; import org.neo4j.kernel.database.NormalizedDatabaseName; import org.neo4j.kernel.impl.coreapi.InternalTransaction; import org.neo4j.kernel.impl.index.schema.IndexImporterFactoryImpl; @@ -148,8 +152,10 @@ import java.util.List; import java.util.Locale; import java.util.Optional; +import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; +import java.util.stream.Stream; import static java.lang.String.format; import static org.neo4j.gds.compat.InternalReadOps.countByIdGenerator; @@ -920,4 +926,30 @@ public boolean isCompositeDatabase(GraphDatabaseService databaseService) { var databaseManager = GraphDatabaseApiProxy.resolveDependency(databaseService, FabricDatabaseManager.class); return databaseManager.isFabricDatabase(GraphDatabaseApiProxy.databaseId(databaseService)); } + + @Override + public T lookupComponentProvider(Context ctx, Class component, boolean safe) throws ProcedureException { + var globalProcedures = GraphDatabaseApiProxy.resolveDependency(ctx.dependencyResolver(), GlobalProcedures.class); + return globalProcedures.lookupComponentProvider(component, safe).apply(ctx); + } + + @Override + public GlobalProcedureRegistry globalProcedureRegistry(GlobalProcedures globalProcedures) { + return new GlobalProcedureRegistry() { + @Override + public Set getAllProcedures() { + return globalProcedures.getAllProcedures(); + } + + @Override + public Stream getAllNonAggregatingFunctions() { + return globalProcedures.getAllNonAggregatingFunctions(); + } + + @Override + public Stream getAllAggregatingFunctions() { + return globalProcedures.getAllAggregatingFunctions(); + } + }; + } } diff --git a/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/Neo4jProxyImpl.java b/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/Neo4jProxyImpl.java index 03dd860406..b53c1bdf1a 100644 --- a/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/Neo4jProxyImpl.java +++ b/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/Neo4jProxyImpl.java @@ -44,6 +44,7 @@ import org.neo4j.gds.compat.GdsDatabaseLayout; import org.neo4j.gds.compat.GdsDatabaseManagementServiceBuilder; import org.neo4j.gds.compat.GdsGraphDatabaseAPI; +import org.neo4j.gds.compat.GlobalProcedureRegistry; import org.neo4j.gds.compat.GraphDatabaseApiProxy; import org.neo4j.gds.compat.InputEntityIdVisitor; import org.neo4j.gds.compat.Neo4jProxyApi; @@ -87,6 +88,7 @@ import org.neo4j.internal.kernel.api.Scan; import org.neo4j.internal.kernel.api.TokenPredicate; import org.neo4j.internal.kernel.api.connectioninfo.ClientConnectionInfo; +import org.neo4j.internal.kernel.api.exceptions.ProcedureException; import org.neo4j.internal.kernel.api.procs.FieldSignature; import org.neo4j.internal.kernel.api.procs.Neo4jTypes; import org.neo4j.internal.kernel.api.procs.ProcedureSignature; @@ -113,6 +115,8 @@ import org.neo4j.kernel.api.KernelTransactionHandle; import org.neo4j.kernel.api.procedure.CallableProcedure; import org.neo4j.kernel.api.procedure.CallableUserAggregationFunction; +import org.neo4j.kernel.api.procedure.Context; +import org.neo4j.kernel.api.procedure.GlobalProcedures; import org.neo4j.kernel.database.NormalizedDatabaseName; import org.neo4j.kernel.impl.coreapi.InternalTransaction; import org.neo4j.kernel.impl.index.schema.IndexImporterFactoryImpl; @@ -148,8 +152,10 @@ import java.util.List; import java.util.Locale; import java.util.Optional; +import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; +import java.util.stream.Stream; import static java.lang.String.format; import static org.neo4j.gds.compat.InternalReadOps.countByIdGenerator; @@ -915,4 +921,30 @@ public boolean isCompositeDatabase(GraphDatabaseService databaseService) { var databaseManager = GraphDatabaseApiProxy.resolveDependency(databaseService, FabricDatabaseManager.class); return databaseManager.isFabricDatabase(GraphDatabaseApiProxy.databaseId(databaseService)); } + + @Override + public T lookupComponentProvider(Context ctx, Class component, boolean safe) throws ProcedureException { + var globalProcedures = GraphDatabaseApiProxy.resolveDependency(ctx.dependencyResolver(), GlobalProcedures.class); + return globalProcedures.lookupComponentProvider(component, safe).apply(ctx); + } + + @Override + public GlobalProcedureRegistry globalProcedureRegistry(GlobalProcedures globalProcedures) { + return new GlobalProcedureRegistry() { + @Override + public Set getAllProcedures() { + return globalProcedures.getAllProcedures(); + } + + @Override + public Stream getAllNonAggregatingFunctions() { + return globalProcedures.getAllNonAggregatingFunctions(); + } + + @Override + public Stream getAllAggregatingFunctions() { + return globalProcedures.getAllAggregatingFunctions(); + } + }; + } } diff --git a/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/Neo4jProxyImpl.java b/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/Neo4jProxyImpl.java index a9db437b3e..e0762819ac 100644 --- a/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/Neo4jProxyImpl.java +++ b/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/Neo4jProxyImpl.java @@ -44,6 +44,7 @@ import org.neo4j.gds.compat.GdsDatabaseLayout; import org.neo4j.gds.compat.GdsDatabaseManagementServiceBuilder; import org.neo4j.gds.compat.GdsGraphDatabaseAPI; +import org.neo4j.gds.compat.GlobalProcedureRegistry; import org.neo4j.gds.compat.GraphDatabaseApiProxy; import org.neo4j.gds.compat.InputEntityIdVisitor; import org.neo4j.gds.compat.Neo4jProxyApi; @@ -87,6 +88,7 @@ import org.neo4j.internal.kernel.api.Scan; import org.neo4j.internal.kernel.api.TokenPredicate; import org.neo4j.internal.kernel.api.connectioninfo.ClientConnectionInfo; +import org.neo4j.internal.kernel.api.exceptions.ProcedureException; import org.neo4j.internal.kernel.api.procs.FieldSignature; import org.neo4j.internal.kernel.api.procs.Neo4jTypes; import org.neo4j.internal.kernel.api.procs.ProcedureSignature; @@ -113,6 +115,8 @@ import org.neo4j.kernel.api.KernelTransactionHandle; import org.neo4j.kernel.api.procedure.CallableProcedure; import org.neo4j.kernel.api.procedure.CallableUserAggregationFunction; +import org.neo4j.kernel.api.procedure.Context; +import org.neo4j.kernel.api.procedure.GlobalProcedures; import org.neo4j.kernel.database.NormalizedDatabaseName; import org.neo4j.kernel.impl.coreapi.InternalTransaction; import org.neo4j.kernel.impl.index.schema.IndexImporterFactoryImpl; @@ -148,8 +152,10 @@ import java.util.List; import java.util.Locale; import java.util.Optional; +import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; +import java.util.stream.Stream; import static java.lang.String.format; import static org.neo4j.gds.compat.InternalReadOps.countByIdGenerator; @@ -915,4 +921,30 @@ public boolean isCompositeDatabase(GraphDatabaseService databaseService) { var databaseManager = GraphDatabaseApiProxy.resolveDependency(databaseService, FabricDatabaseManager.class); return databaseManager.isFabricDatabase(GraphDatabaseApiProxy.databaseId(databaseService)); } + + @Override + public T lookupComponentProvider(Context ctx, Class component, boolean safe) throws ProcedureException { + var globalProcedures = GraphDatabaseApiProxy.resolveDependency(ctx.dependencyResolver(), GlobalProcedures.class); + return globalProcedures.lookupComponentProvider(component, safe).apply(ctx); + } + + @Override + public GlobalProcedureRegistry globalProcedureRegistry(GlobalProcedures globalProcedures) { + return new GlobalProcedureRegistry() { + @Override + public Set getAllProcedures() { + return globalProcedures.getAllProcedures(); + } + + @Override + public Stream getAllNonAggregatingFunctions() { + return globalProcedures.getAllNonAggregatingFunctions(); + } + + @Override + public Stream getAllAggregatingFunctions() { + return globalProcedures.getAllAggregatingFunctions(); + } + }; + } } diff --git a/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/Neo4jProxyImpl.java b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/Neo4jProxyImpl.java index 243d4e32cd..cbd431f918 100644 --- a/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/Neo4jProxyImpl.java +++ b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/Neo4jProxyImpl.java @@ -44,6 +44,7 @@ import org.neo4j.gds.compat.GdsDatabaseLayout; import org.neo4j.gds.compat.GdsDatabaseManagementServiceBuilder; import org.neo4j.gds.compat.GdsGraphDatabaseAPI; +import org.neo4j.gds.compat.GlobalProcedureRegistry; import org.neo4j.gds.compat.GraphDatabaseApiProxy; import org.neo4j.gds.compat.InputEntityIdVisitor; import org.neo4j.gds.compat.Neo4jProxyApi; @@ -87,6 +88,7 @@ import org.neo4j.internal.kernel.api.Scan; import org.neo4j.internal.kernel.api.TokenPredicate; import org.neo4j.internal.kernel.api.connectioninfo.ClientConnectionInfo; +import org.neo4j.internal.kernel.api.exceptions.ProcedureException; import org.neo4j.internal.kernel.api.procs.FieldSignature; import org.neo4j.internal.kernel.api.procs.Neo4jTypes; import org.neo4j.internal.kernel.api.procs.ProcedureSignature; @@ -113,6 +115,8 @@ import org.neo4j.kernel.api.KernelTransactionHandle; import org.neo4j.kernel.api.procedure.CallableProcedure; import org.neo4j.kernel.api.procedure.CallableUserAggregationFunction; +import org.neo4j.kernel.api.procedure.Context; +import org.neo4j.kernel.api.procedure.GlobalProcedures; import org.neo4j.kernel.database.NormalizedDatabaseName; import org.neo4j.kernel.impl.coreapi.InternalTransaction; import org.neo4j.kernel.impl.index.schema.IndexImporterFactoryImpl; @@ -148,8 +152,10 @@ import java.util.List; import java.util.Locale; import java.util.Optional; +import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; +import java.util.stream.Stream; import static java.lang.String.format; import static org.neo4j.gds.compat.InternalReadOps.countByIdGenerator; @@ -915,4 +921,30 @@ public boolean isCompositeDatabase(GraphDatabaseService databaseService) { var databaseManager = GraphDatabaseApiProxy.resolveDependency(databaseService, FabricDatabaseManager.class); return databaseManager.isFabricDatabase(GraphDatabaseApiProxy.databaseId(databaseService)); } + + @Override + public T lookupComponentProvider(Context ctx, Class component, boolean safe) throws ProcedureException { + var globalProcedures = GraphDatabaseApiProxy.resolveDependency(ctx.dependencyResolver(), GlobalProcedures.class); + return globalProcedures.lookupComponentProvider(component, safe).apply(ctx); + } + + @Override + public GlobalProcedureRegistry globalProcedureRegistry(GlobalProcedures globalProcedures) { + return new GlobalProcedureRegistry() { + @Override + public Set getAllProcedures() { + return globalProcedures.getAllProcedures(); + } + + @Override + public Stream getAllNonAggregatingFunctions() { + return globalProcedures.getAllNonAggregatingFunctions(); + } + + @Override + public Stream getAllAggregatingFunctions() { + return globalProcedures.getAllAggregatingFunctions(); + } + }; + } } diff --git a/compatibility/api/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/GlobalProcedureRegistry.java b/compatibility/api/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/GlobalProcedureRegistry.java new file mode 100644 index 0000000000..fd22cf2c8e --- /dev/null +++ b/compatibility/api/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/GlobalProcedureRegistry.java @@ -0,0 +1,32 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat; + +import org.neo4j.internal.kernel.api.procs.ProcedureSignature; +import org.neo4j.internal.kernel.api.procs.UserFunctionSignature; + +import java.util.Set; +import java.util.stream.Stream; + +public interface GlobalProcedureRegistry { + Set getAllProcedures(); + Stream getAllNonAggregatingFunctions(); + Stream getAllAggregatingFunctions(); +} diff --git a/compatibility/api/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/Neo4jProxyApi.java b/compatibility/api/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/Neo4jProxyApi.java index 81a7b2f6d7..abe644512a 100644 --- a/compatibility/api/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/Neo4jProxyApi.java +++ b/compatibility/api/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/Neo4jProxyApi.java @@ -52,6 +52,7 @@ import org.neo4j.internal.kernel.api.Read; import org.neo4j.internal.kernel.api.RelationshipScanCursor; import org.neo4j.internal.kernel.api.Scan; +import org.neo4j.internal.kernel.api.exceptions.ProcedureException; import org.neo4j.internal.kernel.api.procs.FieldSignature; import org.neo4j.internal.kernel.api.procs.Neo4jTypes; import org.neo4j.internal.kernel.api.procs.ProcedureSignature; @@ -72,6 +73,8 @@ import org.neo4j.kernel.api.KernelTransactionHandle; import org.neo4j.kernel.api.procedure.CallableProcedure; import org.neo4j.kernel.api.procedure.CallableUserAggregationFunction; +import org.neo4j.kernel.api.procedure.Context; +import org.neo4j.kernel.api.procedure.GlobalProcedures; import org.neo4j.kernel.impl.coreapi.InternalTransaction; import org.neo4j.kernel.impl.query.TransactionalContext; import org.neo4j.kernel.impl.query.TransactionalContextFactory; @@ -320,4 +323,8 @@ TransactionalContext newQueryContext( ); boolean isCompositeDatabase(GraphDatabaseService databaseService); + + T lookupComponentProvider(Context ctx, Class component, boolean safe) throws ProcedureException; + + GlobalProcedureRegistry globalProcedureRegistry(GlobalProcedures globalProcedures); } diff --git a/compatibility/common/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/Neo4jProxy.java b/compatibility/common/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/Neo4jProxy.java index 5eee84233e..923dfa94af 100644 --- a/compatibility/common/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/Neo4jProxy.java +++ b/compatibility/common/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/Neo4jProxy.java @@ -52,6 +52,7 @@ import org.neo4j.internal.kernel.api.Read; import org.neo4j.internal.kernel.api.RelationshipScanCursor; import org.neo4j.internal.kernel.api.Scan; +import org.neo4j.internal.kernel.api.exceptions.ProcedureException; import org.neo4j.internal.kernel.api.procs.FieldSignature; import org.neo4j.internal.kernel.api.procs.Neo4jTypes; import org.neo4j.internal.kernel.api.procs.ProcedureSignature; @@ -72,6 +73,8 @@ import org.neo4j.kernel.api.KernelTransactionHandle; import org.neo4j.kernel.api.procedure.CallableProcedure; import org.neo4j.kernel.api.procedure.CallableUserAggregationFunction; +import org.neo4j.kernel.api.procedure.Context; +import org.neo4j.kernel.api.procedure.GlobalProcedures; import org.neo4j.kernel.impl.coreapi.InternalTransaction; import org.neo4j.kernel.impl.query.TransactionalContext; import org.neo4j.kernel.impl.query.TransactionalContextFactory; @@ -90,6 +93,7 @@ import java.nio.file.Path; import java.util.List; import java.util.Optional; +import java.util.stream.Collectors; public final class Neo4jProxy { @@ -481,6 +485,27 @@ public static boolean isCompositeDatabase(GraphDatabaseService databaseService) return IMPL.isCompositeDatabase(databaseService); } + public static T lookupComponentProvider(Context ctx, Class component, boolean safe) throws + ProcedureException { + return IMPL.lookupComponentProvider(ctx, component, safe); + } + + public static GlobalProcedureRegistry globalProcedureRegistry(GlobalProcedures globalProcedures) { + return IMPL.globalProcedureRegistry(globalProcedures); + } + + public static void registerUserAggregationFunction(GraphDatabaseService db, CallableUserAggregationFunction function) throws + KernelException { + var globalProcedures = GraphDatabaseApiProxy.resolveDependency(db, GlobalProcedures.class); + var alreadyExists = globalProcedureRegistry(globalProcedures).getAllAggregatingFunctions() + .collect(Collectors.toSet()) + .contains(function.signature()); + + if (!alreadyExists) { + GraphDatabaseApiProxy.register(globalProcedures, function); + } + } + private Neo4jProxy() { throw new UnsupportedOperationException("No instances"); } diff --git a/cypher-aggregation/src/main/java/org/neo4j/gds/projection/AlphaCypherAggregation.java b/cypher-aggregation/src/main/java/org/neo4j/gds/projection/AlphaCypherAggregation.java index 0087b7fd11..870a2f449b 100644 --- a/cypher-aggregation/src/main/java/org/neo4j/gds/projection/AlphaCypherAggregation.java +++ b/cypher-aggregation/src/main/java/org/neo4j/gds/projection/AlphaCypherAggregation.java @@ -19,7 +19,6 @@ */ package org.neo4j.gds.projection; -import com.neo4j.gds.internal.CustomProceduresUtil; import org.neo4j.gds.annotation.CustomProcedure; import org.neo4j.gds.api.DatabaseId; import org.neo4j.gds.compat.CompatUserAggregationFunction; @@ -103,8 +102,8 @@ public static CompatUserAggregationFunction newInstance() { @Override public CompatUserAggregator create(Context ctx) throws ProcedureException { - var databaseService = CustomProceduresUtil.lookupSafeComponentProvider(ctx, GraphDatabaseService.class); - var username = CustomProceduresUtil.lookupSafeComponentProvider(ctx, Username.class); + var databaseService = Neo4jProxy.lookupComponentProvider(ctx, GraphDatabaseService.class, true); + var username = Neo4jProxy.lookupComponentProvider(ctx, Username.class, true); var runsOnCompositeDatabase = Neo4jProxy.isCompositeDatabase(databaseService); var writeMode = runsOnCompositeDatabase diff --git a/cypher-aggregation/src/main/java/org/neo4j/gds/projection/CypherAggregation.java b/cypher-aggregation/src/main/java/org/neo4j/gds/projection/CypherAggregation.java index 9e5f9aea08..357e80550d 100644 --- a/cypher-aggregation/src/main/java/org/neo4j/gds/projection/CypherAggregation.java +++ b/cypher-aggregation/src/main/java/org/neo4j/gds/projection/CypherAggregation.java @@ -19,7 +19,6 @@ */ package org.neo4j.gds.projection; -import com.neo4j.gds.internal.CustomProceduresUtil; import org.neo4j.gds.annotation.CustomProcedure; import org.neo4j.gds.api.DatabaseId; import org.neo4j.gds.compat.CompatUserAggregationFunction; @@ -102,8 +101,8 @@ public static CompatUserAggregationFunction newInstance() { @Override public CompatUserAggregator create(Context ctx) throws ProcedureException { - var databaseService = CustomProceduresUtil.lookupSafeComponentProvider(ctx, GraphDatabaseService.class); - var username = CustomProceduresUtil.lookupSafeComponentProvider(ctx, Username.class); + var databaseService = Neo4jProxy.lookupComponentProvider(ctx, GraphDatabaseService.class, true); + var username = Neo4jProxy.lookupComponentProvider(ctx, Username.class, true); var runsOnCompositeDatabase = Neo4jProxy.isCompositeDatabase(databaseService); var writeMode = runsOnCompositeDatabase diff --git a/cypher-aggregation/src/test/java/org/neo4j/gds/projection/CypherAggregationTest.java b/cypher-aggregation/src/test/java/org/neo4j/gds/projection/CypherAggregationTest.java index 02e0e2d7ae..3e4f6c5287 100644 --- a/cypher-aggregation/src/test/java/org/neo4j/gds/projection/CypherAggregationTest.java +++ b/cypher-aggregation/src/test/java/org/neo4j/gds/projection/CypherAggregationTest.java @@ -85,7 +85,7 @@ class CypherAggregationTest extends BaseProcTest { @BeforeEach void setup() throws Exception { - GraphDatabaseApiProxy.register(db, Neo4jProxy.callableUserAggregationFunction(CypherAggregation.newInstance())); + Neo4jProxy.registerUserAggregationFunction(db, Neo4jProxy.callableUserAggregationFunction(CypherAggregation.newInstance())); registerProcedures(GraphDropProc.class, GraphListProc.class, WccStreamProc.class); } diff --git a/neo4j-adapter/src/main/java/org/neo4j/gds/compat/GraphDatabaseApiProxy.java b/neo4j-adapter/src/main/java/org/neo4j/gds/compat/GraphDatabaseApiProxy.java index 0eca1c93c9..12d2746168 100644 --- a/neo4j-adapter/src/main/java/org/neo4j/gds/compat/GraphDatabaseApiProxy.java +++ b/neo4j-adapter/src/main/java/org/neo4j/gds/compat/GraphDatabaseApiProxy.java @@ -32,7 +32,9 @@ import org.neo4j.internal.kernel.api.security.LoginContext; import org.neo4j.io.layout.DatabaseLayout; import org.neo4j.kernel.api.KernelTransaction; +import org.neo4j.kernel.api.procedure.CallableProcedure; import org.neo4j.kernel.api.procedure.CallableUserAggregationFunction; +import org.neo4j.kernel.api.procedure.CallableUserFunction; import org.neo4j.kernel.api.procedure.GlobalProcedures; import org.neo4j.kernel.database.NamedDatabaseId; import org.neo4j.kernel.impl.coreapi.InternalTransaction; @@ -107,7 +109,23 @@ public static void registerAggregationFunctions(GraphDatabaseService db, Class clazz, Object obj) { var globalProceduresClass = globalProcedures.getClass(); var registerProcedureMethod = Arrays.stream(globalProceduresClass.getDeclaredMethods()) .filter(method -> method.getName().equals("register")) - .filter(method -> method.getParameterCount() == 1 && method.getParameterTypes()[0].equals(CallableUserAggregationFunction.class)) + .filter(method -> (method.getParameterCount() == 1 || method.getParameterCount() == 2) + && method.getParameterTypes()[0].equals(clazz)) .findFirst() .orElseThrow(() -> new RuntimeException("Could not find registerAggregationFunction method")); try { switch (registerProcedureMethod.getParameterCount()) { case 1: - registerProcedureMethod.invoke(globalProcedures, function); + registerProcedureMethod.invoke(globalProcedures, obj); break; case 2: { - registerProcedureMethod.invoke(globalProcedures, function, true); + registerProcedureMethod.invoke(globalProcedures, obj, true); break; } default: diff --git a/proc/common/src/main/java/com/neo4j/gds/internal/CustomProceduresUtil.java b/proc/common/src/main/java/com/neo4j/gds/internal/CustomProceduresUtil.java deleted file mode 100644 index a0fd82baf7..0000000000 --- a/proc/common/src/main/java/com/neo4j/gds/internal/CustomProceduresUtil.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright (c) "Neo4j" - * Neo4j Sweden AB [http://neo4j.com] - * - * This file is part of Neo4j. - * - * Neo4j is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -package com.neo4j.gds.internal; - -import org.neo4j.gds.compat.GraphDatabaseApiProxy; -import org.neo4j.internal.kernel.api.exceptions.ProcedureException; -import org.neo4j.kernel.api.procedure.Context; -import org.neo4j.kernel.api.procedure.GlobalProcedures; - -public final class CustomProceduresUtil { - - public static T resolveDependency(Context ctx, Class dependency) { - return GraphDatabaseApiProxy.resolveDependency(ctx.dependencyResolver(), dependency); - } - - public static T lookupComponentProvider(Context ctx, Class component) throws ProcedureException { - var globalProcedures = resolveDependency(ctx, GlobalProcedures.class); - return globalProcedures.lookupComponentProvider(component, false).apply(ctx); - } - - /** - * Like {@link #lookupComponentProvider(Context, Class)} but only allows safe components. - * Safe components are those that are not sandboxed by the kernel and can be used without - * setting the {@code dbms.security.procedures.unrestricted} setting. - */ - public static T lookupSafeComponentProvider(Context ctx, Class component) throws ProcedureException { - var globalProcedures = resolveDependency(ctx, GlobalProcedures.class); - return globalProcedures.lookupComponentProvider(component, true).apply(ctx); - } - - private CustomProceduresUtil() {} -} From 869babb7ba5cbbba75bf85606b0595990946ddcd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B6ren=20Reichardt?= Date: Fri, 28 Jul 2023 13:51:04 +0200 Subject: [PATCH 189/273] Refactor and fix compat register method calls --- .../java/org/neo4j/gds/compat/Neo4jProxy.java | 13 ---- .../gds/projection/CypherAggregationTest.java | 18 ++++- .../gds/compat/GraphDatabaseApiProxy.java | 69 +++++++++---------- 3 files changed, 50 insertions(+), 50 deletions(-) diff --git a/compatibility/common/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/Neo4jProxy.java b/compatibility/common/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/Neo4jProxy.java index 923dfa94af..0cfb540669 100644 --- a/compatibility/common/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/Neo4jProxy.java +++ b/compatibility/common/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/Neo4jProxy.java @@ -93,7 +93,6 @@ import java.nio.file.Path; import java.util.List; import java.util.Optional; -import java.util.stream.Collectors; public final class Neo4jProxy { @@ -494,18 +493,6 @@ public static GlobalProcedureRegistry globalProcedureRegistry(GlobalProcedures g return IMPL.globalProcedureRegistry(globalProcedures); } - public static void registerUserAggregationFunction(GraphDatabaseService db, CallableUserAggregationFunction function) throws - KernelException { - var globalProcedures = GraphDatabaseApiProxy.resolveDependency(db, GlobalProcedures.class); - var alreadyExists = globalProcedureRegistry(globalProcedures).getAllAggregatingFunctions() - .collect(Collectors.toSet()) - .contains(function.signature()); - - if (!alreadyExists) { - GraphDatabaseApiProxy.register(globalProcedures, function); - } - } - private Neo4jProxy() { throw new UnsupportedOperationException("No instances"); } diff --git a/cypher-aggregation/src/test/java/org/neo4j/gds/projection/CypherAggregationTest.java b/cypher-aggregation/src/test/java/org/neo4j/gds/projection/CypherAggregationTest.java index 3e4f6c5287..7c17947675 100644 --- a/cypher-aggregation/src/test/java/org/neo4j/gds/projection/CypherAggregationTest.java +++ b/cypher-aggregation/src/test/java/org/neo4j/gds/projection/CypherAggregationTest.java @@ -28,6 +28,7 @@ import org.junit.jupiter.params.provider.CsvSource; import org.junit.jupiter.params.provider.MethodSource; import org.junit.jupiter.params.provider.ValueSource; +import org.neo4j.exceptions.KernelException; import org.neo4j.gds.BaseProcTest; import org.neo4j.gds.BaseTest; import org.neo4j.gds.NodeLabel; @@ -39,6 +40,7 @@ import org.neo4j.gds.api.nodeproperties.ValueType; import org.neo4j.gds.catalog.GraphDropProc; import org.neo4j.gds.catalog.GraphListProc; +import org.neo4j.gds.compat.CompatUserAggregationFunction; import org.neo4j.gds.compat.GraphDatabaseApiProxy; import org.neo4j.gds.compat.Neo4jProxy; import org.neo4j.gds.core.RandomGraphTestCase; @@ -46,8 +48,10 @@ import org.neo4j.gds.extension.Neo4jGraph; import org.neo4j.gds.wcc.WccStreamProc; import org.neo4j.graphdb.Direction; +import org.neo4j.graphdb.GraphDatabaseService; import org.neo4j.graphdb.RelationshipType; import org.neo4j.graphdb.Result; +import org.neo4j.kernel.api.procedure.GlobalProcedures; import java.util.List; import java.util.Map; @@ -85,7 +89,7 @@ class CypherAggregationTest extends BaseProcTest { @BeforeEach void setup() throws Exception { - Neo4jProxy.registerUserAggregationFunction(db, Neo4jProxy.callableUserAggregationFunction(CypherAggregation.newInstance())); + registerUserAggregationFunction(db, CypherAggregation.newInstance()); registerProcedures(GraphDropProc.class, GraphListProc.class, WccStreamProc.class); } @@ -1053,4 +1057,16 @@ void testExplicitlyMissingSourceOrTargetNodeInformation(String presentKey, Strin assertThatCode(() -> runQuery(query)) .doesNotThrowAnyException(); } + + private static void registerUserAggregationFunction(GraphDatabaseService db, CompatUserAggregationFunction function) throws + KernelException { + var globalProcedures = GraphDatabaseApiProxy.resolveDependency(db, GlobalProcedures.class); + var alreadyExists = Neo4jProxy.globalProcedureRegistry(globalProcedures).getAllAggregatingFunctions() + .collect(Collectors.toSet()) + .contains(function.signature()); + + if (!alreadyExists) { + GraphDatabaseApiProxy.register(globalProcedures, Neo4jProxy.callableUserAggregationFunction(function)); + } + } } diff --git a/neo4j-adapter/src/main/java/org/neo4j/gds/compat/GraphDatabaseApiProxy.java b/neo4j-adapter/src/main/java/org/neo4j/gds/compat/GraphDatabaseApiProxy.java index 12d2746168..471c8acf3b 100644 --- a/neo4j-adapter/src/main/java/org/neo4j/gds/compat/GraphDatabaseApiProxy.java +++ b/neo4j-adapter/src/main/java/org/neo4j/gds/compat/GraphDatabaseApiProxy.java @@ -43,11 +43,13 @@ import org.neo4j.storageengine.api.StorageEngineFactory; import java.lang.reflect.InvocationTargetException; -import java.util.Arrays; +import java.lang.reflect.Method; import java.util.Map; +import java.util.Optional; import java.util.function.Consumer; import java.util.function.Function; + public final class GraphDatabaseApiProxy { public static Neo4jVersion neo4jVersion() { @@ -123,6 +125,7 @@ public static void register(GlobalProcedures procedures, CallableUserFunction fu register(procedures, CallableUserFunction.class, function); } + @SuppressForbidden(reason = "We're not implementing CallableProcedure, just passing it on") public static void register(GlobalProcedures procedures, CallableProcedure procedure) throws KernelException { register(procedures, CallableProcedure.class, procedure); @@ -248,49 +251,43 @@ private static void registerProcedures( boolean overrideCurrentImplementation ) { var globalProceduresClass = globalProcedures.getClass(); - var registerProcedureMethod = Arrays.stream(globalProceduresClass.getDeclaredMethods()) - .filter(method -> method.getName().equals("registerProcedure")) - .findFirst() - .orElseThrow(() -> new RuntimeException("Could not find registerProcedure method")); - try { - switch (registerProcedureMethod.getParameterCount()) { - case 1: - registerProcedureMethod.invoke(globalProcedures, procedureClass); - break; - case 2: { - registerProcedureMethod.invoke(globalProcedures, procedureClass, overrideCurrentImplementation); - break; - } - default: - throw new RuntimeException("Unexpected number of parameters for registerProcedure method"); - } - } catch (IllegalAccessException | InvocationTargetException e) { - throw new RuntimeException(e); + var legacySignatureMethod = getMethod(globalProceduresClass, "registerProcedure", Class.class, boolean.class); + var newSignatureMethod = getMethod(globalProceduresClass, "registerProcedure", Class.class); + if (legacySignatureMethod.isPresent()) { + invokeMethod(globalProcedures, legacySignatureMethod.get(), procedureClass, overrideCurrentImplementation); + } else if (newSignatureMethod.isPresent()){ + invokeMethod(globalProcedures, newSignatureMethod.get(), procedureClass); + } else { + throw new RuntimeException("Could not find registerProcedure method"); } } private static void register(GlobalProcedures globalProcedures, Class clazz, Object obj) { var globalProceduresClass = globalProcedures.getClass(); - var registerProcedureMethod = Arrays.stream(globalProceduresClass.getDeclaredMethods()) - .filter(method -> method.getName().equals("register")) - .filter(method -> (method.getParameterCount() == 1 || method.getParameterCount() == 2) - && method.getParameterTypes()[0].equals(clazz)) - .findFirst() - .orElseThrow(() -> new RuntimeException("Could not find registerAggregationFunction method")); + var legacySignatureMethod = getMethod(globalProceduresClass, "register", clazz, boolean.class); + var newSignatureMethod = getMethod(globalProceduresClass, "register", clazz); + if (legacySignatureMethod.isPresent()) { + invokeMethod(globalProcedures, legacySignatureMethod.get(), obj, true); + } else if (newSignatureMethod.isPresent()){ + invokeMethod(globalProcedures, newSignatureMethod.get(), obj); + } else { + throw new RuntimeException("Could not find registerProcedure method"); + } + } + + private static Optional getMethod(Class clazz, String name, Class... parameterTypes) { try { - switch (registerProcedureMethod.getParameterCount()) { - case 1: - registerProcedureMethod.invoke(globalProcedures, obj); - break; - case 2: { - registerProcedureMethod.invoke(globalProcedures, obj, true); - break; - } - default: - throw new RuntimeException("Unexpected number of parameters for register method"); - } + return Optional.of(clazz.getMethod(name, parameterTypes)); + } catch (NoSuchMethodException e) { + return Optional.empty(); + } + } + + private static void invokeMethod(Object obj, Method method, Object... arguments) { + try { + method.invoke(obj, arguments); } catch (IllegalAccessException | InvocationTargetException e) { throw new RuntimeException(e); } From 2c0af223822f60e57884aa8af36778c4cf941d67 Mon Sep 17 00:00:00 2001 From: Paul Horn Date: Wed, 2 Aug 2023 11:29:06 +0200 Subject: [PATCH 190/273] let rc be 5.11 --- gradle/dependencies.gradle | 1 + 1 file changed, 1 insertion(+) diff --git a/gradle/dependencies.gradle b/gradle/dependencies.gradle index f80dbfb555..8672b3474e 100644 --- a/gradle/dependencies.gradle +++ b/gradle/dependencies.gradle @@ -11,6 +11,7 @@ ext { '5.8': properties.getOrDefault('neo4jVersion58', '5.8.0'), '5.9': properties.getOrDefault('neo4jVersion59', '5.9.0'), '5.10': properties.getOrDefault('neo4jVersion510', '5.10.0'), + '5.11': properties.getOrDefault('neo4jVersion511', '5.11.0'), ] neo4jDefault = neos.'4.4' From 8191370459abf9e8d5d5fb2bb13f44fbdff8b106 Mon Sep 17 00:00:00 2001 From: Paul Horn Date: Wed, 2 Aug 2023 18:14:48 +0200 Subject: [PATCH 191/273] Only reset clock if it was set --- .../org/neo4j/gds/extension/FakeClockSupportExtension.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test-utils/src/main/java/org/neo4j/gds/extension/FakeClockSupportExtension.java b/test-utils/src/main/java/org/neo4j/gds/extension/FakeClockSupportExtension.java index 44bbe1e5ba..237a72a6aa 100644 --- a/test-utils/src/main/java/org/neo4j/gds/extension/FakeClockSupportExtension.java +++ b/test-utils/src/main/java/org/neo4j/gds/extension/FakeClockSupportExtension.java @@ -47,7 +47,9 @@ public void beforeEach(ExtensionContext context) throws Exception { @Override public void afterEach(ExtensionContext context) throws Exception { - ClockService.setClock(clockBefore); - clockBefore = null; + if (clockBefore != null) { + ClockService.setClock(clockBefore); + clockBefore = null; + } } } From 34b5ff43c8c62078e3ad3845192a0d023403ec43 Mon Sep 17 00:00:00 2001 From: Paul Horn Date: Wed, 2 Aug 2023 18:18:45 +0200 Subject: [PATCH 192/273] Ignore procedure exception when a function was already registered --- .../java/org/neo4j/gds/compat/GraphDatabaseApiProxy.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/neo4j-adapter/src/main/java/org/neo4j/gds/compat/GraphDatabaseApiProxy.java b/neo4j-adapter/src/main/java/org/neo4j/gds/compat/GraphDatabaseApiProxy.java index 471c8acf3b..120edc064d 100644 --- a/neo4j-adapter/src/main/java/org/neo4j/gds/compat/GraphDatabaseApiProxy.java +++ b/neo4j-adapter/src/main/java/org/neo4j/gds/compat/GraphDatabaseApiProxy.java @@ -29,6 +29,7 @@ import org.neo4j.graphdb.NotFoundException; import org.neo4j.graphdb.Result; import org.neo4j.graphdb.Transaction; +import org.neo4j.internal.kernel.api.exceptions.ProcedureException; import org.neo4j.internal.kernel.api.security.LoginContext; import org.neo4j.io.layout.DatabaseLayout; import org.neo4j.kernel.api.KernelTransaction; @@ -288,7 +289,11 @@ private static Optional getMethod(Class clazz, String name, Class. private static void invokeMethod(Object obj, Method method, Object... arguments) { try { method.invoke(obj, arguments); - } catch (IllegalAccessException | InvocationTargetException e) { + } catch (InvocationTargetException e) { + if (e.getTargetException().getClass() != ProcedureException.class) { + throw new RuntimeException(e.getTargetException()); + } + } catch (IllegalAccessException e) { throw new RuntimeException(e); } } From acd131efc58deafe0ff28de82cea08412b84ada3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Thu, 3 Aug 2023 16:18:22 +0200 Subject: [PATCH 193/273] Use more stable ui-bundle url --- doc/preview.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/preview.yml b/doc/preview.yml index 1901f52d69..7a25f90bdd 100644 --- a/doc/preview.yml +++ b/doc/preview.yml @@ -16,7 +16,7 @@ content: - '!**/README.adoc' ui: bundle: - url: https://s3-eu-west-1.amazonaws.com/static-content.neo4j.com/build/ui-bundle-latest.zip + url: https://static-content.neo4j.com/build/ui-bundle-latest.zip snapshot: true output_dir: /assets From b9febccb12a79bdf99c882048a1c925636933a0e Mon Sep 17 00:00:00 2001 From: Paul Horn Date: Thu, 3 Aug 2023 16:56:06 +0200 Subject: [PATCH 194/273] Add proper throws declarations --- .../gds/compat/GraphDatabaseApiProxy.java | 21 ++++++++++++------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/neo4j-adapter/src/main/java/org/neo4j/gds/compat/GraphDatabaseApiProxy.java b/neo4j-adapter/src/main/java/org/neo4j/gds/compat/GraphDatabaseApiProxy.java index 120edc064d..0301c3aac4 100644 --- a/neo4j-adapter/src/main/java/org/neo4j/gds/compat/GraphDatabaseApiProxy.java +++ b/neo4j-adapter/src/main/java/org/neo4j/gds/compat/GraphDatabaseApiProxy.java @@ -250,7 +250,7 @@ private static void registerProcedures( GlobalProcedures globalProcedures, Class procedureClass, boolean overrideCurrentImplementation - ) { + ) throws KernelException { var globalProceduresClass = globalProcedures.getClass(); var legacySignatureMethod = getMethod(globalProceduresClass, "registerProcedure", Class.class, boolean.class); @@ -264,7 +264,7 @@ private static void registerProcedures( } } - private static void register(GlobalProcedures globalProcedures, Class clazz, Object obj) { + private static void register(GlobalProcedures globalProcedures, Class clazz, Object obj) throws KernelException { var globalProceduresClass = globalProcedures.getClass(); var legacySignatureMethod = getMethod(globalProceduresClass, "register", clazz, boolean.class); @@ -286,14 +286,19 @@ private static Optional getMethod(Class clazz, String name, Class. } } - private static void invokeMethod(Object obj, Method method, Object... arguments) { + private static void invokeMethod(Object obj, Method method, Object... arguments) + throws KernelException { try { - method.invoke(obj, arguments); - } catch (InvocationTargetException e) { - if (e.getTargetException().getClass() != ProcedureException.class) { - throw new RuntimeException(e.getTargetException()); + try { + method.invoke(obj, arguments); + } catch (InvocationTargetException e) { + if (e.getTargetException().getClass() != ProcedureException.class) { + throw e.getTargetException(); + } } - } catch (IllegalAccessException e) { + } catch (KernelException | RuntimeException e) { + throw e; + } catch (Throwable e) { throw new RuntimeException(e); } } From 8831f8feaf04d7103426c204663a29e45e30bae4 Mon Sep 17 00:00:00 2001 From: Paul Horn Date: Thu, 3 Aug 2023 17:02:00 +0200 Subject: [PATCH 195/273] Configure scala and log4j deps for 5.11 --- gradle/dependencies.gradle | 2 ++ 1 file changed, 2 insertions(+) diff --git a/gradle/dependencies.gradle b/gradle/dependencies.gradle index 8672b3474e..b4ab440724 100644 --- a/gradle/dependencies.gradle +++ b/gradle/dependencies.gradle @@ -31,6 +31,7 @@ ext { '5.8': '2.13.10', '5.9': '2.13.10', '5.10': '2.13.10', + '5.11': '2.13.10', ] log4js = [ @@ -45,6 +46,7 @@ ext { '5.8': '2.20.0', '5.9': '2.20.0', '5.10': '2.20.0', + '5.11': '2.20.0', ] ver = [ From 3d460ea1063332659d75b76c38fea177acd48eec Mon Sep 17 00:00:00 2001 From: yuval Date: Fri, 4 Aug 2023 16:25:10 +0200 Subject: [PATCH 196/273] fix the node label mapping docs issue on 2.4 --- doc/modules/ROOT/pages/management-ops/backup-restore.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/modules/ROOT/pages/management-ops/backup-restore.adoc b/doc/modules/ROOT/pages/management-ops/backup-restore.adoc index c57290a97b..017b0e807e 100644 --- a/doc/modules/ROOT/pages/management-ops/backup-restore.adoc +++ b/doc/modules/ROOT/pages/management-ops/backup-restore.adoc @@ -47,7 +47,7 @@ YIELD | Name | Type | Default | Description | concurrency | Integer | 4 | The number of concurrent threads used for performing the backup. | includeGraphs | Boolean | true | Flag to decide whether only models or also graphs should be backed up. -| useLabelMapping | Boolean | true | Flag to decide whether to use node label mapping when exporting the graph. See xref:management-ops/graph-export/graph-export-csv.adoc#node_label_mapping[exporting graphs to csv] for details. +| useLabelMapping | Boolean | true | Flag to decide whether to use node label mapping when exporting the graph. See xref:graph-catalog-export-ops.adoc#node_label_mapping[exporting graphs to csv] for details. |=== .Results From db83d3e4d7ac3f9908bdcb1a26d065c242f2da53 Mon Sep 17 00:00:00 2001 From: Mats Rydberg Date: Fri, 4 Aug 2023 17:37:33 +0200 Subject: [PATCH 197/273] Bump 4.4 to 4.4.24 --- README.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.adoc b/README.adoc index c07f431c18..d991496d89 100644 --- a/README.adoc +++ b/README.adoc @@ -38,7 +38,7 @@ When installing GDS manually, please refer to the below compatibility matrix: |Neo4j 5.4.0 |Neo4j 5.2.0 |Neo4j 5.1.0 -|Neo4j 4.4.9 - 4.4.23 +|Neo4j 4.4.9 - 4.4.24 .1+.^|Java 11 .9+<.^|GDS 2.3.x |Neo4j 5.8.0 From e8ba002bb76dae65dfabcd457b0a92a5d4edc264 Mon Sep 17 00:00:00 2001 From: Mats Rydberg Date: Fri, 4 Aug 2023 17:38:05 +0200 Subject: [PATCH 198/273] Bump 4.4 to 4.4.24 --- gradle/dependencies.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle/dependencies.gradle b/gradle/dependencies.gradle index b4ab440724..edad2b2b92 100644 --- a/gradle/dependencies.gradle +++ b/gradle/dependencies.gradle @@ -1,6 +1,6 @@ ext { neos = [ - '4.4': properties.getOrDefault('neo4jVersion44', '4.4.22'), + '4.4': properties.getOrDefault('neo4jVersion44', '4.4.24'), '5.1': properties.getOrDefault('neo4jVersion51', '5.1.0'), '5.2': properties.getOrDefault('neo4jVersion52', '5.2.0'), '5.3': properties.getOrDefault('neo4jVersion53', '5.3.0'), From ca26e08c6e8a6848a4a0770aec42af6f2d2feb46 Mon Sep 17 00:00:00 2001 From: ioannispan Date: Mon, 7 Aug 2023 11:06:01 +0200 Subject: [PATCH 199/273] no-dir trait --- doc/modules/ROOT/pages/algorithms/k-core.adoc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/modules/ROOT/pages/algorithms/k-core.adoc b/doc/modules/ROOT/pages/algorithms/k-core.adoc index 2b30a5d59f..d8d54d7ffc 100644 --- a/doc/modules/ROOT/pages/algorithms/k-core.adoc +++ b/doc/modules/ROOT/pages/algorithms/k-core.adoc @@ -5,7 +5,9 @@ :result: core value :algorithm: K-Core Decomposition +:no-directed: :undirected: + include::partial$/algorithms/shared/algorithm-traits.adoc[] [[algorithms-k-core-intro]] From 42f5ffc602e822f295eedb8549db1484ffab6d66 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonatan=20J=C3=A4derberg?= Date: Tue, 8 Aug 2023 11:04:59 +0200 Subject: [PATCH 200/273] Publish compat layer for DB V5.11 --- .../5.11/neo4j-kernel-adapter/build.gradle | 65 ++ .../compat/_511/Neo4jProxyFactoryImpl.java | 44 + .../compat/_511/SettingProxyFactoryImpl.java | 44 + .../_511/BoltTransactionRunnerImpl.java | 97 ++ .../compat/_511/CallableProcedureImpl.java | 53 + .../CallableUserAggregationFunctionImpl.java | 77 ++ .../gds/compat/_511/CompatAccessModeImpl.java | 44 + .../_511/CompatGraphDatabaseAPIImpl.java | 74 ++ .../gds/compat/_511/CompatIndexQueryImpl.java | 31 + .../_511/CompatUsernameAuthSubjectImpl.java | 35 + .../compat/_511/CompositeNodeCursorImpl.java | 32 + .../compat/_511/GdsDatabaseLayoutImpl.java | 50 + ...sDatabaseManagementServiceBuilderImpl.java | 54 + .../compat/_511/Neo4jProxyFactoryImpl.java | 44 + .../neo4j/gds/compat/_511/Neo4jProxyImpl.java | 951 ++++++++++++++++++ .../compat/_511/NodeLabelIndexLookupImpl.java | 68 ++ .../gds/compat/_511/PartitionedStoreScan.java | 58 ++ .../_511/ReferencePropertyReference.java | 49 + .../compat/_511/ScanBasedStoreScanImpl.java | 40 + .../compat/_511/SettingProxyFactoryImpl.java | 44 + .../gds/compat/_511/SettingProxyImpl.java | 87 ++ .../neo4j/gds/compat/_511/TestLogImpl.java | 146 +++ .../compat/_511/VirtualRelationshipImpl.java | 41 + .../5.11/storage-engine-adapter/build.gradle | 68 ++ .../_511/InMemoryStorageEngineFactory.java | 268 +++++ .../_511/StorageEngineProxyFactoryImpl.java | 44 + .../InMemoryCommandCreationContextImpl.java | 107 ++ .../compat/_511/InMemoryCountsStoreImpl.java | 111 ++ .../_511/InMemoryMetaDataProviderImpl.java | 201 ++++ .../gds/compat/_511/InMemoryNodeCursor.java | 82 ++ .../_511/InMemoryNodePropertyCursor.java | 45 + .../compat/_511/InMemoryPropertyCursor.java | 71 ++ .../_511/InMemoryPropertySelectionImpl.java | 55 + .../InMemoryRelationshipPropertyCursor.java | 60 ++ .../_511/InMemoryRelationshipScanCursor.java | 61 ++ .../InMemoryRelationshipTraversalCursor.java | 47 + .../_511/InMemoryStorageEngineFactory.java | 571 +++++++++++ .../_511/InMemoryStorageEngineImpl.java | 366 +++++++ .../compat/_511/InMemoryStorageLocksImpl.java | 86 ++ .../gds/compat/_511/InMemoryStoreVersion.java | 70 ++ .../_511/InMemoryTransactionIdStoreImpl.java | 117 +++ .../gds/compat/_511/InMemoryVersionCheck.java | 61 ++ .../_511/StorageEngineProxyFactoryImpl.java | 44 + .../compat/_511/StorageEngineProxyImpl.java | 140 +++ .../InMemoryLogVersionRepository511.java | 71 ++ ...nMemoryStorageCommandReaderFactory511.java | 43 + .../InMemoryStorageReader511.java | 333 ++++++ gradle/dependencies.gradle | 1 + .../org/neo4j/gds/compat/Neo4jVersion.java | 7 +- .../neo4j/gds/compat/Neo4jVersionTest.java | 3 +- .../java/org/neo4j/gds/SysInfoProcTest.java | 13 + 51 files changed, 5372 insertions(+), 2 deletions(-) create mode 100644 compatibility/5.11/neo4j-kernel-adapter/build.gradle create mode 100644 compatibility/5.11/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_511/Neo4jProxyFactoryImpl.java create mode 100644 compatibility/5.11/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_511/SettingProxyFactoryImpl.java create mode 100644 compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/BoltTransactionRunnerImpl.java create mode 100644 compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/CallableProcedureImpl.java create mode 100644 compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/CallableUserAggregationFunctionImpl.java create mode 100644 compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/CompatAccessModeImpl.java create mode 100644 compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/CompatGraphDatabaseAPIImpl.java create mode 100644 compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/CompatIndexQueryImpl.java create mode 100644 compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/CompatUsernameAuthSubjectImpl.java create mode 100644 compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/CompositeNodeCursorImpl.java create mode 100644 compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/GdsDatabaseLayoutImpl.java create mode 100644 compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/GdsDatabaseManagementServiceBuilderImpl.java create mode 100644 compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/Neo4jProxyFactoryImpl.java create mode 100644 compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/Neo4jProxyImpl.java create mode 100644 compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/NodeLabelIndexLookupImpl.java create mode 100644 compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/PartitionedStoreScan.java create mode 100644 compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/ReferencePropertyReference.java create mode 100644 compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/ScanBasedStoreScanImpl.java create mode 100644 compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/SettingProxyFactoryImpl.java create mode 100644 compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/SettingProxyImpl.java create mode 100644 compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/TestLogImpl.java create mode 100644 compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/VirtualRelationshipImpl.java create mode 100644 compatibility/5.11/storage-engine-adapter/build.gradle create mode 100644 compatibility/5.11/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_511/InMemoryStorageEngineFactory.java create mode 100644 compatibility/5.11/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_511/StorageEngineProxyFactoryImpl.java create mode 100644 compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/InMemoryCommandCreationContextImpl.java create mode 100644 compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/InMemoryCountsStoreImpl.java create mode 100644 compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/InMemoryMetaDataProviderImpl.java create mode 100644 compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/InMemoryNodeCursor.java create mode 100644 compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/InMemoryNodePropertyCursor.java create mode 100644 compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/InMemoryPropertyCursor.java create mode 100644 compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/InMemoryPropertySelectionImpl.java create mode 100644 compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/InMemoryRelationshipPropertyCursor.java create mode 100644 compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/InMemoryRelationshipScanCursor.java create mode 100644 compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/InMemoryRelationshipTraversalCursor.java create mode 100644 compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/InMemoryStorageEngineFactory.java create mode 100644 compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/InMemoryStorageEngineImpl.java create mode 100644 compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/InMemoryStorageLocksImpl.java create mode 100644 compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/InMemoryStoreVersion.java create mode 100644 compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/InMemoryTransactionIdStoreImpl.java create mode 100644 compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/InMemoryVersionCheck.java create mode 100644 compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/StorageEngineProxyFactoryImpl.java create mode 100644 compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/StorageEngineProxyImpl.java create mode 100644 compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository511.java create mode 100644 compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory511.java create mode 100644 compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageReader511.java diff --git a/compatibility/5.11/neo4j-kernel-adapter/build.gradle b/compatibility/5.11/neo4j-kernel-adapter/build.gradle new file mode 100644 index 0000000000..927eee8f07 --- /dev/null +++ b/compatibility/5.11/neo4j-kernel-adapter/build.gradle @@ -0,0 +1,65 @@ +apply plugin: 'java-library' +apply plugin: 'me.champeau.mrjar' + +description = 'Neo4j Graph Data Science :: Neo4j Kernel Adapter 5.11' + +group = 'org.neo4j.gds' + +// for all 5.x versions +if (ver.'neo4j'.startsWith('5.')) { + sourceSets { + main { + java { + srcDirs = ['src/main/java17'] + } + } + } + + dependencies { + annotationProcessor project(':annotations') + annotationProcessor group: 'org.immutables', name: 'value', version: ver.'immutables' + annotationProcessor group: 'org.neo4j', name: 'annotations', version: neos.'5.11' + + compileOnly project(':annotations') + compileOnly group: 'com.github.spotbugs', name: 'spotbugs-annotations', version: ver.'spotbugsToolVersion' + compileOnly group: 'org.immutables', name: 'value-annotations', version: ver.'immutables' + compileOnly group: 'org.neo4j', name: 'annotations', version: neos.'5.11' + compileOnly group: 'org.neo4j', name: 'neo4j', version: neos.'5.11' + compileOnly group: 'org.neo4j', name: 'neo4j-record-storage-engine', version: neos.'5.11' + compileOnly group: 'org.neo4j.community', name: 'it-test-support', version: neos.'5.11' + + implementation project(':neo4j-kernel-adapter-api') + } +} else { + multiRelease { + targetVersions 11, 17 + } + + if (!project.hasProperty('no-forbidden-apis')) { + forbiddenApisJava17 { + exclude('**') + } + } + + dependencies { + annotationProcessor group: 'org.neo4j', name: 'annotations', version: ver.'neo4j' + + compileOnly project(':annotations') + compileOnly group: 'org.neo4j', name: 'annotations', version: ver.'neo4j' + + implementation project(':neo4j-kernel-adapter-api') + + java17AnnotationProcessor project(':annotations') + java17AnnotationProcessor group: 'org.immutables', name: 'value', version: ver.'immutables' + java17AnnotationProcessor group: 'org.neo4j', name: 'annotations', version: neos.'5.11' + + java17CompileOnly project(':annotations') + java17CompileOnly group: 'org.immutables', name: 'value-annotations', version: ver.'immutables' + java17CompileOnly group: 'org.neo4j', name: 'neo4j', version: neos.'5.11' + java17CompileOnly group: 'org.neo4j', name: 'neo4j-record-storage-engine', version: neos.'5.11' + java17CompileOnly group: 'org.neo4j.community', name: 'it-test-support', version: neos.'5.11' + java17CompileOnly group: 'com.github.spotbugs', name: 'spotbugs-annotations', version: ver.'spotbugsToolVersion' + + java17Implementation project(':neo4j-kernel-adapter-api') + } +} diff --git a/compatibility/5.11/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_511/Neo4jProxyFactoryImpl.java b/compatibility/5.11/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_511/Neo4jProxyFactoryImpl.java new file mode 100644 index 0000000000..32ae5e87d8 --- /dev/null +++ b/compatibility/5.11/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_511/Neo4jProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._511; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jProxyApi; +import org.neo4j.gds.compat.Neo4jProxyFactory; +import org.neo4j.gds.compat.Neo4jVersion; + +@ServiceProvider +public final class Neo4jProxyFactoryImpl implements Neo4jProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return false; + } + + @Override + public Neo4jProxyApi load() { + throw new UnsupportedOperationException("5.11 compatibility requires JDK17"); + } + + @Override + public String description() { + return "Neo4j 5.11 (placeholder)"; + } +} diff --git a/compatibility/5.11/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_511/SettingProxyFactoryImpl.java b/compatibility/5.11/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_511/SettingProxyFactoryImpl.java new file mode 100644 index 0000000000..2e0c1a9e2b --- /dev/null +++ b/compatibility/5.11/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_511/SettingProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._511; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jVersion; +import org.neo4j.gds.compat.SettingProxyApi; +import org.neo4j.gds.compat.SettingProxyFactory; + +@ServiceProvider +public final class SettingProxyFactoryImpl implements SettingProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return false; + } + + @Override + public SettingProxyApi load() { + throw new UnsupportedOperationException("5.11 compatibility requires JDK17"); + } + + @Override + public String description() { + return "Neo4j Settings 5.11 (placeholder)"; + } +} diff --git a/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/BoltTransactionRunnerImpl.java b/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/BoltTransactionRunnerImpl.java new file mode 100644 index 0000000000..f76c638288 --- /dev/null +++ b/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/BoltTransactionRunnerImpl.java @@ -0,0 +1,97 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._511; + +import org.neo4j.bolt.dbapi.BoltGraphDatabaseServiceSPI; +import org.neo4j.bolt.dbapi.BoltTransaction; +import org.neo4j.bolt.protocol.common.message.AccessMode; +import org.neo4j.bolt.protocol.common.message.request.connection.RoutingContext; +import org.neo4j.bolt.tx.statement.StatementQuerySubscriber; +import org.neo4j.exceptions.KernelException; +import org.neo4j.gds.compat.BoltQuerySubscriber; +import org.neo4j.gds.compat.BoltTransactionRunner; +import org.neo4j.graphdb.QueryStatistics; +import org.neo4j.internal.kernel.api.connectioninfo.ClientConnectionInfo; +import org.neo4j.internal.kernel.api.security.LoginContext; +import org.neo4j.kernel.api.KernelTransaction; +import org.neo4j.kernel.impl.query.QueryExecutionConfiguration; +import org.neo4j.kernel.impl.query.QueryExecutionKernelException; +import org.neo4j.values.virtual.MapValue; + +import java.time.Duration; +import java.util.List; +import java.util.Map; + +public class BoltTransactionRunnerImpl extends BoltTransactionRunner { + + @Override + protected BoltQuerySubscriber boltQuerySubscriber() { + var subscriber = new StatementQuerySubscriber(); + return new BoltQuerySubscriber<>() { + @Override + public void assertSucceeded() throws KernelException { + subscriber.assertSuccess(); + } + + @Override + public QueryStatistics queryStatistics() { + return subscriber.getStatistics(); + } + + @Override + public StatementQuerySubscriber innerSubscriber() { + return subscriber; + } + }; + } + + @Override + protected void executeQuery( + BoltTransaction boltTransaction, + String query, + MapValue parameters, + StatementQuerySubscriber querySubscriber + ) throws QueryExecutionKernelException { + boltTransaction.executeQuery(query, parameters, true, querySubscriber); + } + + @Override + protected BoltTransaction beginBoltWriteTransaction( + BoltGraphDatabaseServiceSPI fabricDb, + LoginContext loginContext, + KernelTransaction.Type kernelTransactionType, + ClientConnectionInfo clientConnectionInfo, + List bookmarks, + Duration txTimeout, + Map txMetadata + ) { + return fabricDb.beginTransaction( + kernelTransactionType, + loginContext, + clientConnectionInfo, + bookmarks, + txTimeout, + AccessMode.WRITE, + txMetadata, + new RoutingContext(true, Map.of()), + QueryExecutionConfiguration.DEFAULT_CONFIG + ); + } +} diff --git a/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/CallableProcedureImpl.java b/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/CallableProcedureImpl.java new file mode 100644 index 0000000000..ac60ee006b --- /dev/null +++ b/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/CallableProcedureImpl.java @@ -0,0 +1,53 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._511; + +import org.neo4j.collection.RawIterator; +import org.neo4j.gds.annotation.SuppressForbidden; +import org.neo4j.gds.compat.CompatCallableProcedure; +import org.neo4j.internal.kernel.api.exceptions.ProcedureException; +import org.neo4j.internal.kernel.api.procs.ProcedureSignature; +import org.neo4j.kernel.api.ResourceMonitor; +import org.neo4j.kernel.api.procedure.CallableProcedure; +import org.neo4j.kernel.api.procedure.Context; +import org.neo4j.values.AnyValue; + +@SuppressForbidden(reason = "This is the compat API") +public final class CallableProcedureImpl implements CallableProcedure { + private final CompatCallableProcedure procedure; + + CallableProcedureImpl(CompatCallableProcedure procedure) { + this.procedure = procedure; + } + + @Override + public ProcedureSignature signature() { + return this.procedure.signature(); + } + + @Override + public RawIterator apply( + Context ctx, + AnyValue[] input, + ResourceMonitor resourceMonitor + ) throws ProcedureException { + return this.procedure.apply(ctx, input); + } +} diff --git a/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/CallableUserAggregationFunctionImpl.java b/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/CallableUserAggregationFunctionImpl.java new file mode 100644 index 0000000000..b7a79d76f3 --- /dev/null +++ b/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/CallableUserAggregationFunctionImpl.java @@ -0,0 +1,77 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._511; + +import org.neo4j.gds.annotation.SuppressForbidden; +import org.neo4j.gds.compat.CompatUserAggregationFunction; +import org.neo4j.gds.compat.CompatUserAggregator; +import org.neo4j.internal.kernel.api.exceptions.ProcedureException; +import org.neo4j.internal.kernel.api.procs.UserAggregationReducer; +import org.neo4j.internal.kernel.api.procs.UserAggregationUpdater; +import org.neo4j.internal.kernel.api.procs.UserFunctionSignature; +import org.neo4j.kernel.api.procedure.CallableUserAggregationFunction; +import org.neo4j.kernel.api.procedure.Context; +import org.neo4j.values.AnyValue; + +@SuppressForbidden(reason = "This is the compat API") +public final class CallableUserAggregationFunctionImpl implements CallableUserAggregationFunction { + private final CompatUserAggregationFunction function; + + CallableUserAggregationFunctionImpl(CompatUserAggregationFunction function) { + this.function = function; + } + + @Override + public UserFunctionSignature signature() { + return this.function.signature(); + } + + @Override + public UserAggregationReducer createReducer(Context ctx) throws ProcedureException { + return new UserAggregatorImpl(this.function.create(ctx)); + } + + private static final class UserAggregatorImpl implements UserAggregationReducer, UserAggregationUpdater { + private final CompatUserAggregator aggregator; + + private UserAggregatorImpl(CompatUserAggregator aggregator) { + this.aggregator = aggregator; + } + + @Override + public UserAggregationUpdater newUpdater() { + return this; + } + + @Override + public void update(AnyValue[] input) throws ProcedureException { + this.aggregator.update(input); + } + + @Override + public void applyUpdates() { + } + + @Override + public AnyValue result() throws ProcedureException { + return this.aggregator.result(); + } + } +} diff --git a/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/CompatAccessModeImpl.java b/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/CompatAccessModeImpl.java new file mode 100644 index 0000000000..4af537fe9c --- /dev/null +++ b/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/CompatAccessModeImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._511; + +import org.neo4j.gds.compat.CompatAccessMode; +import org.neo4j.gds.compat.CustomAccessMode; +import org.neo4j.internal.kernel.api.RelTypeSupplier; +import org.neo4j.internal.kernel.api.TokenSet; + +import java.util.function.Supplier; + +public final class CompatAccessModeImpl extends CompatAccessMode { + + CompatAccessModeImpl(CustomAccessMode custom) { + super(custom); + } + + @Override + public boolean allowsReadNodeProperty(Supplier labels, int propertyKey) { + return custom.allowsReadNodeProperty(propertyKey); + } + + @Override + public boolean allowsReadRelationshipProperty(RelTypeSupplier relType, int propertyKey) { + return custom.allowsReadRelationshipProperty(propertyKey); + } +} diff --git a/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/CompatGraphDatabaseAPIImpl.java b/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/CompatGraphDatabaseAPIImpl.java new file mode 100644 index 0000000000..a4c2fc7a6d --- /dev/null +++ b/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/CompatGraphDatabaseAPIImpl.java @@ -0,0 +1,74 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._511; + +import org.neo4j.dbms.api.DatabaseManagementService; +import org.neo4j.dbms.systemgraph.TopologyGraphDbmsModel; +import org.neo4j.gds.compat.GdsGraphDatabaseAPI; +import org.neo4j.internal.kernel.api.connectioninfo.ClientConnectionInfo; +import org.neo4j.internal.kernel.api.security.LoginContext; +import org.neo4j.kernel.api.KernelTransaction; +import org.neo4j.kernel.api.exceptions.Status; +import org.neo4j.kernel.impl.coreapi.InternalTransaction; +import org.neo4j.kernel.impl.coreapi.TransactionExceptionMapper; + +import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; + +final class CompatGraphDatabaseAPIImpl extends GdsGraphDatabaseAPI { + + CompatGraphDatabaseAPIImpl(DatabaseManagementService dbms) { + super(dbms); + } + + @Override + public boolean isAvailable() { + return api.isAvailable(); + } + + @Override + public TopologyGraphDbmsModel.HostedOnMode mode() { + // NOTE: This means we can never start clusters locally, which is probably fine since: + // 1) We never did this before + // 2) We only use this for tests and benchmarks + return TopologyGraphDbmsModel.HostedOnMode.SINGLE; + } + + @Override + public InternalTransaction beginTransaction( + KernelTransaction.Type type, + LoginContext loginContext, + ClientConnectionInfo clientInfo, + long timeout, + TimeUnit unit, + Consumer terminationCallback, + TransactionExceptionMapper transactionExceptionMapper + ) { + return api.beginTransaction( + type, + loginContext, + clientInfo, + timeout, + unit, + terminationCallback, + transactionExceptionMapper + ); + } +} diff --git a/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/CompatIndexQueryImpl.java b/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/CompatIndexQueryImpl.java new file mode 100644 index 0000000000..11d2514b95 --- /dev/null +++ b/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/CompatIndexQueryImpl.java @@ -0,0 +1,31 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._511; + +import org.neo4j.gds.compat.CompatIndexQuery; +import org.neo4j.internal.kernel.api.PropertyIndexQuery; + +final class CompatIndexQueryImpl implements CompatIndexQuery { + final PropertyIndexQuery indexQuery; + + CompatIndexQueryImpl(PropertyIndexQuery indexQuery) { + this.indexQuery = indexQuery; + } +} diff --git a/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/CompatUsernameAuthSubjectImpl.java b/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/CompatUsernameAuthSubjectImpl.java new file mode 100644 index 0000000000..79a8b581ef --- /dev/null +++ b/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/CompatUsernameAuthSubjectImpl.java @@ -0,0 +1,35 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._511; + +import org.neo4j.gds.compat.CompatUsernameAuthSubject; +import org.neo4j.internal.kernel.api.security.AuthSubject; + +final class CompatUsernameAuthSubjectImpl extends CompatUsernameAuthSubject { + + CompatUsernameAuthSubjectImpl(String username, AuthSubject authSubject) { + super(username, authSubject); + } + + @Override + public String executingUser() { + return username; + } +} diff --git a/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/CompositeNodeCursorImpl.java b/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/CompositeNodeCursorImpl.java new file mode 100644 index 0000000000..98978df157 --- /dev/null +++ b/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/CompositeNodeCursorImpl.java @@ -0,0 +1,32 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._511; + +import org.neo4j.gds.compat.CompositeNodeCursor; +import org.neo4j.internal.kernel.api.NodeLabelIndexCursor; + +import java.util.List; + +public final class CompositeNodeCursorImpl extends CompositeNodeCursor { + + CompositeNodeCursorImpl(List cursors, int[] labelIds) { + super(cursors, labelIds); + } +} diff --git a/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/GdsDatabaseLayoutImpl.java b/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/GdsDatabaseLayoutImpl.java new file mode 100644 index 0000000000..1dda86e46e --- /dev/null +++ b/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/GdsDatabaseLayoutImpl.java @@ -0,0 +1,50 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._511; + +import org.neo4j.gds.compat.GdsDatabaseLayout; +import org.neo4j.io.layout.DatabaseLayout; + +import java.nio.file.Path; + +public class GdsDatabaseLayoutImpl implements GdsDatabaseLayout { + private final DatabaseLayout databaseLayout; + + public GdsDatabaseLayoutImpl(DatabaseLayout databaseLayout) {this.databaseLayout = databaseLayout;} + + @Override + public Path databaseDirectory() { + return databaseLayout.databaseDirectory(); + } + + @Override + public Path getTransactionLogsDirectory() { + return databaseLayout.getTransactionLogsDirectory(); + } + + @Override + public Path metadataStore() { + return databaseLayout.metadataStore(); + } + + public DatabaseLayout databaseLayout() { + return databaseLayout; + } +} diff --git a/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/GdsDatabaseManagementServiceBuilderImpl.java b/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/GdsDatabaseManagementServiceBuilderImpl.java new file mode 100644 index 0000000000..665ae3d6cd --- /dev/null +++ b/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/GdsDatabaseManagementServiceBuilderImpl.java @@ -0,0 +1,54 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._511; + +import org.neo4j.dbms.api.DatabaseManagementService; +import org.neo4j.dbms.api.DatabaseManagementServiceBuilderImplementation; +import org.neo4j.gds.compat.GdsDatabaseManagementServiceBuilder; +import org.neo4j.graphdb.config.Setting; + +import java.nio.file.Path; +import java.util.Map; + +public class GdsDatabaseManagementServiceBuilderImpl implements GdsDatabaseManagementServiceBuilder { + + private final DatabaseManagementServiceBuilderImplementation dbmsBuilder; + + GdsDatabaseManagementServiceBuilderImpl(Path storeDir) { + this.dbmsBuilder = new DatabaseManagementServiceBuilderImplementation(storeDir); + } + + @Override + public GdsDatabaseManagementServiceBuilder setConfigRaw(Map configMap) { + dbmsBuilder.setConfigRaw(configMap); + return this; + } + + @Override + public GdsDatabaseManagementServiceBuilder setConfig(Setting setting, S value) { + dbmsBuilder.setConfig(setting, value); + return this; + } + + @Override + public DatabaseManagementService build() { + return dbmsBuilder.build(); + } +} diff --git a/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/Neo4jProxyFactoryImpl.java b/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/Neo4jProxyFactoryImpl.java new file mode 100644 index 0000000000..7719f97a64 --- /dev/null +++ b/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/Neo4jProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._511; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jProxyApi; +import org.neo4j.gds.compat.Neo4jProxyFactory; +import org.neo4j.gds.compat.Neo4jVersion; + +@ServiceProvider +public final class Neo4jProxyFactoryImpl implements Neo4jProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return version == Neo4jVersion.V_5_11; + } + + @Override + public Neo4jProxyApi load() { + return new Neo4jProxyImpl(); + } + + @Override + public String description() { + return "Neo4j 5.11"; + } +} diff --git a/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/Neo4jProxyImpl.java b/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/Neo4jProxyImpl.java new file mode 100644 index 0000000000..28eb91d620 --- /dev/null +++ b/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/Neo4jProxyImpl.java @@ -0,0 +1,951 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._511; + +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import org.neo4j.common.DependencyResolver; +import org.neo4j.common.EntityType; +import org.neo4j.configuration.BootloaderSettings; +import org.neo4j.configuration.Config; +import org.neo4j.configuration.GraphDatabaseSettings; +import org.neo4j.configuration.SettingValueParsers; +import org.neo4j.configuration.connectors.ConnectorPortRegister; +import org.neo4j.configuration.connectors.ConnectorType; +import org.neo4j.configuration.helpers.DatabaseNameValidator; +import org.neo4j.dbms.api.DatabaseManagementService; +import org.neo4j.exceptions.KernelException; +import org.neo4j.fabric.FabricDatabaseManager; +import org.neo4j.gds.annotation.SuppressForbidden; +import org.neo4j.gds.compat.BoltTransactionRunner; +import org.neo4j.gds.compat.CompatCallableProcedure; +import org.neo4j.gds.compat.CompatExecutionMonitor; +import org.neo4j.gds.compat.CompatIndexQuery; +import org.neo4j.gds.compat.CompatInput; +import org.neo4j.gds.compat.CompatUserAggregationFunction; +import org.neo4j.gds.compat.CompositeNodeCursor; +import org.neo4j.gds.compat.CustomAccessMode; +import org.neo4j.gds.compat.GdsDatabaseLayout; +import org.neo4j.gds.compat.GdsDatabaseManagementServiceBuilder; +import org.neo4j.gds.compat.GdsGraphDatabaseAPI; +import org.neo4j.gds.compat.GlobalProcedureRegistry; +import org.neo4j.gds.compat.GraphDatabaseApiProxy; +import org.neo4j.gds.compat.InputEntityIdVisitor; +import org.neo4j.gds.compat.Neo4jProxyApi; +import org.neo4j.gds.compat.PropertyReference; +import org.neo4j.gds.compat.StoreScan; +import org.neo4j.gds.compat.TestLog; +import org.neo4j.graphdb.GraphDatabaseService; +import org.neo4j.graphdb.Node; +import org.neo4j.graphdb.Relationship; +import org.neo4j.graphdb.RelationshipType; +import org.neo4j.graphdb.config.Setting; +import org.neo4j.internal.batchimport.AdditionalInitialIds; +import org.neo4j.internal.batchimport.BatchImporter; +import org.neo4j.internal.batchimport.BatchImporterFactory; +import org.neo4j.internal.batchimport.Configuration; +import org.neo4j.internal.batchimport.IndexConfig; +import org.neo4j.internal.batchimport.InputIterable; +import org.neo4j.internal.batchimport.Monitor; +import org.neo4j.internal.batchimport.input.Collector; +import org.neo4j.internal.batchimport.input.IdType; +import org.neo4j.internal.batchimport.input.Input; +import org.neo4j.internal.batchimport.input.InputEntityVisitor; +import org.neo4j.internal.batchimport.input.PropertySizeCalculator; +import org.neo4j.internal.batchimport.input.ReadableGroups; +import org.neo4j.internal.batchimport.staging.ExecutionMonitor; +import org.neo4j.internal.batchimport.staging.StageExecution; +import org.neo4j.internal.helpers.HostnamePort; +import org.neo4j.internal.id.IdGenerator; +import org.neo4j.internal.id.IdGeneratorFactory; +import org.neo4j.internal.kernel.api.Cursor; +import org.neo4j.internal.kernel.api.IndexQueryConstraints; +import org.neo4j.internal.kernel.api.IndexReadSession; +import org.neo4j.internal.kernel.api.NodeCursor; +import org.neo4j.internal.kernel.api.NodeLabelIndexCursor; +import org.neo4j.internal.kernel.api.NodeValueIndexCursor; +import org.neo4j.internal.kernel.api.PropertyCursor; +import org.neo4j.internal.kernel.api.PropertyIndexQuery; +import org.neo4j.internal.kernel.api.QueryContext; +import org.neo4j.internal.kernel.api.Read; +import org.neo4j.internal.kernel.api.RelationshipScanCursor; +import org.neo4j.internal.kernel.api.Scan; +import org.neo4j.internal.kernel.api.TokenPredicate; +import org.neo4j.internal.kernel.api.connectioninfo.ClientConnectionInfo; +import org.neo4j.internal.kernel.api.exceptions.ProcedureException; +import org.neo4j.internal.kernel.api.procs.FieldSignature; +import org.neo4j.internal.kernel.api.procs.Neo4jTypes; +import org.neo4j.internal.kernel.api.procs.ProcedureSignature; +import org.neo4j.internal.kernel.api.procs.QualifiedName; +import org.neo4j.internal.kernel.api.procs.UserFunctionSignature; +import org.neo4j.internal.kernel.api.security.AccessMode; +import org.neo4j.internal.kernel.api.security.AuthSubject; +import org.neo4j.internal.kernel.api.security.SecurityContext; +import org.neo4j.internal.recordstorage.RecordIdType; +import org.neo4j.internal.schema.IndexCapability; +import org.neo4j.internal.schema.IndexDescriptor; +import org.neo4j.internal.schema.IndexOrder; +import org.neo4j.internal.schema.SchemaDescriptors; +import org.neo4j.io.fs.FileSystemAbstraction; +import org.neo4j.io.layout.DatabaseLayout; +import org.neo4j.io.layout.Neo4jLayout; +import org.neo4j.io.layout.recordstorage.RecordDatabaseLayout; +import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.context.CursorContextFactory; +import org.neo4j.io.pagecache.context.EmptyVersionContextSupplier; +import org.neo4j.io.pagecache.tracing.PageCacheTracer; +import org.neo4j.kernel.api.KernelTransaction; +import org.neo4j.kernel.api.KernelTransactionHandle; +import org.neo4j.kernel.api.procedure.CallableProcedure; +import org.neo4j.kernel.api.procedure.CallableUserAggregationFunction; +import org.neo4j.kernel.api.procedure.Context; +import org.neo4j.kernel.api.procedure.GlobalProcedures; +import org.neo4j.kernel.database.NormalizedDatabaseName; +import org.neo4j.kernel.impl.coreapi.InternalTransaction; +import org.neo4j.kernel.impl.index.schema.IndexImporterFactoryImpl; +import org.neo4j.kernel.impl.query.QueryExecutionConfiguration; +import org.neo4j.kernel.impl.query.TransactionalContext; +import org.neo4j.kernel.impl.query.TransactionalContextFactory; +import org.neo4j.kernel.impl.store.RecordStore; +import org.neo4j.kernel.impl.store.format.RecordFormatSelector; +import org.neo4j.kernel.impl.store.format.RecordFormats; +import org.neo4j.kernel.impl.store.record.AbstractBaseRecord; +import org.neo4j.kernel.impl.transaction.log.EmptyLogTailMetadata; +import org.neo4j.kernel.impl.transaction.log.files.TransactionLogInitializer; +import org.neo4j.logging.Log; +import org.neo4j.logging.internal.LogService; +import org.neo4j.memory.EmptyMemoryTracker; +import org.neo4j.procedure.Mode; +import org.neo4j.scheduler.JobScheduler; +import org.neo4j.ssl.config.SslPolicyLoader; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.StorageEngineFactory; +import org.neo4j.util.Bits; +import org.neo4j.values.storable.TextArray; +import org.neo4j.values.storable.ValueCategory; +import org.neo4j.values.storable.Values; +import org.neo4j.values.virtual.MapValue; +import org.neo4j.values.virtual.NodeValue; +import org.neo4j.values.virtual.VirtualValues; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Locale; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static java.lang.String.format; +import static org.neo4j.gds.compat.InternalReadOps.countByIdGenerator; +import static org.neo4j.io.pagecache.context.EmptyVersionContextSupplier.EMPTY; + +public final class Neo4jProxyImpl implements Neo4jProxyApi { + + @Override + public GdsGraphDatabaseAPI newDb(DatabaseManagementService dbms) { + return new CompatGraphDatabaseAPIImpl(dbms); + } + + @Override + public String validateExternalDatabaseName(String databaseName) { + var normalizedName = new NormalizedDatabaseName(databaseName); + DatabaseNameValidator.validateExternalDatabaseName(normalizedName); + return normalizedName.name(); + } + + @Override + public AccessMode accessMode(CustomAccessMode customAccessMode) { + return new CompatAccessModeImpl(customAccessMode); + } + + @Override + public String username(AuthSubject subject) { + return subject.executingUser(); + } + + @Override + public SecurityContext securityContext( + String username, + AuthSubject authSubject, + AccessMode mode, + String databaseName + ) { + return new SecurityContext( + new CompatUsernameAuthSubjectImpl(username, authSubject), + mode, + // GDS is always operating from an embedded context + ClientConnectionInfo.EMBEDDED_CONNECTION, + databaseName + ); + } + + @Override + public long getHighId(RecordStore recordStore) { + return recordStore.getIdGenerator().getHighId(); + } + + @Override + public List> entityCursorScan( + KernelTransaction transaction, + int[] labelIds, + int batchSize, + boolean allowPartitionedScan + ) { + if (allowPartitionedScan) { + return partitionedNodeLabelIndexScan(transaction, batchSize, labelIds); + } else { + var read = transaction.dataRead(); + return Arrays + .stream(labelIds) + .mapToObj(read::nodeLabelScan) + .map(scan -> scanToStoreScan(scan, batchSize)) + .collect(Collectors.toList()); + } + } + + @Override + public PropertyCursor allocatePropertyCursor(KernelTransaction kernelTransaction) { + return kernelTransaction + .cursors() + .allocatePropertyCursor(kernelTransaction.cursorContext(), kernelTransaction.memoryTracker()); + } + + @Override + public PropertyReference propertyReference(NodeCursor nodeCursor) { + return ReferencePropertyReference.of(nodeCursor.propertiesReference()); + } + + @Override + public PropertyReference propertyReference(RelationshipScanCursor relationshipScanCursor) { + return ReferencePropertyReference.of(relationshipScanCursor.propertiesReference()); + } + + @Override + public PropertyReference noPropertyReference() { + return ReferencePropertyReference.empty(); + } + + @Override + public void nodeProperties( + KernelTransaction kernelTransaction, + long nodeReference, + PropertyReference reference, + PropertyCursor cursor + ) { + var neoReference = ((ReferencePropertyReference) reference).reference; + kernelTransaction + .dataRead() + .nodeProperties(nodeReference, neoReference, PropertySelection.ALL_PROPERTIES, cursor); + } + + @Override + public void relationshipProperties( + KernelTransaction kernelTransaction, + long relationshipReference, + PropertyReference reference, + PropertyCursor cursor + ) { + var neoReference = ((ReferencePropertyReference) reference).reference; + kernelTransaction + .dataRead() + .relationshipProperties(relationshipReference, neoReference, PropertySelection.ALL_PROPERTIES, cursor); + } + + @Override + public NodeCursor allocateNodeCursor(KernelTransaction kernelTransaction) { + return kernelTransaction.cursors().allocateNodeCursor(kernelTransaction.cursorContext()); + } + + @Override + public RelationshipScanCursor allocateRelationshipScanCursor(KernelTransaction kernelTransaction) { + return kernelTransaction.cursors().allocateRelationshipScanCursor(kernelTransaction.cursorContext()); + } + + @Override + public NodeLabelIndexCursor allocateNodeLabelIndexCursor(KernelTransaction kernelTransaction) { + return kernelTransaction.cursors().allocateNodeLabelIndexCursor(kernelTransaction.cursorContext()); + } + + @Override + public NodeValueIndexCursor allocateNodeValueIndexCursor(KernelTransaction kernelTransaction) { + return kernelTransaction + .cursors() + .allocateNodeValueIndexCursor(kernelTransaction.cursorContext(), kernelTransaction.memoryTracker()); + } + + @Override + public boolean hasNodeLabelIndex(KernelTransaction kernelTransaction) { + return NodeLabelIndexLookupImpl.hasNodeLabelIndex(kernelTransaction); + } + + @Override + public StoreScan nodeLabelIndexScan( + KernelTransaction transaction, + int labelId, + int batchSize, + boolean allowPartitionedScan + ) { + if (allowPartitionedScan) { + return partitionedNodeLabelIndexScan(transaction, batchSize, labelId).get(0); + } else { + var read = transaction.dataRead(); + return scanToStoreScan(read.nodeLabelScan(labelId), batchSize); + } + } + + @Override + public StoreScan scanToStoreScan(Scan scan, int batchSize) { + return new ScanBasedStoreScanImpl<>(scan, batchSize); + } + + private List> partitionedNodeLabelIndexScan( + KernelTransaction transaction, + int batchSize, + int... labelIds + ) { + var indexDescriptor = NodeLabelIndexLookupImpl.findUsableMatchingIndex( + transaction, + SchemaDescriptors.forAnyEntityTokens(EntityType.NODE) + ); + + if (indexDescriptor == IndexDescriptor.NO_INDEX) { + throw new IllegalStateException("There is no index that can back a node label scan."); + } + + var read = transaction.dataRead(); + + // Our current strategy is to select the token with the highest count + // and use that one as the driving partitioned index scan. The partitions + // of all other partitioned index scans will be aligned to that one. + int maxToken = labelIds[0]; + long maxCount = read.countsForNodeWithoutTxState(labelIds[0]); + + for (int i = 1; i < labelIds.length; i++) { + long count = read.countsForNodeWithoutTxState(labelIds[i]); + if (count > maxCount) { + maxCount = count; + maxToken = labelIds[i]; + } + } + + int numberOfPartitions = PartitionedStoreScan.getNumberOfPartitions(maxCount, batchSize); + + try { + var session = read.tokenReadSession(indexDescriptor); + + var partitionedScan = read.nodeLabelScan( + session, + numberOfPartitions, + transaction.cursorContext(), + new TokenPredicate(maxToken) + ); + + var scans = new ArrayList>(labelIds.length); + scans.add(new PartitionedStoreScan(partitionedScan)); + + // Initialize the remaining index scans with the partitioning of the first scan. + for (int labelToken : labelIds) { + if (labelToken != maxToken) { + var scan = read.nodeLabelScan(session, partitionedScan, new TokenPredicate(labelToken)); + scans.add(new PartitionedStoreScan(scan)); + } + } + + return scans; + } catch (KernelException e) { + // should not happen, we check for the index existence and applicability + // before reading it + throw new RuntimeException("Unexpected error while initialising reading from node label index", e); + } + } + + @Override + public CompatIndexQuery rangeIndexQuery( + int propertyKeyId, + double from, + boolean fromInclusive, + double to, + boolean toInclusive + ) { + return new CompatIndexQueryImpl(PropertyIndexQuery.range(propertyKeyId, from, fromInclusive, to, toInclusive)); + } + + @Override + public CompatIndexQuery rangeAllIndexQuery(int propertyKeyId) { + var rangePredicate = PropertyIndexQuery.range( + propertyKeyId, + Values.doubleValue(Double.NEGATIVE_INFINITY), + true, + Values.doubleValue(Double.POSITIVE_INFINITY), + true + ); + return new CompatIndexQueryImpl(rangePredicate); + } + + @Override + public void nodeIndexSeek( + Read dataRead, + IndexReadSession index, + NodeValueIndexCursor cursor, + IndexOrder indexOrder, + boolean needsValues, + CompatIndexQuery query + ) throws KernelException { + var indexQueryConstraints = indexOrder == IndexOrder.NONE + ? IndexQueryConstraints.unordered(needsValues) + : IndexQueryConstraints.constrained(indexOrder, needsValues); + + dataRead.nodeIndexSeek( + (QueryContext) dataRead, + index, + cursor, + indexQueryConstraints, + ((CompatIndexQueryImpl) query).indexQuery + ); + } + + @Override + public CompositeNodeCursor compositeNodeCursor(List cursors, int[] labelIds) { + return new CompositeNodeCursorImpl(cursors, labelIds); + } + + @Override + public Configuration batchImporterConfig( + int batchSize, + int writeConcurrency, + Optional pageCacheMemory, + boolean highIO, + IndexConfig indexConfig + ) { + return new org.neo4j.internal.batchimport.Configuration() { + @Override + public int batchSize() { + return batchSize; + } + + @Override + public int maxNumberOfWorkerThreads() { + return writeConcurrency; + } + + @Override + public boolean highIO() { + return highIO; + } + + @Override + public IndexConfig indexConfig() { + return indexConfig; + } + }; + } + + @Override + public int writeConcurrency(Configuration batchImportConfiguration) { + return batchImportConfiguration.maxNumberOfWorkerThreads(); + } + + @Override + public BatchImporter instantiateBatchImporter( + BatchImporterFactory factory, + GdsDatabaseLayout directoryStructure, + FileSystemAbstraction fileSystem, + PageCacheTracer pageCacheTracer, + Configuration configuration, + LogService logService, + ExecutionMonitor executionMonitor, + AdditionalInitialIds additionalInitialIds, + Config dbConfig, + RecordFormats recordFormats, + JobScheduler jobScheduler, + Collector badCollector + ) { + dbConfig.set(GraphDatabaseSettings.db_format, recordFormats.name()); + var databaseLayout = ((GdsDatabaseLayoutImpl) directoryStructure).databaseLayout(); + return factory.instantiate( + databaseLayout, + fileSystem, + pageCacheTracer, + configuration, + logService, + executionMonitor, + additionalInitialIds, + new EmptyLogTailMetadata(dbConfig), + dbConfig, + Monitor.NO_MONITOR, + jobScheduler, + badCollector, + TransactionLogInitializer.getLogFilesInitializer(), + new IndexImporterFactoryImpl(), + EmptyMemoryTracker.INSTANCE, + new CursorContextFactory(PageCacheTracer.NULL, EmptyVersionContextSupplier.EMPTY) + ); + } + + @Override + public Input batchInputFrom(CompatInput compatInput) { + return new InputFromCompatInput(compatInput); + } + + @Override + public InputEntityIdVisitor.Long inputEntityLongIdVisitor(IdType idType, ReadableGroups groups) { + switch (idType) { + case ACTUAL -> { + return new InputEntityIdVisitor.Long() { + @Override + public void visitNodeId(InputEntityVisitor visitor, long id) { + visitor.id(id); + } + + @Override + public void visitSourceId(InputEntityVisitor visitor, long id) { + visitor.startId(id); + } + + @Override + public void visitTargetId(InputEntityVisitor visitor, long id) { + visitor.endId(id); + } + }; + } + case INTEGER -> { + var globalGroup = groups.get(null); + + return new InputEntityIdVisitor.Long() { + @Override + public void visitNodeId(InputEntityVisitor visitor, long id) { + visitor.id(id, globalGroup); + } + + @Override + public void visitSourceId(InputEntityVisitor visitor, long id) { + visitor.startId(id, globalGroup); + } + + @Override + public void visitTargetId(InputEntityVisitor visitor, long id) { + visitor.endId(id, globalGroup); + } + }; + } + default -> throw new IllegalStateException("Unexpected value: " + idType); + } + } + + @Override + public InputEntityIdVisitor.String inputEntityStringIdVisitor(ReadableGroups groups) { + var globalGroup = groups.get(null); + + return new InputEntityIdVisitor.String() { + @Override + public void visitNodeId(InputEntityVisitor visitor, String id) { + visitor.id(id, globalGroup); + } + + @Override + public void visitSourceId(InputEntityVisitor visitor, String id) { + visitor.startId(id, globalGroup); + } + + @Override + public void visitTargetId(InputEntityVisitor visitor, String id) { + visitor.endId(id, globalGroup); + } + }; + } + + @Override + public Setting additionalJvm() { + return BootloaderSettings.additional_jvm; + } + + @Override + public Setting pageCacheMemory() { + return GraphDatabaseSettings.pagecache_memory; + } + + @Override + public Long pageCacheMemoryValue(String value) { + return SettingValueParsers.BYTES.parse(value); + } + + @Override + public ProcedureSignature procedureSignature( + QualifiedName name, + List inputSignature, + List outputSignature, + Mode mode, + boolean admin, + String deprecated, + String description, + String warning, + boolean eager, + boolean caseInsensitive, + boolean systemProcedure, + boolean internal, + boolean allowExpiredCredentials, + boolean threadSafe + ) { + return new ProcedureSignature( + name, + inputSignature, + outputSignature, + mode, + admin, + deprecated, + description, + warning, + eager, + caseInsensitive, + systemProcedure, + internal, + allowExpiredCredentials, + threadSafe + ); + } + + @Override + public long getHighestPossibleNodeCount( + Read read, IdGeneratorFactory idGeneratorFactory + ) { + return countByIdGenerator(idGeneratorFactory, RecordIdType.NODE).orElseGet(read::nodesGetCount); + } + + @Override + public long getHighestPossibleRelationshipCount( + Read read, IdGeneratorFactory idGeneratorFactory + ) { + return countByIdGenerator(idGeneratorFactory, RecordIdType.RELATIONSHIP).orElseGet(read::relationshipsGetCount); + } + + @Override + public String versionLongToString(long storeVersion) { + // copied from org.neo4j.kernel.impl.store.LegacyMetadataHandler.versionLongToString which is private + if (storeVersion == -1) { + return "Unknown"; + } + Bits bits = Bits.bitsFromLongs(new long[]{storeVersion}); + int length = bits.getShort(8); + if (length == 0 || length > 7) { + throw new IllegalArgumentException(format(Locale.ENGLISH, "The read version string length %d is not proper.", length)); + } + char[] result = new char[length]; + for (int i = 0; i < length; i++) { + result[i] = (char) bits.getShort(8); + } + return new String(result); + } + + private static final class InputFromCompatInput implements Input { + private final CompatInput delegate; + + private InputFromCompatInput(CompatInput delegate) { + this.delegate = delegate; + } + + @Override + public InputIterable nodes(Collector badCollector) { + return delegate.nodes(badCollector); + } + + @Override + public InputIterable relationships(Collector badCollector) { + return delegate.relationships(badCollector); + } + + @Override + public IdType idType() { + return delegate.idType(); + } + + @Override + public ReadableGroups groups() { + return delegate.groups(); + } + + @Override + public Estimates calculateEstimates(PropertySizeCalculator propertySizeCalculator) throws IOException { + return delegate.calculateEstimates((values, kernelTransaction) -> propertySizeCalculator.calculateSize( + values, + kernelTransaction.cursorContext(), + kernelTransaction.memoryTracker() + )); + } + } + + @Override + public TestLog testLog() { + return new TestLogImpl(); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public Log getUserLog(LogService logService, Class loggingClass) { + return logService.getUserLog(loggingClass); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public Log getInternalLog(LogService logService, Class loggingClass) { + return logService.getInternalLog(loggingClass); + } + + @Override + public NodeValue nodeValue(long id, TextArray labels, MapValue properties) { + return VirtualValues.nodeValue(id, String.valueOf(id), labels, properties); + } + + @Override + public Relationship virtualRelationship(long id, Node startNode, Node endNode, RelationshipType type) { + return new VirtualRelationshipImpl(id, startNode, endNode, type); + } + + @Override + public GdsDatabaseManagementServiceBuilder databaseManagementServiceBuilder(Path storeDir) { + return new GdsDatabaseManagementServiceBuilderImpl(storeDir); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public RecordFormats selectRecordFormatForStore( + DatabaseLayout databaseLayout, + FileSystemAbstraction fs, + PageCache pageCache, + LogService logService, + PageCacheTracer pageCacheTracer + ) { + return RecordFormatSelector.selectForStore( + (RecordDatabaseLayout) databaseLayout, + fs, + pageCache, + logService.getInternalLogProvider(), + new CursorContextFactory(pageCacheTracer, EMPTY) + ); + } + + @Override + public boolean isNotNumericIndex(IndexCapability indexCapability) { + return !indexCapability.areValueCategoriesAccepted(ValueCategory.NUMBER); + } + + @Override + public void setAllowUpgrades(Config.Builder configBuilder, boolean value) { + } + + @Override + public String defaultRecordFormatSetting() { + return GraphDatabaseSettings.db_format.defaultValue(); + } + + @Override + public void configureRecordFormat(Config.Builder configBuilder, String recordFormat) { + var databaseRecordFormat = recordFormat.toLowerCase(Locale.ENGLISH); + configBuilder.set(GraphDatabaseSettings.db_format, databaseRecordFormat); + } + + @Override + public GdsDatabaseLayout databaseLayout(Config config, String databaseName) { + var storageEngineFactory = StorageEngineFactory.selectStorageEngine(config); + var dbLayout = neo4jLayout(config).databaseLayout(databaseName); + var databaseLayout = storageEngineFactory.formatSpecificDatabaseLayout(dbLayout); + return new GdsDatabaseLayoutImpl(databaseLayout); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public Neo4jLayout neo4jLayout(Config config) { + return Neo4jLayout.of(config); + } + + @Override + public BoltTransactionRunner boltTransactionRunner() { + return new BoltTransactionRunnerImpl(); + } + + @Override + public HostnamePort getLocalBoltAddress(ConnectorPortRegister connectorPortRegister) { + return connectorPortRegister.getLocalAddress(ConnectorType.BOLT); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public SslPolicyLoader createSllPolicyLoader( + FileSystemAbstraction fileSystem, + Config config, + LogService logService + ) { + return SslPolicyLoader.create(fileSystem, config, logService.getInternalLogProvider()); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public RecordFormats recordFormatSelector( + String databaseName, + Config databaseConfig, + FileSystemAbstraction fs, + LogService logService, + GraphDatabaseService databaseService + ) { + var neo4jLayout = Neo4jLayout.of(databaseConfig); + var recordDatabaseLayout = RecordDatabaseLayout.of(neo4jLayout, databaseName); + return RecordFormatSelector.selectForStoreOrConfigForNewDbs( + databaseConfig, + recordDatabaseLayout, + fs, + GraphDatabaseApiProxy.resolveDependency(databaseService, PageCache.class), + logService.getInternalLogProvider(), + GraphDatabaseApiProxy.resolveDependency(databaseService, CursorContextFactory.class) + ); + } + + @Override + public ExecutionMonitor executionMonitor(CompatExecutionMonitor compatExecutionMonitor) { + return new ExecutionMonitor.Adapter( + compatExecutionMonitor.checkIntervalMillis(), + TimeUnit.MILLISECONDS + ) { + + @Override + public void initialize(DependencyResolver dependencyResolver) { + compatExecutionMonitor.initialize(dependencyResolver); + } + + @Override + public void start(StageExecution execution) { + compatExecutionMonitor.start(execution); + } + + @Override + public void end(StageExecution execution, long totalTimeMillis) { + compatExecutionMonitor.end(execution, totalTimeMillis); + } + + @Override + public void done(boolean successful, long totalTimeMillis, String additionalInformation) { + compatExecutionMonitor.done(successful, totalTimeMillis, additionalInformation); + } + + @Override + public void check(StageExecution execution) { + compatExecutionMonitor.check(execution); + } + }; + } + + @Override + @SuppressFBWarnings("NP_LOAD_OF_KNOWN_NULL_VALUE") // We assign nulls because it makes the code more readable + public UserFunctionSignature userFunctionSignature( + QualifiedName name, + List inputSignature, + Neo4jTypes.AnyType type, + String description, + boolean internal, + boolean threadSafe, + Optional deprecatedBy + ) { + String category = null; // No predefined categpry (like temporal or math) + var caseInsensitive = false; // case sensitive name match + var isBuiltIn = false; // is built in; never true for GDS + + return new UserFunctionSignature( + name, + inputSignature, + type, + deprecatedBy.orElse(null), + description, + category, + caseInsensitive, + isBuiltIn, + internal, + threadSafe + ); + } + + @Override + @SuppressForbidden(reason = "This is the compat API") + public CallableProcedure callableProcedure(CompatCallableProcedure procedure) { + return new CallableProcedureImpl(procedure); + } + + @Override + @SuppressForbidden(reason = "This is the compat API") + public CallableUserAggregationFunction callableUserAggregationFunction(CompatUserAggregationFunction function) { + return new CallableUserAggregationFunctionImpl(function); + } + + @Override + public long transactionId(KernelTransactionHandle kernelTransactionHandle) { + return kernelTransactionHandle.getTransactionSequenceNumber(); + } + + @Override + public void reserveNeo4jIds(IdGeneratorFactory generatorFactory, int size, CursorContext cursorContext) { + IdGenerator idGenerator = generatorFactory.get(RecordIdType.NODE); + + idGenerator.nextConsecutiveIdRange(size, false, cursorContext); + } + + @Override + public TransactionalContext newQueryContext( + TransactionalContextFactory contextFactory, + InternalTransaction tx, + String queryText, + MapValue queryParameters + ) { + return contextFactory.newContext(tx, queryText, queryParameters, QueryExecutionConfiguration.DEFAULT_CONFIG); + } + + @Override + public boolean isCompositeDatabase(GraphDatabaseService databaseService) { + var databaseManager = GraphDatabaseApiProxy.resolveDependency(databaseService, FabricDatabaseManager.class); + return databaseManager.isFabricDatabase(GraphDatabaseApiProxy.databaseId(databaseService)); + } + + @Override + public T lookupComponentProvider(Context ctx, Class component, boolean safe) throws ProcedureException { + var globalProcedures = GraphDatabaseApiProxy.resolveDependency(ctx.dependencyResolver(), GlobalProcedures.class); + return globalProcedures.getCurrentView().lookupComponentProvider(component, safe).apply(ctx); + } + + @Override + public GlobalProcedureRegistry globalProcedureRegistry(GlobalProcedures globalProcedures) { + return new GlobalProcedureRegistry() { + @Override + public Set getAllProcedures() { + return globalProcedures.getCurrentView().getAllProcedures(); + } + + @Override + public Stream getAllNonAggregatingFunctions() { + return globalProcedures.getCurrentView().getAllNonAggregatingFunctions(); + } + + @Override + public Stream getAllAggregatingFunctions() { + return globalProcedures.getCurrentView().getAllAggregatingFunctions(); + } + }; + } +} diff --git a/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/NodeLabelIndexLookupImpl.java b/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/NodeLabelIndexLookupImpl.java new file mode 100644 index 0000000000..5465d3261a --- /dev/null +++ b/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/NodeLabelIndexLookupImpl.java @@ -0,0 +1,68 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._511; + +import org.neo4j.common.EntityType; +import org.neo4j.internal.kernel.api.InternalIndexState; +import org.neo4j.internal.kernel.api.SchemaRead; +import org.neo4j.internal.kernel.api.exceptions.schema.IndexNotFoundKernelException; +import org.neo4j.internal.schema.IndexDescriptor; +import org.neo4j.internal.schema.IndexType; +import org.neo4j.internal.schema.SchemaDescriptor; +import org.neo4j.internal.schema.SchemaDescriptors; +import org.neo4j.kernel.api.KernelTransaction; + +final class NodeLabelIndexLookupImpl { + + static boolean hasNodeLabelIndex(KernelTransaction transaction) { + return NodeLabelIndexLookupImpl.findUsableMatchingIndex( + transaction, + SchemaDescriptors.forAnyEntityTokens(EntityType.NODE) + ) != IndexDescriptor.NO_INDEX; + } + + static IndexDescriptor findUsableMatchingIndex( + KernelTransaction transaction, + SchemaDescriptor schemaDescriptor + ) { + var schemaRead = transaction.schemaRead(); + var iterator = schemaRead.index(schemaDescriptor); + while (iterator.hasNext()) { + var index = iterator.next(); + if (index.getIndexType() == IndexType.LOOKUP && indexIsOnline(schemaRead, index)) { + return index; + } + } + return IndexDescriptor.NO_INDEX; + } + + private static boolean indexIsOnline(SchemaRead schemaRead, IndexDescriptor index) { + var state = InternalIndexState.FAILED; + try { + state = schemaRead.indexGetState(index); + } catch (IndexNotFoundKernelException e) { + // Well the index should always exist here, but if we didn't find it while checking the state, + // then we obviously don't want to use it. + } + return state == InternalIndexState.ONLINE; + } + + private NodeLabelIndexLookupImpl() {} +} diff --git a/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/PartitionedStoreScan.java b/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/PartitionedStoreScan.java new file mode 100644 index 0000000000..cbd2f63fcf --- /dev/null +++ b/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/PartitionedStoreScan.java @@ -0,0 +1,58 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._511; + +import org.neo4j.gds.compat.StoreScan; +import org.neo4j.internal.kernel.api.NodeLabelIndexCursor; +import org.neo4j.internal.kernel.api.PartitionedScan; +import org.neo4j.kernel.api.KernelTransaction; + +final class PartitionedStoreScan implements StoreScan { + private final PartitionedScan scan; + + PartitionedStoreScan(PartitionedScan scan) { + this.scan = scan; + } + + static int getNumberOfPartitions(long nodeCount, int batchSize) { + int numberOfPartitions; + if (nodeCount > 0) { + // ceil div to try to get enough partitions so a single one does + // not include more nodes than batchSize + long partitions = ((nodeCount - 1) / batchSize) + 1; + + // value must be positive + if (partitions < 1) { + partitions = 1; + } + + numberOfPartitions = (int) Long.min(Integer.MAX_VALUE, partitions); + } else { + // we have no partitions to scan, but the value must still be positive + numberOfPartitions = 1; + } + return numberOfPartitions; + } + + @Override + public boolean reserveBatch(NodeLabelIndexCursor cursor, KernelTransaction ktx) { + return scan.reservePartition(cursor, ktx.cursorContext(), ktx.securityContext().mode()); + } +} diff --git a/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/ReferencePropertyReference.java b/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/ReferencePropertyReference.java new file mode 100644 index 0000000000..7ec85a1d19 --- /dev/null +++ b/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/ReferencePropertyReference.java @@ -0,0 +1,49 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._511; + +import org.neo4j.gds.compat.PropertyReference; +import org.neo4j.storageengine.api.Reference; + +import java.util.Objects; + +public final class ReferencePropertyReference implements PropertyReference { + + private static final PropertyReference EMPTY = new ReferencePropertyReference(null); + + public final Reference reference; + + private ReferencePropertyReference(Reference reference) { + this.reference = reference; + } + + public static PropertyReference of(Reference reference) { + return new ReferencePropertyReference(Objects.requireNonNull(reference)); + } + + public static PropertyReference empty() { + return EMPTY; + } + + @Override + public boolean isEmpty() { + return reference == null; + } +} diff --git a/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/ScanBasedStoreScanImpl.java b/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/ScanBasedStoreScanImpl.java new file mode 100644 index 0000000000..fbee3ca33b --- /dev/null +++ b/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/ScanBasedStoreScanImpl.java @@ -0,0 +1,40 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._511; + +import org.neo4j.gds.compat.StoreScan; +import org.neo4j.internal.kernel.api.Cursor; +import org.neo4j.internal.kernel.api.Scan; +import org.neo4j.kernel.api.KernelTransaction; + +public final class ScanBasedStoreScanImpl implements StoreScan { + private final Scan scan; + private final int batchSize; + + public ScanBasedStoreScanImpl(Scan scan, int batchSize) { + this.scan = scan; + this.batchSize = batchSize; + } + + @Override + public boolean reserveBatch(C cursor, KernelTransaction ktx) { + return scan.reserveBatch(cursor, batchSize, ktx.cursorContext(), ktx.securityContext().mode()); + } +} diff --git a/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/SettingProxyFactoryImpl.java b/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/SettingProxyFactoryImpl.java new file mode 100644 index 0000000000..0782eebff8 --- /dev/null +++ b/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/SettingProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._511; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jVersion; +import org.neo4j.gds.compat.SettingProxyApi; +import org.neo4j.gds.compat.SettingProxyFactory; + +@ServiceProvider +public final class SettingProxyFactoryImpl implements SettingProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return version == Neo4jVersion.V_5_11; + } + + @Override + public SettingProxyApi load() { + return new SettingProxyImpl(); + } + + @Override + public String description() { + return "Neo4j Settings 5.11"; + } +} diff --git a/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/SettingProxyImpl.java b/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/SettingProxyImpl.java new file mode 100644 index 0000000000..965caa2d66 --- /dev/null +++ b/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/SettingProxyImpl.java @@ -0,0 +1,87 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._511; + +import org.neo4j.configuration.Config; +import org.neo4j.configuration.SettingBuilder; +import org.neo4j.dbms.systemgraph.TopologyGraphDbmsModel; +import org.neo4j.gds.compat.DatabaseMode; +import org.neo4j.gds.compat.SettingProxyApi; +import org.neo4j.graphdb.GraphDatabaseService; +import org.neo4j.graphdb.config.Setting; +import org.neo4j.kernel.impl.factory.GraphDatabaseFacade; +import org.neo4j.kernel.internal.GraphDatabaseAPI; + +public class SettingProxyImpl implements SettingProxyApi { + + @Override + public Setting setting(org.neo4j.gds.compat.Setting setting) { + var builder = SettingBuilder.newBuilder(setting.name(), setting.parser(), setting.defaultValue()); + if (setting.dynamic()) { + builder = builder.dynamic(); + } + if (setting.immutable()) { + builder = builder.immutable(); + } + setting.dependency().ifPresent(builder::setDependency); + setting.constraints().forEach(builder::addConstraint); + return builder.build(); + } + + @Override + public DatabaseMode databaseMode(Config config, GraphDatabaseService databaseService) { + return switch (((GraphDatabaseAPI) databaseService).mode()) { + case RAFT -> DatabaseMode.CORE; + case REPLICA -> DatabaseMode.READ_REPLICA; + case SINGLE -> DatabaseMode.SINGLE; + case VIRTUAL -> throw new UnsupportedOperationException("What's a virtual database anyway?"); + }; + } + + @Override + public void setDatabaseMode(Config config, DatabaseMode databaseMode, GraphDatabaseService databaseService) { + // super hacky, there is no way to set the mode of a database without restarting it + if (!(databaseService instanceof GraphDatabaseFacade db)) { + throw new IllegalArgumentException( + "Cannot set database mode on a database that is not a GraphDatabaseFacade"); + } + try { + var modeField = GraphDatabaseFacade.class.getDeclaredField("mode"); + modeField.setAccessible(true); + modeField.set(db, switch (databaseMode) { + case CORE -> TopologyGraphDbmsModel.HostedOnMode.RAFT; + case READ_REPLICA -> TopologyGraphDbmsModel.HostedOnMode.REPLICA; + case SINGLE -> TopologyGraphDbmsModel.HostedOnMode.SINGLE; + }); + } catch (NoSuchFieldException e) { + throw new RuntimeException( + "Could not set the mode field because it no longer exists. This compat layer needs to be updated.", + e + ); + } catch (IllegalAccessException e) { + throw new RuntimeException("Could not get the permissions to set the mode field.", e); + } + } + + @Override + public String secondaryModeName() { + return "Secondary"; + } +} diff --git a/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/TestLogImpl.java b/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/TestLogImpl.java new file mode 100644 index 0000000000..cbacd42e0d --- /dev/null +++ b/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/TestLogImpl.java @@ -0,0 +1,146 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._511; + +import org.neo4j.gds.annotation.SuppressForbidden; +import org.neo4j.gds.compat.TestLog; + +import java.util.ArrayList; +import java.util.Locale; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.ConcurrentMap; + +public class TestLogImpl implements TestLog { + + private final ConcurrentMap> messages; + + TestLogImpl() { + messages = new ConcurrentHashMap<>(3); + } + + @Override + public void assertContainsMessage(String level, String fragment) { + if (!containsMessage(level, fragment)) { + throw new RuntimeException( + String.format( + Locale.US, + "Expected log output to contain `%s` for log level `%s`%nLog messages:%n%s", + fragment, + level, + String.join("\n", messages.get(level)) + ) + ); + } + } + + @Override + public boolean containsMessage(String level, String fragment) { + ConcurrentLinkedQueue messageList = messages.getOrDefault(level, new ConcurrentLinkedQueue<>()); + return messageList.stream().anyMatch((message) -> message.contains(fragment)); + } + + @Override + public boolean hasMessages(String level) { + return !messages.getOrDefault(level, new ConcurrentLinkedQueue<>()).isEmpty(); + } + + @Override + public ArrayList getMessages(String level) { + return new ArrayList<>(messages.getOrDefault(level, new ConcurrentLinkedQueue<>())); + } + + @SuppressForbidden(reason = "test log can print") + public void printMessages() { + System.out.println("TestLog Messages: " + messages); + } + + @Override + public boolean isDebugEnabled() { + return true; + } + + @Override + public void debug(String message) { + logMessage(DEBUG, message); + } + + @Override + public void debug(String message, Throwable throwable) { + debug(String.format(Locale.US, "%s - %s", message, throwable.getMessage())); + } + + @Override + public void debug(String format, Object... arguments) { + debug(String.format(Locale.US, format, arguments)); + } + + @Override + public void info(String message) { + logMessage(INFO, message); + } + + @Override + public void info(String message, Throwable throwable) { + info(String.format(Locale.US, "%s - %s", message, throwable.getMessage())); + } + + @Override + public void info(String format, Object... arguments) { + info(String.format(Locale.US, format, arguments)); + } + + @Override + public void warn(String message) { + logMessage(WARN, message); + } + + @Override + public void warn(String message, Throwable throwable) { + warn(String.format(Locale.US, "%s - %s", message, throwable.getMessage())); + } + + @Override + public void warn(String format, Object... arguments) { + warn(String.format(Locale.US, format, arguments)); + } + + @Override + public void error(String message) { + logMessage(ERROR, message); + } + + @Override + public void error(String message, Throwable throwable) { + error(String.format(Locale.US, "%s - %s", message, throwable.getMessage())); + } + + @Override + public void error(String format, Object... arguments) { + error(String.format(Locale.US, format, arguments)); + } + + private void logMessage(String level, String message) { + messages.computeIfAbsent( + level, + (ignore) -> new ConcurrentLinkedQueue<>() + ).add(message); + } +} diff --git a/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/VirtualRelationshipImpl.java b/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/VirtualRelationshipImpl.java new file mode 100644 index 0000000000..e14ab2d222 --- /dev/null +++ b/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/VirtualRelationshipImpl.java @@ -0,0 +1,41 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._511; + +import org.neo4j.gds.compat.VirtualRelationship; +import org.neo4j.graphdb.Node; +import org.neo4j.graphdb.RelationshipType; + +public class VirtualRelationshipImpl extends VirtualRelationship { + + VirtualRelationshipImpl( + long id, + Node startNode, + Node endNode, + RelationshipType type + ) { + super(id, startNode, endNode, type); + } + + @Override + public String getElementId() { + return Long.toString(getId()); + } +} diff --git a/compatibility/5.11/storage-engine-adapter/build.gradle b/compatibility/5.11/storage-engine-adapter/build.gradle new file mode 100644 index 0000000000..0a21467ee3 --- /dev/null +++ b/compatibility/5.11/storage-engine-adapter/build.gradle @@ -0,0 +1,68 @@ +apply plugin: 'java-library' +apply plugin: 'me.champeau.mrjar' + +description = 'Neo4j Graph Data Science :: Storage Engine Adapter 5.11' + +group = 'org.neo4j.gds' + +// for all 5.x versions +if (ver.'neo4j'.startsWith('5.')) { + sourceSets { + main { + java { + srcDirs = ['src/main/java17'] + } + } + } + + dependencies { + annotationProcessor project(':annotations') + annotationProcessor group: 'org.immutables', name: 'value', version: ver.'immutables' + annotationProcessor group: 'org.neo4j', name: 'annotations', version: neos.'5.11' + + compileOnly project(':annotations') + compileOnly project(':progress-tracking') + compileOnly group: 'org.immutables', name: 'value-annotations', version: ver.'immutables' + compileOnly group: 'org.neo4j', name: 'neo4j', version: neos.'5.11' + compileOnly group: 'org.neo4j', name: 'neo4j-record-storage-engine', version: neos.'5.11' + + implementation project(':core') + implementation project(':storage-engine-adapter-api') + implementation project(':config-api') + implementation project(':string-formatting') + } +} else { + multiRelease { + targetVersions 11, 17 + } + + if (!project.hasProperty('no-forbidden-apis')) { + forbiddenApisJava17 { + exclude('**') + } + } + + dependencies { + annotationProcessor group: 'org.neo4j', name: 'annotations', version: ver.'neo4j' + compileOnly group: 'org.neo4j', name: 'annotations', version: ver.'neo4j' + compileOnly group: 'org.neo4j', name: 'neo4j-kernel-api', version: ver.'neo4j' + + implementation project(':neo4j-adapter') + implementation project(':storage-engine-adapter-api') + + java17AnnotationProcessor project(':annotations') + java17AnnotationProcessor group: 'org.immutables', name: 'value', version: ver.'immutables' + java17AnnotationProcessor group: 'org.neo4j', name: 'annotations', version: neos.'5.11' + + java17CompileOnly project(':annotations') + java17CompileOnly project(':progress-tracking') + java17CompileOnly group: 'org.immutables', name: 'value-annotations', version: ver.'immutables' + java17CompileOnly group: 'org.neo4j', name: 'neo4j', version: neos.'5.11' + java17CompileOnly group: 'org.neo4j', name: 'neo4j-record-storage-engine', version: neos.'5.11' + + java17Implementation project(':core') + java17Implementation project(':storage-engine-adapter-api') + java17Implementation project(':config-api') + java17Implementation project(':string-formatting') + } +} diff --git a/compatibility/5.11/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_511/InMemoryStorageEngineFactory.java b/compatibility/5.11/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_511/InMemoryStorageEngineFactory.java new file mode 100644 index 0000000000..6b85ad7150 --- /dev/null +++ b/compatibility/5.11/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_511/InMemoryStorageEngineFactory.java @@ -0,0 +1,268 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._511; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.configuration.Config; +import org.neo4j.dbms.database.readonly.DatabaseReadOnlyChecker; +import org.neo4j.index.internal.gbptree.RecoveryCleanupWorkCollector; +import org.neo4j.internal.id.IdController; +import org.neo4j.internal.id.IdGeneratorFactory; +import org.neo4j.internal.schema.IndexConfigCompleter; +import org.neo4j.internal.schema.SchemaRule; +import org.neo4j.internal.schema.SchemaState; +import org.neo4j.io.fs.FileSystemAbstraction; +import org.neo4j.io.layout.DatabaseLayout; +import org.neo4j.io.layout.Neo4jLayout; +import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.tracing.PageCacheTracer; +import org.neo4j.lock.LockService; +import org.neo4j.logging.LogProvider; +import org.neo4j.logging.internal.LogService; +import org.neo4j.memory.MemoryTracker; +import org.neo4j.monitoring.DatabaseHealth; +import org.neo4j.scheduler.JobScheduler; +import org.neo4j.storageengine.api.CommandReaderFactory; +import org.neo4j.storageengine.api.ConstraintRuleAccessor; +import org.neo4j.storageengine.api.LogVersionRepository; +import org.neo4j.storageengine.api.MetadataProvider; +import org.neo4j.storageengine.api.StorageEngine; +import org.neo4j.storageengine.api.StorageEngineFactory; +import org.neo4j.storageengine.api.StorageFilesState; +import org.neo4j.storageengine.api.StoreId; +import org.neo4j.storageengine.api.StoreVersion; +import org.neo4j.storageengine.api.StoreVersionCheck; +import org.neo4j.storageengine.api.TransactionIdStore; +import org.neo4j.storageengine.migration.RollingUpgradeCompatibility; +import org.neo4j.storageengine.migration.SchemaRuleMigrationAccess; +import org.neo4j.storageengine.migration.StoreMigrationParticipant; +import org.neo4j.token.TokenHolders; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.List; +import java.util.Optional; +import java.util.UUID; + +@ServiceProvider +public class InMemoryStorageEngineFactory implements StorageEngineFactory { + + @Override + public String name() { + return "unsupported511"; + } + + @Override + public StoreVersionCheck versionCheck( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + LogService logService, + PageCacheTracer pageCacheTracer + ) { + throw new UnsupportedOperationException("5.11 storage engine requires JDK17"); + } + + @Override + public StoreVersion versionInformation(String storeVersion) { + throw new UnsupportedOperationException("5.11 storage engine requires JDK17"); + } + + @Override + public StoreVersion versionInformation(StoreId storeId) { + throw new UnsupportedOperationException("5.11 storage engine requires JDK17"); + } + + @Override + public RollingUpgradeCompatibility rollingUpgradeCompatibility() { + throw new UnsupportedOperationException("5.11 storage engine requires JDK17"); + } + + @Override + public List migrationParticipants( + FileSystemAbstraction fs, + Config config, + PageCache pageCache, + JobScheduler jobScheduler, + LogService logService, + PageCacheTracer cacheTracer, + MemoryTracker memoryTracker + ) { + throw new UnsupportedOperationException("5.11 storage engine requires JDK17"); + } + + @Override + public StorageEngine instantiate( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + TokenHolders tokenHolders, + SchemaState schemaState, + ConstraintRuleAccessor constraintSemantics, + IndexConfigCompleter indexConfigCompleter, + LockService lockService, + IdGeneratorFactory idGeneratorFactory, + IdController idController, + DatabaseHealth databaseHealth, + LogProvider internalLogProvider, + LogProvider userLogProvider, + RecoveryCleanupWorkCollector recoveryCleanupWorkCollector, + PageCacheTracer cacheTracer, + boolean createStoreIfNotExists, + DatabaseReadOnlyChecker readOnlyChecker, + MemoryTracker memoryTracker + ) { + throw new UnsupportedOperationException("5.11 storage engine requires JDK17"); + } + + @Override + public List listStorageFiles(FileSystemAbstraction fileSystem, DatabaseLayout databaseLayout) throws + IOException { + throw new UnsupportedOperationException("5.11 storage engine requires JDK17"); + } + + @Override + public boolean storageExists(FileSystemAbstraction fileSystem, DatabaseLayout databaseLayout, PageCache pageCache) { + return false; + } + + @Override + public TransactionIdStore readOnlyTransactionIdStore( + FileSystemAbstraction filySystem, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) throws IOException { + throw new UnsupportedOperationException("5.11 storage engine requires JDK17"); + } + + @Override + public LogVersionRepository readOnlyLogVersionRepository( + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) throws IOException { + throw new UnsupportedOperationException("5.11 storage engine requires JDK17"); + } + + @Override + public MetadataProvider transactionMetaDataStore( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + PageCacheTracer cacheTracer, + DatabaseReadOnlyChecker readOnlyChecker + ) throws IOException { + throw new UnsupportedOperationException("5.11 storage engine requires JDK17"); + } + + @Override + public StoreId storeId( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) throws IOException { + throw new UnsupportedOperationException("5.11 storage engine requires JDK17"); + } + + @Override + public void setStoreId( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext, + StoreId storeId, + long upgradeTxChecksum, + long upgradeTxCommitTimestamp + ) throws IOException { + throw new UnsupportedOperationException("5.11 storage engine requires JDK17"); + } + + @Override + public void setExternalStoreUUID( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext, + UUID externalStoreId + ) throws IOException { + throw new UnsupportedOperationException("5.11 storage engine requires JDK17"); + } + + @Override + public Optional databaseIdUuid( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) { + throw new UnsupportedOperationException("5.11 storage engine requires JDK17"); + } + + @Override + public SchemaRuleMigrationAccess schemaRuleMigrationAccess( + FileSystemAbstraction fs, + PageCache pageCache, + Config config, + DatabaseLayout databaseLayout, + LogService logService, + String recordFormats, + PageCacheTracer cacheTracer, + CursorContext cursorContext, + MemoryTracker memoryTracker + ) { + throw new UnsupportedOperationException("5.11 storage engine requires JDK17"); + } + + @Override + public List loadSchemaRules( + FileSystemAbstraction fs, + PageCache pageCache, + Config config, + DatabaseLayout databaseLayout, + CursorContext cursorContext + ) { + throw new UnsupportedOperationException("5.11 storage engine requires JDK17"); + } + + @Override + public StorageFilesState checkStoreFileState( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache + ) { + throw new UnsupportedOperationException("5.11 storage engine requires JDK17"); + } + + @Override + public CommandReaderFactory commandReaderFactory() { + throw new UnsupportedOperationException("5.11 storage engine requires JDK17"); + } + + @Override + public DatabaseLayout databaseLayout(Neo4jLayout neo4jLayout, String databaseName) { + throw new UnsupportedOperationException("5.11 storage engine requires JDK17"); + } +} diff --git a/compatibility/5.11/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_511/StorageEngineProxyFactoryImpl.java b/compatibility/5.11/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_511/StorageEngineProxyFactoryImpl.java new file mode 100644 index 0000000000..6d2f321742 --- /dev/null +++ b/compatibility/5.11/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_511/StorageEngineProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._511; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jVersion; +import org.neo4j.gds.compat.StorageEngineProxyApi; +import org.neo4j.gds.compat.StorageEngineProxyFactory; + +@ServiceProvider +public class StorageEngineProxyFactoryImpl implements StorageEngineProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return false; + } + + @Override + public StorageEngineProxyApi load() { + throw new UnsupportedOperationException("5.11 storage engine requires JDK17"); + } + + @Override + public String description() { + return "Storage Engine 5.11"; + } +} diff --git a/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/InMemoryCommandCreationContextImpl.java b/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/InMemoryCommandCreationContextImpl.java new file mode 100644 index 0000000000..da81b6bbe3 --- /dev/null +++ b/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/InMemoryCommandCreationContextImpl.java @@ -0,0 +1,107 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._511; + +import org.neo4j.configuration.Config; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.kernel.KernelVersion; +import org.neo4j.kernel.KernelVersionProvider; +import org.neo4j.lock.LockTracer; +import org.neo4j.lock.ResourceLocker; +import org.neo4j.storageengine.api.CommandCreationContext; +import org.neo4j.storageengine.api.cursor.StoreCursors; + +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Supplier; + +public class InMemoryCommandCreationContextImpl implements CommandCreationContext { + + private final AtomicLong schemaTokens; + private final AtomicInteger propertyTokens; + private final AtomicInteger labelTokens; + private final AtomicInteger typeTokens; + + InMemoryCommandCreationContextImpl() { + this.schemaTokens = new AtomicLong(0); + this.propertyTokens = new AtomicInteger(0); + this.labelTokens = new AtomicInteger(0); + this.typeTokens = new AtomicInteger(0); + } + + @Override + public long reserveNode() { + throw new UnsupportedOperationException("Creating nodes is not supported"); + } + + @Override + public long reserveRelationship( + long sourceNode, + long targetNode, + int relationshipType, + boolean sourceNodeAddedInTx, + boolean targetNodeAddedInTx + ) { + throw new UnsupportedOperationException("Creating relationships is not supported"); + } + + @Override + public long reserveSchema() { + return schemaTokens.getAndIncrement(); + } + + @Override + public int reserveLabelTokenId() { + return labelTokens.getAndIncrement(); + } + + @Override + public int reservePropertyKeyTokenId() { + return propertyTokens.getAndIncrement(); + } + + @Override + public int reserveRelationshipTypeTokenId() { + return typeTokens.getAndIncrement(); + } + + @Override + public void close() { + + } + + @Override + public void initialize( + KernelVersionProvider kernelVersionProvider, + CursorContext cursorContext, + StoreCursors storeCursors, + Supplier oldestActiveTransactionSequenceNumber, + ResourceLocker locks, + Supplier lockTracer + ) { + + } + + @Override + public KernelVersion kernelVersion() { + // NOTE: Double-check if this is still correct when you copy this into a new compat layer + return KernelVersion.getLatestVersion(Config.newBuilder().build()); + } +} diff --git a/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/InMemoryCountsStoreImpl.java b/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/InMemoryCountsStoreImpl.java new file mode 100644 index 0000000000..6e3814395c --- /dev/null +++ b/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/InMemoryCountsStoreImpl.java @@ -0,0 +1,111 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._511; + +import org.neo4j.annotations.documented.ReporterFactory; +import org.neo4j.counts.CountsStore; +import org.neo4j.counts.CountsUpdater; +import org.neo4j.counts.CountsVisitor; +import org.neo4j.gds.NodeLabel; +import org.neo4j.gds.api.GraphStore; +import org.neo4j.internal.helpers.progress.ProgressMonitorFactory; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.context.CursorContextFactory; +import org.neo4j.io.pagecache.tracing.FileFlushEvent; +import org.neo4j.memory.MemoryTracker; +import org.neo4j.token.TokenHolders; +import org.neo4j.token.api.TokenNotFoundException; + +import java.io.IOException; + +public class InMemoryCountsStoreImpl implements CountsStore { + + private final GraphStore graphStore; + private final TokenHolders tokenHolders; + + public InMemoryCountsStoreImpl( + GraphStore graphStore, + TokenHolders tokenHolders + ) { + + this.graphStore = graphStore; + this.tokenHolders = tokenHolders; + } + + @Override + public void start(CursorContext cursorContext, MemoryTracker memoryTracker) throws IOException { + + } + + @Override + public CountsUpdater updater(long txId, boolean isLast, CursorContext cursorContext) { + throw new UnsupportedOperationException("Updates are not supported"); + } + + @Override + public void checkpoint(FileFlushEvent fileFlushEvent, CursorContext cursorContext) { + + } + + @Override + public long nodeCount(int labelId, CursorContext cursorContext) { + if (labelId == -1) { + return graphStore.nodeCount(); + } + + String nodeLabel; + try { + nodeLabel = tokenHolders.labelTokens().getTokenById(labelId).name(); + } catch (TokenNotFoundException e) { + throw new RuntimeException(e); + } + return graphStore.nodes().nodeCount(NodeLabel.of(nodeLabel)); + } + + @Override + public long relationshipCount(int startLabelId, int typeId, int endLabelId, CursorContext cursorContext) { + // TODO: this is quite wrong + return graphStore.relationshipCount(); + } + + @Override + public boolean consistencyCheck( + ReporterFactory reporterFactory, + CursorContextFactory cursorContextFactory, + int i, + ProgressMonitorFactory progressMonitorFactory + ) { + return true; + } + + @Override + public void close() { + + } + + @Override + public void accept(CountsVisitor visitor, CursorContext cursorContext) { + tokenHolders.labelTokens().getAllTokens().forEach(labelToken -> { + visitor.visitNodeCount(labelToken.id(), nodeCount(labelToken.id(), cursorContext)); + }); + + visitor.visitRelationshipCount(-1, -1, -1, graphStore.relationshipCount()); + } +} diff --git a/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/InMemoryMetaDataProviderImpl.java b/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/InMemoryMetaDataProviderImpl.java new file mode 100644 index 0000000000..23790cccb4 --- /dev/null +++ b/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/InMemoryMetaDataProviderImpl.java @@ -0,0 +1,201 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._511; + +import org.neo4j.internal.recordstorage.InMemoryLogVersionRepository511; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.context.TransactionIdSnapshot; +import org.neo4j.storageengine.api.ClosedTransactionMetadata; +import org.neo4j.storageengine.api.ExternalStoreId; +import org.neo4j.storageengine.api.MetadataProvider; +import org.neo4j.storageengine.api.StoreId; +import org.neo4j.storageengine.api.TransactionId; + +import java.io.IOException; +import java.util.Optional; +import java.util.UUID; + +public class InMemoryMetaDataProviderImpl implements MetadataProvider { + + private final ExternalStoreId externalStoreId; + private final InMemoryLogVersionRepository511 logVersionRepository; + private final InMemoryTransactionIdStoreImpl transactionIdStore; + + InMemoryMetaDataProviderImpl() { + this.logVersionRepository = new InMemoryLogVersionRepository511(); + this.externalStoreId = new ExternalStoreId(UUID.randomUUID()); + this.transactionIdStore = new InMemoryTransactionIdStoreImpl(); + } + + @Override + public ExternalStoreId getExternalStoreId() { + return this.externalStoreId; + } + + @Override + public ClosedTransactionMetadata getLastClosedTransaction() { + return this.transactionIdStore.getLastClosedTransaction(); + } + + @Override + public void setCurrentLogVersion(long version) { + logVersionRepository.setCurrentLogVersion(version); + } + + @Override + public long incrementAndGetVersion() { + return logVersionRepository.incrementAndGetVersion(); + } + + @Override + public void setCheckpointLogVersion(long version) { + logVersionRepository.setCheckpointLogVersion(version); + } + + @Override + public long incrementAndGetCheckpointLogVersion() { + return logVersionRepository.incrementAndGetCheckpointLogVersion(); + } + + @Override + public void transactionCommitted(long transactionId, int checksum, long commitTimestamp, long consensusIndex) { + transactionIdStore.transactionCommitted(transactionId, checksum, commitTimestamp, consensusIndex); + } + + @Override + public void setLastCommittedAndClosedTransactionId( + long transactionId, + int checksum, + long commitTimestamp, + long consensusIndex, + long byteOffset, + long logVersion + ) { + transactionIdStore.setLastCommittedAndClosedTransactionId( + transactionId, + checksum, + commitTimestamp, + consensusIndex, + byteOffset, + logVersion + ); + } + + @Override + public void transactionClosed( + long transactionId, + long logVersion, + long byteOffset, + int checksum, + long commitTimestamp, + long consensusIndex + ) { + this.transactionIdStore.transactionClosed( + transactionId, + logVersion, + byteOffset, + checksum, + commitTimestamp, + consensusIndex + ); + } + + @Override + public void resetLastClosedTransaction( + long transactionId, + long logVersion, + long byteOffset, + int checksum, + long commitTimestamp, + long consensusIndex + ) { + this.transactionIdStore.resetLastClosedTransaction( + transactionId, + logVersion, + byteOffset, + checksum, + commitTimestamp, + consensusIndex + ); + } + + @Override + public TransactionIdSnapshot getClosedTransactionSnapshot() { + return new TransactionIdSnapshot(this.getLastClosedTransactionId()); + } + + @Override + public void regenerateMetadata(StoreId storeId, UUID externalStoreUUID, CursorContext cursorContext) { + } + + @Override + public StoreId getStoreId() { + return StoreId.UNKNOWN; + } + + @Override + public void close() throws IOException { + } + + @Override + public long getCurrentLogVersion() { + return this.logVersionRepository.getCurrentLogVersion(); + } + + @Override + public long getCheckpointLogVersion() { + return this.logVersionRepository.getCheckpointLogVersion(); + } + + @Override + public long nextCommittingTransactionId() { + return this.transactionIdStore.nextCommittingTransactionId(); + } + + @Override + public long committingTransactionId() { + return this.transactionIdStore.committingTransactionId(); + } + + @Override + public long getLastCommittedTransactionId() { + return this.transactionIdStore.getLastCommittedTransactionId(); + } + + @Override + public TransactionId getLastCommittedTransaction() { + return this.transactionIdStore.getLastCommittedTransaction(); + } + + @Override + public long getLastClosedTransactionId() { + return this.transactionIdStore.getLastClosedTransactionId(); + } + + @Override + public Optional getDatabaseIdUuid(CursorContext cursorTracer) { + throw new IllegalStateException("Not supported"); + } + + @Override + public void setDatabaseIdUuid(UUID uuid, CursorContext cursorContext) { + throw new IllegalStateException("Not supported"); + } +} diff --git a/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/InMemoryNodeCursor.java b/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/InMemoryNodeCursor.java new file mode 100644 index 0000000000..5f4bb3cdd0 --- /dev/null +++ b/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/InMemoryNodeCursor.java @@ -0,0 +1,82 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._511; + +import org.neo4j.gds.api.GraphStore; +import org.neo4j.gds.compat.AbstractInMemoryNodeCursor; +import org.neo4j.storageengine.api.AllNodeScan; +import org.neo4j.storageengine.api.Degrees; +import org.neo4j.storageengine.api.LongReference; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.storageengine.api.RelationshipSelection; +import org.neo4j.storageengine.api.StoragePropertyCursor; +import org.neo4j.storageengine.api.StorageRelationshipTraversalCursor; +import org.neo4j.token.TokenHolders; + +public class InMemoryNodeCursor extends AbstractInMemoryNodeCursor { + + public InMemoryNodeCursor(GraphStore graphStore, TokenHolders tokenHolders) { + super(graphStore, tokenHolders); + } + + @Override + public boolean hasLabel() { + return hasAtLeastOneLabelForCurrentNode(); + } + + @Override + public Reference propertiesReference() { + return LongReference.longReference(getId()); + } + + @Override + public void properties(StoragePropertyCursor propertyCursor, PropertySelection selection) { + propertyCursor.initNodeProperties(propertiesReference(), selection); + } + + @Override + public void properties(StoragePropertyCursor propertyCursor) { + properties(propertyCursor, PropertySelection.ALL_PROPERTIES); + } + + @Override + public boolean supportsFastRelationshipsTo() { + return false; + } + + @Override + public void relationshipsTo( + StorageRelationshipTraversalCursor storageRelationshipTraversalCursor, + RelationshipSelection relationshipSelection, + long neighbourNodeReference + ) { + throw new UnsupportedOperationException(); + } + + @Override + public void degrees(RelationshipSelection selection, Degrees.Mutator mutator) { + } + + @Override + public boolean scanBatch(AllNodeScan allNodeScan, long sizeHint) { + return super.scanBatch(allNodeScan, (int) sizeHint); + } +} diff --git a/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/InMemoryNodePropertyCursor.java b/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/InMemoryNodePropertyCursor.java new file mode 100644 index 0000000000..4463339e2b --- /dev/null +++ b/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/InMemoryNodePropertyCursor.java @@ -0,0 +1,45 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._511; + +import org.neo4j.gds.compat.AbstractInMemoryNodePropertyCursor; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.storageengine.api.LongReference; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.token.TokenHolders; + +public class InMemoryNodePropertyCursor extends AbstractInMemoryNodePropertyCursor { + + public InMemoryNodePropertyCursor(CypherGraphStore graphStore, TokenHolders tokenHolders) { + super(graphStore, tokenHolders); + } + + @Override + public void initNodeProperties(Reference reference, PropertySelection selection, long ownerReference) { + reset(); + setId(((LongReference) reference).id); + setPropertySelection(new InMemoryPropertySelectionImpl(selection)); + } + + @Override + public void initRelationshipProperties(Reference reference, PropertySelection selection, long ownerReference) { + } +} diff --git a/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/InMemoryPropertyCursor.java b/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/InMemoryPropertyCursor.java new file mode 100644 index 0000000000..502fda65b2 --- /dev/null +++ b/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/InMemoryPropertyCursor.java @@ -0,0 +1,71 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._511; + +import org.neo4j.gds.compat.AbstractInMemoryPropertyCursor; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.storageengine.api.StorageNodeCursor; +import org.neo4j.storageengine.api.StorageRelationshipCursor; +import org.neo4j.token.TokenHolders; + +public class InMemoryPropertyCursor extends AbstractInMemoryPropertyCursor { + + public InMemoryPropertyCursor(CypherGraphStore graphStore, TokenHolders tokenHolders) { + super(graphStore, tokenHolders); + } + + @Override + public void initNodeProperties(Reference reference, PropertySelection selection, long ownerReference) { + if (this.delegate == null || !(this.delegate instanceof InMemoryNodePropertyCursor)) { + this.delegate = new InMemoryNodePropertyCursor(graphStore, tokenHolders); + } + + ((InMemoryNodePropertyCursor) delegate).initNodeProperties(reference, selection); + } + + @Override + public void initNodeProperties(StorageNodeCursor nodeCursor, PropertySelection selection) { + if (this.delegate == null || !(this.delegate instanceof InMemoryNodePropertyCursor)) { + this.delegate = new InMemoryNodePropertyCursor(graphStore, tokenHolders); + } + + ((InMemoryNodePropertyCursor) delegate).initNodeProperties(nodeCursor, selection); + } + + @Override + public void initRelationshipProperties(StorageRelationshipCursor relationshipCursor, PropertySelection selection) { + if (this.delegate == null || !(this.delegate instanceof InMemoryRelationshipPropertyCursor)) { + this.delegate = new InMemoryRelationshipPropertyCursor(graphStore, tokenHolders); + } + + ((InMemoryRelationshipPropertyCursor) delegate).initRelationshipProperties(relationshipCursor, selection); + } + + @Override + public void initRelationshipProperties(Reference reference, PropertySelection selection, long ownerReference) { + if (this.delegate == null || !(this.delegate instanceof InMemoryRelationshipPropertyCursor)) { + this.delegate = new InMemoryRelationshipPropertyCursor(graphStore, tokenHolders); + } + + ((InMemoryRelationshipPropertyCursor) delegate).initRelationshipProperties(reference, selection); + } +} diff --git a/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/InMemoryPropertySelectionImpl.java b/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/InMemoryPropertySelectionImpl.java new file mode 100644 index 0000000000..839a0b7fb3 --- /dev/null +++ b/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/InMemoryPropertySelectionImpl.java @@ -0,0 +1,55 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._511; + +import org.neo4j.gds.compat.InMemoryPropertySelection; +import org.neo4j.storageengine.api.PropertySelection; + +public class InMemoryPropertySelectionImpl implements InMemoryPropertySelection { + + private final PropertySelection propertySelection; + + public InMemoryPropertySelectionImpl(PropertySelection propertySelection) {this.propertySelection = propertySelection;} + + @Override + public boolean isLimited() { + return propertySelection.isLimited(); + } + + @Override + public int numberOfKeys() { + return propertySelection.numberOfKeys(); + } + + @Override + public int key(int index) { + return propertySelection.key(index); + } + + @Override + public boolean test(int key) { + return propertySelection.test(key); + } + + @Override + public boolean isKeysOnly() { + return propertySelection.isKeysOnly(); + } +} diff --git a/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/InMemoryRelationshipPropertyCursor.java b/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/InMemoryRelationshipPropertyCursor.java new file mode 100644 index 0000000000..43d155aba6 --- /dev/null +++ b/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/InMemoryRelationshipPropertyCursor.java @@ -0,0 +1,60 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._511; + +import org.neo4j.gds.compat.AbstractInMemoryRelationshipPropertyCursor; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.gds.storageengine.InMemoryRelationshipCursor; +import org.neo4j.storageengine.api.LongReference; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.storageengine.api.StorageRelationshipCursor; +import org.neo4j.token.TokenHolders; + +public class InMemoryRelationshipPropertyCursor extends AbstractInMemoryRelationshipPropertyCursor { + + InMemoryRelationshipPropertyCursor(CypherGraphStore graphStore, TokenHolders tokenHolders) { + super(graphStore, tokenHolders); + } + + @Override + public void initNodeProperties( + Reference reference, PropertySelection propertySelection, long ownerReference + ) { + + } + + @Override + public void initRelationshipProperties( + Reference reference, PropertySelection propertySelection, long ownerReference + ) { + var relationshipId = ((LongReference) reference).id; + var relationshipCursor = new InMemoryRelationshipScanCursor(graphStore, tokenHolders); + relationshipCursor.single(relationshipId); + relationshipCursor.next(); + relationshipCursor.properties(this, new InMemoryPropertySelectionImpl(propertySelection)); + } + + @Override + public void initRelationshipProperties(StorageRelationshipCursor relationshipCursor, PropertySelection selection) { + var inMemoryRelationshipCursor = (InMemoryRelationshipCursor) relationshipCursor; + inMemoryRelationshipCursor.properties(this, selection); + } +} diff --git a/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/InMemoryRelationshipScanCursor.java b/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/InMemoryRelationshipScanCursor.java new file mode 100644 index 0000000000..f64ac350f1 --- /dev/null +++ b/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/InMemoryRelationshipScanCursor.java @@ -0,0 +1,61 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._511; + +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.internal.recordstorage.AbstractInMemoryRelationshipScanCursor; +import org.neo4j.storageengine.api.AllRelationshipsScan; +import org.neo4j.storageengine.api.LongReference; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.storageengine.api.StoragePropertyCursor; +import org.neo4j.token.TokenHolders; + +public class InMemoryRelationshipScanCursor extends AbstractInMemoryRelationshipScanCursor { + + public InMemoryRelationshipScanCursor( + CypherGraphStore graphStore, + TokenHolders tokenHolders + ) { + super(graphStore, tokenHolders); + } + + @Override + public void single(long reference, long sourceNodeReference, int type, long targetNodeReference) { + single(reference); + } + + @Override + public Reference propertiesReference() { + return LongReference.longReference(getId()); + } + + @Override + public void properties( + StoragePropertyCursor storagePropertyCursor, PropertySelection propertySelection + ) { + properties(storagePropertyCursor, new InMemoryPropertySelectionImpl(propertySelection)); + } + + @Override + public boolean scanBatch(AllRelationshipsScan allRelationshipsScan, long sizeHint) { + return super.scanBatch(allRelationshipsScan, (int) sizeHint); + } +} diff --git a/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/InMemoryRelationshipTraversalCursor.java b/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/InMemoryRelationshipTraversalCursor.java new file mode 100644 index 0000000000..7be8a43625 --- /dev/null +++ b/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/InMemoryRelationshipTraversalCursor.java @@ -0,0 +1,47 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._511; + +import org.neo4j.gds.compat.AbstractInMemoryRelationshipTraversalCursor; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.storageengine.api.LongReference; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.storageengine.api.StoragePropertyCursor; +import org.neo4j.token.TokenHolders; + +public class InMemoryRelationshipTraversalCursor extends AbstractInMemoryRelationshipTraversalCursor { + + public InMemoryRelationshipTraversalCursor(CypherGraphStore graphStore, TokenHolders tokenHolders) { + super(graphStore, tokenHolders); + } + + @Override + public Reference propertiesReference() { + return LongReference.longReference(getId()); + } + + @Override + public void properties( + StoragePropertyCursor propertyCursor, PropertySelection selection + ) { + properties(propertyCursor, new InMemoryPropertySelectionImpl(selection)); + } +} diff --git a/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/InMemoryStorageEngineFactory.java b/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/InMemoryStorageEngineFactory.java new file mode 100644 index 0000000000..d08d8b633a --- /dev/null +++ b/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/InMemoryStorageEngineFactory.java @@ -0,0 +1,571 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._511; + +import org.eclipse.collections.api.factory.Sets; +import org.eclipse.collections.api.set.ImmutableSet; +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.configuration.Config; +import org.neo4j.consistency.checking.ConsistencyFlags; +import org.neo4j.consistency.report.ConsistencySummaryStatistics; +import org.neo4j.dbms.database.readonly.DatabaseReadOnlyChecker; +import org.neo4j.function.ThrowingSupplier; +import org.neo4j.gds.annotation.SuppressForbidden; +import org.neo4j.gds.compat.Neo4jVersion; +import org.neo4j.gds.compat.StorageEngineFactoryIdProvider; +import org.neo4j.gds.compat.StorageEngineProxyApi; +import org.neo4j.index.internal.gbptree.RecoveryCleanupWorkCollector; +import org.neo4j.internal.batchimport.AdditionalInitialIds; +import org.neo4j.internal.batchimport.BatchImporter; +import org.neo4j.internal.batchimport.Configuration; +import org.neo4j.internal.batchimport.IncrementalBatchImporter; +import org.neo4j.internal.batchimport.IndexImporterFactory; +import org.neo4j.internal.batchimport.Monitor; +import org.neo4j.internal.batchimport.ReadBehaviour; +import org.neo4j.internal.batchimport.input.Collector; +import org.neo4j.internal.batchimport.input.Input; +import org.neo4j.internal.batchimport.input.LenientStoreInput; +import org.neo4j.internal.id.IdGeneratorFactory; +import org.neo4j.internal.id.ScanOnOpenReadOnlyIdGeneratorFactory; +import org.neo4j.internal.recordstorage.InMemoryStorageCommandReaderFactory511; +import org.neo4j.internal.recordstorage.StoreTokens; +import org.neo4j.internal.schema.IndexConfigCompleter; +import org.neo4j.internal.schema.SchemaRule; +import org.neo4j.internal.schema.SchemaState; +import org.neo4j.io.fs.FileSystemAbstraction; +import org.neo4j.io.layout.DatabaseLayout; +import org.neo4j.io.layout.Neo4jLayout; +import org.neo4j.io.layout.recordstorage.RecordDatabaseLayout; +import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.context.CursorContextFactory; +import org.neo4j.io.pagecache.tracing.PageCacheTracer; +import org.neo4j.kernel.KernelVersionRepository; +import org.neo4j.kernel.api.index.IndexProvidersAccess; +import org.neo4j.kernel.impl.api.index.IndexProviderMap; +import org.neo4j.kernel.impl.locking.LockManager; +import org.neo4j.kernel.impl.store.MetaDataStore; +import org.neo4j.kernel.impl.store.NeoStores; +import org.neo4j.kernel.impl.store.StoreFactory; +import org.neo4j.kernel.impl.store.StoreType; +import org.neo4j.kernel.impl.store.cursor.CachedStoreCursors; +import org.neo4j.kernel.impl.transaction.log.LogTailLogVersionsMetadata; +import org.neo4j.kernel.impl.transaction.log.LogTailMetadata; +import org.neo4j.lock.LockService; +import org.neo4j.logging.InternalLog; +import org.neo4j.logging.InternalLogProvider; +import org.neo4j.logging.NullLogProvider; +import org.neo4j.logging.internal.LogService; +import org.neo4j.memory.MemoryTracker; +import org.neo4j.monitoring.DatabaseHealth; +import org.neo4j.scheduler.JobScheduler; +import org.neo4j.storageengine.api.CommandReaderFactory; +import org.neo4j.storageengine.api.ConstraintRuleAccessor; +import org.neo4j.storageengine.api.LogFilesInitializer; +import org.neo4j.storageengine.api.MetadataProvider; +import org.neo4j.storageengine.api.SchemaRule44; +import org.neo4j.storageengine.api.StorageEngine; +import org.neo4j.storageengine.api.StorageEngineFactory; +import org.neo4j.storageengine.api.StorageFilesState; +import org.neo4j.storageengine.api.StoreId; +import org.neo4j.storageengine.api.StoreVersion; +import org.neo4j.storageengine.api.StoreVersionCheck; +import org.neo4j.storageengine.api.StoreVersionIdentifier; +import org.neo4j.storageengine.migration.SchemaRuleMigrationAccessExtended; +import org.neo4j.storageengine.migration.StoreMigrationParticipant; +import org.neo4j.time.SystemNanoClock; +import org.neo4j.token.DelegatingTokenHolder; +import org.neo4j.token.ReadOnlyTokenCreator; +import org.neo4j.token.TokenHolders; +import org.neo4j.token.api.NamedToken; +import org.neo4j.token.api.TokenHolder; +import org.neo4j.token.api.TokensLoader; + +import java.io.IOException; +import java.io.OutputStream; +import java.io.PrintStream; +import java.io.UncheckedIOException; +import java.nio.file.OpenOption; +import java.nio.file.Path; +import java.time.Clock; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.UUID; +import java.util.function.Function; + +@ServiceProvider +public class InMemoryStorageEngineFactory implements StorageEngineFactory { + + static final String IN_MEMORY_STORAGE_ENGINE_NAME = "in-memory-511"; + + public InMemoryStorageEngineFactory() { + StorageEngineProxyApi.requireNeo4jVersion(Neo4jVersion.V_5_11, StorageEngineFactory.class); + } + + @Override + public byte id() { + return StorageEngineFactoryIdProvider.ID; + } + + @Override + public boolean storageExists(FileSystemAbstraction fileSystem, DatabaseLayout databaseLayout) { + return false; + } + + @Override + public StorageEngine instantiate( + FileSystemAbstraction fs, + Clock clock, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + TokenHolders tokenHolders, + SchemaState schemaState, + ConstraintRuleAccessor constraintSemantics, + IndexConfigCompleter indexConfigCompleter, + LockService lockService, + IdGeneratorFactory idGeneratorFactory, + DatabaseHealth databaseHealth, + InternalLogProvider internalLogProvider, + InternalLogProvider userLogProvider, + RecoveryCleanupWorkCollector recoveryCleanupWorkCollector, + LogTailMetadata logTailMetadata, + KernelVersionRepository kernelVersionRepository, + MemoryTracker memoryTracker, + CursorContextFactory contextFactory, + PageCacheTracer pageCacheTracer + ) { + StoreFactory factory = new StoreFactory( + databaseLayout, + config, + idGeneratorFactory, + pageCache, + pageCacheTracer, + fs, + internalLogProvider, + contextFactory, + false, + logTailMetadata + ); + + factory.openNeoStores(StoreType.LABEL_TOKEN).close(); + + return new InMemoryStorageEngineImpl( + databaseLayout, + tokenHolders + ); + } + + @Override + public Optional databaseIdUuid( + FileSystemAbstraction fs, DatabaseLayout databaseLayout, PageCache pageCache, CursorContext cursorContext + ) { + var fieldAccess = MetaDataStore.getFieldAccess( + pageCache, + RecordDatabaseLayout.convert(databaseLayout).metadataStore(), + databaseLayout.getDatabaseName(), + cursorContext + ); + + try { + return fieldAccess.readDatabaseUUID(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + @Override + public List migrationParticipants( + FileSystemAbstraction fileSystemAbstraction, + Config config, + PageCache pageCache, + JobScheduler jobScheduler, + LogService logService, + MemoryTracker memoryTracker, + PageCacheTracer pageCacheTracer, + CursorContextFactory cursorContextFactory, + boolean b + ) { + return List.of(); + } + + @Override + public DatabaseLayout databaseLayout( + Neo4jLayout neo4jLayout, String databaseName + ) { + return RecordDatabaseLayout.of(neo4jLayout, databaseName); + } + + @Override + public DatabaseLayout formatSpecificDatabaseLayout(DatabaseLayout plainLayout) { + return databaseLayout(plainLayout.getNeo4jLayout(), plainLayout.getDatabaseName()); + } + + @SuppressForbidden(reason = "This is the compat layer and we don't really need to go through the proxy") + @Override + public BatchImporter batchImporter( + DatabaseLayout databaseLayout, + FileSystemAbstraction fileSystemAbstraction, + PageCacheTracer pageCacheTracer, + Configuration configuration, + LogService logService, + PrintStream printStream, + boolean b, + AdditionalInitialIds additionalInitialIds, + Config config, + Monitor monitor, + JobScheduler jobScheduler, + Collector collector, + LogFilesInitializer logFilesInitializer, + IndexImporterFactory indexImporterFactory, + MemoryTracker memoryTracker, + CursorContextFactory cursorContextFactory + ) { + throw new UnsupportedOperationException("Batch Import into GDS is not supported"); + } + + @Override + public Input asBatchImporterInput( + DatabaseLayout databaseLayout, + FileSystemAbstraction fileSystemAbstraction, + PageCache pageCache, + PageCacheTracer pageCacheTracer, + Config config, + MemoryTracker memoryTracker, + ReadBehaviour readBehaviour, + boolean b, + CursorContextFactory cursorContextFactory, + LogTailMetadata logTailMetadata + ) { + NeoStores neoStores = (new StoreFactory( + databaseLayout, + config, + new ScanOnOpenReadOnlyIdGeneratorFactory(), + pageCache, + pageCacheTracer, + fileSystemAbstraction, + NullLogProvider.getInstance(), + cursorContextFactory, + false, + logTailMetadata + )).openAllNeoStores(); + return new LenientStoreInput( + neoStores, + readBehaviour.decorateTokenHolders(this.loadReadOnlyTokens(neoStores, true, cursorContextFactory)), + true, + cursorContextFactory, + readBehaviour + ); + } + + @Override + public long optimalAvailableConsistencyCheckerMemory( + FileSystemAbstraction fileSystemAbstraction, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache + ) { + return 0; + } + + @Override + public String name() { + return IN_MEMORY_STORAGE_ENGINE_NAME; + } + + @Override + public Set supportedFormats(boolean includeFormatsUnderDevelopment) { + return Set.of(IN_MEMORY_STORAGE_ENGINE_NAME); + } + + @Override + public boolean supportedFormat(String format, boolean includeFormatsUnderDevelopment) { + return format.equals(IN_MEMORY_STORAGE_ENGINE_NAME); + } + + @Override + public MetadataProvider transactionMetaDataStore( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + DatabaseReadOnlyChecker readOnlyChecker, + CursorContextFactory contextFactory, + LogTailLogVersionsMetadata logTailMetadata, + PageCacheTracer pageCacheTracer + ) throws IOException { + return new InMemoryMetaDataProviderImpl(); + } + + @Override + public StoreVersionCheck versionCheck( + FileSystemAbstraction fileSystemAbstraction, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + LogService logService, + CursorContextFactory cursorContextFactory + ) { + return new InMemoryVersionCheck(); + } + + @Override + public List loadSchemaRules( + FileSystemAbstraction fileSystemAbstraction, + PageCache pageCache, + PageCacheTracer pageCacheTracer, + Config config, + DatabaseLayout databaseLayout, + boolean b, + Function function, + CursorContextFactory cursorContextFactory + ) { + return List.of(); + } + + @Override + public List load44SchemaRules( + FileSystemAbstraction fs, + PageCache pageCache, + PageCacheTracer pageCacheTracer, + Config config, + DatabaseLayout databaseLayout, + CursorContextFactory contextFactory, + LogTailLogVersionsMetadata logTailMetadata + ) { + return List.of(); + } + + @Override + public TokenHolders loadReadOnlyTokens( + FileSystemAbstraction fileSystemAbstraction, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + PageCacheTracer pageCacheTracer, + boolean lenient, + CursorContextFactory cursorContextFactory + ) { + StoreFactory factory = new StoreFactory( + databaseLayout, + config, + new ScanOnOpenReadOnlyIdGeneratorFactory(), + pageCache, + pageCacheTracer, + fileSystemAbstraction, + NullLogProvider.getInstance(), + cursorContextFactory, + false, + LogTailMetadata.EMPTY_LOG_TAIL + ); + try ( + NeoStores stores = factory.openNeoStores( + StoreType.PROPERTY_KEY_TOKEN, StoreType.PROPERTY_KEY_TOKEN_NAME, + StoreType.LABEL_TOKEN, StoreType.LABEL_TOKEN_NAME, + StoreType.RELATIONSHIP_TYPE_TOKEN, StoreType.RELATIONSHIP_TYPE_TOKEN_NAME + ) + ) { + return loadReadOnlyTokens(stores, lenient, cursorContextFactory); + } + } + + @Override + public SchemaRuleMigrationAccessExtended schemaRuleMigrationAccess( + FileSystemAbstraction fs, + PageCache pageCache, + PageCacheTracer pageCacheTracer, + Config config, + DatabaseLayout databaseLayout, + CursorContextFactory contextFactory, + MemoryTracker memoryTracker + ) { + // this is used by store copy, which is not supported for GDS storage engine + return null; + } + + private TokenHolders loadReadOnlyTokens( + NeoStores stores, + boolean lenient, + CursorContextFactory cursorContextFactory + ) { + try ( + var cursorContext = cursorContextFactory.create("loadReadOnlyTokens"); + var storeCursors = new CachedStoreCursors(stores, cursorContext) + ) { + stores.start( cursorContext ); + TokensLoader loader = lenient ? StoreTokens.allReadableTokens( stores ) : StoreTokens.allTokens( stores ); + TokenHolder propertyKeys = new DelegatingTokenHolder( ReadOnlyTokenCreator.READ_ONLY, TokenHolder.TYPE_PROPERTY_KEY ); + TokenHolder labels = new DelegatingTokenHolder( ReadOnlyTokenCreator.READ_ONLY, TokenHolder.TYPE_LABEL ); + TokenHolder relationshipTypes = new DelegatingTokenHolder( ReadOnlyTokenCreator.READ_ONLY, TokenHolder.TYPE_RELATIONSHIP_TYPE ); + + propertyKeys.setInitialTokens( lenient ? unique( loader.getPropertyKeyTokens( storeCursors ) ) : loader.getPropertyKeyTokens( storeCursors ) ); + labels.setInitialTokens( lenient ? unique( loader.getLabelTokens( storeCursors ) ) : loader.getLabelTokens( storeCursors ) ); + relationshipTypes.setInitialTokens( + lenient ? unique( loader.getRelationshipTypeTokens( storeCursors ) ) : loader.getRelationshipTypeTokens( storeCursors ) ); + return new TokenHolders( propertyKeys, labels, relationshipTypes ); + } + catch ( IOException e ) + { + throw new UncheckedIOException( e ); + } + } + + private static List unique( List tokens ) + { + if ( !tokens.isEmpty() ) + { + Set names = new HashSet<>( tokens.size() ); + int i = 0; + while ( i < tokens.size() ) + { + if ( names.add( tokens.get( i ).name() ) ) + { + i++; + } + else + { + // Remove the token at the given index, by replacing it with the last token in the list. + // This changes the order of elements, but can be done in constant time instead of linear time. + int lastIndex = tokens.size() - 1; + NamedToken endToken = tokens.remove( lastIndex ); + if ( i < lastIndex ) + { + tokens.set( i, endToken ); + } + } + } + } + return tokens; + } + + @Override + public CommandReaderFactory commandReaderFactory() { + return InMemoryStorageCommandReaderFactory511.INSTANCE; + } + @Override + public void consistencyCheck( + FileSystemAbstraction fileSystem, + DatabaseLayout layout, + Config config, + PageCache pageCache, + IndexProviderMap indexProviders, + InternalLog reportLog, + InternalLog verboseLog, + ConsistencySummaryStatistics summary, + int numberOfThreads, + long maxOffHeapCachingMemory, + OutputStream progressOutput, + boolean verbose, + ConsistencyFlags flags, + CursorContextFactory contextFactory, + PageCacheTracer pageCacheTracer, + LogTailMetadata logTailMetadata + ) { + // we can do no-op, since our "database" is _always_ consistent + } + + @Override + public ImmutableSet getStoreOpenOptions( + FileSystemAbstraction fs, + PageCache pageCache, + DatabaseLayout layout, + CursorContextFactory contextFactory + ) { + // Not sure about this, empty set is returned when the store files are in `little-endian` format + // See: `org.neo4j.kernel.impl.store.format.PageCacheOptionsSelector.select` + return Sets.immutable.empty(); + } + + @Override + public StoreId retrieveStoreId( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) throws IOException { + return StoreId.retrieveFromStore(fs, databaseLayout, pageCache, cursorContext); + } + + + @Override + public Optional versionInformation(StoreVersionIdentifier storeVersionIdentifier) { + return Optional.of(new InMemoryStoreVersion()); + } + + @Override + public void resetMetadata( + FileSystemAbstraction fileSystemAbstraction, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + CursorContextFactory cursorContextFactory, + PageCacheTracer pageCacheTracer, + StoreId storeId, + UUID externalStoreId + ) { + throw new UnsupportedOperationException(); + } + + @Override + public IncrementalBatchImporter incrementalBatchImporter( + DatabaseLayout databaseLayout, + FileSystemAbstraction fileSystemAbstraction, + PageCacheTracer pageCacheTracer, + Configuration configuration, + LogService logService, + PrintStream printStream, + boolean b, + AdditionalInitialIds additionalInitialIds, + ThrowingSupplier throwingSupplier, + Config config, + Monitor monitor, + JobScheduler jobScheduler, + Collector collector, + LogFilesInitializer logFilesInitializer, + IndexImporterFactory indexImporterFactory, + MemoryTracker memoryTracker, + CursorContextFactory cursorContextFactory, + IndexProvidersAccess indexProvidersAccess + ) { + throw new UnsupportedOperationException(); + } + + @Override + public LockManager createLockManager(Config config, SystemNanoClock systemNanoClock) { + return LockManager.NO_LOCKS_LOCK_MANAGER; + } + + @Override + public List listStorageFiles( + FileSystemAbstraction fileSystem, DatabaseLayout databaseLayout + ) { + return Collections.emptyList(); + } + + @Override + public StorageFilesState checkStoreFileState( + FileSystemAbstraction fs, DatabaseLayout databaseLayout, PageCache pageCache + ) { + return StorageFilesState.recoveredState(); + } +} diff --git a/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/InMemoryStorageEngineImpl.java b/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/InMemoryStorageEngineImpl.java new file mode 100644 index 0000000000..7a80d4e67b --- /dev/null +++ b/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/InMemoryStorageEngineImpl.java @@ -0,0 +1,366 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._511; + +import org.neo4j.configuration.Config; +import org.neo4j.counts.CountsStore; +import org.neo4j.exceptions.KernelException; +import org.neo4j.gds.compat.TokenManager; +import org.neo4j.gds.config.GraphProjectConfig; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.gds.core.loading.GraphStoreCatalog; +import org.neo4j.gds.storageengine.InMemoryDatabaseCreationCatalog; +import org.neo4j.gds.storageengine.InMemoryTransactionStateVisitor; +import org.neo4j.internal.diagnostics.DiagnosticsLogger; +import org.neo4j.internal.recordstorage.InMemoryStorageReader511; +import org.neo4j.internal.schema.StorageEngineIndexingBehaviour; +import org.neo4j.io.fs.WritableChannel; +import org.neo4j.io.layout.DatabaseLayout; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.tracing.DatabaseFlushEvent; +import org.neo4j.kernel.KernelVersion; +import org.neo4j.kernel.impl.store.stats.StoreEntityCounters; +import org.neo4j.kernel.lifecycle.Lifecycle; +import org.neo4j.kernel.lifecycle.LifecycleAdapter; +import org.neo4j.lock.LockGroup; +import org.neo4j.lock.LockService; +import org.neo4j.lock.LockTracer; +import org.neo4j.lock.ResourceLocker; +import org.neo4j.logging.InternalLog; +import org.neo4j.memory.MemoryTracker; +import org.neo4j.storageengine.api.CommandBatchToApply; +import org.neo4j.storageengine.api.CommandCreationContext; +import org.neo4j.storageengine.api.CommandStream; +import org.neo4j.storageengine.api.IndexUpdateListener; +import org.neo4j.storageengine.api.InternalErrorTracer; +import org.neo4j.storageengine.api.MetadataProvider; +import org.neo4j.storageengine.api.StorageCommand; +import org.neo4j.storageengine.api.StorageEngine; +import org.neo4j.storageengine.api.StorageEngineFactory; +import org.neo4j.storageengine.api.StorageLocks; +import org.neo4j.storageengine.api.StorageReader; +import org.neo4j.storageengine.api.StoreFileMetadata; +import org.neo4j.storageengine.api.StoreId; +import org.neo4j.storageengine.api.TransactionApplicationMode; +import org.neo4j.storageengine.api.cursor.StoreCursors; +import org.neo4j.storageengine.api.enrichment.Enrichment; +import org.neo4j.storageengine.api.enrichment.EnrichmentCommand; +import org.neo4j.storageengine.api.txstate.ReadableTransactionState; +import org.neo4j.storageengine.api.txstate.TxStateVisitor; +import org.neo4j.storageengine.api.txstate.validation.TransactionValidatorFactory; +import org.neo4j.time.SystemNanoClock; +import org.neo4j.token.TokenHolders; + +import java.util.Collection; +import java.util.List; +import java.util.stream.Collectors; + +import static org.neo4j.gds.utils.StringFormatting.formatWithLocale; + +public final class InMemoryStorageEngineImpl implements StorageEngine { + + public static final byte ID = 42; + private final MetadataProvider metadataProvider; + private final CypherGraphStore graphStore; + private final DatabaseLayout databaseLayout; + private final InMemoryTransactionStateVisitor txStateVisitor; + + private final CommandCreationContext commandCreationContext; + + private final TokenManager tokenManager; + private final InMemoryCountsStoreImpl countsStore; + + private static final StorageEngineIndexingBehaviour INDEXING_BEHAVIOUR = new StorageEngineIndexingBehaviour() { + @Override + public boolean useNodeIdsInRelationshipTokenIndex() { + return false; + } + + @Override + public boolean requireCoordinationLocks() { + return false; + } + + @Override + public int nodesPerPage() { + return 0; + } + + @Override + public int relationshipsPerPage() { + return 0; + } + }; + + InMemoryStorageEngineImpl( + DatabaseLayout databaseLayout, + TokenHolders tokenHolders + ) { + this.databaseLayout = databaseLayout; + this.graphStore = getGraphStoreFromCatalog(databaseLayout.getDatabaseName()); + this.txStateVisitor = new InMemoryTransactionStateVisitor(graphStore, tokenHolders); + this.commandCreationContext = new InMemoryCommandCreationContextImpl(); + this.tokenManager = new TokenManager( + tokenHolders, + InMemoryStorageEngineImpl.this.txStateVisitor, + InMemoryStorageEngineImpl.this.graphStore, + commandCreationContext + ); + InMemoryStorageEngineImpl.this.graphStore.initialize(tokenHolders); + this.countsStore = new InMemoryCountsStoreImpl(graphStore, tokenHolders); + this.metadataProvider = new InMemoryMetaDataProviderImpl(); + } + + private static CypherGraphStore getGraphStoreFromCatalog(String databaseName) { + var graphName = InMemoryDatabaseCreationCatalog.getRegisteredDbCreationGraphName(databaseName); + return (CypherGraphStore) GraphStoreCatalog.getAllGraphStores() + .filter(graphStoreWithUserNameAndConfig -> graphStoreWithUserNameAndConfig + .config() + .graphName() + .equals(graphName)) + .findFirst() + .orElseThrow(() -> new IllegalArgumentException(formatWithLocale( + "No graph with name `%s` was found in GraphStoreCatalog. Available graph names are %s", + graphName, + GraphStoreCatalog.getAllGraphStores() + .map(GraphStoreCatalog.GraphStoreWithUserNameAndConfig::config) + .map(GraphProjectConfig::graphName) + .collect(Collectors.toList()) + ))) + .graphStore(); + } + + @Override + public StoreEntityCounters storeEntityCounters() { + return new StoreEntityCounters() { + @Override + public long nodes() { + return graphStore.nodeCount(); + } + + @Override + public long relationships() { + return graphStore.relationshipCount(); + } + + @Override + public long properties() { + return graphStore.nodePropertyKeys().size() + graphStore.relationshipPropertyKeys().size(); + } + + @Override + public long relationshipTypes() { + return graphStore.relationshipTypes().size(); + } + + @Override + public long allNodesCountStore(CursorContext cursorContext) { + return graphStore.nodeCount(); + } + + @Override + public long allRelationshipsCountStore(CursorContext cursorContext) { + return graphStore.relationshipCount(); + } + }; + } + + @Override + public StoreCursors createStorageCursors(CursorContext initialContext) { + return StoreCursors.NULL; + } + + @Override + public StorageLocks createStorageLocks(ResourceLocker locker) { + return new InMemoryStorageLocksImpl(locker); + } + + @Override + public List createCommands( + ReadableTransactionState state, + StorageReader storageReader, + CommandCreationContext creationContext, + LockTracer lockTracer, + TxStateVisitor.Decorator additionalTxStateVisitor, + CursorContext cursorContext, + StoreCursors storeCursors, + MemoryTracker memoryTracker + ) throws KernelException { + state.accept(txStateVisitor); + return List.of(); + } + + @Override + public void dumpDiagnostics(InternalLog internalLog, DiagnosticsLogger diagnosticsLogger) { + } + + @Override + public List createUpgradeCommands( + KernelVersion versionToUpgradeFrom, + KernelVersion versionToUpgradeTo + ) { + return List.of(); + } + + @Override + public StoreId retrieveStoreId() { + return metadataProvider.getStoreId(); + } + + @Override + public StorageEngineIndexingBehaviour indexingBehaviour() { + return INDEXING_BEHAVIOUR; + } + + @Override + public StorageReader newReader() { + return new InMemoryStorageReader511(graphStore, tokenManager.tokenHolders(), countsStore); + } + + @Override + public void addIndexUpdateListener(IndexUpdateListener listener) { + + } + + @Override + public void apply(CommandBatchToApply batch, TransactionApplicationMode mode) { + } + + @Override + public void init() { + } + + @Override + public void start() { + + } + + @Override + public void stop() { + shutdown(); + } + + @Override + public void shutdown() { + InMemoryDatabaseCreationCatalog.removeDatabaseEntry(databaseLayout.getDatabaseName()); + } + + @Override + public void listStorageFiles( + Collection atomic, Collection replayable + ) { + + } + + @Override + public Lifecycle schemaAndTokensLifecycle() { + return new LifecycleAdapter() { + @Override + public void init() { + + } + }; + } + + @Override + public CountsStore countsAccessor() { + return countsStore; + } + + @Override + public MetadataProvider metadataProvider() { + return metadataProvider; + } + + @Override + public String name() { + return "gds in-memory storage engine"; + } + + @Override + public byte id() { + return ID; + } + + @Override + public CommandCreationContext newCommandCreationContext(boolean multiVersioned) { + return commandCreationContext; + } + + @Override + public TransactionValidatorFactory createTransactionValidatorFactory( + StorageEngineFactory storageEngineFactory, + Config config, + SystemNanoClock systemNanoClock + ) { + return TransactionValidatorFactory.EMPTY_VALIDATOR_FACTORY; + } + + @Override + public void lockRecoveryCommands( + CommandStream commands, LockService lockService, LockGroup lockGroup, TransactionApplicationMode mode + ) { + + } + + @Override + public void rollback(ReadableTransactionState txState, CursorContext cursorContext) { + // rollback is not supported but it is also called when we fail for something else + // that we do not support, such as removing node properties + // TODO: do we want to inspect the txState to infer if rollback was called explicitly or not? + } + + @Override + public void checkpoint(DatabaseFlushEvent flushEvent, CursorContext cursorContext) { + // checkpoint is not supported but it is also called when we fail for something else + // that we do not support, such as removing node properties + } + + @Override + public void preAllocateStoreFilesForCommands(CommandBatchToApply batch, TransactionApplicationMode mode) { + // GDS has its own mechanism of memory allocation, so we don't need this + } + + @Override + public EnrichmentCommand createEnrichmentCommand(KernelVersion kernelVersion, Enrichment enrichment) { + return new EnrichmentCommand() { + + @Override + public Enrichment enrichment() { + return null; + } + + @Override + public void serialize(WritableChannel channel) { + + } + + @Override + public KernelVersion kernelVersion() { + return kernelVersion; + } + }; + } + + @Override + public InternalErrorTracer internalErrorTracer() { + return InternalErrorTracer.NO_TRACER; + } +} diff --git a/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/InMemoryStorageLocksImpl.java b/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/InMemoryStorageLocksImpl.java new file mode 100644 index 0000000000..a7ac028ce8 --- /dev/null +++ b/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/InMemoryStorageLocksImpl.java @@ -0,0 +1,86 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._511; + +import org.neo4j.lock.LockTracer; +import org.neo4j.lock.ResourceLocker; +import org.neo4j.storageengine.api.StorageLocks; +import org.neo4j.storageengine.api.txstate.ReadableTransactionState; + +public class InMemoryStorageLocksImpl implements StorageLocks { + + InMemoryStorageLocksImpl(ResourceLocker locker) {} + + @Override + public void acquireExclusiveNodeLock(LockTracer lockTracer, long... ids) {} + + @Override + public void releaseExclusiveNodeLock(long... ids) {} + + @Override + public void acquireSharedNodeLock(LockTracer lockTracer, long... ids) {} + + @Override + public void releaseSharedNodeLock(long... ids) {} + + @Override + public void acquireExclusiveRelationshipLock(LockTracer lockTracer, long... ids) {} + + @Override + public void releaseExclusiveRelationshipLock(long... ids) {} + + @Override + public void acquireSharedRelationshipLock(LockTracer lockTracer, long... ids) {} + + @Override + public void releaseSharedRelationshipLock(long... ids) {} + + @Override + public void acquireRelationshipCreationLock( + LockTracer lockTracer, + long sourceNode, + long targetNode, + boolean sourceNodeAddedInTx, + boolean targetNodeAddedInTx + ) { + } + + @Override + public void acquireRelationshipDeletionLock( + LockTracer lockTracer, + long sourceNode, + long targetNode, + long relationship, + boolean relationshipAddedInTx, + boolean sourceNodeAddedInTx, + boolean targetNodeAddedInTx + ) { + } + + @Override + public void acquireNodeDeletionLock( + ReadableTransactionState readableTransactionState, + LockTracer lockTracer, + long node + ) {} + + @Override + public void acquireNodeLabelChangeLock(LockTracer lockTracer, long node, int labelId) {} +} diff --git a/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/InMemoryStoreVersion.java b/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/InMemoryStoreVersion.java new file mode 100644 index 0000000000..946e757906 --- /dev/null +++ b/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/InMemoryStoreVersion.java @@ -0,0 +1,70 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._511; + +import org.neo4j.configuration.Config; +import org.neo4j.storageengine.api.StoreVersion; +import org.neo4j.storageengine.api.format.Capability; +import org.neo4j.storageengine.api.format.CapabilityType; + +import java.util.Optional; + +public class InMemoryStoreVersion implements StoreVersion { + + public static final String STORE_VERSION = "gds-experimental"; + + @Override + public String getStoreVersionUserString() { + return "Unknown"; + } + + @Override + public Optional successorStoreVersion(Config config) { + return Optional.empty(); + } + + @Override + public String formatName() { + return getClass().getSimpleName(); + } + + @Override + public boolean onlyForMigration() { + return false; + } + + @Override + public boolean hasCapability(Capability capability) { + return false; + } + + @Override + public boolean hasCompatibleCapabilities( + StoreVersion otherVersion, CapabilityType type + ) { + return false; + } + + @Override + public String introductionNeo4jVersion() { + return "foo"; + } + +} diff --git a/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/InMemoryTransactionIdStoreImpl.java b/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/InMemoryTransactionIdStoreImpl.java new file mode 100644 index 0000000000..c95a6524be --- /dev/null +++ b/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/InMemoryTransactionIdStoreImpl.java @@ -0,0 +1,117 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._511; + +import org.neo4j.internal.recordstorage.AbstractTransactionIdStore; +import org.neo4j.io.pagecache.context.TransactionIdSnapshot; +import org.neo4j.kernel.impl.transaction.log.LogPosition; +import org.neo4j.storageengine.api.ClosedTransactionMetadata; +import org.neo4j.storageengine.api.TransactionId; +import org.neo4j.storageengine.api.TransactionIdStore; + +public class InMemoryTransactionIdStoreImpl extends AbstractTransactionIdStore { + + @Override + protected void initLastCommittedAndClosedTransactionId( + long previouslyCommittedTxId, + int checksum, + long previouslyCommittedTxCommitTimestamp, + long previouslyCommittedTxLogByteOffset, + long previouslyCommittedTxLogVersion + ) { + this.setLastCommittedAndClosedTransactionId( + previouslyCommittedTxId, + checksum, + previouslyCommittedTxCommitTimestamp, + TransactionIdStore.UNKNOWN_CONSENSUS_INDEX, + previouslyCommittedTxLogByteOffset, + previouslyCommittedTxLogVersion + ); + } + + @Override + public ClosedTransactionMetadata getLastClosedTransaction() { + long[] metaData = this.closedTransactionId.get(); + return new ClosedTransactionMetadata( + metaData[0], + new LogPosition(metaData[1], metaData[2]), + (int) metaData[3], + metaData[4], + metaData[5] + ); + } + + @Override + public TransactionIdSnapshot getClosedTransactionSnapshot() { + return new TransactionIdSnapshot(this.getLastClosedTransactionId()); + } + + @Override + protected TransactionId transactionId(long transactionId, int checksum, long commitTimestamp) { + return new TransactionId(transactionId, checksum, commitTimestamp, TransactionIdStore.UNKNOWN_CONSENSUS_INDEX); + } + + @Override + public void transactionCommitted(long transactionId, int checksum, long commitTimestamp, long consensusIndex) { + + } + + @Override + public void setLastCommittedAndClosedTransactionId( + long transactionId, + int checksum, + long commitTimestamp, + long consensusIndex, + long byteOffset, + long logVersion + ) { + + } + + @Override + public void transactionClosed( + long transactionId, + long logVersion, + long byteOffset, + int checksum, + long commitTimestamp, + long consensusIndex + ) { + this.closedTransactionId.offer( + transactionId, + new long[]{logVersion, byteOffset, checksum, commitTimestamp, consensusIndex} + ); + } + + @Override + public void resetLastClosedTransaction( + long transactionId, + long logVersion, + long byteOffset, + int checksum, + long commitTimestamp, + long consensusIndex + ) { + this.closedTransactionId.set( + transactionId, + new long[]{logVersion, byteOffset, checksum, commitTimestamp, consensusIndex} + ); + } +} diff --git a/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/InMemoryVersionCheck.java b/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/InMemoryVersionCheck.java new file mode 100644 index 0000000000..8cb999f8c7 --- /dev/null +++ b/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/InMemoryVersionCheck.java @@ -0,0 +1,61 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._511; + +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.kernel.impl.store.format.FormatFamily; +import org.neo4j.storageengine.api.StoreVersionCheck; +import org.neo4j.storageengine.api.StoreVersionIdentifier; + +import static org.neo4j.gds.compat._511.InMemoryStoreVersion.STORE_VERSION; + +public class InMemoryVersionCheck implements StoreVersionCheck { + + private static final StoreVersionIdentifier STORE_IDENTIFIER = new StoreVersionIdentifier( + STORE_VERSION, + FormatFamily.STANDARD.name(), + 0, + 0 + ); + + @Override + public boolean isCurrentStoreVersionFullySupported(CursorContext cursorContext) { + return true; + } + + @Override + public MigrationCheckResult getAndCheckMigrationTargetVersion(String formatFamily, CursorContext cursorContext) { + return new StoreVersionCheck.MigrationCheckResult(MigrationOutcome.NO_OP, STORE_IDENTIFIER, null, null); + } + + @Override + public UpgradeCheckResult getAndCheckUpgradeTargetVersion(CursorContext cursorContext) { + return new StoreVersionCheck.UpgradeCheckResult(UpgradeOutcome.NO_OP, STORE_IDENTIFIER, null, null); + } + + @Override + public String getIntroductionVersionFromVersion(StoreVersionIdentifier storeVersionIdentifier) { + return STORE_VERSION; + } + + public StoreVersionIdentifier findLatestVersion(String s) { + return STORE_IDENTIFIER; + } +} diff --git a/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/StorageEngineProxyFactoryImpl.java b/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/StorageEngineProxyFactoryImpl.java new file mode 100644 index 0000000000..687468d4e4 --- /dev/null +++ b/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/StorageEngineProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._511; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jVersion; +import org.neo4j.gds.compat.StorageEngineProxyApi; +import org.neo4j.gds.compat.StorageEngineProxyFactory; + +@ServiceProvider +public class StorageEngineProxyFactoryImpl implements StorageEngineProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return version == Neo4jVersion.V_5_11; + } + + @Override + public StorageEngineProxyApi load() { + return new StorageEngineProxyImpl(); + } + + @Override + public String description() { + return "Storage Engine 5.11"; + } +} diff --git a/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/StorageEngineProxyImpl.java b/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/StorageEngineProxyImpl.java new file mode 100644 index 0000000000..d2688f8a37 --- /dev/null +++ b/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/StorageEngineProxyImpl.java @@ -0,0 +1,140 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._511; + +import org.neo4j.common.Edition; +import org.neo4j.configuration.Config; +import org.neo4j.configuration.GraphDatabaseInternalSettings; +import org.neo4j.dbms.api.DatabaseManagementService; +import org.neo4j.gds.compat.AbstractInMemoryNodeCursor; +import org.neo4j.gds.compat.AbstractInMemoryNodePropertyCursor; +import org.neo4j.gds.compat.AbstractInMemoryRelationshipPropertyCursor; +import org.neo4j.gds.compat.AbstractInMemoryRelationshipTraversalCursor; +import org.neo4j.gds.compat.GdsDatabaseManagementServiceBuilder; +import org.neo4j.gds.compat.GraphDatabaseApiProxy; +import org.neo4j.gds.compat.StorageEngineProxyApi; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.graphdb.Direction; +import org.neo4j.graphdb.GraphDatabaseService; +import org.neo4j.internal.recordstorage.AbstractInMemoryRelationshipScanCursor; +import org.neo4j.io.layout.DatabaseLayout; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.RelationshipSelection; +import org.neo4j.storageengine.api.StorageEngine; +import org.neo4j.storageengine.api.StorageEntityCursor; +import org.neo4j.storageengine.api.StoragePropertyCursor; +import org.neo4j.storageengine.api.StorageRelationshipTraversalCursor; +import org.neo4j.token.TokenHolders; + +import static org.neo4j.configuration.GraphDatabaseSettings.db_format; + +public class StorageEngineProxyImpl implements StorageEngineProxyApi { + + @Override + public void initRelationshipTraversalCursorForRelType( + StorageRelationshipTraversalCursor cursor, + long sourceNodeId, + int relTypeToken + ) { + var relationshipSelection = RelationshipSelection.selection( + relTypeToken, + Direction.OUTGOING + ); + cursor.init(sourceNodeId, -1, relationshipSelection); + } + + @Override + public StorageEngine createInMemoryStorageEngine(DatabaseLayout databaseLayout, TokenHolders tokenHolders) { + return new InMemoryStorageEngineImpl(databaseLayout, tokenHolders); + } + + @Override + public void createInMemoryDatabase( + DatabaseManagementService dbms, + String dbName, + Config config + ) { + config.set(db_format, InMemoryStorageEngineFactory.IN_MEMORY_STORAGE_ENGINE_NAME); + dbms.createDatabase(dbName, config); + } + + @Override + public GraphDatabaseService startAndGetInMemoryDatabase(DatabaseManagementService dbms, String dbName) { + dbms.startDatabase(dbName); + return dbms.database(dbName); + } + + @Override + public GdsDatabaseManagementServiceBuilder setSkipDefaultIndexesOnCreationSetting(GdsDatabaseManagementServiceBuilder dbmsBuilder) { + return dbmsBuilder.setConfig(GraphDatabaseInternalSettings.skip_default_indexes_on_creation, true); + } + + @Override + public AbstractInMemoryNodeCursor inMemoryNodeCursor(CypherGraphStore graphStore, TokenHolders tokenHolders) { + return new InMemoryNodeCursor(graphStore, tokenHolders); + } + + @Override + public AbstractInMemoryNodePropertyCursor inMemoryNodePropertyCursor( + CypherGraphStore graphStore, + TokenHolders tokenHolders + ) { + return new InMemoryNodePropertyCursor(graphStore, tokenHolders); + } + + @Override + public AbstractInMemoryRelationshipTraversalCursor inMemoryRelationshipTraversalCursor( + CypherGraphStore graphStore, TokenHolders tokenHolders + ) { + return new InMemoryRelationshipTraversalCursor(graphStore, tokenHolders); + } + + @Override + public AbstractInMemoryRelationshipScanCursor inMemoryRelationshipScanCursor( + CypherGraphStore graphStore, TokenHolders tokenHolders + ) { + return new InMemoryRelationshipScanCursor(graphStore, tokenHolders); + } + + @Override + public AbstractInMemoryRelationshipPropertyCursor inMemoryRelationshipPropertyCursor( + CypherGraphStore graphStore, TokenHolders tokenHolders + ) { + return new InMemoryRelationshipPropertyCursor(graphStore, tokenHolders); + } + + @Override + public void properties( + StorageEntityCursor storageCursor, StoragePropertyCursor propertyCursor, int[] propertySelection + ) { + PropertySelection selection; + if (propertySelection.length == 0) { + selection = PropertySelection.ALL_PROPERTIES; + } else { + selection = PropertySelection.selection(propertySelection); + } + storageCursor.properties(propertyCursor, selection); + } + + @Override + public Edition dbmsEdition(GraphDatabaseService databaseService) { + return GraphDatabaseApiProxy.dbmsInfo(databaseService).edition; + } +} diff --git a/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository511.java b/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository511.java new file mode 100644 index 0000000000..542a9239cd --- /dev/null +++ b/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository511.java @@ -0,0 +1,71 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.internal.recordstorage; + +import org.neo4j.storageengine.api.LogVersionRepository; + +import java.util.concurrent.atomic.AtomicLong; + +public class InMemoryLogVersionRepository511 implements LogVersionRepository { + + private final AtomicLong logVersion; + private final AtomicLong checkpointLogVersion; + + public InMemoryLogVersionRepository511() { + this(0, 0); + } + + private InMemoryLogVersionRepository511(long initialLogVersion, long initialCheckpointLogVersion) { + this.logVersion = new AtomicLong(); + this.checkpointLogVersion = new AtomicLong(); + this.logVersion.set(initialLogVersion); + this.checkpointLogVersion.set(initialCheckpointLogVersion); + } + + @Override + public void setCurrentLogVersion(long version) { + this.logVersion.set(version); + } + + @Override + public long incrementAndGetVersion() { + return this.logVersion.incrementAndGet(); + } + + @Override + public void setCheckpointLogVersion(long version) { + this.checkpointLogVersion.set(version); + } + + @Override + public long incrementAndGetCheckpointLogVersion() { + return this.checkpointLogVersion.incrementAndGet(); + } + + @Override + public long getCurrentLogVersion() { + return this.logVersion.get(); + } + + @Override + public long getCheckpointLogVersion() { + return this.checkpointLogVersion.get(); + } +} diff --git a/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory511.java b/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory511.java new file mode 100644 index 0000000000..004ffea98d --- /dev/null +++ b/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory511.java @@ -0,0 +1,43 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.internal.recordstorage; + +import org.neo4j.kernel.KernelVersion; +import org.neo4j.storageengine.api.CommandReader; +import org.neo4j.storageengine.api.CommandReaderFactory; + +public class InMemoryStorageCommandReaderFactory511 implements CommandReaderFactory { + + public static final CommandReaderFactory INSTANCE = new InMemoryStorageCommandReaderFactory511(); + + @Override + public CommandReader get(KernelVersion kernelVersion) { + switch (kernelVersion) { + case V4_2: + return LogCommandSerializationV4_2.INSTANCE; + case V4_3_D4: + return LogCommandSerializationV4_3_D3.INSTANCE; + case V5_0: + return LogCommandSerializationV5_0.INSTANCE; + default: + throw new IllegalArgumentException("Unsupported kernel version " + kernelVersion); + } + } +} diff --git a/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageReader511.java b/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageReader511.java new file mode 100644 index 0000000000..9ba49899c7 --- /dev/null +++ b/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageReader511.java @@ -0,0 +1,333 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.internal.recordstorage; + +import org.eclipse.collections.api.set.primitive.IntSet; +import org.eclipse.collections.impl.set.immutable.primitive.ImmutableIntSetFactoryImpl; +import org.neo4j.common.EntityType; +import org.neo4j.common.TokenNameLookup; +import org.neo4j.counts.CountsStore; +import org.neo4j.gds.compat._511.InMemoryNodeCursor; +import org.neo4j.gds.compat._511.InMemoryPropertyCursor; +import org.neo4j.gds.compat._511.InMemoryRelationshipScanCursor; +import org.neo4j.gds.compat._511.InMemoryRelationshipTraversalCursor; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.internal.schema.ConstraintDescriptor; +import org.neo4j.internal.schema.IndexDescriptor; +import org.neo4j.internal.schema.IndexType; +import org.neo4j.internal.schema.SchemaDescriptor; +import org.neo4j.internal.schema.constraints.IndexBackedConstraintDescriptor; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.memory.MemoryTracker; +import org.neo4j.storageengine.api.AllNodeScan; +import org.neo4j.storageengine.api.AllRelationshipsScan; +import org.neo4j.storageengine.api.StorageNodeCursor; +import org.neo4j.storageengine.api.StoragePropertyCursor; +import org.neo4j.storageengine.api.StorageReader; +import org.neo4j.storageengine.api.StorageRelationshipScanCursor; +import org.neo4j.storageengine.api.StorageRelationshipTraversalCursor; +import org.neo4j.storageengine.api.StorageSchemaReader; +import org.neo4j.storageengine.api.cursor.StoreCursors; +import org.neo4j.token.TokenHolders; + +import java.util.Collection; +import java.util.Collections; +import java.util.Iterator; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Function; + +public class InMemoryStorageReader511 implements StorageReader { + + protected final CypherGraphStore graphStore; + protected final TokenHolders tokenHolders; + protected final CountsStore counts; + private final Map, Object> dependantState; + private boolean closed; + + public InMemoryStorageReader511( + CypherGraphStore graphStore, + TokenHolders tokenHolders, + CountsStore counts + ) { + this.graphStore = graphStore; + + this.tokenHolders = tokenHolders; + this.counts = counts; + this.dependantState = new ConcurrentHashMap<>(); + } + + @Override + public Collection uniquenessConstraintsGetRelated( + long[] changedLabels, + long[] unchangedLabels, + int[] propertyKeyIds, + boolean propertyKeyListIsComplete, + EntityType entityType + ) { + return Collections.emptyList(); + } + + @Override + public long relationshipsGetCount(CursorContext cursorTracer) { + return graphStore.relationshipCount(); + } + + @Override + public boolean nodeExists(long id, StoreCursors storeCursors) { + var originalId = graphStore.nodes().toOriginalNodeId(id); + return graphStore.nodes().containsOriginalId(originalId); + } + + @Override + public boolean relationshipExists(long id, StoreCursors storeCursors) { + return true; + } + + @Override + public StorageNodeCursor allocateNodeCursor( + CursorContext cursorContext, StoreCursors storeCursors + ) { + return new InMemoryNodeCursor(graphStore, tokenHolders); + } + + @Override + public StoragePropertyCursor allocatePropertyCursor( + CursorContext cursorContext, StoreCursors storeCursors, MemoryTracker memoryTracker + ) { + return new InMemoryPropertyCursor(graphStore, tokenHolders); + } + + @Override + public StorageRelationshipTraversalCursor allocateRelationshipTraversalCursor( + CursorContext cursorContext, StoreCursors storeCursors + ) { + return new InMemoryRelationshipTraversalCursor(graphStore, tokenHolders); + } + + @Override + public StorageRelationshipScanCursor allocateRelationshipScanCursor( + CursorContext cursorContext, StoreCursors storeCursors + ) { + return new InMemoryRelationshipScanCursor(graphStore, tokenHolders); + } + + @Override + public IndexDescriptor indexGetForSchemaAndType( + SchemaDescriptor descriptor, IndexType type + ) { + return null; + } + + @Override + public AllRelationshipsScan allRelationshipScan() { + return new AbstractInMemoryAllRelationshipScan() { + @Override + boolean scanRange(AbstractInMemoryRelationshipScanCursor cursor, long start, long stopInclusive) { + return cursor.scanRange(start, stopInclusive); + } + + @Override + public boolean scanBatch(long sizeHint, AbstractInMemoryRelationshipScanCursor cursor) { + return super.scanBatch(sizeHint, cursor); + } + }; + } + + @Override + public Iterator indexGetForSchema(SchemaDescriptor descriptor) { + return Collections.emptyIterator(); + } + + @Override + public Iterator indexesGetForLabel(int labelId) { + return Collections.emptyIterator(); + } + + @Override + public Iterator indexesGetForRelationshipType(int relationshipType) { + return Collections.emptyIterator(); + } + + @Override + public IndexDescriptor indexGetForName(String name) { + return null; + } + + @Override + public ConstraintDescriptor constraintGetForName(String name) { + return null; + } + + @Override + public boolean indexExists(IndexDescriptor index) { + return false; + } + + @Override + public Iterator indexesGetAll() { + return Collections.emptyIterator(); + } + + @Override + public Collection valueIndexesGetRelated( + long[] tokens, int propertyKeyId, EntityType entityType + ) { + return valueIndexesGetRelated(tokens, new int[]{propertyKeyId}, entityType); + } + + @Override + public Collection valueIndexesGetRelated( + long[] tokens, int[] propertyKeyIds, EntityType entityType + ) { + return Collections.emptyList(); + } + + @Override + public Collection uniquenessConstraintsGetRelated( + long[] labels, + int propertyKeyId, + EntityType entityType + ) { + return Collections.emptyList(); + } + + @Override + public Collection uniquenessConstraintsGetRelated( + long[] tokens, + int[] propertyKeyIds, + EntityType entityType + ) { + return Collections.emptyList(); + } + + @Override + public boolean hasRelatedSchema(long[] labels, int propertyKey, EntityType entityType) { + return false; + } + + @Override + public boolean hasRelatedSchema(int label, EntityType entityType) { + return false; + } + + @Override + public Iterator constraintsGetForSchema(SchemaDescriptor descriptor) { + return Collections.emptyIterator(); + } + + @Override + public boolean constraintExists(ConstraintDescriptor descriptor) { + return false; + } + + @Override + public Iterator constraintsGetForLabel(int labelId) { + return Collections.emptyIterator(); + } + + @Override + public Iterator constraintsGetForRelationshipType(int typeId) { + return Collections.emptyIterator(); + } + + @Override + public Iterator constraintsGetAll() { + return Collections.emptyIterator(); + } + + @Override + public IntSet constraintsGetPropertyTokensForLogicalKey(int token, EntityType entityType) { + return ImmutableIntSetFactoryImpl.INSTANCE.empty(); + } + + @Override + public Long indexGetOwningUniquenessConstraintId(IndexDescriptor index) { + return null; + } + + @Override + public long countsForNode(int labelId, CursorContext cursorContext) { + return counts.nodeCount(labelId, cursorContext); + } + + @Override + public long countsForRelationship(int startLabelId, int typeId, int endLabelId, CursorContext cursorContext) { + return counts.relationshipCount(startLabelId, typeId, endLabelId, cursorContext); + } + + @Override + public long nodesGetCount(CursorContext cursorContext) { + return graphStore.nodeCount(); + } + + @Override + public int labelCount() { + return graphStore.nodes().availableNodeLabels().size(); + } + + @Override + public int propertyKeyCount() { + int nodePropertyCount = graphStore + .schema() + .nodeSchema() + .unionProperties() + .size(); + int relPropertyCount = graphStore + .schema() + .relationshipSchema() + .unionProperties() + .size(); + + return nodePropertyCount + relPropertyCount; + } + + @Override + public int relationshipTypeCount() { + return graphStore.schema().relationshipSchema().entries().size(); + } + + @Override + public T getOrCreateSchemaDependantState(Class type, Function factory) { + return type.cast(dependantState.computeIfAbsent(type, key -> factory.apply(this))); + } + + @Override + public AllNodeScan allNodeScan() { + return new InMemoryNodeScan(); + } + + @Override + public void close() { + assert !closed; + closed = true; + } + + @Override + public StorageSchemaReader schemaSnapshot() { + return this; + } + + @Override + public TokenNameLookup tokenNameLookup() { + return tokenHolders; + } + +} diff --git a/gradle/dependencies.gradle b/gradle/dependencies.gradle index edad2b2b92..2d139e4348 100644 --- a/gradle/dependencies.gradle +++ b/gradle/dependencies.gradle @@ -12,6 +12,7 @@ ext { '5.9': properties.getOrDefault('neo4jVersion59', '5.9.0'), '5.10': properties.getOrDefault('neo4jVersion510', '5.10.0'), '5.11': properties.getOrDefault('neo4jVersion511', '5.11.0'), + '5.11': properties.getOrDefault('neo4jVersion511', '5.11.0'), ] neo4jDefault = neos.'4.4' diff --git a/neo4j-adapter/src/main/java/org/neo4j/gds/compat/Neo4jVersion.java b/neo4j-adapter/src/main/java/org/neo4j/gds/compat/Neo4jVersion.java index 066e56831f..10dfe4b431 100644 --- a/neo4j-adapter/src/main/java/org/neo4j/gds/compat/Neo4jVersion.java +++ b/neo4j-adapter/src/main/java/org/neo4j/gds/compat/Neo4jVersion.java @@ -38,6 +38,7 @@ public enum Neo4jVersion { V_5_8, V_5_9, V_5_10, + V_5_11, V_RC; @Override @@ -65,6 +66,8 @@ public String toString() { return "5.9"; case V_5_10: return "5.10"; + case V_5_11: + return "5.11"; case V_RC: return "rc"; default: @@ -74,7 +77,7 @@ public String toString() { public MajorMinorVersion semanticVersion() { if (this == V_RC) { - return ImmutableMajorMinorVersion.of(5, 11); + return ImmutableMajorMinorVersion.of(5, 12); } String version = toString(); @@ -158,6 +161,8 @@ static Neo4jVersion parse(String version) { } else if (minorVersion == 10) { return Neo4jVersion.V_5_10; } else if (minorVersion == 11) { + return Neo4jVersion.V_5_11; + } else if (minorVersion == 12) { return Neo4jVersion.V_RC; } } diff --git a/neo4j-adapter/src/test/java/org/neo4j/gds/compat/Neo4jVersionTest.java b/neo4j-adapter/src/test/java/org/neo4j/gds/compat/Neo4jVersionTest.java index a7170a9868..be948ec070 100644 --- a/neo4j-adapter/src/test/java/org/neo4j/gds/compat/Neo4jVersionTest.java +++ b/neo4j-adapter/src/test/java/org/neo4j/gds/compat/Neo4jVersionTest.java @@ -51,7 +51,8 @@ class Neo4jVersionTest { "5.8.0, V_5_8", "5.9.0, V_5_9", "5.10.0, V_5_10", - "5.11.0, V_RC", + "5.11.0, V_5_11", + "5.12.0, V_RC", }) void testParse(String input, Neo4jVersion expected) { assertEquals(expected.name(), Neo4jVersion.parse(input).name()); diff --git a/proc/sysinfo/src/test/java/org/neo4j/gds/SysInfoProcTest.java b/proc/sysinfo/src/test/java/org/neo4j/gds/SysInfoProcTest.java index 85e28e0bf2..786b0ca3c5 100644 --- a/proc/sysinfo/src/test/java/org/neo4j/gds/SysInfoProcTest.java +++ b/proc/sysinfo/src/test/java/org/neo4j/gds/SysInfoProcTest.java @@ -102,6 +102,11 @@ class SysInfoProcTest extends BaseProcTest { "Neo4j Settings 5.10", "Neo4j Settings 5.10 (placeholder)", + "Neo4j 5.11", + "Neo4j 5.11 (placeholder)", + "Neo4j Settings 5.11", + "Neo4j Settings 5.11 (placeholder)", + "Neo4j DEV", "Neo4j DEV (placeholder)", "Neo4j Settings DEV", @@ -242,6 +247,14 @@ void testSysInfoProc() throws IOException { "Neo4j 5.10" ); break; + case V_5_11: + expectedCompatibilities = Set.of( + "Neo4j Settings 5.11 (placeholder)", + "Neo4j Settings 5.11", + "Neo4j 5.11 (placeholder)", + "Neo4j 5.11" + ); + break; case V_RC: expectedCompatibilities = Set.of( "Neo4j Settings RC", From d4e2009efa077a7eebf5bf67c4e0ff157f33ddc6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonatan=20J=C3=A4derberg?= Date: Wed, 9 Aug 2023 18:13:17 +0200 Subject: [PATCH 201/273] Incremend GDS Aura version --- gradle/version.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle/version.gradle b/gradle/version.gradle index 3c3b5e41ce..fc2cd0d57f 100644 --- a/gradle/version.gradle +++ b/gradle/version.gradle @@ -1,6 +1,6 @@ ext { gdsBaseVersion = '2.4.4' - gdsAuraVersion = '29' + gdsAuraVersion = '30' gdsVersion = gdsBaseVersion + (rootProject.hasProperty('aurads') ? "+${gdsAuraVersion}" : "") } From b930282385ca3e757bb2ced8d38921afffdff4de Mon Sep 17 00:00:00 2001 From: ioannispan Date: Mon, 14 Aug 2023 17:36:54 +0200 Subject: [PATCH 202/273] Fix all shortest path docs --- .../AllPairsShortestPathSyntaxTest.java | 40 +++++++++++++++++++ .../all-pairs-shortest-path.adoc | 26 +++++++++--- 2 files changed, 60 insertions(+), 6 deletions(-) create mode 100644 doc-test/src/test/java/org/neo4j/gds/doc/syntax/AllPairsShortestPathSyntaxTest.java diff --git a/doc-test/src/test/java/org/neo4j/gds/doc/syntax/AllPairsShortestPathSyntaxTest.java b/doc-test/src/test/java/org/neo4j/gds/doc/syntax/AllPairsShortestPathSyntaxTest.java new file mode 100644 index 0000000000..a933f8d4e6 --- /dev/null +++ b/doc-test/src/test/java/org/neo4j/gds/doc/syntax/AllPairsShortestPathSyntaxTest.java @@ -0,0 +1,40 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.doc.syntax; + +import java.util.List; + +import static org.neo4j.gds.doc.syntax.SyntaxMode.STREAM; + +class AllPairsShortestPathSyntaxTest extends SyntaxTestBase { + + @Override + protected Iterable syntaxModes() { + return List.of( + SyntaxModeMeta.of(STREAM) + ); + } + + @Override + protected String adocFile() { + return "pages/alpha-algorithms/all-pairs-shortest-path.adoc"; + } + +} diff --git a/doc/modules/ROOT/pages/alpha-algorithms/all-pairs-shortest-path.adoc b/doc/modules/ROOT/pages/alpha-algorithms/all-pairs-shortest-path.adoc index a596d8111b..7f6ca93a25 100644 --- a/doc/modules/ROOT/pages/alpha-algorithms/all-pairs-shortest-path.adoc +++ b/doc/modules/ROOT/pages/alpha-algorithms/all-pairs-shortest-path.adoc @@ -43,6 +43,8 @@ Plain cypher does not support filtering `Infinity` values, so `gds.util.isFinite [[algorithm-all-pairs-shortest-path-syntax]] == Syntax +[.include-with-stream] +====== .The following will run the algorithm and stream results: [source, cypher, role=noplay] ---- @@ -50,18 +52,30 @@ CALL gds.alpha.allShortestPaths.stream( graphName: string, configuration: map ) -YIELD startNodeId, targetNodeId, distance +YIELD sourceNodeId, targetNodeId, distance ---- -.Parameters -[opts="header",cols="1,1,1,1,4"] +include::partial$/algorithms/common-configuration/common-parameters.adoc[] + +.Configuration +[opts="header",cols="3,2,3m,2,8"] |=== -| Name | Type | Default | Optional | Description +| Name | Type | Default | Optional | Description +include::partial$/algorithms/common-configuration/common-stream-stats-configuration-entries.adoc[] | xref:common-usage/running-algos.adoc#common-configuration-relationship-weight-property[relationshipWeightProperty] | String | null | yes | Name of the relationship property to use as weights. If unspecified, the algorithm runs unweighted. -| concurrency | Integer | 4 | yes | The number of concurrent threads used for running the algorithm. Also provides the default value for 'readConcurrency' and 'writeConcurrency'. This is dependent on the Neo4j edition; for more information, see xref:installation/System-requirements.adoc#system-requirements-cpu[CPU]. -| readConcurrency | Integer | value of 'concurrency' | yes | The number of concurrent threads used for reading the graph. + +|=== + +.Results +[opts="header",cols="1,1,6"] +|=== +| Name | Type | Description +| sourceNodeId | Integer | The source node. +| targetNodeId | Integer | The target node. +| distance | Float | The distance of the shortest path from source to target. |=== +====== [[algorithm-all-pairs-shortest-path-sample]] == All Pairs Shortest Path algorithm sample From 0a125ac5f5ff41e0f6c04d168c3d11264b52c837 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonatan=20J=C3=A4derberg?= Date: Tue, 15 Aug 2023 12:46:20 +0200 Subject: [PATCH 203/273] Publish compat for DB V5.11 --- build.gradle | 2 ++ settings.gradle | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/build.gradle b/build.gradle index a11e882cc0..2638b2315b 100644 --- a/build.gradle +++ b/build.gradle @@ -37,6 +37,7 @@ ext { project(':neo4j-kernel-adapter-5.8'), project(':neo4j-kernel-adapter-5.9'), project(':neo4j-kernel-adapter-5.10'), + project(':neo4j-kernel-adapter-5.11'), ], 'storage-engine-adapter': [ project(':storage-engine-adapter-4.4'), @@ -50,6 +51,7 @@ ext { project(':storage-engine-adapter-5.8'), project(':storage-engine-adapter-5.9'), project(':storage-engine-adapter-5.10'), + project(':storage-engine-adapter-5.11'), ] ] } diff --git a/settings.gradle b/settings.gradle index dc27ab8a74..1ea5e666db 100644 --- a/settings.gradle +++ b/settings.gradle @@ -160,6 +160,9 @@ project(':neo4j-kernel-adapter-5.9').projectDir = file('compatibility/5.9/neo4j- include('neo4j-kernel-adapter-5.10') project(':neo4j-kernel-adapter-5.10').projectDir = file('compatibility/5.10/neo4j-kernel-adapter') +include('neo4j-kernel-adapter-5.11') +project(':neo4j-kernel-adapter-5.11').projectDir = file('compatibility/5.11/neo4j-kernel-adapter') + include('neo4j-kernel-adapter-api') project(':neo4j-kernel-adapter-api').projectDir = file('compatibility/api/neo4j-kernel-adapter') @@ -268,6 +271,9 @@ project(':storage-engine-adapter-5.9').projectDir = file('compatibility/5.9/stor include('storage-engine-adapter-5.10') project(':storage-engine-adapter-5.10').projectDir = file('compatibility/5.10/storage-engine-adapter') +include('storage-engine-adapter-5.11') +project(':storage-engine-adapter-5.11').projectDir = file('compatibility/5.11/storage-engine-adapter') + include('storage-engine-adapter-api') project(':storage-engine-adapter-api').projectDir = file('compatibility/api/storage-engine-adapter') From cb8f748c6ff1a84c00e51eb99890f2e7f44dc0c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonatan=20J=C3=A4derberg?= Date: Tue, 15 Aug 2023 14:59:07 +0200 Subject: [PATCH 204/273] Record 2.4.4 in readme, supported versions --- README.adoc | 19 ++++++++++--------- .../supported-neo4j-versions.adoc | 1 + 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/README.adoc b/README.adoc index d991496d89..ed9abadf19 100644 --- a/README.adoc +++ b/README.adoc @@ -27,8 +27,9 @@ When installing GDS manually, please refer to the below compatibility matrix: |=== |GDS version | Neo4j version | Java Version .11+<.^|_GDS 2.4.x_ +|Neo4j 5.11.0 +.11+.^|Java 17 |Neo4j 5.10.0 -.10+.^|Java 17 |Neo4j 5.9.0 |Neo4j 5.8.0 |Neo4j 5.7.0 @@ -95,7 +96,7 @@ For the most basic set of features, like graph loading and the graph representat org.neo4j.gds core - 2.4.3 + 2.4.4 ---- @@ -107,21 +108,21 @@ The algorithms are located in the `algo-common`, `algo` and `alpha-algo` modules org.neo4j.gds algo-common - 2.4.3 + 2.4.4 org.neo4j.gds algo - 2.4.3 + 2.4.4 org.neo4j.gds alpha-algo - 2.4.3 + 2.4.4 ---- @@ -133,28 +134,28 @@ The procedures are located in the `proc-common`, `proc` and `alpha-proc` modules org.neo4j.gds proc-common - 2.4.3 + 2.4.4 org.neo4j.gds proc - 2.4.3 + 2.4.4 org.neo4j.gds alpha-proc - 2.4.3 + 2.4.4 org.neo4j.gds open-write-services - 2.4.3 + 2.4.4 ---- diff --git a/doc/modules/ROOT/pages/installation/supported-neo4j-versions.adoc b/doc/modules/ROOT/pages/installation/supported-neo4j-versions.adoc index eefdd0c98b..37de15a2ca 100644 --- a/doc/modules/ROOT/pages/installation/supported-neo4j-versions.adoc +++ b/doc/modules/ROOT/pages/installation/supported-neo4j-versions.adoc @@ -10,6 +10,7 @@ If your version of GDS or Neo4j is not listed in the matrix, you should upgrade. [opts=header] |=== | Neo4j version | Neo4j Graph Data Science +| `5.11` | `2.4.4` or later | `5.10` | `2.4.2` or later | `5.9` | `2.4`, `2.3.9` or later footnote:eol[This version series is end-of-life and will not receive further patches. Please use a later version.] | `5.8` | `2.4`, `2.3.6` or later footnote:eol[] From e418fc893661cab09a1770d2f1f5f57446590b36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonatan=20J=C3=A4derberg?= Date: Tue, 15 Aug 2023 15:56:59 +0200 Subject: [PATCH 205/273] Bump version to 2.4.5 --- doc/modules/ROOT/pages/management-ops/utility-functions.adoc | 2 +- gradle/version.gradle | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/modules/ROOT/pages/management-ops/utility-functions.adoc b/doc/modules/ROOT/pages/management-ops/utility-functions.adoc index cd84872878..6c8ce148a7 100644 --- a/doc/modules/ROOT/pages/management-ops/utility-functions.adoc +++ b/doc/modules/ROOT/pages/management-ops/utility-functions.adoc @@ -27,7 +27,7 @@ RETURN gds.version() AS version [opts="header"] |=== | version -| "2.4.4" +| "2.4.5" |=== -- diff --git a/gradle/version.gradle b/gradle/version.gradle index fc2cd0d57f..fe0caf36e0 100644 --- a/gradle/version.gradle +++ b/gradle/version.gradle @@ -1,5 +1,5 @@ ext { - gdsBaseVersion = '2.4.4' + gdsBaseVersion = '2.4.5' gdsAuraVersion = '30' gdsVersion = gdsBaseVersion + (rootProject.hasProperty('aurads') ? "+${gdsAuraVersion}" : "") From ef73f806a99d3e1e5ed786868d9f3f818b77ecba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonatan=20J=C3=A4derberg?= Date: Thu, 17 Aug 2023 11:59:52 +0200 Subject: [PATCH 206/273] Update RC to 5.12.0 --- gradle/dependencies.gradle | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/gradle/dependencies.gradle b/gradle/dependencies.gradle index 2d139e4348..01cf1382f0 100644 --- a/gradle/dependencies.gradle +++ b/gradle/dependencies.gradle @@ -12,7 +12,7 @@ ext { '5.9': properties.getOrDefault('neo4jVersion59', '5.9.0'), '5.10': properties.getOrDefault('neo4jVersion510', '5.10.0'), '5.11': properties.getOrDefault('neo4jVersion511', '5.11.0'), - '5.11': properties.getOrDefault('neo4jVersion511', '5.11.0'), + '5.12': properties.getOrDefault('neo4jVersion511', '5.12.0'), ] neo4jDefault = neos.'4.4' @@ -33,6 +33,7 @@ ext { '5.9': '2.13.10', '5.10': '2.13.10', '5.11': '2.13.10', + '5.12': '2.13.10', ] log4js = [ @@ -48,6 +49,7 @@ ext { '5.9': '2.20.0', '5.10': '2.20.0', '5.11': '2.20.0', + '5.12': '2.20.0', ] ver = [ From 45239937dfe3ea5bb9de37a0548e0eb651b14cd6 Mon Sep 17 00:00:00 2001 From: ioannispan Date: Thu, 17 Aug 2023 12:28:31 +0200 Subject: [PATCH 207/273] Correct priority list of cursors --- .../triangle/intersect/GraphIntersect.java | 7 +- .../UnionGraphTriangleCountingTest.java | 103 ++++++++++++++++++ .../core/huge/CompositeAdjacencyCursor.java | 5 +- .../gds/core/huge/CompositeAdjacencyList.java | 5 +- 4 files changed, 117 insertions(+), 3 deletions(-) create mode 100644 algo/src/test/java/org/neo4j/gds/triangle/UnionGraphTriangleCountingTest.java diff --git a/algo/src/main/java/org/neo4j/gds/triangle/intersect/GraphIntersect.java b/algo/src/main/java/org/neo4j/gds/triangle/intersect/GraphIntersect.java index bd83ac7bda..a4d2b460f4 100644 --- a/algo/src/main/java/org/neo4j/gds/triangle/intersect/GraphIntersect.java +++ b/algo/src/main/java/org/neo4j/gds/triangle/intersect/GraphIntersect.java @@ -86,7 +86,12 @@ public void intersectAll(long nodeA, IntersectionConsumer consumer) { // check the second node's degree int degreeB = degree(nodeB); if (degreeFilter.test(degreeB)) { - neighboursB = cursorForNode(neighboursB, nodeB, degreeB); + neighboursB = cursorForNode( + neighboursB, + nodeB, + degreeB + ); + // find first neighbour Cb of B with id > B nodeCFromB = neighboursB.skipUntil(nodeB); diff --git a/algo/src/test/java/org/neo4j/gds/triangle/UnionGraphTriangleCountingTest.java b/algo/src/test/java/org/neo4j/gds/triangle/UnionGraphTriangleCountingTest.java new file mode 100644 index 0000000000..06ce5d9233 --- /dev/null +++ b/algo/src/test/java/org/neo4j/gds/triangle/UnionGraphTriangleCountingTest.java @@ -0,0 +1,103 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.triangle; + +import org.junit.jupiter.api.Test; +import org.neo4j.gds.Orientation; +import org.neo4j.gds.api.Graph; +import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.extension.GdlExtension; +import org.neo4j.gds.extension.GdlGraph; +import org.neo4j.gds.extension.Inject; + +import static org.assertj.core.api.AssertionsForClassTypes.assertThat; + +@GdlExtension +class UnionGraphTriangleCountingTest { + @GdlGraph(orientation = Orientation.UNDIRECTED) + static String DB_CYPHER="CREATE "+ + "(a0:L4)," + + "(a1:L1)," + + "(a2:L3)," + + "(a3:L1)," + + "(a5:L3)," + + "(a6:L1)," + + "(a7:L4)," + + "(a8:L3)," + + "(a9:L4)," + + "(a10:L1)," + + "(a11:L1)," + + "(a12:L4)," + + "(a13:L4)," + + "(a14:L3)," + + "(a15:L3)," + + "(a17:L4)," + + "(a18:L0)," + + "(a19:L3)," + + "(a20:L1)," + + "(a21:L1)," + + "(a22:L1)," + + "(a23:L2)," + + "(a24:L4)," + + "(a25:L3)," + + "(a26:L0)," + + "(a28:L4)," + + "(a29:L4)," + + "(a11)-[:T1]->(a15), "+ + "(a0)-[:T1]->(a21), "+ + "(a8)-[:T2]->(a28), "+ + "(a12)-[:T3]->(a12), "+ + "(a9)-[:T4]->(a10), "+ + "(a3)-[:T2]->(a26), "+ + "(a7)-[:T0]->(a21), "+ + "(a11)-[:T3]->(a29), "+ + "(a1)-[:T3]->(a14), "+ + "(a14)-[:T0]->(a22), "+ + "(a10)-[:T1]->(a13), "+ + "(a3)-[:T0]->(a21), "+ + "(a5)-[:T3]->(a28), "+ + "(a10)-[:T3]->(a25), "+ + "(a8)-[:T1]->(a14), "+ + "(a11)-[:T3]->(a15), "+ + "(a13)-[:T2]->(a18), "+ + "(a13)-[:T4]->(a20), "+ + "(a6)-[:T1]->(a29), "+ + "(a12)-[:T3]->(a14), "+ + "(a3)-[:T2]->(a21), "+ + "(a2)-[:T1]->(a21), "+ + "(a0)-[:T0]->(a20), "+ + "(a24)-[:T0]->(a29), "+ + "(a10)-[:T4]->(a19), "+ + "(a0)-[:T1]->(a28), "+ + "(a9)-[:T4]->(a17), "+ + "(a15)-[:T4]->(a21), "+ + "(a21)-[:T1]->(a24) "; + + @Inject + Graph graph; + + @Test + void shouldWorkWithUnionGraphs() { + var config=TriangleCountStreamConfigImpl.builder().concurrency(1).build(); + var a=IntersectingTriangleCount.create(graph,config, Pools.DEFAULT); + var result=a.compute(); + assertThat(result.globalTriangles()).isEqualTo(0); + } +} diff --git a/core/src/main/java/org/neo4j/gds/core/huge/CompositeAdjacencyCursor.java b/core/src/main/java/org/neo4j/gds/core/huge/CompositeAdjacencyCursor.java index 652fc35a8c..7cd5f57e39 100644 --- a/core/src/main/java/org/neo4j/gds/core/huge/CompositeAdjacencyCursor.java +++ b/core/src/main/java/org/neo4j/gds/core/huge/CompositeAdjacencyCursor.java @@ -29,7 +29,6 @@ import java.util.PriorityQueue; public class CompositeAdjacencyCursor implements AdjacencyCursor { - private final PriorityQueue cursorQueue; private final List cursors; @@ -52,6 +51,10 @@ public List cursors() { return cursors; } + void exchangeCursor(AdjacencyCursor oldCursor, AdjacencyCursor newCursor) { + cursorQueue.remove(oldCursor); //remove the old cursor + cursorQueue.add(newCursor); // add the new one + } @Override public int size() { int sum = 0; diff --git a/core/src/main/java/org/neo4j/gds/core/huge/CompositeAdjacencyList.java b/core/src/main/java/org/neo4j/gds/core/huge/CompositeAdjacencyList.java index c359606ac9..b75652e594 100644 --- a/core/src/main/java/org/neo4j/gds/core/huge/CompositeAdjacencyList.java +++ b/core/src/main/java/org/neo4j/gds/core/huge/CompositeAdjacencyList.java @@ -130,7 +130,10 @@ public CompositeAdjacencyCursor adjacencyCursor(@Nullable AdjacencyCursor reuse, var cursor = iter.next(); var newCursor = adjacencyLists.get(index).adjacencyCursor(cursor, node, fallbackValue); if (newCursor != cursor) { - iter.set(adjacencyCursorWrapperFactory.create(newCursor)); + var newCursor1 = adjacencyCursorWrapperFactory.create(newCursor); + iter.set(newCursor1); + compositeReuse.exchangeCursor(cursor, newCursor1); + } } return compositeReuse; From 5a921ca9db8f3e6159e73e1b1da4b31749d4c06a Mon Sep 17 00:00:00 2001 From: ioannispan Date: Thu, 17 Aug 2023 15:30:59 +0200 Subject: [PATCH 208/273] Introduce a cursor queue tracking function --- .../neo4j/gds/core/huge/CompositeAdjacencyCursor.java | 6 +++--- .../neo4j/gds/core/huge/CompositeAdjacencyList.java | 10 ++++++---- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/core/src/main/java/org/neo4j/gds/core/huge/CompositeAdjacencyCursor.java b/core/src/main/java/org/neo4j/gds/core/huge/CompositeAdjacencyCursor.java index 7cd5f57e39..427b80dfae 100644 --- a/core/src/main/java/org/neo4j/gds/core/huge/CompositeAdjacencyCursor.java +++ b/core/src/main/java/org/neo4j/gds/core/huge/CompositeAdjacencyCursor.java @@ -51,9 +51,9 @@ public List cursors() { return cursors; } - void exchangeCursor(AdjacencyCursor oldCursor, AdjacencyCursor newCursor) { - cursorQueue.remove(oldCursor); //remove the old cursor - cursorQueue.add(newCursor); // add the new one + void updateCursorsQueue() { + cursorQueue.clear(); + cursorQueue.addAll(cursors); } @Override public int size() { diff --git a/core/src/main/java/org/neo4j/gds/core/huge/CompositeAdjacencyList.java b/core/src/main/java/org/neo4j/gds/core/huge/CompositeAdjacencyList.java index b75652e594..14658b4c70 100644 --- a/core/src/main/java/org/neo4j/gds/core/huge/CompositeAdjacencyList.java +++ b/core/src/main/java/org/neo4j/gds/core/huge/CompositeAdjacencyList.java @@ -125,17 +125,19 @@ public CompositeAdjacencyCursor adjacencyCursor(@Nullable AdjacencyCursor reuse, if (reuse instanceof CompositeAdjacencyCursor) { var compositeReuse = (CompositeAdjacencyCursor) reuse; var iter = compositeReuse.cursors().listIterator(); + boolean cursorsAreUpdated = false; while (iter.hasNext()) { var index = iter.nextIndex(); var cursor = iter.next(); var newCursor = adjacencyLists.get(index).adjacencyCursor(cursor, node, fallbackValue); if (newCursor != cursor) { - var newCursor1 = adjacencyCursorWrapperFactory.create(newCursor); - iter.set(newCursor1); - compositeReuse.exchangeCursor(cursor, newCursor1); - + iter.set(adjacencyCursorWrapperFactory.create(newCursor)); + cursorsAreUpdated = true; } } + if (cursorsAreUpdated) { + compositeReuse.updateCursorsQueue(); + } return compositeReuse; } return adjacencyCursor(node, fallbackValue); From 183d62ef024402e10484355257618d09e11677a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B6ren=20Reichardt?= Date: Wed, 23 Aug 2023 15:29:32 +0200 Subject: [PATCH 209/273] Fix rc compat --- .../gds/executor/MemoryEstimationExecutorTest.java | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/executor/src/test/java/org/neo4j/gds/executor/MemoryEstimationExecutorTest.java b/executor/src/test/java/org/neo4j/gds/executor/MemoryEstimationExecutorTest.java index f72a836fab..95936b218b 100644 --- a/executor/src/test/java/org/neo4j/gds/executor/MemoryEstimationExecutorTest.java +++ b/executor/src/test/java/org/neo4j/gds/executor/MemoryEstimationExecutorTest.java @@ -25,12 +25,12 @@ import org.neo4j.gds.BaseTest; import org.neo4j.gds.GdsCypher; import org.neo4j.gds.NodeProjections; -import org.neo4j.gds.ProcedureCallContextReturnColumns; import org.neo4j.gds.RelationshipProjections; import org.neo4j.gds.api.AlgorithmMetaDataSetter; import org.neo4j.gds.api.CloseableResourceRegistry; import org.neo4j.gds.api.DatabaseId; import org.neo4j.gds.api.NodeLookup; +import org.neo4j.gds.api.ProcedureReturnColumns; import org.neo4j.gds.api.TerminationMonitor; import org.neo4j.gds.catalog.GraphProjectProc; import org.neo4j.gds.compat.GraphDatabaseApiProxy; @@ -47,7 +47,6 @@ import org.neo4j.gds.test.TestMutateConfig; import org.neo4j.gds.transaction.DatabaseTransactionContext; import org.neo4j.graphdb.Transaction; -import org.neo4j.internal.kernel.api.procs.ProcedureCallContext; import java.util.List; import java.util.Map; @@ -74,13 +73,7 @@ void setup() throws Exception { .builder() .databaseId(DatabaseId.of(db)) .dependencyResolver(GraphDatabaseApiProxy.dependencyResolver(db)) - .returnColumns(new ProcedureCallContextReturnColumns(new ProcedureCallContext( - 42, - new String[0], - false, - "neo4j", - false - ))) + .returnColumns(ProcedureReturnColumns.EMPTY) .log(Neo4jProxy.testLog()) .taskRegistryFactory(EmptyTaskRegistryFactory.INSTANCE) .userLogRegistryFactory(EmptyUserLogRegistryFactory.INSTANCE) From 35735ca129d5e1062e2888966124ffeb37749a56 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B6ren=20Reichardt?= Date: Thu, 24 Aug 2023 11:56:38 +0200 Subject: [PATCH 210/273] Post 2.4.5 release actions --- README.adoc | 16 ++++++++-------- .../pages/management-ops/utility-functions.adoc | 2 +- examples/pregel-bootstrap/build.gradle | 2 +- gradle/version.gradle | 2 +- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/README.adoc b/README.adoc index ed9abadf19..d8639759a6 100644 --- a/README.adoc +++ b/README.adoc @@ -96,7 +96,7 @@ For the most basic set of features, like graph loading and the graph representat org.neo4j.gds core - 2.4.4 + 2.4.5 ---- @@ -108,21 +108,21 @@ The algorithms are located in the `algo-common`, `algo` and `alpha-algo` modules org.neo4j.gds algo-common - 2.4.4 + 2.4.5 org.neo4j.gds algo - 2.4.4 + 2.4.5 org.neo4j.gds alpha-algo - 2.4.4 + 2.4.5 ---- @@ -134,28 +134,28 @@ The procedures are located in the `proc-common`, `proc` and `alpha-proc` modules org.neo4j.gds proc-common - 2.4.4 + 2.4.5 org.neo4j.gds proc - 2.4.4 + 2.4.5 org.neo4j.gds alpha-proc - 2.4.4 + 2.4.5 org.neo4j.gds open-write-services - 2.4.4 + 2.4.5 ---- diff --git a/doc/modules/ROOT/pages/management-ops/utility-functions.adoc b/doc/modules/ROOT/pages/management-ops/utility-functions.adoc index 6c8ce148a7..ac95df788a 100644 --- a/doc/modules/ROOT/pages/management-ops/utility-functions.adoc +++ b/doc/modules/ROOT/pages/management-ops/utility-functions.adoc @@ -27,7 +27,7 @@ RETURN gds.version() AS version [opts="header"] |=== | version -| "2.4.5" +| "2.4.6" |=== -- diff --git a/examples/pregel-bootstrap/build.gradle b/examples/pregel-bootstrap/build.gradle index 5d45ce4568..867250985f 100644 --- a/examples/pregel-bootstrap/build.gradle +++ b/examples/pregel-bootstrap/build.gradle @@ -7,7 +7,7 @@ plugins { ext { // Make sure these are the same as your installation of GDS and Neo4j - gdsVersion = '2.4.4' + gdsVersion = '2.4.5' neo4jVersion = '5.10.0' // Necessary to generate value classes for Pregel configs diff --git a/gradle/version.gradle b/gradle/version.gradle index fe0caf36e0..9b87184150 100644 --- a/gradle/version.gradle +++ b/gradle/version.gradle @@ -1,5 +1,5 @@ ext { - gdsBaseVersion = '2.4.5' + gdsBaseVersion = '2.4.6' gdsAuraVersion = '30' gdsVersion = gdsBaseVersion + (rootProject.hasProperty('aurads') ? "+${gdsAuraVersion}" : "") From 5564e3bfc2eb375a60cd3759e59d4908f22dd16b Mon Sep 17 00:00:00 2001 From: Ioannis Pan <74839024+IoannisPanagiotas@users.noreply.github.com> Date: Thu, 24 Aug 2023 12:53:02 +0200 Subject: [PATCH 211/273] Changing Aura version here to avoid new pr for oneline --- gradle/version.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle/version.gradle b/gradle/version.gradle index 9b87184150..c9abfd3acc 100644 --- a/gradle/version.gradle +++ b/gradle/version.gradle @@ -1,6 +1,6 @@ ext { gdsBaseVersion = '2.4.6' - gdsAuraVersion = '30' + gdsAuraVersion = '31' gdsVersion = gdsBaseVersion + (rootProject.hasProperty('aurads') ? "+${gdsAuraVersion}" : "") } From d658084363b0687e36f5b1b03bb4c4b431a5f462 Mon Sep 17 00:00:00 2001 From: yuval Date: Mon, 14 Aug 2023 18:15:54 +0200 Subject: [PATCH 212/273] Disallowing empty property names 1. PropertyMapping is the proper place for this validation. the reason is that all graph creations should be using it. 2. Creating another validation flow to fail "faster" in new cypher projections doesn't really make sense, because we will always be failing on the first relationship we see anyway (all properties are equal through all relationships). Co-authored-by: Mats Rydberg --- .../neo4j/gds/core/loading/CypherQueryEstimator.java | 3 +++ .../src/main/java/org/neo4j/gds/PropertyMapping.java | 7 +++++++ .../test/java/org/neo4j/gds/PropertyMappingTest.java | 12 ++++++++++++ 3 files changed, 22 insertions(+) diff --git a/core/src/main/java/org/neo4j/gds/core/loading/CypherQueryEstimator.java b/core/src/main/java/org/neo4j/gds/core/loading/CypherQueryEstimator.java index 2f9f603b58..5b69a1e26e 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/CypherQueryEstimator.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/CypherQueryEstimator.java @@ -19,6 +19,7 @@ */ package org.neo4j.gds.core.loading; +import org.neo4j.gds.PropertyMapping; import org.neo4j.gds.annotation.ValueClass; import org.neo4j.gds.transaction.TransactionContext; @@ -55,6 +56,8 @@ public EstimationResult runEstimationQuery(String query, Collection rese var propertyColumns = new ArrayList<>(result.columns()); propertyColumns.removeAll(reservedColumns); + propertyColumns.forEach(property -> PropertyMapping.validatePropertyKey(property)); + return ImmutableEstimationResult.of(estimatedRows.longValue(), propertyColumns.size()); } }); diff --git a/graph-projection-api/src/main/java/org/neo4j/gds/PropertyMapping.java b/graph-projection-api/src/main/java/org/neo4j/gds/PropertyMapping.java index a021bf97d1..4cc2c2f140 100644 --- a/graph-projection-api/src/main/java/org/neo4j/gds/PropertyMapping.java +++ b/graph-projection-api/src/main/java/org/neo4j/gds/PropertyMapping.java @@ -67,6 +67,13 @@ public void validateProperties() { if (neoPropertyKey().equals(ElementProjection.PROJECT_ALL) && aggregation() != Aggregation.COUNT) { throw new IllegalArgumentException("A '*' property key can only be used in combination with count aggregation."); } + validatePropertyKey(propertyKey()); + } + + public static void validatePropertyKey(String propertyKey) { + if (propertyKey.isEmpty()) { + throw new IllegalArgumentException("Property key must not be empty."); + } } public static PropertyMapping fromObject(String propertyKey, Object stringOrMap) { diff --git a/graph-projection-api/src/test/java/org/neo4j/gds/PropertyMappingTest.java b/graph-projection-api/src/test/java/org/neo4j/gds/PropertyMappingTest.java index 7d46118601..bf1ce5ec42 100644 --- a/graph-projection-api/src/test/java/org/neo4j/gds/PropertyMappingTest.java +++ b/graph-projection-api/src/test/java/org/neo4j/gds/PropertyMappingTest.java @@ -97,4 +97,16 @@ void failsOnWrongKeyType() { containsString("Expected the value of 'property' to be of type String, but was 'Integer'.") ); } + + @Test + void failsOnEmptyPropertyKey() { + IllegalArgumentException ex = assertThrows( + IllegalArgumentException.class, () -> PropertyMapping.fromObject("", Map.of( + "neoKey", 42 + ))); + assertThat( + ex.getMessage(), + containsString("Property key must not be empty") + ); + } } From 8a7b51de979d7917c7d128ac12ca3af37ae8d6bd Mon Sep 17 00:00:00 2001 From: yuval Date: Mon, 21 Aug 2023 09:58:05 +0200 Subject: [PATCH 213/273] Using a method reference instead of lambda Co-authored-by: Martin Junghanns --- .../java/org/neo4j/gds/core/loading/CypherQueryEstimator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/org/neo4j/gds/core/loading/CypherQueryEstimator.java b/core/src/main/java/org/neo4j/gds/core/loading/CypherQueryEstimator.java index 5b69a1e26e..5d11e1a105 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/CypherQueryEstimator.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/CypherQueryEstimator.java @@ -56,7 +56,7 @@ public EstimationResult runEstimationQuery(String query, Collection rese var propertyColumns = new ArrayList<>(result.columns()); propertyColumns.removeAll(reservedColumns); - propertyColumns.forEach(property -> PropertyMapping.validatePropertyKey(property)); + propertyColumns.forEach(PropertyMapping::validatePropertyKey); return ImmutableEstimationResult.of(estimatedRows.longValue(), propertyColumns.size()); } From 2f32a7a605c9e039d8669cf1e8b8bf9df62ba6c2 Mon Sep 17 00:00:00 2001 From: yuval Date: Mon, 21 Aug 2023 10:09:48 +0200 Subject: [PATCH 214/273] Not having a local var in assertions Co-authored-by: Martin Junghanns --- .../org/neo4j/gds/PropertyMappingTest.java | 28 ++++++++----------- 1 file changed, 11 insertions(+), 17 deletions(-) diff --git a/graph-projection-api/src/test/java/org/neo4j/gds/PropertyMappingTest.java b/graph-projection-api/src/test/java/org/neo4j/gds/PropertyMappingTest.java index bf1ce5ec42..6e29abc1de 100644 --- a/graph-projection-api/src/test/java/org/neo4j/gds/PropertyMappingTest.java +++ b/graph-projection-api/src/test/java/org/neo4j/gds/PropertyMappingTest.java @@ -25,10 +25,8 @@ import java.util.Map; -import static org.hamcrest.CoreMatchers.containsString; -import static org.hamcrest.MatcherAssert.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; class PropertyMappingTest { @@ -88,25 +86,21 @@ void shouldSupportCaseInsensitiveConfigKeys() { @Test void failsOnWrongKeyType() { - IllegalArgumentException ex = assertThrows( - IllegalArgumentException.class, () -> PropertyMapping.fromObject("transaction_count", Map.of( + assertThatThrownBy( + () -> PropertyMapping.fromObject("transaction_count", Map.of( "property", 42 - ))); - assertThat( - ex.getMessage(), - containsString("Expected the value of 'property' to be of type String, but was 'Integer'.") - ); + ))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Expected the value of 'property' to be of type String, but was 'Integer'."); } @Test void failsOnEmptyPropertyKey() { - IllegalArgumentException ex = assertThrows( - IllegalArgumentException.class, () -> PropertyMapping.fromObject("", Map.of( + assertThatThrownBy( + () -> PropertyMapping.fromObject("", Map.of( "neoKey", 42 - ))); - assertThat( - ex.getMessage(), - containsString("Property key must not be empty") - ); + ))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Property key must not be empty"); } } From 0e76f9b389ec0559d42f60f4980d68b7d3424da0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Mon, 28 Aug 2023 09:29:47 +0200 Subject: [PATCH 215/273] Bump neo 4.4 version to 4.4.25 --- README.adoc | 2 +- gradle/dependencies.gradle | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.adoc b/README.adoc index ed9abadf19..0810813b67 100644 --- a/README.adoc +++ b/README.adoc @@ -39,7 +39,7 @@ When installing GDS manually, please refer to the below compatibility matrix: |Neo4j 5.4.0 |Neo4j 5.2.0 |Neo4j 5.1.0 -|Neo4j 4.4.9 - 4.4.24 +|Neo4j 4.4.9 - 4.4.25 .1+.^|Java 11 .9+<.^|GDS 2.3.x |Neo4j 5.8.0 diff --git a/gradle/dependencies.gradle b/gradle/dependencies.gradle index 01cf1382f0..ca19058f2a 100644 --- a/gradle/dependencies.gradle +++ b/gradle/dependencies.gradle @@ -1,6 +1,6 @@ ext { neos = [ - '4.4': properties.getOrDefault('neo4jVersion44', '4.4.24'), + '4.4': properties.getOrDefault('neo4jVersion44', '4.4.25'), '5.1': properties.getOrDefault('neo4jVersion51', '5.1.0'), '5.2': properties.getOrDefault('neo4jVersion52', '5.2.0'), '5.3': properties.getOrDefault('neo4jVersion53', '5.3.0'), From 0ef34d8662cbd637e2ccb1804ffa96b266254518 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Mon, 28 Aug 2023 11:38:36 +0200 Subject: [PATCH 216/273] Fix documentation example around `projectedFeatureDimension` --- .../pages/machine-learning/node-embeddings/graph-sage.adoc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/modules/ROOT/pages/machine-learning/node-embeddings/graph-sage.adoc b/doc/modules/ROOT/pages/machine-learning/node-embeddings/graph-sage.adoc index c4c32353c8..98db126821 100644 --- a/doc/modules/ROOT/pages/machine-learning/node-embeddings/graph-sage.adoc +++ b/doc/modules/ROOT/pages/machine-learning/node-embeddings/graph-sage.adoc @@ -540,7 +540,7 @@ CALL gds.graph.project( We can now run GraphSAGE in multi-label mode on that graph by specifying the `projectedFeatureDimension` parameter. Multi-label GraphSAGE removes the requirement, that each node in the in-memory graph must have all `featureProperties`. However, the projections are independent per label and even if two labels have the same `featureProperty` they are considered as different features before projection. -The `projectedFeatureDimension` equals the maximum length of the feature-array, i.e., `age` and `cost` both are scalar features plus the list feature `heightAndWeight` which has a length of two. +The `projectedFeatureDimension` equals the maximum length of the feature-array, i.e., for persons `age` (1) and `heightAndWeight` (2) sum up to a length of 3 and instruments with only cost leads to a length of 1. Thus, the `projectedFeatureDimension` should be set to 3. For each node its unique labels properties is projected using a label specific projection to vector space of dimension `projectedFeatureDimension`. Note that the `cost` feature is only defined for the instrument nodes, while `age` and `heightAndWeight` are only defined for persons. @@ -551,7 +551,7 @@ CALL gds.beta.graphSage.train( { modelName: 'multiLabelModel', featureProperties: ['age', 'heightAndWeight', 'cost'], - projectedFeatureDimension: 4 + projectedFeatureDimension: 3 } ) ---- From e243629be0886dba1f6e20a64b3fc9e61072e164 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Tue, 29 Aug 2023 15:41:25 +0200 Subject: [PATCH 217/273] Improve wording Co-authored-by: Mats Rydberg --- .../pages/machine-learning/node-embeddings/graph-sage.adoc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/doc/modules/ROOT/pages/machine-learning/node-embeddings/graph-sage.adoc b/doc/modules/ROOT/pages/machine-learning/node-embeddings/graph-sage.adoc index 98db126821..2bfab3ba6e 100644 --- a/doc/modules/ROOT/pages/machine-learning/node-embeddings/graph-sage.adoc +++ b/doc/modules/ROOT/pages/machine-learning/node-embeddings/graph-sage.adoc @@ -540,7 +540,10 @@ CALL gds.graph.project( We can now run GraphSAGE in multi-label mode on that graph by specifying the `projectedFeatureDimension` parameter. Multi-label GraphSAGE removes the requirement, that each node in the in-memory graph must have all `featureProperties`. However, the projections are independent per label and even if two labels have the same `featureProperty` they are considered as different features before projection. -The `projectedFeatureDimension` equals the maximum length of the feature-array, i.e., for persons `age` (1) and `heightAndWeight` (2) sum up to a length of 3 and instruments with only cost leads to a length of 1. Thus, the `projectedFeatureDimension` should be set to 3. +The `projectedFeatureDimension` should equal the maximum length of the feature-array. +In our example, persons have `age` (1) and `heightAndWeight` (2), summing up to a length of 3. +Instruments only have `cost` with length of 1. +Thus, the `projectedFeatureDimension` should be set to 3. For each node its unique labels properties is projected using a label specific projection to vector space of dimension `projectedFeatureDimension`. Note that the `cost` feature is only defined for the instrument nodes, while `age` and `heightAndWeight` are only defined for persons. From 2b2c08930decd802b070ae426294cbf414cd4114 Mon Sep 17 00:00:00 2001 From: Veselin Nikolov Date: Thu, 31 Aug 2023 09:10:28 +0100 Subject: [PATCH 218/273] Bump AuraDS version --- gradle/version.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle/version.gradle b/gradle/version.gradle index c9abfd3acc..bdf585c5e8 100644 --- a/gradle/version.gradle +++ b/gradle/version.gradle @@ -1,6 +1,6 @@ ext { gdsBaseVersion = '2.4.6' - gdsAuraVersion = '31' + gdsAuraVersion = '32' gdsVersion = gdsBaseVersion + (rootProject.hasProperty('aurads') ? "+${gdsAuraVersion}" : "") } From 6d488fa1cbd8ad0de93dfc8318cc2a8218de2311 Mon Sep 17 00:00:00 2001 From: Veselin Nikolov Date: Thu, 7 Sep 2023 12:58:34 +0100 Subject: [PATCH 219/273] Make sure that HITS write and mutate procedures can run --- .../main/java/org/neo4j/gds/pregel/Hits.java | 4 +- .../neo4j/gds/hits/HitsMutateProcTest.java | 86 +++++++++++++++++++ .../org/neo4j/gds/hits/HitsWriteProcTest.java | 86 +++++++++++++++++++ 3 files changed, 175 insertions(+), 1 deletion(-) create mode 100644 alpha/alpha-proc/src/test/java/org/neo4j/gds/hits/HitsMutateProcTest.java create mode 100644 alpha/alpha-proc/src/test/java/org/neo4j/gds/hits/HitsWriteProcTest.java diff --git a/alpha/alpha-proc/src/main/java/org/neo4j/gds/pregel/Hits.java b/alpha/alpha-proc/src/main/java/org/neo4j/gds/pregel/Hits.java index 2dd1a02ea7..9f2e104898 100644 --- a/alpha/alpha-proc/src/main/java/org/neo4j/gds/pregel/Hits.java +++ b/alpha/alpha-proc/src/main/java/org/neo4j/gds/pregel/Hits.java @@ -171,7 +171,9 @@ default String authProperty() { return "auth"; } - @Override + @Value.Default + @Configuration.ConvertWith(method = "org.neo4j.gds.beta.pregel.Partitioning#parse") + @Configuration.ToMapValue("org.neo4j.gds.beta.pregel.Partitioning#toString") default Partitioning partitioning() { return Partitioning.AUTO; } diff --git a/alpha/alpha-proc/src/test/java/org/neo4j/gds/hits/HitsMutateProcTest.java b/alpha/alpha-proc/src/test/java/org/neo4j/gds/hits/HitsMutateProcTest.java new file mode 100644 index 0000000000..043113fc3e --- /dev/null +++ b/alpha/alpha-proc/src/test/java/org/neo4j/gds/hits/HitsMutateProcTest.java @@ -0,0 +1,86 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.hits; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.neo4j.gds.BaseProcTest; +import org.neo4j.gds.catalog.GraphProjectProc; +import org.neo4j.gds.extension.Neo4jGraph; +import org.neo4j.gds.pregel.HitsMutateProc; + +import static org.assertj.core.api.Assertions.assertThatNoException; + +class HitsMutateProcTest extends BaseProcTest { + + @Neo4jGraph + public static final String DB_CYPHER = + "CREATE" + + " (a:Website {name: 'A'})," + + " (b:Website {name: 'B'})," + + " (c:Website {name: 'C'})," + + " (d:Website {name: 'D'})," + + " (e:Website {name: 'E'})," + + " (f:Website {name: 'F'})," + + " (g:Website {name: 'G'})," + + " (h:Website {name: 'H'})," + + " (i:Website {name: 'I'})," + + + " (a)-[:LINK]->(b)," + + " (a)-[:LINK]->(c)," + + " (a)-[:LINK]->(d)," + + " (b)-[:LINK]->(c)," + + " (b)-[:LINK]->(d)," + + " (c)-[:LINK]->(d)," + + + " (e)-[:LINK]->(b)," + + " (e)-[:LINK]->(d)," + + " (e)-[:LINK]->(f)," + + " (e)-[:LINK]->(h)," + + + " (f)-[:LINK]->(g)," + + " (f)-[:LINK]->(i)," + + " (f)-[:LINK]->(h)," + + " (g)-[:LINK]->(h)," + + " (g)-[:LINK]->(i)," + + " (h)-[:LINK]->(i)"; + + @BeforeEach + void setupGraph() throws Exception { + registerProcedures( + HitsMutateProc.class, + GraphProjectProc.class + ); + + runQuery( + "CALL gds.graph.project(" + + " 'myGraph'," + + " 'Website'," + + " {LINK: {indexInverse: true}}" + + ");"); + } + + @Test + void shouldRunWithoutError() { + assertThatNoException() + .as("The `HITS` write procedure should run without raising an exception.") + .isThrownBy(() -> runQuery("CALL gds.alpha.hits.mutate('myGraph', { hitsIterations: 1, writeProperty:'hits' })")); + } +} diff --git a/alpha/alpha-proc/src/test/java/org/neo4j/gds/hits/HitsWriteProcTest.java b/alpha/alpha-proc/src/test/java/org/neo4j/gds/hits/HitsWriteProcTest.java new file mode 100644 index 0000000000..4e8e136fb0 --- /dev/null +++ b/alpha/alpha-proc/src/test/java/org/neo4j/gds/hits/HitsWriteProcTest.java @@ -0,0 +1,86 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.hits; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.neo4j.gds.BaseProcTest; +import org.neo4j.gds.catalog.GraphProjectProc; +import org.neo4j.gds.extension.Neo4jGraph; +import org.neo4j.gds.pregel.HitsWriteProc; + +import static org.assertj.core.api.Assertions.assertThatNoException; + +class HitsWriteProcTest extends BaseProcTest { + + @Neo4jGraph + public static final String DB_CYPHER = + "CREATE" + + " (a:Website {name: 'A'})," + + " (b:Website {name: 'B'})," + + " (c:Website {name: 'C'})," + + " (d:Website {name: 'D'})," + + " (e:Website {name: 'E'})," + + " (f:Website {name: 'F'})," + + " (g:Website {name: 'G'})," + + " (h:Website {name: 'H'})," + + " (i:Website {name: 'I'})," + + + " (a)-[:LINK]->(b)," + + " (a)-[:LINK]->(c)," + + " (a)-[:LINK]->(d)," + + " (b)-[:LINK]->(c)," + + " (b)-[:LINK]->(d)," + + " (c)-[:LINK]->(d)," + + + " (e)-[:LINK]->(b)," + + " (e)-[:LINK]->(d)," + + " (e)-[:LINK]->(f)," + + " (e)-[:LINK]->(h)," + + + " (f)-[:LINK]->(g)," + + " (f)-[:LINK]->(i)," + + " (f)-[:LINK]->(h)," + + " (g)-[:LINK]->(h)," + + " (g)-[:LINK]->(i)," + + " (h)-[:LINK]->(i)"; + + @BeforeEach + void setupGraph() throws Exception { + registerProcedures( + HitsWriteProc.class, + GraphProjectProc.class + ); + + runQuery( + "CALL gds.graph.project(" + + " 'myGraph'," + + " 'Website'," + + " {LINK: {indexInverse: true}}" + + ");"); + } + + @Test + void shouldRunWithoutError() { + assertThatNoException() + .as("The `HITS` mutate procedure should run without raising an exception.") + .isThrownBy(() -> runQuery("CALL gds.alpha.hits.write('myGraph', { hitsIterations: 1, mutateProperty:'hits' })")); + } +} From 4b7d21e072aaf3416dccbfdf7089eb550d404202 Mon Sep 17 00:00:00 2001 From: yuval Date: Thu, 7 Sep 2023 21:04:30 +0300 Subject: [PATCH 220/273] compat script --- .../5.12/neo4j-kernel-adapter/build.gradle | 65 ++ .../compat/_512/Neo4jProxyFactoryImpl.java | 31 + .../compat/_512/SettingProxyFactoryImpl.java | 31 + .../_512/BoltTransactionRunnerImpl.java | 97 ++ .../compat/_512/CallableProcedureImpl.java | 53 + .../CallableUserAggregationFunctionImpl.java | 77 ++ .../gds/compat/_512/CompatAccessModeImpl.java | 44 + .../_512/CompatGraphDatabaseAPIImpl.java | 74 ++ .../gds/compat/_512/CompatIndexQueryImpl.java | 31 + .../_512/CompatUsernameAuthSubjectImpl.java | 35 + .../compat/_512/CompositeNodeCursorImpl.java | 32 + .../compat/_512/GdsDatabaseLayoutImpl.java | 50 + ...sDatabaseManagementServiceBuilderImpl.java | 54 + .../compat/_512/Neo4jProxyFactoryImpl.java | 44 + .../neo4j/gds/compat/_512/Neo4jProxyImpl.java | 950 ++++++++++++++++++ .../compat/_512/NodeLabelIndexLookupImpl.java | 68 ++ .../gds/compat/_512/PartitionedStoreScan.java | 58 ++ .../_512/ReferencePropertyReference.java | 49 + .../compat/_512/ScanBasedStoreScanImpl.java | 40 + .../compat/_512/SettingProxyFactoryImpl.java | 44 + .../gds/compat/_512/SettingProxyImpl.java | 87 ++ .../neo4j/gds/compat/_512/TestLogImpl.java | 146 +++ .../compat/_512/VirtualRelationshipImpl.java | 41 + .../5.12/storage-engine-adapter/build.gradle | 68 ++ .../_512/InMemoryStorageEngineFactory.java | 268 +++++ .../_512/StorageEngineProxyFactoryImpl.java | 31 + .../InMemoryCommandCreationContextImpl.java | 107 ++ .../compat/_512/InMemoryCountsStoreImpl.java | 111 ++ .../_512/InMemoryMetaDataProviderImpl.java | 201 ++++ .../gds/compat/_512/InMemoryNodeCursor.java | 82 ++ .../_512/InMemoryNodePropertyCursor.java | 45 + .../compat/_512/InMemoryPropertyCursor.java | 71 ++ .../_512/InMemoryPropertySelectionImpl.java | 55 + .../InMemoryRelationshipPropertyCursor.java | 60 ++ .../_512/InMemoryRelationshipScanCursor.java | 61 ++ .../InMemoryRelationshipTraversalCursor.java | 47 + .../_512/InMemoryStorageEngineFactory.java | 571 +++++++++++ .../_512/InMemoryStorageEngineImpl.java | 366 +++++++ .../compat/_512/InMemoryStorageLocksImpl.java | 86 ++ .../gds/compat/_512/InMemoryStoreVersion.java | 70 ++ .../_512/InMemoryTransactionIdStoreImpl.java | 117 +++ .../gds/compat/_512/InMemoryVersionCheck.java | 61 ++ .../_512/StorageEngineProxyFactoryImpl.java | 44 + .../compat/_512/StorageEngineProxyImpl.java | 140 +++ .../InMemoryLogVersionRepository512.java | 71 ++ ...nMemoryStorageCommandReaderFactory512.java | 43 + .../InMemoryStorageReader512.java | 333 ++++++ gradle/dependencies.gradle | 1 + 48 files changed, 5311 insertions(+) create mode 100644 compatibility/5.12/neo4j-kernel-adapter/build.gradle create mode 100644 compatibility/5.12/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_512/Neo4jProxyFactoryImpl.java create mode 100644 compatibility/5.12/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_512/SettingProxyFactoryImpl.java create mode 100644 compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/BoltTransactionRunnerImpl.java create mode 100644 compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/CallableProcedureImpl.java create mode 100644 compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/CallableUserAggregationFunctionImpl.java create mode 100644 compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/CompatAccessModeImpl.java create mode 100644 compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/CompatGraphDatabaseAPIImpl.java create mode 100644 compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/CompatIndexQueryImpl.java create mode 100644 compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/CompatUsernameAuthSubjectImpl.java create mode 100644 compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/CompositeNodeCursorImpl.java create mode 100644 compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/GdsDatabaseLayoutImpl.java create mode 100644 compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/GdsDatabaseManagementServiceBuilderImpl.java create mode 100644 compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/Neo4jProxyFactoryImpl.java create mode 100644 compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/Neo4jProxyImpl.java create mode 100644 compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/NodeLabelIndexLookupImpl.java create mode 100644 compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/PartitionedStoreScan.java create mode 100644 compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/ReferencePropertyReference.java create mode 100644 compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/ScanBasedStoreScanImpl.java create mode 100644 compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/SettingProxyFactoryImpl.java create mode 100644 compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/SettingProxyImpl.java create mode 100644 compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/TestLogImpl.java create mode 100644 compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/VirtualRelationshipImpl.java create mode 100644 compatibility/5.12/storage-engine-adapter/build.gradle create mode 100644 compatibility/5.12/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_512/InMemoryStorageEngineFactory.java create mode 100644 compatibility/5.12/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_512/StorageEngineProxyFactoryImpl.java create mode 100644 compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/InMemoryCommandCreationContextImpl.java create mode 100644 compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/InMemoryCountsStoreImpl.java create mode 100644 compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/InMemoryMetaDataProviderImpl.java create mode 100644 compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/InMemoryNodeCursor.java create mode 100644 compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/InMemoryNodePropertyCursor.java create mode 100644 compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/InMemoryPropertyCursor.java create mode 100644 compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/InMemoryPropertySelectionImpl.java create mode 100644 compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/InMemoryRelationshipPropertyCursor.java create mode 100644 compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/InMemoryRelationshipScanCursor.java create mode 100644 compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/InMemoryRelationshipTraversalCursor.java create mode 100644 compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/InMemoryStorageEngineFactory.java create mode 100644 compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/InMemoryStorageEngineImpl.java create mode 100644 compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/InMemoryStorageLocksImpl.java create mode 100644 compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/InMemoryStoreVersion.java create mode 100644 compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/InMemoryTransactionIdStoreImpl.java create mode 100644 compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/InMemoryVersionCheck.java create mode 100644 compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/StorageEngineProxyFactoryImpl.java create mode 100644 compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/StorageEngineProxyImpl.java create mode 100644 compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository512.java create mode 100644 compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory512.java create mode 100644 compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageReader512.java diff --git a/compatibility/5.12/neo4j-kernel-adapter/build.gradle b/compatibility/5.12/neo4j-kernel-adapter/build.gradle new file mode 100644 index 0000000000..498b905b8b --- /dev/null +++ b/compatibility/5.12/neo4j-kernel-adapter/build.gradle @@ -0,0 +1,65 @@ +apply plugin: 'java-library' +apply plugin: 'me.champeau.mrjar' + +description = 'Neo4j Graph Data Science :: Neo4j Kernel Adapter 5.12' + +group = 'org.neo4j.gds' + +// for all 5.x versions +if (ver.'neo4j'.startsWith('5.')) { + sourceSets { + main { + java { + srcDirs = ['src/main/java17'] + } + } + } + + dependencies { + annotationProcessor project(':annotations') + annotationProcessor group: 'org.immutables', name: 'value', version: ver.'immutables' + annotationProcessor group: 'org.neo4j', name: 'annotations', version: neos.'5.12' + + compileOnly project(':annotations') + compileOnly group: 'com.github.spotbugs', name: 'spotbugs-annotations', version: ver.'spotbugsToolVersion' + compileOnly group: 'org.immutables', name: 'value-annotations', version: ver.'immutables' + compileOnly group: 'org.neo4j', name: 'annotations', version: neos.'5.12' + compileOnly group: 'org.neo4j', name: 'neo4j', version: neos.'5.12' + compileOnly group: 'org.neo4j', name: 'neo4j-record-storage-engine', version: neos.'5.12' + compileOnly group: 'org.neo4j.community', name: 'it-test-support', version: neos.'5.12' + + implementation project(':neo4j-kernel-adapter-api') + } +} else { + multiRelease { + targetVersions 11, 17 + } + + if (!project.hasProperty('no-forbidden-apis')) { + forbiddenApisJava17 { + exclude('**') + } + } + + dependencies { + annotationProcessor group: 'org.neo4j', name: 'annotations', version: ver.'neo4j' + + compileOnly project(':annotations') + compileOnly group: 'org.neo4j', name: 'annotations', version: ver.'neo4j' + + implementation project(':neo4j-kernel-adapter-api') + + java17AnnotationProcessor project(':annotations') + java17AnnotationProcessor group: 'org.immutables', name: 'value', version: ver.'immutables' + java17AnnotationProcessor group: 'org.neo4j', name: 'annotations', version: neos.'5.12' + + java17CompileOnly project(':annotations') + java17CompileOnly group: 'org.immutables', name: 'value-annotations', version: ver.'immutables' + java17CompileOnly group: 'org.neo4j', name: 'neo4j', version: neos.'5.12' + java17CompileOnly group: 'org.neo4j', name: 'neo4j-record-storage-engine', version: neos.'5.12' + java17CompileOnly group: 'org.neo4j.community', name: 'it-test-support', version: neos.'5.12' + java17CompileOnly group: 'com.github.spotbugs', name: 'spotbugs-annotations', version: ver.'spotbugsToolVersion' + + java17Implementation project(':neo4j-kernel-adapter-api') + } +} diff --git a/compatibility/5.12/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_512/Neo4jProxyFactoryImpl.java b/compatibility/5.12/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_512/Neo4jProxyFactoryImpl.java new file mode 100644 index 0000000000..664ac89c8e --- /dev/null +++ b/compatibility/5.12/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_512/Neo4jProxyFactoryImpl.java @@ -0,0 +1,31 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * This file contains proprietary code that is only available via a commercial license from Neo4j. + * For more information, see https://neo4j.com/contact-us/ + */ +package org.neo4j.gds.compat._512; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jProxyApi; +import org.neo4j.gds.compat.Neo4jProxyFactory; +import org.neo4j.gds.compat.Neo4jVersion; + +@ServiceProvider +public final class Neo4jProxyFactoryImpl implements Neo4jProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return false; + } + + @Override + public Neo4jProxyApi load() { + throw new UnsupportedOperationException("5.12 compatibility requires JDK17"); + } + + @Override + public String description() { + return "Neo4j 5.12 (placeholder)"; + } +} diff --git a/compatibility/5.12/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_512/SettingProxyFactoryImpl.java b/compatibility/5.12/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_512/SettingProxyFactoryImpl.java new file mode 100644 index 0000000000..06032e87fa --- /dev/null +++ b/compatibility/5.12/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_512/SettingProxyFactoryImpl.java @@ -0,0 +1,31 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * This file contains proprietary code that is only available via a commercial license from Neo4j. + * For more information, see https://neo4j.com/contact-us/ + */ +package org.neo4j.gds.compat._512; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jVersion; +import org.neo4j.gds.compat.SettingProxyApi; +import org.neo4j.gds.compat.SettingProxyFactory; + +@ServiceProvider +public final class SettingProxyFactoryImpl implements SettingProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return false; + } + + @Override + public SettingProxyApi load() { + throw new UnsupportedOperationException("5.12 compatibility requires JDK17"); + } + + @Override + public String description() { + return "Neo4j Settings 5.12 (placeholder)"; + } +} diff --git a/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/BoltTransactionRunnerImpl.java b/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/BoltTransactionRunnerImpl.java new file mode 100644 index 0000000000..6148b17ef9 --- /dev/null +++ b/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/BoltTransactionRunnerImpl.java @@ -0,0 +1,97 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._512; + +import org.neo4j.bolt.dbapi.BoltGraphDatabaseServiceSPI; +import org.neo4j.bolt.dbapi.BoltTransaction; +import org.neo4j.bolt.protocol.common.message.AccessMode; +import org.neo4j.bolt.protocol.common.message.request.connection.RoutingContext; +import org.neo4j.bolt.tx.statement.StatementQuerySubscriber; +import org.neo4j.exceptions.KernelException; +import org.neo4j.gds.compat.BoltQuerySubscriber; +import org.neo4j.gds.compat.BoltTransactionRunner; +import org.neo4j.graphdb.QueryStatistics; +import org.neo4j.internal.kernel.api.connectioninfo.ClientConnectionInfo; +import org.neo4j.internal.kernel.api.security.LoginContext; +import org.neo4j.kernel.api.KernelTransaction; +import org.neo4j.kernel.impl.query.QueryExecutionConfiguration; +import org.neo4j.kernel.impl.query.QueryExecutionKernelException; +import org.neo4j.values.virtual.MapValue; + +import java.time.Duration; +import java.util.List; +import java.util.Map; + +public class BoltTransactionRunnerImpl extends BoltTransactionRunner { + + @Override + protected BoltQuerySubscriber boltQuerySubscriber() { + var subscriber = new StatementQuerySubscriber(); + return new BoltQuerySubscriber<>() { + @Override + public void assertSucceeded() throws KernelException { + subscriber.assertSuccess(); + } + + @Override + public QueryStatistics queryStatistics() { + return subscriber.getStatistics(); + } + + @Override + public StatementQuerySubscriber innerSubscriber() { + return subscriber; + } + }; + } + + @Override + protected void executeQuery( + BoltTransaction boltTransaction, + String query, + MapValue parameters, + StatementQuerySubscriber querySubscriber + ) throws QueryExecutionKernelException { + boltTransaction.executeQuery(query, parameters, true, querySubscriber); + } + + @Override + protected BoltTransaction beginBoltWriteTransaction( + BoltGraphDatabaseServiceSPI fabricDb, + LoginContext loginContext, + KernelTransaction.Type kernelTransactionType, + ClientConnectionInfo clientConnectionInfo, + List bookmarks, + Duration txTimeout, + Map txMetadata + ) { + return fabricDb.beginTransaction( + kernelTransactionType, + loginContext, + clientConnectionInfo, + bookmarks, + txTimeout, + AccessMode.WRITE, + txMetadata, + new RoutingContext(true, Map.of()), + QueryExecutionConfiguration.DEFAULT_CONFIG + ); + } +} diff --git a/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/CallableProcedureImpl.java b/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/CallableProcedureImpl.java new file mode 100644 index 0000000000..c940486e6d --- /dev/null +++ b/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/CallableProcedureImpl.java @@ -0,0 +1,53 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._512; + +import org.neo4j.collection.RawIterator; +import org.neo4j.gds.annotation.SuppressForbidden; +import org.neo4j.gds.compat.CompatCallableProcedure; +import org.neo4j.internal.kernel.api.exceptions.ProcedureException; +import org.neo4j.internal.kernel.api.procs.ProcedureSignature; +import org.neo4j.kernel.api.ResourceMonitor; +import org.neo4j.kernel.api.procedure.CallableProcedure; +import org.neo4j.kernel.api.procedure.Context; +import org.neo4j.values.AnyValue; + +@SuppressForbidden(reason = "This is the compat API") +public final class CallableProcedureImpl implements CallableProcedure { + private final CompatCallableProcedure procedure; + + CallableProcedureImpl(CompatCallableProcedure procedure) { + this.procedure = procedure; + } + + @Override + public ProcedureSignature signature() { + return this.procedure.signature(); + } + + @Override + public RawIterator apply( + Context ctx, + AnyValue[] input, + ResourceMonitor resourceMonitor + ) throws ProcedureException { + return this.procedure.apply(ctx, input); + } +} diff --git a/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/CallableUserAggregationFunctionImpl.java b/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/CallableUserAggregationFunctionImpl.java new file mode 100644 index 0000000000..68e4cd838a --- /dev/null +++ b/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/CallableUserAggregationFunctionImpl.java @@ -0,0 +1,77 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._512; + +import org.neo4j.gds.annotation.SuppressForbidden; +import org.neo4j.gds.compat.CompatUserAggregationFunction; +import org.neo4j.gds.compat.CompatUserAggregator; +import org.neo4j.internal.kernel.api.exceptions.ProcedureException; +import org.neo4j.internal.kernel.api.procs.UserAggregationReducer; +import org.neo4j.internal.kernel.api.procs.UserAggregationUpdater; +import org.neo4j.internal.kernel.api.procs.UserFunctionSignature; +import org.neo4j.kernel.api.procedure.CallableUserAggregationFunction; +import org.neo4j.kernel.api.procedure.Context; +import org.neo4j.values.AnyValue; + +@SuppressForbidden(reason = "This is the compat API") +public final class CallableUserAggregationFunctionImpl implements CallableUserAggregationFunction { + private final CompatUserAggregationFunction function; + + CallableUserAggregationFunctionImpl(CompatUserAggregationFunction function) { + this.function = function; + } + + @Override + public UserFunctionSignature signature() { + return this.function.signature(); + } + + @Override + public UserAggregationReducer createReducer(Context ctx) throws ProcedureException { + return new UserAggregatorImpl(this.function.create(ctx)); + } + + private static final class UserAggregatorImpl implements UserAggregationReducer, UserAggregationUpdater { + private final CompatUserAggregator aggregator; + + private UserAggregatorImpl(CompatUserAggregator aggregator) { + this.aggregator = aggregator; + } + + @Override + public UserAggregationUpdater newUpdater() { + return this; + } + + @Override + public void update(AnyValue[] input) throws ProcedureException { + this.aggregator.update(input); + } + + @Override + public void applyUpdates() { + } + + @Override + public AnyValue result() throws ProcedureException { + return this.aggregator.result(); + } + } +} diff --git a/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/CompatAccessModeImpl.java b/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/CompatAccessModeImpl.java new file mode 100644 index 0000000000..7ff84d9939 --- /dev/null +++ b/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/CompatAccessModeImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._512; + +import org.neo4j.gds.compat.CompatAccessMode; +import org.neo4j.gds.compat.CustomAccessMode; +import org.neo4j.internal.kernel.api.RelTypeSupplier; +import org.neo4j.internal.kernel.api.TokenSet; + +import java.util.function.Supplier; + +public final class CompatAccessModeImpl extends CompatAccessMode { + + CompatAccessModeImpl(CustomAccessMode custom) { + super(custom); + } + + @Override + public boolean allowsReadNodeProperty(Supplier labels, int propertyKey) { + return custom.allowsReadNodeProperty(propertyKey); + } + + @Override + public boolean allowsReadRelationshipProperty(RelTypeSupplier relType, int propertyKey) { + return custom.allowsReadRelationshipProperty(propertyKey); + } +} diff --git a/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/CompatGraphDatabaseAPIImpl.java b/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/CompatGraphDatabaseAPIImpl.java new file mode 100644 index 0000000000..f4995ac8ba --- /dev/null +++ b/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/CompatGraphDatabaseAPIImpl.java @@ -0,0 +1,74 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._512; + +import org.neo4j.dbms.api.DatabaseManagementService; +import org.neo4j.dbms.systemgraph.TopologyGraphDbmsModel; +import org.neo4j.gds.compat.GdsGraphDatabaseAPI; +import org.neo4j.internal.kernel.api.connectioninfo.ClientConnectionInfo; +import org.neo4j.internal.kernel.api.security.LoginContext; +import org.neo4j.kernel.api.KernelTransaction; +import org.neo4j.kernel.api.exceptions.Status; +import org.neo4j.kernel.impl.coreapi.InternalTransaction; +import org.neo4j.kernel.impl.coreapi.TransactionExceptionMapper; + +import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; + +final class CompatGraphDatabaseAPIImpl extends GdsGraphDatabaseAPI { + + CompatGraphDatabaseAPIImpl(DatabaseManagementService dbms) { + super(dbms); + } + + @Override + public boolean isAvailable() { + return api.isAvailable(); + } + + @Override + public TopologyGraphDbmsModel.HostedOnMode mode() { + // NOTE: This means we can never start clusters locally, which is probably fine since: + // 1) We never did this before + // 2) We only use this for tests and benchmarks + return TopologyGraphDbmsModel.HostedOnMode.SINGLE; + } + + @Override + public InternalTransaction beginTransaction( + KernelTransaction.Type type, + LoginContext loginContext, + ClientConnectionInfo clientInfo, + long timeout, + TimeUnit unit, + Consumer terminationCallback, + TransactionExceptionMapper transactionExceptionMapper + ) { + return api.beginTransaction( + type, + loginContext, + clientInfo, + timeout, + unit, + terminationCallback, + transactionExceptionMapper + ); + } +} diff --git a/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/CompatIndexQueryImpl.java b/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/CompatIndexQueryImpl.java new file mode 100644 index 0000000000..143ccbdbdb --- /dev/null +++ b/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/CompatIndexQueryImpl.java @@ -0,0 +1,31 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._512; + +import org.neo4j.gds.compat.CompatIndexQuery; +import org.neo4j.internal.kernel.api.PropertyIndexQuery; + +final class CompatIndexQueryImpl implements CompatIndexQuery { + final PropertyIndexQuery indexQuery; + + CompatIndexQueryImpl(PropertyIndexQuery indexQuery) { + this.indexQuery = indexQuery; + } +} diff --git a/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/CompatUsernameAuthSubjectImpl.java b/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/CompatUsernameAuthSubjectImpl.java new file mode 100644 index 0000000000..67ffd47576 --- /dev/null +++ b/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/CompatUsernameAuthSubjectImpl.java @@ -0,0 +1,35 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._512; + +import org.neo4j.gds.compat.CompatUsernameAuthSubject; +import org.neo4j.internal.kernel.api.security.AuthSubject; + +final class CompatUsernameAuthSubjectImpl extends CompatUsernameAuthSubject { + + CompatUsernameAuthSubjectImpl(String username, AuthSubject authSubject) { + super(username, authSubject); + } + + @Override + public String executingUser() { + return username; + } +} diff --git a/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/CompositeNodeCursorImpl.java b/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/CompositeNodeCursorImpl.java new file mode 100644 index 0000000000..c63636a254 --- /dev/null +++ b/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/CompositeNodeCursorImpl.java @@ -0,0 +1,32 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._512; + +import org.neo4j.gds.compat.CompositeNodeCursor; +import org.neo4j.internal.kernel.api.NodeLabelIndexCursor; + +import java.util.List; + +public final class CompositeNodeCursorImpl extends CompositeNodeCursor { + + CompositeNodeCursorImpl(List cursors, int[] labelIds) { + super(cursors, labelIds); + } +} diff --git a/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/GdsDatabaseLayoutImpl.java b/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/GdsDatabaseLayoutImpl.java new file mode 100644 index 0000000000..a013d2ed0d --- /dev/null +++ b/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/GdsDatabaseLayoutImpl.java @@ -0,0 +1,50 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._512; + +import org.neo4j.gds.compat.GdsDatabaseLayout; +import org.neo4j.io.layout.DatabaseLayout; + +import java.nio.file.Path; + +public class GdsDatabaseLayoutImpl implements GdsDatabaseLayout { + private final DatabaseLayout databaseLayout; + + public GdsDatabaseLayoutImpl(DatabaseLayout databaseLayout) {this.databaseLayout = databaseLayout;} + + @Override + public Path databaseDirectory() { + return databaseLayout.databaseDirectory(); + } + + @Override + public Path getTransactionLogsDirectory() { + return databaseLayout.getTransactionLogsDirectory(); + } + + @Override + public Path metadataStore() { + return databaseLayout.metadataStore(); + } + + public DatabaseLayout databaseLayout() { + return databaseLayout; + } +} diff --git a/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/GdsDatabaseManagementServiceBuilderImpl.java b/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/GdsDatabaseManagementServiceBuilderImpl.java new file mode 100644 index 0000000000..d1556f00ea --- /dev/null +++ b/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/GdsDatabaseManagementServiceBuilderImpl.java @@ -0,0 +1,54 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._512; + +import org.neo4j.dbms.api.DatabaseManagementService; +import org.neo4j.dbms.api.DatabaseManagementServiceBuilderImplementation; +import org.neo4j.gds.compat.GdsDatabaseManagementServiceBuilder; +import org.neo4j.graphdb.config.Setting; + +import java.nio.file.Path; +import java.util.Map; + +public class GdsDatabaseManagementServiceBuilderImpl implements GdsDatabaseManagementServiceBuilder { + + private final DatabaseManagementServiceBuilderImplementation dbmsBuilder; + + GdsDatabaseManagementServiceBuilderImpl(Path storeDir) { + this.dbmsBuilder = new DatabaseManagementServiceBuilderImplementation(storeDir); + } + + @Override + public GdsDatabaseManagementServiceBuilder setConfigRaw(Map configMap) { + dbmsBuilder.setConfigRaw(configMap); + return this; + } + + @Override + public GdsDatabaseManagementServiceBuilder setConfig(Setting setting, S value) { + dbmsBuilder.setConfig(setting, value); + return this; + } + + @Override + public DatabaseManagementService build() { + return dbmsBuilder.build(); + } +} diff --git a/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/Neo4jProxyFactoryImpl.java b/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/Neo4jProxyFactoryImpl.java new file mode 100644 index 0000000000..4b7a863afd --- /dev/null +++ b/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/Neo4jProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._512; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jProxyApi; +import org.neo4j.gds.compat.Neo4jProxyFactory; +import org.neo4j.gds.compat.Neo4jVersion; + +@ServiceProvider +public final class Neo4jProxyFactoryImpl implements Neo4jProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return version == Neo4jVersion.V_5_12; + } + + @Override + public Neo4jProxyApi load() { + return new Neo4jProxyImpl(); + } + + @Override + public String description() { + return "Neo4j 5.12"; + } +} diff --git a/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/Neo4jProxyImpl.java b/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/Neo4jProxyImpl.java new file mode 100644 index 0000000000..9a02e66c34 --- /dev/null +++ b/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/Neo4jProxyImpl.java @@ -0,0 +1,950 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._512; + +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import org.neo4j.common.DependencyResolver; +import org.neo4j.common.EntityType; +import org.neo4j.configuration.BootloaderSettings; +import org.neo4j.configuration.Config; +import org.neo4j.configuration.GraphDatabaseSettings; +import org.neo4j.configuration.SettingValueParsers; +import org.neo4j.configuration.connectors.ConnectorPortRegister; +import org.neo4j.configuration.connectors.ConnectorType; +import org.neo4j.configuration.helpers.DatabaseNameValidator; +import org.neo4j.dbms.api.DatabaseManagementService; +import org.neo4j.exceptions.KernelException; +import org.neo4j.fabric.FabricDatabaseManager; +import org.neo4j.gds.annotation.SuppressForbidden; +import org.neo4j.gds.compat.BoltTransactionRunner; +import org.neo4j.gds.compat.CompatCallableProcedure; +import org.neo4j.gds.compat.CompatExecutionMonitor; +import org.neo4j.gds.compat.CompatIndexQuery; +import org.neo4j.gds.compat.CompatInput; +import org.neo4j.gds.compat.CompatUserAggregationFunction; +import org.neo4j.gds.compat.CompositeNodeCursor; +import org.neo4j.gds.compat.CustomAccessMode; +import org.neo4j.gds.compat.GdsDatabaseLayout; +import org.neo4j.gds.compat.GdsDatabaseManagementServiceBuilder; +import org.neo4j.gds.compat.GdsGraphDatabaseAPI; +import org.neo4j.gds.compat.GlobalProcedureRegistry; +import org.neo4j.gds.compat.GraphDatabaseApiProxy; +import org.neo4j.gds.compat.InputEntityIdVisitor; +import org.neo4j.gds.compat.Neo4jProxyApi; +import org.neo4j.gds.compat.PropertyReference; +import org.neo4j.gds.compat.StoreScan; +import org.neo4j.gds.compat.TestLog; +import org.neo4j.graphdb.GraphDatabaseService; +import org.neo4j.graphdb.Node; +import org.neo4j.graphdb.Relationship; +import org.neo4j.graphdb.RelationshipType; +import org.neo4j.graphdb.config.Setting; +import org.neo4j.internal.batchimport.AdditionalInitialIds; +import org.neo4j.internal.batchimport.BatchImporter; +import org.neo4j.internal.batchimport.BatchImporterFactory; +import org.neo4j.internal.batchimport.Configuration; +import org.neo4j.internal.batchimport.IndexConfig; +import org.neo4j.internal.batchimport.InputIterable; +import org.neo4j.internal.batchimport.Monitor; +import org.neo4j.internal.batchimport.input.Collector; +import org.neo4j.internal.batchimport.input.IdType; +import org.neo4j.internal.batchimport.input.Input; +import org.neo4j.internal.batchimport.input.InputEntityVisitor; +import org.neo4j.internal.batchimport.input.PropertySizeCalculator; +import org.neo4j.internal.batchimport.input.ReadableGroups; +import org.neo4j.internal.batchimport.staging.ExecutionMonitor; +import org.neo4j.internal.batchimport.staging.StageExecution; +import org.neo4j.internal.helpers.HostnamePort; +import org.neo4j.internal.id.IdGenerator; +import org.neo4j.internal.id.IdGeneratorFactory; +import org.neo4j.internal.kernel.api.Cursor; +import org.neo4j.internal.kernel.api.IndexQueryConstraints; +import org.neo4j.internal.kernel.api.IndexReadSession; +import org.neo4j.internal.kernel.api.NodeCursor; +import org.neo4j.internal.kernel.api.NodeLabelIndexCursor; +import org.neo4j.internal.kernel.api.NodeValueIndexCursor; +import org.neo4j.internal.kernel.api.PropertyCursor; +import org.neo4j.internal.kernel.api.PropertyIndexQuery; +import org.neo4j.internal.kernel.api.QueryContext; +import org.neo4j.internal.kernel.api.Read; +import org.neo4j.internal.kernel.api.RelationshipScanCursor; +import org.neo4j.internal.kernel.api.Scan; +import org.neo4j.internal.kernel.api.TokenPredicate; +import org.neo4j.internal.kernel.api.connectioninfo.ClientConnectionInfo; +import org.neo4j.internal.kernel.api.exceptions.ProcedureException; +import org.neo4j.internal.kernel.api.procs.FieldSignature; +import org.neo4j.internal.kernel.api.procs.Neo4jTypes; +import org.neo4j.internal.kernel.api.procs.ProcedureSignature; +import org.neo4j.internal.kernel.api.procs.QualifiedName; +import org.neo4j.internal.kernel.api.procs.UserFunctionSignature; +import org.neo4j.internal.kernel.api.security.AccessMode; +import org.neo4j.internal.kernel.api.security.AuthSubject; +import org.neo4j.internal.kernel.api.security.SecurityContext; +import org.neo4j.internal.recordstorage.RecordIdType; +import org.neo4j.internal.schema.IndexCapability; +import org.neo4j.internal.schema.IndexDescriptor; +import org.neo4j.internal.schema.IndexOrder; +import org.neo4j.internal.schema.SchemaDescriptors; +import org.neo4j.io.fs.FileSystemAbstraction; +import org.neo4j.io.layout.DatabaseLayout; +import org.neo4j.io.layout.Neo4jLayout; +import org.neo4j.io.layout.recordstorage.RecordDatabaseLayout; +import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.context.CursorContextFactory; +import org.neo4j.io.pagecache.context.FixedVersionContextSupplier; +import org.neo4j.io.pagecache.tracing.PageCacheTracer; +import org.neo4j.kernel.api.KernelTransaction; +import org.neo4j.kernel.api.KernelTransactionHandle; +import org.neo4j.kernel.api.procedure.CallableProcedure; +import org.neo4j.kernel.api.procedure.CallableUserAggregationFunction; +import org.neo4j.kernel.api.procedure.Context; +import org.neo4j.kernel.api.procedure.GlobalProcedures; +import org.neo4j.kernel.database.NormalizedDatabaseName; +import org.neo4j.kernel.impl.coreapi.InternalTransaction; +import org.neo4j.kernel.impl.index.schema.IndexImporterFactoryImpl; +import org.neo4j.kernel.impl.query.QueryExecutionConfiguration; +import org.neo4j.kernel.impl.query.TransactionalContext; +import org.neo4j.kernel.impl.query.TransactionalContextFactory; +import org.neo4j.kernel.impl.store.RecordStore; +import org.neo4j.kernel.impl.store.format.RecordFormatSelector; +import org.neo4j.kernel.impl.store.format.RecordFormats; +import org.neo4j.kernel.impl.store.record.AbstractBaseRecord; +import org.neo4j.kernel.impl.transaction.log.EmptyLogTailMetadata; +import org.neo4j.kernel.impl.transaction.log.files.TransactionLogInitializer; +import org.neo4j.logging.Log; +import org.neo4j.logging.internal.LogService; +import org.neo4j.memory.EmptyMemoryTracker; +import org.neo4j.procedure.Mode; +import org.neo4j.scheduler.JobScheduler; +import org.neo4j.ssl.config.SslPolicyLoader; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.StorageEngineFactory; +import org.neo4j.util.BitBuffer; +import org.neo4j.values.storable.TextArray; +import org.neo4j.values.storable.ValueCategory; +import org.neo4j.values.storable.Values; +import org.neo4j.values.virtual.MapValue; +import org.neo4j.values.virtual.NodeValue; +import org.neo4j.values.virtual.VirtualValues; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Locale; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static java.lang.String.format; +import static org.neo4j.gds.compat.InternalReadOps.countByIdGenerator; + +public final class Neo4jProxyImpl implements Neo4jProxyApi { + + @Override + public GdsGraphDatabaseAPI newDb(DatabaseManagementService dbms) { + return new CompatGraphDatabaseAPIImpl(dbms); + } + + @Override + public String validateExternalDatabaseName(String databaseName) { + var normalizedName = new NormalizedDatabaseName(databaseName); + DatabaseNameValidator.validateExternalDatabaseName(normalizedName); + return normalizedName.name(); + } + + @Override + public AccessMode accessMode(CustomAccessMode customAccessMode) { + return new CompatAccessModeImpl(customAccessMode); + } + + @Override + public String username(AuthSubject subject) { + return subject.executingUser(); + } + + @Override + public SecurityContext securityContext( + String username, + AuthSubject authSubject, + AccessMode mode, + String databaseName + ) { + return new SecurityContext( + new CompatUsernameAuthSubjectImpl(username, authSubject), + mode, + // GDS is always operating from an embedded context + ClientConnectionInfo.EMBEDDED_CONNECTION, + databaseName + ); + } + + @Override + public long getHighId(RecordStore recordStore) { + return recordStore.getIdGenerator().getHighId(); + } + + @Override + public List> entityCursorScan( + KernelTransaction transaction, + int[] labelIds, + int batchSize, + boolean allowPartitionedScan + ) { + if (allowPartitionedScan) { + return partitionedNodeLabelIndexScan(transaction, batchSize, labelIds); + } else { + var read = transaction.dataRead(); + return Arrays + .stream(labelIds) + .mapToObj(read::nodeLabelScan) + .map(scan -> scanToStoreScan(scan, batchSize)) + .collect(Collectors.toList()); + } + } + + @Override + public PropertyCursor allocatePropertyCursor(KernelTransaction kernelTransaction) { + return kernelTransaction + .cursors() + .allocatePropertyCursor(kernelTransaction.cursorContext(), kernelTransaction.memoryTracker()); + } + + @Override + public PropertyReference propertyReference(NodeCursor nodeCursor) { + return ReferencePropertyReference.of(nodeCursor.propertiesReference()); + } + + @Override + public PropertyReference propertyReference(RelationshipScanCursor relationshipScanCursor) { + return ReferencePropertyReference.of(relationshipScanCursor.propertiesReference()); + } + + @Override + public PropertyReference noPropertyReference() { + return ReferencePropertyReference.empty(); + } + + @Override + public void nodeProperties( + KernelTransaction kernelTransaction, + long nodeReference, + PropertyReference reference, + PropertyCursor cursor + ) { + var neoReference = ((ReferencePropertyReference) reference).reference; + kernelTransaction + .dataRead() + .nodeProperties(nodeReference, neoReference, PropertySelection.ALL_PROPERTIES, cursor); + } + + @Override + public void relationshipProperties( + KernelTransaction kernelTransaction, + long relationshipReference, + PropertyReference reference, + PropertyCursor cursor + ) { + var neoReference = ((ReferencePropertyReference) reference).reference; + kernelTransaction + .dataRead() + .relationshipProperties(relationshipReference, neoReference, PropertySelection.ALL_PROPERTIES, cursor); + } + + @Override + public NodeCursor allocateNodeCursor(KernelTransaction kernelTransaction) { + return kernelTransaction.cursors().allocateNodeCursor(kernelTransaction.cursorContext()); + } + + @Override + public RelationshipScanCursor allocateRelationshipScanCursor(KernelTransaction kernelTransaction) { + return kernelTransaction.cursors().allocateRelationshipScanCursor(kernelTransaction.cursorContext()); + } + + @Override + public NodeLabelIndexCursor allocateNodeLabelIndexCursor(KernelTransaction kernelTransaction) { + return kernelTransaction.cursors().allocateNodeLabelIndexCursor(kernelTransaction.cursorContext()); + } + + @Override + public NodeValueIndexCursor allocateNodeValueIndexCursor(KernelTransaction kernelTransaction) { + return kernelTransaction + .cursors() + .allocateNodeValueIndexCursor(kernelTransaction.cursorContext(), kernelTransaction.memoryTracker()); + } + + @Override + public boolean hasNodeLabelIndex(KernelTransaction kernelTransaction) { + return NodeLabelIndexLookupImpl.hasNodeLabelIndex(kernelTransaction); + } + + @Override + public StoreScan nodeLabelIndexScan( + KernelTransaction transaction, + int labelId, + int batchSize, + boolean allowPartitionedScan + ) { + if (allowPartitionedScan) { + return partitionedNodeLabelIndexScan(transaction, batchSize, labelId).get(0); + } else { + var read = transaction.dataRead(); + return scanToStoreScan(read.nodeLabelScan(labelId), batchSize); + } + } + + @Override + public StoreScan scanToStoreScan(Scan scan, int batchSize) { + return new ScanBasedStoreScanImpl<>(scan, batchSize); + } + + private List> partitionedNodeLabelIndexScan( + KernelTransaction transaction, + int batchSize, + int... labelIds + ) { + var indexDescriptor = NodeLabelIndexLookupImpl.findUsableMatchingIndex( + transaction, + SchemaDescriptors.forAnyEntityTokens(EntityType.NODE) + ); + + if (indexDescriptor == IndexDescriptor.NO_INDEX) { + throw new IllegalStateException("There is no index that can back a node label scan."); + } + + var read = transaction.dataRead(); + + // Our current strategy is to select the token with the highest count + // and use that one as the driving partitioned index scan. The partitions + // of all other partitioned index scans will be aligned to that one. + int maxToken = labelIds[0]; + long maxCount = read.countsForNodeWithoutTxState(labelIds[0]); + + for (int i = 1; i < labelIds.length; i++) { + long count = read.countsForNodeWithoutTxState(labelIds[i]); + if (count > maxCount) { + maxCount = count; + maxToken = labelIds[i]; + } + } + + int numberOfPartitions = PartitionedStoreScan.getNumberOfPartitions(maxCount, batchSize); + + try { + var session = read.tokenReadSession(indexDescriptor); + + var partitionedScan = read.nodeLabelScan( + session, + numberOfPartitions, + transaction.cursorContext(), + new TokenPredicate(maxToken) + ); + + var scans = new ArrayList>(labelIds.length); + scans.add(new PartitionedStoreScan(partitionedScan)); + + // Initialize the remaining index scans with the partitioning of the first scan. + for (int labelToken : labelIds) { + if (labelToken != maxToken) { + var scan = read.nodeLabelScan(session, partitionedScan, new TokenPredicate(labelToken)); + scans.add(new PartitionedStoreScan(scan)); + } + } + + return scans; + } catch (KernelException e) { + // should not happen, we check for the index existence and applicability + // before reading it + throw new RuntimeException("Unexpected error while initialising reading from node label index", e); + } + } + + @Override + public CompatIndexQuery rangeIndexQuery( + int propertyKeyId, + double from, + boolean fromInclusive, + double to, + boolean toInclusive + ) { + return new CompatIndexQueryImpl(PropertyIndexQuery.range(propertyKeyId, from, fromInclusive, to, toInclusive)); + } + + @Override + public CompatIndexQuery rangeAllIndexQuery(int propertyKeyId) { + var rangePredicate = PropertyIndexQuery.range( + propertyKeyId, + Values.doubleValue(Double.NEGATIVE_INFINITY), + true, + Values.doubleValue(Double.POSITIVE_INFINITY), + true + ); + return new CompatIndexQueryImpl(rangePredicate); + } + + @Override + public void nodeIndexSeek( + Read dataRead, + IndexReadSession index, + NodeValueIndexCursor cursor, + IndexOrder indexOrder, + boolean needsValues, + CompatIndexQuery query + ) throws KernelException { + var indexQueryConstraints = indexOrder == IndexOrder.NONE + ? IndexQueryConstraints.unordered(needsValues) + : IndexQueryConstraints.constrained(indexOrder, needsValues); + + dataRead.nodeIndexSeek( + (QueryContext) dataRead, + index, + cursor, + indexQueryConstraints, + ((CompatIndexQueryImpl) query).indexQuery + ); + } + + @Override + public CompositeNodeCursor compositeNodeCursor(List cursors, int[] labelIds) { + return new CompositeNodeCursorImpl(cursors, labelIds); + } + + @Override + public Configuration batchImporterConfig( + int batchSize, + int writeConcurrency, + Optional pageCacheMemory, + boolean highIO, + IndexConfig indexConfig + ) { + return new org.neo4j.internal.batchimport.Configuration() { + @Override + public int batchSize() { + return batchSize; + } + + @Override + public int maxNumberOfWorkerThreads() { + return writeConcurrency; + } + + @Override + public boolean highIO() { + return highIO; + } + + @Override + public IndexConfig indexConfig() { + return indexConfig; + } + }; + } + + @Override + public int writeConcurrency(Configuration batchImportConfiguration) { + return batchImportConfiguration.maxNumberOfWorkerThreads(); + } + + @Override + public BatchImporter instantiateBatchImporter( + BatchImporterFactory factory, + GdsDatabaseLayout directoryStructure, + FileSystemAbstraction fileSystem, + PageCacheTracer pageCacheTracer, + Configuration configuration, + LogService logService, + ExecutionMonitor executionMonitor, + AdditionalInitialIds additionalInitialIds, + Config dbConfig, + RecordFormats recordFormats, + JobScheduler jobScheduler, + Collector badCollector + ) { + dbConfig.set(GraphDatabaseSettings.db_format, recordFormats.name()); + var databaseLayout = ((GdsDatabaseLayoutImpl) directoryStructure).databaseLayout(); + return factory.instantiate( + databaseLayout, + fileSystem, + pageCacheTracer, + configuration, + logService, + executionMonitor, + additionalInitialIds, + new EmptyLogTailMetadata(dbConfig), + dbConfig, + Monitor.NO_MONITOR, + jobScheduler, + badCollector, + TransactionLogInitializer.getLogFilesInitializer(), + new IndexImporterFactoryImpl(), + EmptyMemoryTracker.INSTANCE, + new CursorContextFactory(PageCacheTracer.NULL, FixedVersionContextSupplier.EMPTY_CONTEXT_SUPPLIER) + ); + } + + @Override + public Input batchInputFrom(CompatInput compatInput) { + return new InputFromCompatInput(compatInput); + } + + @Override + public InputEntityIdVisitor.Long inputEntityLongIdVisitor(IdType idType, ReadableGroups groups) { + switch (idType) { + case ACTUAL -> { + return new InputEntityIdVisitor.Long() { + @Override + public void visitNodeId(InputEntityVisitor visitor, long id) { + visitor.id(id); + } + + @Override + public void visitSourceId(InputEntityVisitor visitor, long id) { + visitor.startId(id); + } + + @Override + public void visitTargetId(InputEntityVisitor visitor, long id) { + visitor.endId(id); + } + }; + } + case INTEGER -> { + var globalGroup = groups.get(null); + + return new InputEntityIdVisitor.Long() { + @Override + public void visitNodeId(InputEntityVisitor visitor, long id) { + visitor.id(id, globalGroup); + } + + @Override + public void visitSourceId(InputEntityVisitor visitor, long id) { + visitor.startId(id, globalGroup); + } + + @Override + public void visitTargetId(InputEntityVisitor visitor, long id) { + visitor.endId(id, globalGroup); + } + }; + } + default -> throw new IllegalStateException("Unexpected value: " + idType); + } + } + + @Override + public InputEntityIdVisitor.String inputEntityStringIdVisitor(ReadableGroups groups) { + var globalGroup = groups.get(null); + + return new InputEntityIdVisitor.String() { + @Override + public void visitNodeId(InputEntityVisitor visitor, String id) { + visitor.id(id, globalGroup); + } + + @Override + public void visitSourceId(InputEntityVisitor visitor, String id) { + visitor.startId(id, globalGroup); + } + + @Override + public void visitTargetId(InputEntityVisitor visitor, String id) { + visitor.endId(id, globalGroup); + } + }; + } + + @Override + public Setting additionalJvm() { + return BootloaderSettings.additional_jvm; + } + + @Override + public Setting pageCacheMemory() { + return GraphDatabaseSettings.pagecache_memory; + } + + @Override + public Long pageCacheMemoryValue(String value) { + return SettingValueParsers.BYTES.parse(value); + } + + @Override + public ProcedureSignature procedureSignature( + QualifiedName name, + List inputSignature, + List outputSignature, + Mode mode, + boolean admin, + String deprecated, + String description, + String warning, + boolean eager, + boolean caseInsensitive, + boolean systemProcedure, + boolean internal, + boolean allowExpiredCredentials, + boolean threadSafe + ) { + return new ProcedureSignature( + name, + inputSignature, + outputSignature, + mode, + admin, + deprecated, + description, + warning, + eager, + caseInsensitive, + systemProcedure, + internal, + allowExpiredCredentials, + threadSafe + ); + } + + @Override + public long getHighestPossibleNodeCount( + Read read, IdGeneratorFactory idGeneratorFactory + ) { + return countByIdGenerator(idGeneratorFactory, RecordIdType.NODE).orElseGet(read::nodesGetCount); + } + + @Override + public long getHighestPossibleRelationshipCount( + Read read, IdGeneratorFactory idGeneratorFactory + ) { + return countByIdGenerator(idGeneratorFactory, RecordIdType.RELATIONSHIP).orElseGet(read::relationshipsGetCount); + } + + @Override + public String versionLongToString(long storeVersion) { + // copied from org.neo4j.kernel.impl.store.LegacyMetadataHandler.versionLongToString which is private + if (storeVersion == -1) { + return "Unknown"; + } + BitBuffer bits = BitBuffer.bitsFromLongs(new long[]{storeVersion}); + int length = bits.getShort(8); + if (length == 0 || length > 7) { + throw new IllegalArgumentException(format(Locale.ENGLISH, "The read version string length %d is not proper.", length)); + } + char[] result = new char[length]; + for (int i = 0; i < length; i++) { + result[i] = (char) bits.getShort(8); + } + return new String(result); + } + + private static final class InputFromCompatInput implements Input { + private final CompatInput delegate; + + private InputFromCompatInput(CompatInput delegate) { + this.delegate = delegate; + } + + @Override + public InputIterable nodes(Collector badCollector) { + return delegate.nodes(badCollector); + } + + @Override + public InputIterable relationships(Collector badCollector) { + return delegate.relationships(badCollector); + } + + @Override + public IdType idType() { + return delegate.idType(); + } + + @Override + public ReadableGroups groups() { + return delegate.groups(); + } + + @Override + public Estimates calculateEstimates(PropertySizeCalculator propertySizeCalculator) throws IOException { + return delegate.calculateEstimates((values, kernelTransaction) -> propertySizeCalculator.calculateSize( + values, + kernelTransaction.cursorContext(), + kernelTransaction.memoryTracker() + )); + } + } + + @Override + public TestLog testLog() { + return new TestLogImpl(); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public Log getUserLog(LogService logService, Class loggingClass) { + return logService.getUserLog(loggingClass); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public Log getInternalLog(LogService logService, Class loggingClass) { + return logService.getInternalLog(loggingClass); + } + + @Override + public NodeValue nodeValue(long id, TextArray labels, MapValue properties) { + return VirtualValues.nodeValue(id, String.valueOf(id), labels, properties); + } + + @Override + public Relationship virtualRelationship(long id, Node startNode, Node endNode, RelationshipType type) { + return new VirtualRelationshipImpl(id, startNode, endNode, type); + } + + @Override + public GdsDatabaseManagementServiceBuilder databaseManagementServiceBuilder(Path storeDir) { + return new GdsDatabaseManagementServiceBuilderImpl(storeDir); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public RecordFormats selectRecordFormatForStore( + DatabaseLayout databaseLayout, + FileSystemAbstraction fs, + PageCache pageCache, + LogService logService, + PageCacheTracer pageCacheTracer + ) { + return RecordFormatSelector.selectForStore( + (RecordDatabaseLayout) databaseLayout, + fs, + pageCache, + logService.getInternalLogProvider(), + new CursorContextFactory(pageCacheTracer, FixedVersionContextSupplier.EMPTY_CONTEXT_SUPPLIER) + ); + } + + @Override + public boolean isNotNumericIndex(IndexCapability indexCapability) { + return !indexCapability.areValueCategoriesAccepted(ValueCategory.NUMBER); + } + + @Override + public void setAllowUpgrades(Config.Builder configBuilder, boolean value) { + } + + @Override + public String defaultRecordFormatSetting() { + return GraphDatabaseSettings.db_format.defaultValue(); + } + + @Override + public void configureRecordFormat(Config.Builder configBuilder, String recordFormat) { + var databaseRecordFormat = recordFormat.toLowerCase(Locale.ENGLISH); + configBuilder.set(GraphDatabaseSettings.db_format, databaseRecordFormat); + } + + @Override + public GdsDatabaseLayout databaseLayout(Config config, String databaseName) { + var storageEngineFactory = StorageEngineFactory.selectStorageEngine(config); + var dbLayout = neo4jLayout(config).databaseLayout(databaseName); + var databaseLayout = storageEngineFactory.formatSpecificDatabaseLayout(dbLayout); + return new GdsDatabaseLayoutImpl(databaseLayout); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public Neo4jLayout neo4jLayout(Config config) { + return Neo4jLayout.of(config); + } + + @Override + public BoltTransactionRunner boltTransactionRunner() { + return new BoltTransactionRunnerImpl(); + } + + @Override + public HostnamePort getLocalBoltAddress(ConnectorPortRegister connectorPortRegister) { + return connectorPortRegister.getLocalAddress(ConnectorType.BOLT); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public SslPolicyLoader createSllPolicyLoader( + FileSystemAbstraction fileSystem, + Config config, + LogService logService + ) { + return SslPolicyLoader.create(fileSystem, config, logService.getInternalLogProvider()); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public RecordFormats recordFormatSelector( + String databaseName, + Config databaseConfig, + FileSystemAbstraction fs, + LogService logService, + GraphDatabaseService databaseService + ) { + var neo4jLayout = Neo4jLayout.of(databaseConfig); + var recordDatabaseLayout = RecordDatabaseLayout.of(neo4jLayout, databaseName); + return RecordFormatSelector.selectForStoreOrConfigForNewDbs( + databaseConfig, + recordDatabaseLayout, + fs, + GraphDatabaseApiProxy.resolveDependency(databaseService, PageCache.class), + logService.getInternalLogProvider(), + GraphDatabaseApiProxy.resolveDependency(databaseService, CursorContextFactory.class) + ); + } + + @Override + public ExecutionMonitor executionMonitor(CompatExecutionMonitor compatExecutionMonitor) { + return new ExecutionMonitor.Adapter( + compatExecutionMonitor.checkIntervalMillis(), + TimeUnit.MILLISECONDS + ) { + + @Override + public void initialize(DependencyResolver dependencyResolver) { + compatExecutionMonitor.initialize(dependencyResolver); + } + + @Override + public void start(StageExecution execution) { + compatExecutionMonitor.start(execution); + } + + @Override + public void end(StageExecution execution, long totalTimeMillis) { + compatExecutionMonitor.end(execution, totalTimeMillis); + } + + @Override + public void done(boolean successful, long totalTimeMillis, String additionalInformation) { + compatExecutionMonitor.done(successful, totalTimeMillis, additionalInformation); + } + + @Override + public void check(StageExecution execution) { + compatExecutionMonitor.check(execution); + } + }; + } + + @Override + @SuppressFBWarnings("NP_LOAD_OF_KNOWN_NULL_VALUE") // We assign nulls because it makes the code more readable + public UserFunctionSignature userFunctionSignature( + QualifiedName name, + List inputSignature, + Neo4jTypes.AnyType type, + String description, + boolean internal, + boolean threadSafe, + Optional deprecatedBy + ) { + String category = null; // No predefined categpry (like temporal or math) + var caseInsensitive = false; // case sensitive name match + var isBuiltIn = false; // is built in; never true for GDS + + return new UserFunctionSignature( + name, + inputSignature, + type, + deprecatedBy.orElse(null), + description, + category, + caseInsensitive, + isBuiltIn, + internal, + threadSafe + ); + } + + @Override + @SuppressForbidden(reason = "This is the compat API") + public CallableProcedure callableProcedure(CompatCallableProcedure procedure) { + return new CallableProcedureImpl(procedure); + } + + @Override + @SuppressForbidden(reason = "This is the compat API") + public CallableUserAggregationFunction callableUserAggregationFunction(CompatUserAggregationFunction function) { + return new CallableUserAggregationFunctionImpl(function); + } + + @Override + public long transactionId(KernelTransactionHandle kernelTransactionHandle) { + return kernelTransactionHandle.getTransactionSequenceNumber(); + } + + @Override + public void reserveNeo4jIds(IdGeneratorFactory generatorFactory, int size, CursorContext cursorContext) { + IdGenerator idGenerator = generatorFactory.get(RecordIdType.NODE); + + idGenerator.nextConsecutiveIdRange(size, false, cursorContext); + } + + @Override + public TransactionalContext newQueryContext( + TransactionalContextFactory contextFactory, + InternalTransaction tx, + String queryText, + MapValue queryParameters + ) { + return contextFactory.newContext(tx, queryText, queryParameters, QueryExecutionConfiguration.DEFAULT_CONFIG); + } + + @Override + public boolean isCompositeDatabase(GraphDatabaseService databaseService) { + var databaseManager = GraphDatabaseApiProxy.resolveDependency(databaseService, FabricDatabaseManager.class); + return databaseManager.isFabricDatabase(GraphDatabaseApiProxy.databaseId(databaseService)); + } + + @Override + public T lookupComponentProvider(Context ctx, Class component, boolean safe) throws ProcedureException { + var globalProcedures = GraphDatabaseApiProxy.resolveDependency(ctx.dependencyResolver(), GlobalProcedures.class); + return globalProcedures.getCurrentView().lookupComponentProvider(component, safe).apply(ctx); + } + + @Override + public GlobalProcedureRegistry globalProcedureRegistry(GlobalProcedures globalProcedures) { + return new GlobalProcedureRegistry() { + @Override + public Set getAllProcedures() { + return globalProcedures.getCurrentView().getAllProcedures(); + } + + @Override + public Stream getAllNonAggregatingFunctions() { + return globalProcedures.getCurrentView().getAllNonAggregatingFunctions(); + } + + @Override + public Stream getAllAggregatingFunctions() { + return globalProcedures.getCurrentView().getAllAggregatingFunctions(); + } + }; + } +} diff --git a/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/NodeLabelIndexLookupImpl.java b/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/NodeLabelIndexLookupImpl.java new file mode 100644 index 0000000000..58d54c3806 --- /dev/null +++ b/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/NodeLabelIndexLookupImpl.java @@ -0,0 +1,68 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._512; + +import org.neo4j.common.EntityType; +import org.neo4j.internal.kernel.api.InternalIndexState; +import org.neo4j.internal.kernel.api.SchemaRead; +import org.neo4j.internal.kernel.api.exceptions.schema.IndexNotFoundKernelException; +import org.neo4j.internal.schema.IndexDescriptor; +import org.neo4j.internal.schema.IndexType; +import org.neo4j.internal.schema.SchemaDescriptor; +import org.neo4j.internal.schema.SchemaDescriptors; +import org.neo4j.kernel.api.KernelTransaction; + +final class NodeLabelIndexLookupImpl { + + static boolean hasNodeLabelIndex(KernelTransaction transaction) { + return NodeLabelIndexLookupImpl.findUsableMatchingIndex( + transaction, + SchemaDescriptors.forAnyEntityTokens(EntityType.NODE) + ) != IndexDescriptor.NO_INDEX; + } + + static IndexDescriptor findUsableMatchingIndex( + KernelTransaction transaction, + SchemaDescriptor schemaDescriptor + ) { + var schemaRead = transaction.schemaRead(); + var iterator = schemaRead.index(schemaDescriptor); + while (iterator.hasNext()) { + var index = iterator.next(); + if (index.getIndexType() == IndexType.LOOKUP && indexIsOnline(schemaRead, index)) { + return index; + } + } + return IndexDescriptor.NO_INDEX; + } + + private static boolean indexIsOnline(SchemaRead schemaRead, IndexDescriptor index) { + var state = InternalIndexState.FAILED; + try { + state = schemaRead.indexGetState(index); + } catch (IndexNotFoundKernelException e) { + // Well the index should always exist here, but if we didn't find it while checking the state, + // then we obviously don't want to use it. + } + return state == InternalIndexState.ONLINE; + } + + private NodeLabelIndexLookupImpl() {} +} diff --git a/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/PartitionedStoreScan.java b/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/PartitionedStoreScan.java new file mode 100644 index 0000000000..365e85330d --- /dev/null +++ b/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/PartitionedStoreScan.java @@ -0,0 +1,58 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._512; + +import org.neo4j.gds.compat.StoreScan; +import org.neo4j.internal.kernel.api.NodeLabelIndexCursor; +import org.neo4j.internal.kernel.api.PartitionedScan; +import org.neo4j.kernel.api.KernelTransaction; + +final class PartitionedStoreScan implements StoreScan { + private final PartitionedScan scan; + + PartitionedStoreScan(PartitionedScan scan) { + this.scan = scan; + } + + static int getNumberOfPartitions(long nodeCount, int batchSize) { + int numberOfPartitions; + if (nodeCount > 0) { + // ceil div to try to get enough partitions so a single one does + // not include more nodes than batchSize + long partitions = ((nodeCount - 1) / batchSize) + 1; + + // value must be positive + if (partitions < 1) { + partitions = 1; + } + + numberOfPartitions = (int) Long.min(Integer.MAX_VALUE, partitions); + } else { + // we have no partitions to scan, but the value must still be positive + numberOfPartitions = 1; + } + return numberOfPartitions; + } + + @Override + public boolean reserveBatch(NodeLabelIndexCursor cursor, KernelTransaction ktx) { + return scan.reservePartition(cursor, ktx.cursorContext(), ktx.securityContext().mode()); + } +} diff --git a/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/ReferencePropertyReference.java b/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/ReferencePropertyReference.java new file mode 100644 index 0000000000..ee12df9ca6 --- /dev/null +++ b/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/ReferencePropertyReference.java @@ -0,0 +1,49 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._512; + +import org.neo4j.gds.compat.PropertyReference; +import org.neo4j.storageengine.api.Reference; + +import java.util.Objects; + +public final class ReferencePropertyReference implements PropertyReference { + + private static final PropertyReference EMPTY = new ReferencePropertyReference(null); + + public final Reference reference; + + private ReferencePropertyReference(Reference reference) { + this.reference = reference; + } + + public static PropertyReference of(Reference reference) { + return new ReferencePropertyReference(Objects.requireNonNull(reference)); + } + + public static PropertyReference empty() { + return EMPTY; + } + + @Override + public boolean isEmpty() { + return reference == null; + } +} diff --git a/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/ScanBasedStoreScanImpl.java b/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/ScanBasedStoreScanImpl.java new file mode 100644 index 0000000000..9319f2f423 --- /dev/null +++ b/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/ScanBasedStoreScanImpl.java @@ -0,0 +1,40 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._512; + +import org.neo4j.gds.compat.StoreScan; +import org.neo4j.internal.kernel.api.Cursor; +import org.neo4j.internal.kernel.api.Scan; +import org.neo4j.kernel.api.KernelTransaction; + +public final class ScanBasedStoreScanImpl implements StoreScan { + private final Scan scan; + private final int batchSize; + + public ScanBasedStoreScanImpl(Scan scan, int batchSize) { + this.scan = scan; + this.batchSize = batchSize; + } + + @Override + public boolean reserveBatch(C cursor, KernelTransaction ktx) { + return scan.reserveBatch(cursor, batchSize, ktx.cursorContext(), ktx.securityContext().mode()); + } +} diff --git a/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/SettingProxyFactoryImpl.java b/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/SettingProxyFactoryImpl.java new file mode 100644 index 0000000000..52e994f8e9 --- /dev/null +++ b/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/SettingProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._512; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jVersion; +import org.neo4j.gds.compat.SettingProxyApi; +import org.neo4j.gds.compat.SettingProxyFactory; + +@ServiceProvider +public final class SettingProxyFactoryImpl implements SettingProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return version == Neo4jVersion.V_5_12; + } + + @Override + public SettingProxyApi load() { + return new SettingProxyImpl(); + } + + @Override + public String description() { + return "Neo4j Settings 5.12"; + } +} diff --git a/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/SettingProxyImpl.java b/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/SettingProxyImpl.java new file mode 100644 index 0000000000..8ef4811ea9 --- /dev/null +++ b/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/SettingProxyImpl.java @@ -0,0 +1,87 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._512; + +import org.neo4j.configuration.Config; +import org.neo4j.configuration.SettingBuilder; +import org.neo4j.dbms.systemgraph.TopologyGraphDbmsModel; +import org.neo4j.gds.compat.DatabaseMode; +import org.neo4j.gds.compat.SettingProxyApi; +import org.neo4j.graphdb.GraphDatabaseService; +import org.neo4j.graphdb.config.Setting; +import org.neo4j.kernel.impl.factory.GraphDatabaseFacade; +import org.neo4j.kernel.internal.GraphDatabaseAPI; + +public class SettingProxyImpl implements SettingProxyApi { + + @Override + public Setting setting(org.neo4j.gds.compat.Setting setting) { + var builder = SettingBuilder.newBuilder(setting.name(), setting.parser(), setting.defaultValue()); + if (setting.dynamic()) { + builder = builder.dynamic(); + } + if (setting.immutable()) { + builder = builder.immutable(); + } + setting.dependency().ifPresent(builder::setDependency); + setting.constraints().forEach(builder::addConstraint); + return builder.build(); + } + + @Override + public DatabaseMode databaseMode(Config config, GraphDatabaseService databaseService) { + return switch (((GraphDatabaseAPI) databaseService).mode()) { + case RAFT -> DatabaseMode.CORE; + case REPLICA -> DatabaseMode.READ_REPLICA; + case SINGLE -> DatabaseMode.SINGLE; + case VIRTUAL -> throw new UnsupportedOperationException("What's a virtual database anyway?"); + }; + } + + @Override + public void setDatabaseMode(Config config, DatabaseMode databaseMode, GraphDatabaseService databaseService) { + // super hacky, there is no way to set the mode of a database without restarting it + if (!(databaseService instanceof GraphDatabaseFacade db)) { + throw new IllegalArgumentException( + "Cannot set database mode on a database that is not a GraphDatabaseFacade"); + } + try { + var modeField = GraphDatabaseFacade.class.getDeclaredField("mode"); + modeField.setAccessible(true); + modeField.set(db, switch (databaseMode) { + case CORE -> TopologyGraphDbmsModel.HostedOnMode.RAFT; + case READ_REPLICA -> TopologyGraphDbmsModel.HostedOnMode.REPLICA; + case SINGLE -> TopologyGraphDbmsModel.HostedOnMode.SINGLE; + }); + } catch (NoSuchFieldException e) { + throw new RuntimeException( + "Could not set the mode field because it no longer exists. This compat layer needs to be updated.", + e + ); + } catch (IllegalAccessException e) { + throw new RuntimeException("Could not get the permissions to set the mode field.", e); + } + } + + @Override + public String secondaryModeName() { + return "Secondary"; + } +} diff --git a/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/TestLogImpl.java b/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/TestLogImpl.java new file mode 100644 index 0000000000..ed2748fc96 --- /dev/null +++ b/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/TestLogImpl.java @@ -0,0 +1,146 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._512; + +import org.neo4j.gds.annotation.SuppressForbidden; +import org.neo4j.gds.compat.TestLog; + +import java.util.ArrayList; +import java.util.Locale; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.ConcurrentMap; + +public class TestLogImpl implements TestLog { + + private final ConcurrentMap> messages; + + TestLogImpl() { + messages = new ConcurrentHashMap<>(3); + } + + @Override + public void assertContainsMessage(String level, String fragment) { + if (!containsMessage(level, fragment)) { + throw new RuntimeException( + String.format( + Locale.US, + "Expected log output to contain `%s` for log level `%s`%nLog messages:%n%s", + fragment, + level, + String.join("\n", messages.get(level)) + ) + ); + } + } + + @Override + public boolean containsMessage(String level, String fragment) { + ConcurrentLinkedQueue messageList = messages.getOrDefault(level, new ConcurrentLinkedQueue<>()); + return messageList.stream().anyMatch((message) -> message.contains(fragment)); + } + + @Override + public boolean hasMessages(String level) { + return !messages.getOrDefault(level, new ConcurrentLinkedQueue<>()).isEmpty(); + } + + @Override + public ArrayList getMessages(String level) { + return new ArrayList<>(messages.getOrDefault(level, new ConcurrentLinkedQueue<>())); + } + + @SuppressForbidden(reason = "test log can print") + public void printMessages() { + System.out.println("TestLog Messages: " + messages); + } + + @Override + public boolean isDebugEnabled() { + return true; + } + + @Override + public void debug(String message) { + logMessage(DEBUG, message); + } + + @Override + public void debug(String message, Throwable throwable) { + debug(String.format(Locale.US, "%s - %s", message, throwable.getMessage())); + } + + @Override + public void debug(String format, Object... arguments) { + debug(String.format(Locale.US, format, arguments)); + } + + @Override + public void info(String message) { + logMessage(INFO, message); + } + + @Override + public void info(String message, Throwable throwable) { + info(String.format(Locale.US, "%s - %s", message, throwable.getMessage())); + } + + @Override + public void info(String format, Object... arguments) { + info(String.format(Locale.US, format, arguments)); + } + + @Override + public void warn(String message) { + logMessage(WARN, message); + } + + @Override + public void warn(String message, Throwable throwable) { + warn(String.format(Locale.US, "%s - %s", message, throwable.getMessage())); + } + + @Override + public void warn(String format, Object... arguments) { + warn(String.format(Locale.US, format, arguments)); + } + + @Override + public void error(String message) { + logMessage(ERROR, message); + } + + @Override + public void error(String message, Throwable throwable) { + error(String.format(Locale.US, "%s - %s", message, throwable.getMessage())); + } + + @Override + public void error(String format, Object... arguments) { + error(String.format(Locale.US, format, arguments)); + } + + private void logMessage(String level, String message) { + messages.computeIfAbsent( + level, + (ignore) -> new ConcurrentLinkedQueue<>() + ).add(message); + } +} diff --git a/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/VirtualRelationshipImpl.java b/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/VirtualRelationshipImpl.java new file mode 100644 index 0000000000..938801ad80 --- /dev/null +++ b/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/VirtualRelationshipImpl.java @@ -0,0 +1,41 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._512; + +import org.neo4j.gds.compat.VirtualRelationship; +import org.neo4j.graphdb.Node; +import org.neo4j.graphdb.RelationshipType; + +public class VirtualRelationshipImpl extends VirtualRelationship { + + VirtualRelationshipImpl( + long id, + Node startNode, + Node endNode, + RelationshipType type + ) { + super(id, startNode, endNode, type); + } + + @Override + public String getElementId() { + return Long.toString(getId()); + } +} diff --git a/compatibility/5.12/storage-engine-adapter/build.gradle b/compatibility/5.12/storage-engine-adapter/build.gradle new file mode 100644 index 0000000000..29c2720734 --- /dev/null +++ b/compatibility/5.12/storage-engine-adapter/build.gradle @@ -0,0 +1,68 @@ +apply plugin: 'java-library' +apply plugin: 'me.champeau.mrjar' + +description = 'Neo4j Graph Data Science :: Storage Engine Adapter 5.12' + +group = 'org.neo4j.gds' + +// for all 5.x versions +if (ver.'neo4j'.startsWith('5.')) { + sourceSets { + main { + java { + srcDirs = ['src/main/java17'] + } + } + } + + dependencies { + annotationProcessor project(':annotations') + annotationProcessor group: 'org.immutables', name: 'value', version: ver.'immutables' + annotationProcessor group: 'org.neo4j', name: 'annotations', version: neos.'5.12' + + compileOnly project(':annotations') + compileOnly project(':progress-tracking') + compileOnly group: 'org.immutables', name: 'value-annotations', version: ver.'immutables' + compileOnly group: 'org.neo4j', name: 'neo4j', version: neos.'5.12' + compileOnly group: 'org.neo4j', name: 'neo4j-record-storage-engine', version: neos.'5.12' + + implementation project(':core') + implementation project(':storage-engine-adapter-api') + implementation project(':config-api') + implementation project(':string-formatting') + } +} else { + multiRelease { + targetVersions 11, 17 + } + + if (!project.hasProperty('no-forbidden-apis')) { + forbiddenApisJava17 { + exclude('**') + } + } + + dependencies { + annotationProcessor group: 'org.neo4j', name: 'annotations', version: ver.'neo4j' + compileOnly group: 'org.neo4j', name: 'annotations', version: ver.'neo4j' + compileOnly group: 'org.neo4j', name: 'neo4j-kernel-api', version: ver.'neo4j' + + implementation project(':neo4j-adapter') + implementation project(':storage-engine-adapter-api') + + java17AnnotationProcessor project(':annotations') + java17AnnotationProcessor group: 'org.immutables', name: 'value', version: ver.'immutables' + java17AnnotationProcessor group: 'org.neo4j', name: 'annotations', version: neos.'5.12' + + java17CompileOnly project(':annotations') + java17CompileOnly project(':progress-tracking') + java17CompileOnly group: 'org.immutables', name: 'value-annotations', version: ver.'immutables' + java17CompileOnly group: 'org.neo4j', name: 'neo4j', version: neos.'5.12' + java17CompileOnly group: 'org.neo4j', name: 'neo4j-record-storage-engine', version: neos.'5.12' + + java17Implementation project(':core') + java17Implementation project(':storage-engine-adapter-api') + java17Implementation project(':config-api') + java17Implementation project(':string-formatting') + } +} diff --git a/compatibility/5.12/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_512/InMemoryStorageEngineFactory.java b/compatibility/5.12/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_512/InMemoryStorageEngineFactory.java new file mode 100644 index 0000000000..0c60e8ffa8 --- /dev/null +++ b/compatibility/5.12/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_512/InMemoryStorageEngineFactory.java @@ -0,0 +1,268 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._512; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.configuration.Config; +import org.neo4j.dbms.database.readonly.DatabaseReadOnlyChecker; +import org.neo4j.index.internal.gbptree.RecoveryCleanupWorkCollector; +import org.neo4j.internal.id.IdController; +import org.neo4j.internal.id.IdGeneratorFactory; +import org.neo4j.internal.schema.IndexConfigCompleter; +import org.neo4j.internal.schema.SchemaRule; +import org.neo4j.internal.schema.SchemaState; +import org.neo4j.io.fs.FileSystemAbstraction; +import org.neo4j.io.layout.DatabaseLayout; +import org.neo4j.io.layout.Neo4jLayout; +import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.tracing.PageCacheTracer; +import org.neo4j.lock.LockService; +import org.neo4j.logging.LogProvider; +import org.neo4j.logging.internal.LogService; +import org.neo4j.memory.MemoryTracker; +import org.neo4j.monitoring.DatabaseHealth; +import org.neo4j.scheduler.JobScheduler; +import org.neo4j.storageengine.api.CommandReaderFactory; +import org.neo4j.storageengine.api.ConstraintRuleAccessor; +import org.neo4j.storageengine.api.LogVersionRepository; +import org.neo4j.storageengine.api.MetadataProvider; +import org.neo4j.storageengine.api.StorageEngine; +import org.neo4j.storageengine.api.StorageEngineFactory; +import org.neo4j.storageengine.api.StorageFilesState; +import org.neo4j.storageengine.api.StoreId; +import org.neo4j.storageengine.api.StoreVersion; +import org.neo4j.storageengine.api.StoreVersionCheck; +import org.neo4j.storageengine.api.TransactionIdStore; +import org.neo4j.storageengine.migration.RollingUpgradeCompatibility; +import org.neo4j.storageengine.migration.SchemaRuleMigrationAccess; +import org.neo4j.storageengine.migration.StoreMigrationParticipant; +import org.neo4j.token.TokenHolders; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.List; +import java.util.Optional; +import java.util.UUID; + +@ServiceProvider +public class InMemoryStorageEngineFactory implements StorageEngineFactory { + + @Override + public String name() { + return "unsupported512"; + } + + @Override + public StoreVersionCheck versionCheck( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + LogService logService, + PageCacheTracer pageCacheTracer + ) { + throw new UnsupportedOperationException("5.12 storage engine requires JDK17"); + } + + @Override + public StoreVersion versionInformation(String storeVersion) { + throw new UnsupportedOperationException("5.12 storage engine requires JDK17"); + } + + @Override + public StoreVersion versionInformation(StoreId storeId) { + throw new UnsupportedOperationException("5.12 storage engine requires JDK17"); + } + + @Override + public RollingUpgradeCompatibility rollingUpgradeCompatibility() { + throw new UnsupportedOperationException("5.12 storage engine requires JDK17"); + } + + @Override + public List migrationParticipants( + FileSystemAbstraction fs, + Config config, + PageCache pageCache, + JobScheduler jobScheduler, + LogService logService, + PageCacheTracer cacheTracer, + MemoryTracker memoryTracker + ) { + throw new UnsupportedOperationException("5.12 storage engine requires JDK17"); + } + + @Override + public StorageEngine instantiate( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + TokenHolders tokenHolders, + SchemaState schemaState, + ConstraintRuleAccessor constraintSemantics, + IndexConfigCompleter indexConfigCompleter, + LockService lockService, + IdGeneratorFactory idGeneratorFactory, + IdController idController, + DatabaseHealth databaseHealth, + LogProvider internalLogProvider, + LogProvider userLogProvider, + RecoveryCleanupWorkCollector recoveryCleanupWorkCollector, + PageCacheTracer cacheTracer, + boolean createStoreIfNotExists, + DatabaseReadOnlyChecker readOnlyChecker, + MemoryTracker memoryTracker + ) { + throw new UnsupportedOperationException("5.12 storage engine requires JDK17"); + } + + @Override + public List listStorageFiles(FileSystemAbstraction fileSystem, DatabaseLayout databaseLayout) throws + IOException { + throw new UnsupportedOperationException("5.12 storage engine requires JDK17"); + } + + @Override + public boolean storageExists(FileSystemAbstraction fileSystem, DatabaseLayout databaseLayout, PageCache pageCache) { + return false; + } + + @Override + public TransactionIdStore readOnlyTransactionIdStore( + FileSystemAbstraction filySystem, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) throws IOException { + throw new UnsupportedOperationException("5.12 storage engine requires JDK17"); + } + + @Override + public LogVersionRepository readOnlyLogVersionRepository( + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) throws IOException { + throw new UnsupportedOperationException("5.12 storage engine requires JDK17"); + } + + @Override + public MetadataProvider transactionMetaDataStore( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + PageCacheTracer cacheTracer, + DatabaseReadOnlyChecker readOnlyChecker + ) throws IOException { + throw new UnsupportedOperationException("5.12 storage engine requires JDK17"); + } + + @Override + public StoreId storeId( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) throws IOException { + throw new UnsupportedOperationException("5.12 storage engine requires JDK17"); + } + + @Override + public void setStoreId( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext, + StoreId storeId, + long upgradeTxChecksum, + long upgradeTxCommitTimestamp + ) throws IOException { + throw new UnsupportedOperationException("5.12 storage engine requires JDK17"); + } + + @Override + public void setExternalStoreUUID( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext, + UUID externalStoreId + ) throws IOException { + throw new UnsupportedOperationException("5.12 storage engine requires JDK17"); + } + + @Override + public Optional databaseIdUuid( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) { + throw new UnsupportedOperationException("5.12 storage engine requires JDK17"); + } + + @Override + public SchemaRuleMigrationAccess schemaRuleMigrationAccess( + FileSystemAbstraction fs, + PageCache pageCache, + Config config, + DatabaseLayout databaseLayout, + LogService logService, + String recordFormats, + PageCacheTracer cacheTracer, + CursorContext cursorContext, + MemoryTracker memoryTracker + ) { + throw new UnsupportedOperationException("5.12 storage engine requires JDK17"); + } + + @Override + public List loadSchemaRules( + FileSystemAbstraction fs, + PageCache pageCache, + Config config, + DatabaseLayout databaseLayout, + CursorContext cursorContext + ) { + throw new UnsupportedOperationException("5.12 storage engine requires JDK17"); + } + + @Override + public StorageFilesState checkStoreFileState( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache + ) { + throw new UnsupportedOperationException("5.12 storage engine requires JDK17"); + } + + @Override + public CommandReaderFactory commandReaderFactory() { + throw new UnsupportedOperationException("5.12 storage engine requires JDK17"); + } + + @Override + public DatabaseLayout databaseLayout(Neo4jLayout neo4jLayout, String databaseName) { + throw new UnsupportedOperationException("5.12 storage engine requires JDK17"); + } +} diff --git a/compatibility/5.12/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_512/StorageEngineProxyFactoryImpl.java b/compatibility/5.12/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_512/StorageEngineProxyFactoryImpl.java new file mode 100644 index 0000000000..dbdff6ea60 --- /dev/null +++ b/compatibility/5.12/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_512/StorageEngineProxyFactoryImpl.java @@ -0,0 +1,31 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * This file contains proprietary code that is only available via a commercial license from Neo4j. + * For more information, see https://neo4j.com/contact-us/ + */ +package org.neo4j.gds.compat._512; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jVersion; +import org.neo4j.gds.compat.StorageEngineProxyApi; +import org.neo4j.gds.compat.StorageEngineProxyFactory; + +@ServiceProvider +public class StorageEngineProxyFactoryImpl implements StorageEngineProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return false; + } + + @Override + public StorageEngineProxyApi load() { + throw new UnsupportedOperationException("5.12 storage engine requires JDK17"); + } + + @Override + public String description() { + return "Storage Engine 5.12"; + } +} diff --git a/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/InMemoryCommandCreationContextImpl.java b/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/InMemoryCommandCreationContextImpl.java new file mode 100644 index 0000000000..b881db2eab --- /dev/null +++ b/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/InMemoryCommandCreationContextImpl.java @@ -0,0 +1,107 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._512; + +import org.neo4j.configuration.Config; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.kernel.KernelVersion; +import org.neo4j.kernel.KernelVersionProvider; +import org.neo4j.lock.LockTracer; +import org.neo4j.lock.ResourceLocker; +import org.neo4j.storageengine.api.CommandCreationContext; +import org.neo4j.storageengine.api.cursor.StoreCursors; + +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Supplier; + +public class InMemoryCommandCreationContextImpl implements CommandCreationContext { + + private final AtomicLong schemaTokens; + private final AtomicInteger propertyTokens; + private final AtomicInteger labelTokens; + private final AtomicInteger typeTokens; + + InMemoryCommandCreationContextImpl() { + this.schemaTokens = new AtomicLong(0); + this.propertyTokens = new AtomicInteger(0); + this.labelTokens = new AtomicInteger(0); + this.typeTokens = new AtomicInteger(0); + } + + @Override + public long reserveNode() { + throw new UnsupportedOperationException("Creating nodes is not supported"); + } + + @Override + public long reserveRelationship( + long sourceNode, + long targetNode, + int relationshipType, + boolean sourceNodeAddedInTx, + boolean targetNodeAddedInTx + ) { + throw new UnsupportedOperationException("Creating relationships is not supported"); + } + + @Override + public long reserveSchema() { + return schemaTokens.getAndIncrement(); + } + + @Override + public int reserveLabelTokenId() { + return labelTokens.getAndIncrement(); + } + + @Override + public int reservePropertyKeyTokenId() { + return propertyTokens.getAndIncrement(); + } + + @Override + public int reserveRelationshipTypeTokenId() { + return typeTokens.getAndIncrement(); + } + + @Override + public void close() { + + } + + @Override + public void initialize( + KernelVersionProvider kernelVersionProvider, + CursorContext cursorContext, + StoreCursors storeCursors, + Supplier oldestActiveTransactionSequenceNumber, + ResourceLocker locks, + Supplier lockTracer + ) { + + } + + @Override + public KernelVersion kernelVersion() { + // NOTE: Double-check if this is still correct when you copy this into a new compat layer + return KernelVersion.getLatestVersion(Config.newBuilder().build()); + } +} diff --git a/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/InMemoryCountsStoreImpl.java b/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/InMemoryCountsStoreImpl.java new file mode 100644 index 0000000000..217b653238 --- /dev/null +++ b/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/InMemoryCountsStoreImpl.java @@ -0,0 +1,111 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._512; + +import org.neo4j.annotations.documented.ReporterFactory; +import org.neo4j.counts.CountsStore; +import org.neo4j.counts.CountsUpdater; +import org.neo4j.counts.CountsVisitor; +import org.neo4j.gds.NodeLabel; +import org.neo4j.gds.api.GraphStore; +import org.neo4j.internal.helpers.progress.ProgressMonitorFactory; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.context.CursorContextFactory; +import org.neo4j.io.pagecache.tracing.FileFlushEvent; +import org.neo4j.memory.MemoryTracker; +import org.neo4j.token.TokenHolders; +import org.neo4j.token.api.TokenNotFoundException; + +import java.io.IOException; + +public class InMemoryCountsStoreImpl implements CountsStore { + + private final GraphStore graphStore; + private final TokenHolders tokenHolders; + + public InMemoryCountsStoreImpl( + GraphStore graphStore, + TokenHolders tokenHolders + ) { + + this.graphStore = graphStore; + this.tokenHolders = tokenHolders; + } + + @Override + public void start(CursorContext cursorContext, MemoryTracker memoryTracker) throws IOException { + + } + + @Override + public CountsUpdater updater(long txId, boolean isLast, CursorContext cursorContext) { + throw new UnsupportedOperationException("Updates are not supported"); + } + + @Override + public void checkpoint(FileFlushEvent fileFlushEvent, CursorContext cursorContext) { + + } + + @Override + public long nodeCount(int labelId, CursorContext cursorContext) { + if (labelId == -1) { + return graphStore.nodeCount(); + } + + String nodeLabel; + try { + nodeLabel = tokenHolders.labelTokens().getTokenById(labelId).name(); + } catch (TokenNotFoundException e) { + throw new RuntimeException(e); + } + return graphStore.nodes().nodeCount(NodeLabel.of(nodeLabel)); + } + + @Override + public long relationshipCount(int startLabelId, int typeId, int endLabelId, CursorContext cursorContext) { + // TODO: this is quite wrong + return graphStore.relationshipCount(); + } + + @Override + public boolean consistencyCheck( + ReporterFactory reporterFactory, + CursorContextFactory cursorContextFactory, + int i, + ProgressMonitorFactory progressMonitorFactory + ) { + return true; + } + + @Override + public void close() { + + } + + @Override + public void accept(CountsVisitor visitor, CursorContext cursorContext) { + tokenHolders.labelTokens().getAllTokens().forEach(labelToken -> { + visitor.visitNodeCount(labelToken.id(), nodeCount(labelToken.id(), cursorContext)); + }); + + visitor.visitRelationshipCount(-1, -1, -1, graphStore.relationshipCount()); + } +} diff --git a/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/InMemoryMetaDataProviderImpl.java b/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/InMemoryMetaDataProviderImpl.java new file mode 100644 index 0000000000..43cb7ca0cf --- /dev/null +++ b/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/InMemoryMetaDataProviderImpl.java @@ -0,0 +1,201 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._512; + +import org.neo4j.internal.recordstorage.InMemoryLogVersionRepository512; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.context.TransactionIdSnapshot; +import org.neo4j.storageengine.api.ClosedTransactionMetadata; +import org.neo4j.storageengine.api.ExternalStoreId; +import org.neo4j.storageengine.api.MetadataProvider; +import org.neo4j.storageengine.api.StoreId; +import org.neo4j.storageengine.api.TransactionId; + +import java.io.IOException; +import java.util.Optional; +import java.util.UUID; + +public class InMemoryMetaDataProviderImpl implements MetadataProvider { + + private final ExternalStoreId externalStoreId; + private final InMemoryLogVersionRepository512 logVersionRepository; + private final InMemoryTransactionIdStoreImpl transactionIdStore; + + InMemoryMetaDataProviderImpl() { + this.logVersionRepository = new InMemoryLogVersionRepository512(); + this.externalStoreId = new ExternalStoreId(UUID.randomUUID()); + this.transactionIdStore = new InMemoryTransactionIdStoreImpl(); + } + + @Override + public ExternalStoreId getExternalStoreId() { + return this.externalStoreId; + } + + @Override + public ClosedTransactionMetadata getLastClosedTransaction() { + return this.transactionIdStore.getLastClosedTransaction(); + } + + @Override + public void setCurrentLogVersion(long version) { + logVersionRepository.setCurrentLogVersion(version); + } + + @Override + public long incrementAndGetVersion() { + return logVersionRepository.incrementAndGetVersion(); + } + + @Override + public void setCheckpointLogVersion(long version) { + logVersionRepository.setCheckpointLogVersion(version); + } + + @Override + public long incrementAndGetCheckpointLogVersion() { + return logVersionRepository.incrementAndGetCheckpointLogVersion(); + } + + @Override + public void transactionCommitted(long transactionId, int checksum, long commitTimestamp, long consensusIndex) { + transactionIdStore.transactionCommitted(transactionId, checksum, commitTimestamp, consensusIndex); + } + + @Override + public void setLastCommittedAndClosedTransactionId( + long transactionId, + int checksum, + long commitTimestamp, + long consensusIndex, + long byteOffset, + long logVersion + ) { + transactionIdStore.setLastCommittedAndClosedTransactionId( + transactionId, + checksum, + commitTimestamp, + consensusIndex, + byteOffset, + logVersion + ); + } + + @Override + public void transactionClosed( + long transactionId, + long logVersion, + long byteOffset, + int checksum, + long commitTimestamp, + long consensusIndex + ) { + this.transactionIdStore.transactionClosed( + transactionId, + logVersion, + byteOffset, + checksum, + commitTimestamp, + consensusIndex + ); + } + + @Override + public void resetLastClosedTransaction( + long transactionId, + long logVersion, + long byteOffset, + int checksum, + long commitTimestamp, + long consensusIndex + ) { + this.transactionIdStore.resetLastClosedTransaction( + transactionId, + logVersion, + byteOffset, + checksum, + commitTimestamp, + consensusIndex + ); + } + + @Override + public TransactionIdSnapshot getClosedTransactionSnapshot() { + return new TransactionIdSnapshot(this.getLastClosedTransactionId()); + } + + @Override + public void regenerateMetadata(StoreId storeId, UUID externalStoreUUID, CursorContext cursorContext) { + } + + @Override + public StoreId getStoreId() { + return StoreId.UNKNOWN; + } + + @Override + public void close() throws IOException { + } + + @Override + public long getCurrentLogVersion() { + return this.logVersionRepository.getCurrentLogVersion(); + } + + @Override + public long getCheckpointLogVersion() { + return this.logVersionRepository.getCheckpointLogVersion(); + } + + @Override + public long nextCommittingTransactionId() { + return this.transactionIdStore.nextCommittingTransactionId(); + } + + @Override + public long committingTransactionId() { + return this.transactionIdStore.committingTransactionId(); + } + + @Override + public long getLastCommittedTransactionId() { + return this.transactionIdStore.getLastCommittedTransactionId(); + } + + @Override + public TransactionId getLastCommittedTransaction() { + return this.transactionIdStore.getLastCommittedTransaction(); + } + + @Override + public long getLastClosedTransactionId() { + return this.transactionIdStore.getLastClosedTransactionId(); + } + + @Override + public Optional getDatabaseIdUuid(CursorContext cursorTracer) { + throw new IllegalStateException("Not supported"); + } + + @Override + public void setDatabaseIdUuid(UUID uuid, CursorContext cursorContext) { + throw new IllegalStateException("Not supported"); + } +} diff --git a/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/InMemoryNodeCursor.java b/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/InMemoryNodeCursor.java new file mode 100644 index 0000000000..0872386475 --- /dev/null +++ b/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/InMemoryNodeCursor.java @@ -0,0 +1,82 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._512; + +import org.neo4j.gds.api.GraphStore; +import org.neo4j.gds.compat.AbstractInMemoryNodeCursor; +import org.neo4j.storageengine.api.AllNodeScan; +import org.neo4j.storageengine.api.Degrees; +import org.neo4j.storageengine.api.LongReference; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.storageengine.api.RelationshipSelection; +import org.neo4j.storageengine.api.StoragePropertyCursor; +import org.neo4j.storageengine.api.StorageRelationshipTraversalCursor; +import org.neo4j.token.TokenHolders; + +public class InMemoryNodeCursor extends AbstractInMemoryNodeCursor { + + public InMemoryNodeCursor(GraphStore graphStore, TokenHolders tokenHolders) { + super(graphStore, tokenHolders); + } + + @Override + public boolean hasLabel() { + return hasAtLeastOneLabelForCurrentNode(); + } + + @Override + public Reference propertiesReference() { + return LongReference.longReference(getId()); + } + + @Override + public void properties(StoragePropertyCursor propertyCursor, PropertySelection selection) { + propertyCursor.initNodeProperties(propertiesReference(), selection); + } + + @Override + public void properties(StoragePropertyCursor propertyCursor) { + properties(propertyCursor, PropertySelection.ALL_PROPERTIES); + } + + @Override + public boolean supportsFastRelationshipsTo() { + return false; + } + + @Override + public void relationshipsTo( + StorageRelationshipTraversalCursor storageRelationshipTraversalCursor, + RelationshipSelection relationshipSelection, + long neighbourNodeReference + ) { + throw new UnsupportedOperationException(); + } + + @Override + public void degrees(RelationshipSelection selection, Degrees.Mutator mutator) { + } + + @Override + public boolean scanBatch(AllNodeScan allNodeScan, long sizeHint) { + return super.scanBatch(allNodeScan, (int) sizeHint); + } +} diff --git a/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/InMemoryNodePropertyCursor.java b/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/InMemoryNodePropertyCursor.java new file mode 100644 index 0000000000..b766d0cbc3 --- /dev/null +++ b/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/InMemoryNodePropertyCursor.java @@ -0,0 +1,45 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._512; + +import org.neo4j.gds.compat.AbstractInMemoryNodePropertyCursor; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.storageengine.api.LongReference; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.token.TokenHolders; + +public class InMemoryNodePropertyCursor extends AbstractInMemoryNodePropertyCursor { + + public InMemoryNodePropertyCursor(CypherGraphStore graphStore, TokenHolders tokenHolders) { + super(graphStore, tokenHolders); + } + + @Override + public void initNodeProperties(Reference reference, PropertySelection selection, long ownerReference) { + reset(); + setId(((LongReference) reference).id); + setPropertySelection(new InMemoryPropertySelectionImpl(selection)); + } + + @Override + public void initRelationshipProperties(Reference reference, PropertySelection selection, long ownerReference) { + } +} diff --git a/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/InMemoryPropertyCursor.java b/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/InMemoryPropertyCursor.java new file mode 100644 index 0000000000..4522abddbb --- /dev/null +++ b/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/InMemoryPropertyCursor.java @@ -0,0 +1,71 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._512; + +import org.neo4j.gds.compat.AbstractInMemoryPropertyCursor; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.storageengine.api.StorageNodeCursor; +import org.neo4j.storageengine.api.StorageRelationshipCursor; +import org.neo4j.token.TokenHolders; + +public class InMemoryPropertyCursor extends AbstractInMemoryPropertyCursor { + + public InMemoryPropertyCursor(CypherGraphStore graphStore, TokenHolders tokenHolders) { + super(graphStore, tokenHolders); + } + + @Override + public void initNodeProperties(Reference reference, PropertySelection selection, long ownerReference) { + if (this.delegate == null || !(this.delegate instanceof InMemoryNodePropertyCursor)) { + this.delegate = new InMemoryNodePropertyCursor(graphStore, tokenHolders); + } + + ((InMemoryNodePropertyCursor) delegate).initNodeProperties(reference, selection); + } + + @Override + public void initNodeProperties(StorageNodeCursor nodeCursor, PropertySelection selection) { + if (this.delegate == null || !(this.delegate instanceof InMemoryNodePropertyCursor)) { + this.delegate = new InMemoryNodePropertyCursor(graphStore, tokenHolders); + } + + ((InMemoryNodePropertyCursor) delegate).initNodeProperties(nodeCursor, selection); + } + + @Override + public void initRelationshipProperties(StorageRelationshipCursor relationshipCursor, PropertySelection selection) { + if (this.delegate == null || !(this.delegate instanceof InMemoryRelationshipPropertyCursor)) { + this.delegate = new InMemoryRelationshipPropertyCursor(graphStore, tokenHolders); + } + + ((InMemoryRelationshipPropertyCursor) delegate).initRelationshipProperties(relationshipCursor, selection); + } + + @Override + public void initRelationshipProperties(Reference reference, PropertySelection selection, long ownerReference) { + if (this.delegate == null || !(this.delegate instanceof InMemoryRelationshipPropertyCursor)) { + this.delegate = new InMemoryRelationshipPropertyCursor(graphStore, tokenHolders); + } + + ((InMemoryRelationshipPropertyCursor) delegate).initRelationshipProperties(reference, selection); + } +} diff --git a/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/InMemoryPropertySelectionImpl.java b/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/InMemoryPropertySelectionImpl.java new file mode 100644 index 0000000000..7cbbf1f058 --- /dev/null +++ b/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/InMemoryPropertySelectionImpl.java @@ -0,0 +1,55 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._512; + +import org.neo4j.gds.compat.InMemoryPropertySelection; +import org.neo4j.storageengine.api.PropertySelection; + +public class InMemoryPropertySelectionImpl implements InMemoryPropertySelection { + + private final PropertySelection propertySelection; + + public InMemoryPropertySelectionImpl(PropertySelection propertySelection) {this.propertySelection = propertySelection;} + + @Override + public boolean isLimited() { + return propertySelection.isLimited(); + } + + @Override + public int numberOfKeys() { + return propertySelection.numberOfKeys(); + } + + @Override + public int key(int index) { + return propertySelection.key(index); + } + + @Override + public boolean test(int key) { + return propertySelection.test(key); + } + + @Override + public boolean isKeysOnly() { + return propertySelection.isKeysOnly(); + } +} diff --git a/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/InMemoryRelationshipPropertyCursor.java b/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/InMemoryRelationshipPropertyCursor.java new file mode 100644 index 0000000000..059d09ea71 --- /dev/null +++ b/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/InMemoryRelationshipPropertyCursor.java @@ -0,0 +1,60 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._512; + +import org.neo4j.gds.compat.AbstractInMemoryRelationshipPropertyCursor; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.gds.storageengine.InMemoryRelationshipCursor; +import org.neo4j.storageengine.api.LongReference; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.storageengine.api.StorageRelationshipCursor; +import org.neo4j.token.TokenHolders; + +public class InMemoryRelationshipPropertyCursor extends AbstractInMemoryRelationshipPropertyCursor { + + InMemoryRelationshipPropertyCursor(CypherGraphStore graphStore, TokenHolders tokenHolders) { + super(graphStore, tokenHolders); + } + + @Override + public void initNodeProperties( + Reference reference, PropertySelection propertySelection, long ownerReference + ) { + + } + + @Override + public void initRelationshipProperties( + Reference reference, PropertySelection propertySelection, long ownerReference + ) { + var relationshipId = ((LongReference) reference).id; + var relationshipCursor = new InMemoryRelationshipScanCursor(graphStore, tokenHolders); + relationshipCursor.single(relationshipId); + relationshipCursor.next(); + relationshipCursor.properties(this, new InMemoryPropertySelectionImpl(propertySelection)); + } + + @Override + public void initRelationshipProperties(StorageRelationshipCursor relationshipCursor, PropertySelection selection) { + var inMemoryRelationshipCursor = (InMemoryRelationshipCursor) relationshipCursor; + inMemoryRelationshipCursor.properties(this, selection); + } +} diff --git a/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/InMemoryRelationshipScanCursor.java b/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/InMemoryRelationshipScanCursor.java new file mode 100644 index 0000000000..b4d226ec19 --- /dev/null +++ b/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/InMemoryRelationshipScanCursor.java @@ -0,0 +1,61 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._512; + +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.internal.recordstorage.AbstractInMemoryRelationshipScanCursor; +import org.neo4j.storageengine.api.AllRelationshipsScan; +import org.neo4j.storageengine.api.LongReference; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.storageengine.api.StoragePropertyCursor; +import org.neo4j.token.TokenHolders; + +public class InMemoryRelationshipScanCursor extends AbstractInMemoryRelationshipScanCursor { + + public InMemoryRelationshipScanCursor( + CypherGraphStore graphStore, + TokenHolders tokenHolders + ) { + super(graphStore, tokenHolders); + } + + @Override + public void single(long reference, long sourceNodeReference, int type, long targetNodeReference) { + single(reference); + } + + @Override + public Reference propertiesReference() { + return LongReference.longReference(getId()); + } + + @Override + public void properties( + StoragePropertyCursor storagePropertyCursor, PropertySelection propertySelection + ) { + properties(storagePropertyCursor, new InMemoryPropertySelectionImpl(propertySelection)); + } + + @Override + public boolean scanBatch(AllRelationshipsScan allRelationshipsScan, long sizeHint) { + return super.scanBatch(allRelationshipsScan, (int) sizeHint); + } +} diff --git a/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/InMemoryRelationshipTraversalCursor.java b/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/InMemoryRelationshipTraversalCursor.java new file mode 100644 index 0000000000..197ee873aa --- /dev/null +++ b/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/InMemoryRelationshipTraversalCursor.java @@ -0,0 +1,47 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._512; + +import org.neo4j.gds.compat.AbstractInMemoryRelationshipTraversalCursor; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.storageengine.api.LongReference; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.storageengine.api.StoragePropertyCursor; +import org.neo4j.token.TokenHolders; + +public class InMemoryRelationshipTraversalCursor extends AbstractInMemoryRelationshipTraversalCursor { + + public InMemoryRelationshipTraversalCursor(CypherGraphStore graphStore, TokenHolders tokenHolders) { + super(graphStore, tokenHolders); + } + + @Override + public Reference propertiesReference() { + return LongReference.longReference(getId()); + } + + @Override + public void properties( + StoragePropertyCursor propertyCursor, PropertySelection selection + ) { + properties(propertyCursor, new InMemoryPropertySelectionImpl(selection)); + } +} diff --git a/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/InMemoryStorageEngineFactory.java b/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/InMemoryStorageEngineFactory.java new file mode 100644 index 0000000000..07f04f4285 --- /dev/null +++ b/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/InMemoryStorageEngineFactory.java @@ -0,0 +1,571 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._512; + +import org.eclipse.collections.api.factory.Sets; +import org.eclipse.collections.api.set.ImmutableSet; +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.configuration.Config; +import org.neo4j.consistency.checking.ConsistencyFlags; +import org.neo4j.consistency.report.ConsistencySummaryStatistics; +import org.neo4j.dbms.database.readonly.DatabaseReadOnlyChecker; +import org.neo4j.function.ThrowingSupplier; +import org.neo4j.gds.annotation.SuppressForbidden; +import org.neo4j.gds.compat.Neo4jVersion; +import org.neo4j.gds.compat.StorageEngineFactoryIdProvider; +import org.neo4j.gds.compat.StorageEngineProxyApi; +import org.neo4j.index.internal.gbptree.RecoveryCleanupWorkCollector; +import org.neo4j.internal.batchimport.AdditionalInitialIds; +import org.neo4j.internal.batchimport.BatchImporter; +import org.neo4j.internal.batchimport.Configuration; +import org.neo4j.internal.batchimport.IncrementalBatchImporter; +import org.neo4j.internal.batchimport.IndexImporterFactory; +import org.neo4j.internal.batchimport.Monitor; +import org.neo4j.internal.batchimport.ReadBehaviour; +import org.neo4j.internal.batchimport.input.Collector; +import org.neo4j.internal.batchimport.input.Input; +import org.neo4j.internal.batchimport.input.LenientStoreInput; +import org.neo4j.internal.id.IdGeneratorFactory; +import org.neo4j.internal.id.ScanOnOpenReadOnlyIdGeneratorFactory; +import org.neo4j.internal.recordstorage.InMemoryStorageCommandReaderFactory512; +import org.neo4j.internal.recordstorage.StoreTokens; +import org.neo4j.internal.schema.IndexConfigCompleter; +import org.neo4j.internal.schema.SchemaRule; +import org.neo4j.internal.schema.SchemaState; +import org.neo4j.io.fs.FileSystemAbstraction; +import org.neo4j.io.layout.DatabaseLayout; +import org.neo4j.io.layout.Neo4jLayout; +import org.neo4j.io.layout.recordstorage.RecordDatabaseLayout; +import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.context.CursorContextFactory; +import org.neo4j.io.pagecache.tracing.PageCacheTracer; +import org.neo4j.kernel.KernelVersionRepository; +import org.neo4j.kernel.api.index.IndexProvidersAccess; +import org.neo4j.kernel.impl.api.index.IndexProviderMap; +import org.neo4j.kernel.impl.locking.LockManager; +import org.neo4j.kernel.impl.store.MetaDataStore; +import org.neo4j.kernel.impl.store.NeoStores; +import org.neo4j.kernel.impl.store.StoreFactory; +import org.neo4j.kernel.impl.store.StoreType; +import org.neo4j.kernel.impl.store.cursor.CachedStoreCursors; +import org.neo4j.kernel.impl.transaction.log.LogTailLogVersionsMetadata; +import org.neo4j.kernel.impl.transaction.log.LogTailMetadata; +import org.neo4j.lock.LockService; +import org.neo4j.logging.InternalLog; +import org.neo4j.logging.InternalLogProvider; +import org.neo4j.logging.NullLogProvider; +import org.neo4j.logging.internal.LogService; +import org.neo4j.memory.MemoryTracker; +import org.neo4j.monitoring.DatabaseHealth; +import org.neo4j.scheduler.JobScheduler; +import org.neo4j.storageengine.api.CommandReaderFactory; +import org.neo4j.storageengine.api.ConstraintRuleAccessor; +import org.neo4j.storageengine.api.LogFilesInitializer; +import org.neo4j.storageengine.api.MetadataProvider; +import org.neo4j.storageengine.api.SchemaRule44; +import org.neo4j.storageengine.api.StorageEngine; +import org.neo4j.storageengine.api.StorageEngineFactory; +import org.neo4j.storageengine.api.StorageFilesState; +import org.neo4j.storageengine.api.StoreId; +import org.neo4j.storageengine.api.StoreVersion; +import org.neo4j.storageengine.api.StoreVersionCheck; +import org.neo4j.storageengine.api.StoreVersionIdentifier; +import org.neo4j.storageengine.migration.SchemaRuleMigrationAccessExtended; +import org.neo4j.storageengine.migration.StoreMigrationParticipant; +import org.neo4j.time.SystemNanoClock; +import org.neo4j.token.DelegatingTokenHolder; +import org.neo4j.token.ReadOnlyTokenCreator; +import org.neo4j.token.TokenHolders; +import org.neo4j.token.api.NamedToken; +import org.neo4j.token.api.TokenHolder; +import org.neo4j.token.api.TokensLoader; + +import java.io.IOException; +import java.io.OutputStream; +import java.io.PrintStream; +import java.io.UncheckedIOException; +import java.nio.file.OpenOption; +import java.nio.file.Path; +import java.time.Clock; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.UUID; +import java.util.function.Function; + +@ServiceProvider +public class InMemoryStorageEngineFactory implements StorageEngineFactory { + + static final String IN_MEMORY_STORAGE_ENGINE_NAME = "in-memory-512"; + + public InMemoryStorageEngineFactory() { + StorageEngineProxyApi.requireNeo4jVersion(Neo4jVersion.V_5_12, StorageEngineFactory.class); + } + + @Override + public byte id() { + return StorageEngineFactoryIdProvider.ID; + } + + @Override + public boolean storageExists(FileSystemAbstraction fileSystem, DatabaseLayout databaseLayout) { + return false; + } + + @Override + public StorageEngine instantiate( + FileSystemAbstraction fs, + Clock clock, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + TokenHolders tokenHolders, + SchemaState schemaState, + ConstraintRuleAccessor constraintSemantics, + IndexConfigCompleter indexConfigCompleter, + LockService lockService, + IdGeneratorFactory idGeneratorFactory, + DatabaseHealth databaseHealth, + InternalLogProvider internalLogProvider, + InternalLogProvider userLogProvider, + RecoveryCleanupWorkCollector recoveryCleanupWorkCollector, + LogTailMetadata logTailMetadata, + KernelVersionRepository kernelVersionRepository, + MemoryTracker memoryTracker, + CursorContextFactory contextFactory, + PageCacheTracer pageCacheTracer + ) { + StoreFactory factory = new StoreFactory( + databaseLayout, + config, + idGeneratorFactory, + pageCache, + pageCacheTracer, + fs, + internalLogProvider, + contextFactory, + false, + logTailMetadata + ); + + factory.openNeoStores(StoreType.LABEL_TOKEN).close(); + + return new InMemoryStorageEngineImpl( + databaseLayout, + tokenHolders + ); + } + + @Override + public Optional databaseIdUuid( + FileSystemAbstraction fs, DatabaseLayout databaseLayout, PageCache pageCache, CursorContext cursorContext + ) { + var fieldAccess = MetaDataStore.getFieldAccess( + pageCache, + RecordDatabaseLayout.convert(databaseLayout).metadataStore(), + databaseLayout.getDatabaseName(), + cursorContext + ); + + try { + return fieldAccess.readDatabaseUUID(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + @Override + public List migrationParticipants( + FileSystemAbstraction fileSystemAbstraction, + Config config, + PageCache pageCache, + JobScheduler jobScheduler, + LogService logService, + MemoryTracker memoryTracker, + PageCacheTracer pageCacheTracer, + CursorContextFactory cursorContextFactory, + boolean b + ) { + return List.of(); + } + + @Override + public DatabaseLayout databaseLayout( + Neo4jLayout neo4jLayout, String databaseName + ) { + return RecordDatabaseLayout.of(neo4jLayout, databaseName); + } + + @Override + public DatabaseLayout formatSpecificDatabaseLayout(DatabaseLayout plainLayout) { + return databaseLayout(plainLayout.getNeo4jLayout(), plainLayout.getDatabaseName()); + } + + @SuppressForbidden(reason = "This is the compat layer and we don't really need to go through the proxy") + @Override + public BatchImporter batchImporter( + DatabaseLayout databaseLayout, + FileSystemAbstraction fileSystemAbstraction, + PageCacheTracer pageCacheTracer, + Configuration configuration, + LogService logService, + PrintStream printStream, + boolean b, + AdditionalInitialIds additionalInitialIds, + Config config, + Monitor monitor, + JobScheduler jobScheduler, + Collector collector, + LogFilesInitializer logFilesInitializer, + IndexImporterFactory indexImporterFactory, + MemoryTracker memoryTracker, + CursorContextFactory cursorContextFactory + ) { + throw new UnsupportedOperationException("Batch Import into GDS is not supported"); + } + + @Override + public Input asBatchImporterInput( + DatabaseLayout databaseLayout, + FileSystemAbstraction fileSystemAbstraction, + PageCache pageCache, + PageCacheTracer pageCacheTracer, + Config config, + MemoryTracker memoryTracker, + ReadBehaviour readBehaviour, + boolean b, + CursorContextFactory cursorContextFactory, + LogTailMetadata logTailMetadata + ) { + NeoStores neoStores = (new StoreFactory( + databaseLayout, + config, + new ScanOnOpenReadOnlyIdGeneratorFactory(), + pageCache, + pageCacheTracer, + fileSystemAbstraction, + NullLogProvider.getInstance(), + cursorContextFactory, + false, + logTailMetadata + )).openAllNeoStores(); + return new LenientStoreInput( + neoStores, + readBehaviour.decorateTokenHolders(this.loadReadOnlyTokens(neoStores, true, cursorContextFactory)), + true, + cursorContextFactory, + readBehaviour + ); + } + + @Override + public long optimalAvailableConsistencyCheckerMemory( + FileSystemAbstraction fileSystemAbstraction, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache + ) { + return 0; + } + + @Override + public String name() { + return IN_MEMORY_STORAGE_ENGINE_NAME; + } + + @Override + public Set supportedFormats(boolean includeFormatsUnderDevelopment) { + return Set.of(IN_MEMORY_STORAGE_ENGINE_NAME); + } + + @Override + public boolean supportedFormat(String format, boolean includeFormatsUnderDevelopment) { + return format.equals(IN_MEMORY_STORAGE_ENGINE_NAME); + } + + @Override + public MetadataProvider transactionMetaDataStore( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + DatabaseReadOnlyChecker readOnlyChecker, + CursorContextFactory contextFactory, + LogTailLogVersionsMetadata logTailMetadata, + PageCacheTracer pageCacheTracer + ) throws IOException { + return new InMemoryMetaDataProviderImpl(); + } + + @Override + public StoreVersionCheck versionCheck( + FileSystemAbstraction fileSystemAbstraction, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + LogService logService, + CursorContextFactory cursorContextFactory + ) { + return new InMemoryVersionCheck(); + } + + @Override + public List loadSchemaRules( + FileSystemAbstraction fileSystemAbstraction, + PageCache pageCache, + PageCacheTracer pageCacheTracer, + Config config, + DatabaseLayout databaseLayout, + boolean b, + Function function, + CursorContextFactory cursorContextFactory + ) { + return List.of(); + } + + @Override + public List load44SchemaRules( + FileSystemAbstraction fs, + PageCache pageCache, + PageCacheTracer pageCacheTracer, + Config config, + DatabaseLayout databaseLayout, + CursorContextFactory contextFactory, + LogTailLogVersionsMetadata logTailMetadata + ) { + return List.of(); + } + + @Override + public TokenHolders loadReadOnlyTokens( + FileSystemAbstraction fileSystemAbstraction, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + PageCacheTracer pageCacheTracer, + boolean lenient, + CursorContextFactory cursorContextFactory + ) { + StoreFactory factory = new StoreFactory( + databaseLayout, + config, + new ScanOnOpenReadOnlyIdGeneratorFactory(), + pageCache, + pageCacheTracer, + fileSystemAbstraction, + NullLogProvider.getInstance(), + cursorContextFactory, + false, + LogTailMetadata.EMPTY_LOG_TAIL + ); + try ( + NeoStores stores = factory.openNeoStores( + StoreType.PROPERTY_KEY_TOKEN, StoreType.PROPERTY_KEY_TOKEN_NAME, + StoreType.LABEL_TOKEN, StoreType.LABEL_TOKEN_NAME, + StoreType.RELATIONSHIP_TYPE_TOKEN, StoreType.RELATIONSHIP_TYPE_TOKEN_NAME + ) + ) { + return loadReadOnlyTokens(stores, lenient, cursorContextFactory); + } + } + + @Override + public SchemaRuleMigrationAccessExtended schemaRuleMigrationAccess( + FileSystemAbstraction fs, + PageCache pageCache, + PageCacheTracer pageCacheTracer, + Config config, + DatabaseLayout databaseLayout, + CursorContextFactory contextFactory, + MemoryTracker memoryTracker + ) { + // this is used by store copy, which is not supported for GDS storage engine + return null; + } + + private TokenHolders loadReadOnlyTokens( + NeoStores stores, + boolean lenient, + CursorContextFactory cursorContextFactory + ) { + try ( + var cursorContext = cursorContextFactory.create("loadReadOnlyTokens"); + var storeCursors = new CachedStoreCursors(stores, cursorContext) + ) { + stores.start( cursorContext ); + TokensLoader loader = lenient ? StoreTokens.allReadableTokens( stores ) : StoreTokens.allTokens( stores ); + TokenHolder propertyKeys = new DelegatingTokenHolder( ReadOnlyTokenCreator.READ_ONLY, TokenHolder.TYPE_PROPERTY_KEY ); + TokenHolder labels = new DelegatingTokenHolder( ReadOnlyTokenCreator.READ_ONLY, TokenHolder.TYPE_LABEL ); + TokenHolder relationshipTypes = new DelegatingTokenHolder( ReadOnlyTokenCreator.READ_ONLY, TokenHolder.TYPE_RELATIONSHIP_TYPE ); + + propertyKeys.setInitialTokens( lenient ? unique( loader.getPropertyKeyTokens( storeCursors ) ) : loader.getPropertyKeyTokens( storeCursors ) ); + labels.setInitialTokens( lenient ? unique( loader.getLabelTokens( storeCursors ) ) : loader.getLabelTokens( storeCursors ) ); + relationshipTypes.setInitialTokens( + lenient ? unique( loader.getRelationshipTypeTokens( storeCursors ) ) : loader.getRelationshipTypeTokens( storeCursors ) ); + return new TokenHolders( propertyKeys, labels, relationshipTypes ); + } + catch ( IOException e ) + { + throw new UncheckedIOException( e ); + } + } + + private static List unique( List tokens ) + { + if ( !tokens.isEmpty() ) + { + Set names = new HashSet<>( tokens.size() ); + int i = 0; + while ( i < tokens.size() ) + { + if ( names.add( tokens.get( i ).name() ) ) + { + i++; + } + else + { + // Remove the token at the given index, by replacing it with the last token in the list. + // This changes the order of elements, but can be done in constant time instead of linear time. + int lastIndex = tokens.size() - 1; + NamedToken endToken = tokens.remove( lastIndex ); + if ( i < lastIndex ) + { + tokens.set( i, endToken ); + } + } + } + } + return tokens; + } + + @Override + public CommandReaderFactory commandReaderFactory() { + return InMemoryStorageCommandReaderFactory512.INSTANCE; + } + @Override + public void consistencyCheck( + FileSystemAbstraction fileSystem, + DatabaseLayout layout, + Config config, + PageCache pageCache, + IndexProviderMap indexProviders, + InternalLog reportLog, + InternalLog verboseLog, + ConsistencySummaryStatistics summary, + int numberOfThreads, + long maxOffHeapCachingMemory, + OutputStream progressOutput, + boolean verbose, + ConsistencyFlags flags, + CursorContextFactory contextFactory, + PageCacheTracer pageCacheTracer, + LogTailMetadata logTailMetadata + ) { + // we can do no-op, since our "database" is _always_ consistent + } + + @Override + public ImmutableSet getStoreOpenOptions( + FileSystemAbstraction fs, + PageCache pageCache, + DatabaseLayout layout, + CursorContextFactory contextFactory + ) { + // Not sure about this, empty set is returned when the store files are in `little-endian` format + // See: `org.neo4j.kernel.impl.store.format.PageCacheOptionsSelector.select` + return Sets.immutable.empty(); + } + + @Override + public StoreId retrieveStoreId( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) throws IOException { + return StoreId.retrieveFromStore(fs, databaseLayout, pageCache, cursorContext); + } + + + @Override + public Optional versionInformation(StoreVersionIdentifier storeVersionIdentifier) { + return Optional.of(new InMemoryStoreVersion()); + } + + @Override + public void resetMetadata( + FileSystemAbstraction fileSystemAbstraction, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + CursorContextFactory cursorContextFactory, + PageCacheTracer pageCacheTracer, + StoreId storeId, + UUID externalStoreId + ) { + throw new UnsupportedOperationException(); + } + + @Override + public IncrementalBatchImporter incrementalBatchImporter( + DatabaseLayout databaseLayout, + FileSystemAbstraction fileSystemAbstraction, + PageCacheTracer pageCacheTracer, + Configuration configuration, + LogService logService, + PrintStream printStream, + boolean b, + AdditionalInitialIds additionalInitialIds, + ThrowingSupplier throwingSupplier, + Config config, + Monitor monitor, + JobScheduler jobScheduler, + Collector collector, + LogFilesInitializer logFilesInitializer, + IndexImporterFactory indexImporterFactory, + MemoryTracker memoryTracker, + CursorContextFactory cursorContextFactory, + IndexProvidersAccess indexProvidersAccess + ) { + throw new UnsupportedOperationException(); + } + + @Override + public LockManager createLockManager(Config config, SystemNanoClock systemNanoClock) { + return LockManager.NO_LOCKS_LOCK_MANAGER; + } + + @Override + public List listStorageFiles( + FileSystemAbstraction fileSystem, DatabaseLayout databaseLayout + ) { + return Collections.emptyList(); + } + + @Override + public StorageFilesState checkStoreFileState( + FileSystemAbstraction fs, DatabaseLayout databaseLayout, PageCache pageCache + ) { + return StorageFilesState.recoveredState(); + } +} diff --git a/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/InMemoryStorageEngineImpl.java b/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/InMemoryStorageEngineImpl.java new file mode 100644 index 0000000000..0309c9f476 --- /dev/null +++ b/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/InMemoryStorageEngineImpl.java @@ -0,0 +1,366 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._512; + +import org.neo4j.configuration.Config; +import org.neo4j.counts.CountsStore; +import org.neo4j.exceptions.KernelException; +import org.neo4j.gds.compat.TokenManager; +import org.neo4j.gds.config.GraphProjectConfig; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.gds.core.loading.GraphStoreCatalog; +import org.neo4j.gds.storageengine.InMemoryDatabaseCreationCatalog; +import org.neo4j.gds.storageengine.InMemoryTransactionStateVisitor; +import org.neo4j.internal.diagnostics.DiagnosticsLogger; +import org.neo4j.internal.recordstorage.InMemoryStorageReader512; +import org.neo4j.internal.schema.StorageEngineIndexingBehaviour; +import org.neo4j.io.fs.WritableChannel; +import org.neo4j.io.layout.DatabaseLayout; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.tracing.DatabaseFlushEvent; +import org.neo4j.kernel.KernelVersion; +import org.neo4j.kernel.impl.store.stats.StoreEntityCounters; +import org.neo4j.kernel.lifecycle.Lifecycle; +import org.neo4j.kernel.lifecycle.LifecycleAdapter; +import org.neo4j.lock.LockGroup; +import org.neo4j.lock.LockService; +import org.neo4j.lock.LockTracer; +import org.neo4j.lock.ResourceLocker; +import org.neo4j.logging.InternalLog; +import org.neo4j.memory.MemoryTracker; +import org.neo4j.storageengine.api.CommandBatchToApply; +import org.neo4j.storageengine.api.CommandCreationContext; +import org.neo4j.storageengine.api.CommandStream; +import org.neo4j.storageengine.api.IndexUpdateListener; +import org.neo4j.storageengine.api.InternalErrorTracer; +import org.neo4j.storageengine.api.MetadataProvider; +import org.neo4j.storageengine.api.StorageCommand; +import org.neo4j.storageengine.api.StorageEngine; +import org.neo4j.storageengine.api.StorageEngineFactory; +import org.neo4j.storageengine.api.StorageLocks; +import org.neo4j.storageengine.api.StorageReader; +import org.neo4j.storageengine.api.StoreFileMetadata; +import org.neo4j.storageengine.api.StoreId; +import org.neo4j.storageengine.api.TransactionApplicationMode; +import org.neo4j.storageengine.api.cursor.StoreCursors; +import org.neo4j.storageengine.api.enrichment.Enrichment; +import org.neo4j.storageengine.api.enrichment.EnrichmentCommand; +import org.neo4j.storageengine.api.txstate.ReadableTransactionState; +import org.neo4j.storageengine.api.txstate.TxStateVisitor; +import org.neo4j.storageengine.api.txstate.validation.TransactionValidatorFactory; +import org.neo4j.time.SystemNanoClock; +import org.neo4j.token.TokenHolders; + +import java.util.Collection; +import java.util.List; +import java.util.stream.Collectors; + +import static org.neo4j.gds.utils.StringFormatting.formatWithLocale; + +public final class InMemoryStorageEngineImpl implements StorageEngine { + + public static final byte ID = 42; + private final MetadataProvider metadataProvider; + private final CypherGraphStore graphStore; + private final DatabaseLayout databaseLayout; + private final InMemoryTransactionStateVisitor txStateVisitor; + + private final CommandCreationContext commandCreationContext; + + private final TokenManager tokenManager; + private final InMemoryCountsStoreImpl countsStore; + + private static final StorageEngineIndexingBehaviour INDEXING_BEHAVIOUR = new StorageEngineIndexingBehaviour() { + @Override + public boolean useNodeIdsInRelationshipTokenIndex() { + return false; + } + + @Override + public boolean requireCoordinationLocks() { + return false; + } + + @Override + public int nodesPerPage() { + return 0; + } + + @Override + public int relationshipsPerPage() { + return 0; + } + }; + + InMemoryStorageEngineImpl( + DatabaseLayout databaseLayout, + TokenHolders tokenHolders + ) { + this.databaseLayout = databaseLayout; + this.graphStore = getGraphStoreFromCatalog(databaseLayout.getDatabaseName()); + this.txStateVisitor = new InMemoryTransactionStateVisitor(graphStore, tokenHolders); + this.commandCreationContext = new InMemoryCommandCreationContextImpl(); + this.tokenManager = new TokenManager( + tokenHolders, + InMemoryStorageEngineImpl.this.txStateVisitor, + InMemoryStorageEngineImpl.this.graphStore, + commandCreationContext + ); + InMemoryStorageEngineImpl.this.graphStore.initialize(tokenHolders); + this.countsStore = new InMemoryCountsStoreImpl(graphStore, tokenHolders); + this.metadataProvider = new InMemoryMetaDataProviderImpl(); + } + + private static CypherGraphStore getGraphStoreFromCatalog(String databaseName) { + var graphName = InMemoryDatabaseCreationCatalog.getRegisteredDbCreationGraphName(databaseName); + return (CypherGraphStore) GraphStoreCatalog.getAllGraphStores() + .filter(graphStoreWithUserNameAndConfig -> graphStoreWithUserNameAndConfig + .config() + .graphName() + .equals(graphName)) + .findFirst() + .orElseThrow(() -> new IllegalArgumentException(formatWithLocale( + "No graph with name `%s` was found in GraphStoreCatalog. Available graph names are %s", + graphName, + GraphStoreCatalog.getAllGraphStores() + .map(GraphStoreCatalog.GraphStoreWithUserNameAndConfig::config) + .map(GraphProjectConfig::graphName) + .collect(Collectors.toList()) + ))) + .graphStore(); + } + + @Override + public StoreEntityCounters storeEntityCounters() { + return new StoreEntityCounters() { + @Override + public long nodes() { + return graphStore.nodeCount(); + } + + @Override + public long relationships() { + return graphStore.relationshipCount(); + } + + @Override + public long properties() { + return graphStore.nodePropertyKeys().size() + graphStore.relationshipPropertyKeys().size(); + } + + @Override + public long relationshipTypes() { + return graphStore.relationshipTypes().size(); + } + + @Override + public long allNodesCountStore(CursorContext cursorContext) { + return graphStore.nodeCount(); + } + + @Override + public long allRelationshipsCountStore(CursorContext cursorContext) { + return graphStore.relationshipCount(); + } + }; + } + + @Override + public StoreCursors createStorageCursors(CursorContext initialContext) { + return StoreCursors.NULL; + } + + @Override + public StorageLocks createStorageLocks(ResourceLocker locker) { + return new InMemoryStorageLocksImpl(locker); + } + + @Override + public List createCommands( + ReadableTransactionState state, + StorageReader storageReader, + CommandCreationContext creationContext, + LockTracer lockTracer, + TxStateVisitor.Decorator additionalTxStateVisitor, + CursorContext cursorContext, + StoreCursors storeCursors, + MemoryTracker memoryTracker + ) throws KernelException { + state.accept(txStateVisitor); + return List.of(); + } + + @Override + public void dumpDiagnostics(InternalLog internalLog, DiagnosticsLogger diagnosticsLogger) { + } + + @Override + public List createUpgradeCommands( + KernelVersion versionToUpgradeFrom, + KernelVersion versionToUpgradeTo + ) { + return List.of(); + } + + @Override + public StoreId retrieveStoreId() { + return metadataProvider.getStoreId(); + } + + @Override + public StorageEngineIndexingBehaviour indexingBehaviour() { + return INDEXING_BEHAVIOUR; + } + + @Override + public StorageReader newReader() { + return new InMemoryStorageReader512(graphStore, tokenManager.tokenHolders(), countsStore); + } + + @Override + public void addIndexUpdateListener(IndexUpdateListener listener) { + + } + + @Override + public void apply(CommandBatchToApply batch, TransactionApplicationMode mode) { + } + + @Override + public void init() { + } + + @Override + public void start() { + + } + + @Override + public void stop() { + shutdown(); + } + + @Override + public void shutdown() { + InMemoryDatabaseCreationCatalog.removeDatabaseEntry(databaseLayout.getDatabaseName()); + } + + @Override + public void listStorageFiles( + Collection atomic, Collection replayable + ) { + + } + + @Override + public Lifecycle schemaAndTokensLifecycle() { + return new LifecycleAdapter() { + @Override + public void init() { + + } + }; + } + + @Override + public CountsStore countsAccessor() { + return countsStore; + } + + @Override + public MetadataProvider metadataProvider() { + return metadataProvider; + } + + @Override + public String name() { + return "gds in-memory storage engine"; + } + + @Override + public byte id() { + return ID; + } + + @Override + public CommandCreationContext newCommandCreationContext(boolean multiVersioned) { + return commandCreationContext; + } + + @Override + public TransactionValidatorFactory createTransactionValidatorFactory( + StorageEngineFactory storageEngineFactory, + Config config, + SystemNanoClock systemNanoClock + ) { + return TransactionValidatorFactory.EMPTY_VALIDATOR_FACTORY; + } + + @Override + public void lockRecoveryCommands( + CommandStream commands, LockService lockService, LockGroup lockGroup, TransactionApplicationMode mode + ) { + + } + + @Override + public void rollback(ReadableTransactionState txState, CursorContext cursorContext) { + // rollback is not supported but it is also called when we fail for something else + // that we do not support, such as removing node properties + // TODO: do we want to inspect the txState to infer if rollback was called explicitly or not? + } + + @Override + public void checkpoint(DatabaseFlushEvent flushEvent, CursorContext cursorContext) { + // checkpoint is not supported but it is also called when we fail for something else + // that we do not support, such as removing node properties + } + + @Override + public void preAllocateStoreFilesForCommands(CommandBatchToApply batch, TransactionApplicationMode mode) { + // GDS has its own mechanism of memory allocation, so we don't need this + } + + @Override + public EnrichmentCommand createEnrichmentCommand(KernelVersion kernelVersion, Enrichment enrichment) { + return new EnrichmentCommand() { + + @Override + public Enrichment enrichment() { + return null; + } + + @Override + public void serialize(WritableChannel channel) { + + } + + @Override + public KernelVersion kernelVersion() { + return kernelVersion; + } + }; + } + + @Override + public InternalErrorTracer internalErrorTracer() { + return InternalErrorTracer.NO_TRACER; + } +} diff --git a/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/InMemoryStorageLocksImpl.java b/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/InMemoryStorageLocksImpl.java new file mode 100644 index 0000000000..e31f564272 --- /dev/null +++ b/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/InMemoryStorageLocksImpl.java @@ -0,0 +1,86 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._512; + +import org.neo4j.lock.LockTracer; +import org.neo4j.lock.ResourceLocker; +import org.neo4j.storageengine.api.StorageLocks; +import org.neo4j.storageengine.api.txstate.ReadableTransactionState; + +public class InMemoryStorageLocksImpl implements StorageLocks { + + InMemoryStorageLocksImpl(ResourceLocker locker) {} + + @Override + public void acquireExclusiveNodeLock(LockTracer lockTracer, long... ids) {} + + @Override + public void releaseExclusiveNodeLock(long... ids) {} + + @Override + public void acquireSharedNodeLock(LockTracer lockTracer, long... ids) {} + + @Override + public void releaseSharedNodeLock(long... ids) {} + + @Override + public void acquireExclusiveRelationshipLock(LockTracer lockTracer, long... ids) {} + + @Override + public void releaseExclusiveRelationshipLock(long... ids) {} + + @Override + public void acquireSharedRelationshipLock(LockTracer lockTracer, long... ids) {} + + @Override + public void releaseSharedRelationshipLock(long... ids) {} + + @Override + public void acquireRelationshipCreationLock( + LockTracer lockTracer, + long sourceNode, + long targetNode, + boolean sourceNodeAddedInTx, + boolean targetNodeAddedInTx + ) { + } + + @Override + public void acquireRelationshipDeletionLock( + LockTracer lockTracer, + long sourceNode, + long targetNode, + long relationship, + boolean relationshipAddedInTx, + boolean sourceNodeAddedInTx, + boolean targetNodeAddedInTx + ) { + } + + @Override + public void acquireNodeDeletionLock( + ReadableTransactionState readableTransactionState, + LockTracer lockTracer, + long node + ) {} + + @Override + public void acquireNodeLabelChangeLock(LockTracer lockTracer, long node, int labelId) {} +} diff --git a/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/InMemoryStoreVersion.java b/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/InMemoryStoreVersion.java new file mode 100644 index 0000000000..a587e43619 --- /dev/null +++ b/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/InMemoryStoreVersion.java @@ -0,0 +1,70 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._512; + +import org.neo4j.configuration.Config; +import org.neo4j.storageengine.api.StoreVersion; +import org.neo4j.storageengine.api.format.Capability; +import org.neo4j.storageengine.api.format.CapabilityType; + +import java.util.Optional; + +public class InMemoryStoreVersion implements StoreVersion { + + public static final String STORE_VERSION = "gds-experimental"; + + @Override + public String getStoreVersionUserString() { + return "Unknown"; + } + + @Override + public Optional successorStoreVersion(Config config) { + return Optional.empty(); + } + + @Override + public String formatName() { + return getClass().getSimpleName(); + } + + @Override + public boolean onlyForMigration() { + return false; + } + + @Override + public boolean hasCapability(Capability capability) { + return false; + } + + @Override + public boolean hasCompatibleCapabilities( + StoreVersion otherVersion, CapabilityType type + ) { + return false; + } + + @Override + public String introductionNeo4jVersion() { + return "foo"; + } + +} diff --git a/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/InMemoryTransactionIdStoreImpl.java b/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/InMemoryTransactionIdStoreImpl.java new file mode 100644 index 0000000000..db17b07897 --- /dev/null +++ b/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/InMemoryTransactionIdStoreImpl.java @@ -0,0 +1,117 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._512; + +import org.neo4j.internal.recordstorage.AbstractTransactionIdStore; +import org.neo4j.io.pagecache.context.TransactionIdSnapshot; +import org.neo4j.kernel.impl.transaction.log.LogPosition; +import org.neo4j.storageengine.api.ClosedTransactionMetadata; +import org.neo4j.storageengine.api.TransactionId; +import org.neo4j.storageengine.api.TransactionIdStore; + +public class InMemoryTransactionIdStoreImpl extends AbstractTransactionIdStore { + + @Override + protected void initLastCommittedAndClosedTransactionId( + long previouslyCommittedTxId, + int checksum, + long previouslyCommittedTxCommitTimestamp, + long previouslyCommittedTxLogByteOffset, + long previouslyCommittedTxLogVersion + ) { + this.setLastCommittedAndClosedTransactionId( + previouslyCommittedTxId, + checksum, + previouslyCommittedTxCommitTimestamp, + TransactionIdStore.UNKNOWN_CONSENSUS_INDEX, + previouslyCommittedTxLogByteOffset, + previouslyCommittedTxLogVersion + ); + } + + @Override + public ClosedTransactionMetadata getLastClosedTransaction() { + long[] metaData = this.closedTransactionId.get(); + return new ClosedTransactionMetadata( + metaData[0], + new LogPosition(metaData[1], metaData[2]), + (int) metaData[3], + metaData[4], + metaData[5] + ); + } + + @Override + public TransactionIdSnapshot getClosedTransactionSnapshot() { + return new TransactionIdSnapshot(this.getLastClosedTransactionId()); + } + + @Override + protected TransactionId transactionId(long transactionId, int checksum, long commitTimestamp) { + return new TransactionId(transactionId, checksum, commitTimestamp, TransactionIdStore.UNKNOWN_CONSENSUS_INDEX); + } + + @Override + public void transactionCommitted(long transactionId, int checksum, long commitTimestamp, long consensusIndex) { + + } + + @Override + public void setLastCommittedAndClosedTransactionId( + long transactionId, + int checksum, + long commitTimestamp, + long consensusIndex, + long byteOffset, + long logVersion + ) { + + } + + @Override + public void transactionClosed( + long transactionId, + long logVersion, + long byteOffset, + int checksum, + long commitTimestamp, + long consensusIndex + ) { + this.closedTransactionId.offer( + transactionId, + new long[]{logVersion, byteOffset, checksum, commitTimestamp, consensusIndex} + ); + } + + @Override + public void resetLastClosedTransaction( + long transactionId, + long logVersion, + long byteOffset, + int checksum, + long commitTimestamp, + long consensusIndex + ) { + this.closedTransactionId.set( + transactionId, + new long[]{logVersion, byteOffset, checksum, commitTimestamp, consensusIndex} + ); + } +} diff --git a/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/InMemoryVersionCheck.java b/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/InMemoryVersionCheck.java new file mode 100644 index 0000000000..c23fab7aeb --- /dev/null +++ b/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/InMemoryVersionCheck.java @@ -0,0 +1,61 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._512; + +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.kernel.impl.store.format.FormatFamily; +import org.neo4j.storageengine.api.StoreVersionCheck; +import org.neo4j.storageengine.api.StoreVersionIdentifier; + +import static org.neo4j.gds.compat._512.InMemoryStoreVersion.STORE_VERSION; + +public class InMemoryVersionCheck implements StoreVersionCheck { + + private static final StoreVersionIdentifier STORE_IDENTIFIER = new StoreVersionIdentifier( + STORE_VERSION, + FormatFamily.STANDARD.name(), + 0, + 0 + ); + + @Override + public boolean isCurrentStoreVersionFullySupported(CursorContext cursorContext) { + return true; + } + + @Override + public MigrationCheckResult getAndCheckMigrationTargetVersion(String formatFamily, CursorContext cursorContext) { + return new StoreVersionCheck.MigrationCheckResult(MigrationOutcome.NO_OP, STORE_IDENTIFIER, null, null); + } + + @Override + public UpgradeCheckResult getAndCheckUpgradeTargetVersion(CursorContext cursorContext) { + return new StoreVersionCheck.UpgradeCheckResult(UpgradeOutcome.NO_OP, STORE_IDENTIFIER, null, null); + } + + @Override + public String getIntroductionVersionFromVersion(StoreVersionIdentifier storeVersionIdentifier) { + return STORE_VERSION; + } + + public StoreVersionIdentifier findLatestVersion(String s) { + return STORE_IDENTIFIER; + } +} diff --git a/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/StorageEngineProxyFactoryImpl.java b/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/StorageEngineProxyFactoryImpl.java new file mode 100644 index 0000000000..3b0a804d07 --- /dev/null +++ b/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/StorageEngineProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._512; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jVersion; +import org.neo4j.gds.compat.StorageEngineProxyApi; +import org.neo4j.gds.compat.StorageEngineProxyFactory; + +@ServiceProvider +public class StorageEngineProxyFactoryImpl implements StorageEngineProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return version == Neo4jVersion.V_5_12; + } + + @Override + public StorageEngineProxyApi load() { + return new StorageEngineProxyImpl(); + } + + @Override + public String description() { + return "Storage Engine 5.12"; + } +} diff --git a/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/StorageEngineProxyImpl.java b/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/StorageEngineProxyImpl.java new file mode 100644 index 0000000000..9869947a40 --- /dev/null +++ b/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/StorageEngineProxyImpl.java @@ -0,0 +1,140 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._512; + +import org.neo4j.common.Edition; +import org.neo4j.configuration.Config; +import org.neo4j.configuration.GraphDatabaseInternalSettings; +import org.neo4j.dbms.api.DatabaseManagementService; +import org.neo4j.gds.compat.AbstractInMemoryNodeCursor; +import org.neo4j.gds.compat.AbstractInMemoryNodePropertyCursor; +import org.neo4j.gds.compat.AbstractInMemoryRelationshipPropertyCursor; +import org.neo4j.gds.compat.AbstractInMemoryRelationshipTraversalCursor; +import org.neo4j.gds.compat.GdsDatabaseManagementServiceBuilder; +import org.neo4j.gds.compat.GraphDatabaseApiProxy; +import org.neo4j.gds.compat.StorageEngineProxyApi; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.graphdb.Direction; +import org.neo4j.graphdb.GraphDatabaseService; +import org.neo4j.internal.recordstorage.AbstractInMemoryRelationshipScanCursor; +import org.neo4j.io.layout.DatabaseLayout; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.RelationshipSelection; +import org.neo4j.storageengine.api.StorageEngine; +import org.neo4j.storageengine.api.StorageEntityCursor; +import org.neo4j.storageengine.api.StoragePropertyCursor; +import org.neo4j.storageengine.api.StorageRelationshipTraversalCursor; +import org.neo4j.token.TokenHolders; + +import static org.neo4j.configuration.GraphDatabaseSettings.db_format; + +public class StorageEngineProxyImpl implements StorageEngineProxyApi { + + @Override + public void initRelationshipTraversalCursorForRelType( + StorageRelationshipTraversalCursor cursor, + long sourceNodeId, + int relTypeToken + ) { + var relationshipSelection = RelationshipSelection.selection( + relTypeToken, + Direction.OUTGOING + ); + cursor.init(sourceNodeId, -1, relationshipSelection); + } + + @Override + public StorageEngine createInMemoryStorageEngine(DatabaseLayout databaseLayout, TokenHolders tokenHolders) { + return new InMemoryStorageEngineImpl(databaseLayout, tokenHolders); + } + + @Override + public void createInMemoryDatabase( + DatabaseManagementService dbms, + String dbName, + Config config + ) { + config.set(db_format, InMemoryStorageEngineFactory.IN_MEMORY_STORAGE_ENGINE_NAME); + dbms.createDatabase(dbName, config); + } + + @Override + public GraphDatabaseService startAndGetInMemoryDatabase(DatabaseManagementService dbms, String dbName) { + dbms.startDatabase(dbName); + return dbms.database(dbName); + } + + @Override + public GdsDatabaseManagementServiceBuilder setSkipDefaultIndexesOnCreationSetting(GdsDatabaseManagementServiceBuilder dbmsBuilder) { + return dbmsBuilder.setConfig(GraphDatabaseInternalSettings.skip_default_indexes_on_creation, true); + } + + @Override + public AbstractInMemoryNodeCursor inMemoryNodeCursor(CypherGraphStore graphStore, TokenHolders tokenHolders) { + return new InMemoryNodeCursor(graphStore, tokenHolders); + } + + @Override + public AbstractInMemoryNodePropertyCursor inMemoryNodePropertyCursor( + CypherGraphStore graphStore, + TokenHolders tokenHolders + ) { + return new InMemoryNodePropertyCursor(graphStore, tokenHolders); + } + + @Override + public AbstractInMemoryRelationshipTraversalCursor inMemoryRelationshipTraversalCursor( + CypherGraphStore graphStore, TokenHolders tokenHolders + ) { + return new InMemoryRelationshipTraversalCursor(graphStore, tokenHolders); + } + + @Override + public AbstractInMemoryRelationshipScanCursor inMemoryRelationshipScanCursor( + CypherGraphStore graphStore, TokenHolders tokenHolders + ) { + return new InMemoryRelationshipScanCursor(graphStore, tokenHolders); + } + + @Override + public AbstractInMemoryRelationshipPropertyCursor inMemoryRelationshipPropertyCursor( + CypherGraphStore graphStore, TokenHolders tokenHolders + ) { + return new InMemoryRelationshipPropertyCursor(graphStore, tokenHolders); + } + + @Override + public void properties( + StorageEntityCursor storageCursor, StoragePropertyCursor propertyCursor, int[] propertySelection + ) { + PropertySelection selection; + if (propertySelection.length == 0) { + selection = PropertySelection.ALL_PROPERTIES; + } else { + selection = PropertySelection.selection(propertySelection); + } + storageCursor.properties(propertyCursor, selection); + } + + @Override + public Edition dbmsEdition(GraphDatabaseService databaseService) { + return GraphDatabaseApiProxy.dbmsInfo(databaseService).edition; + } +} diff --git a/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository512.java b/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository512.java new file mode 100644 index 0000000000..16023f54e4 --- /dev/null +++ b/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository512.java @@ -0,0 +1,71 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.internal.recordstorage; + +import org.neo4j.storageengine.api.LogVersionRepository; + +import java.util.concurrent.atomic.AtomicLong; + +public class InMemoryLogVersionRepository512 implements LogVersionRepository { + + private final AtomicLong logVersion; + private final AtomicLong checkpointLogVersion; + + public InMemoryLogVersionRepository512() { + this(0, 0); + } + + private InMemoryLogVersionRepository512(long initialLogVersion, long initialCheckpointLogVersion) { + this.logVersion = new AtomicLong(); + this.checkpointLogVersion = new AtomicLong(); + this.logVersion.set(initialLogVersion); + this.checkpointLogVersion.set(initialCheckpointLogVersion); + } + + @Override + public void setCurrentLogVersion(long version) { + this.logVersion.set(version); + } + + @Override + public long incrementAndGetVersion() { + return this.logVersion.incrementAndGet(); + } + + @Override + public void setCheckpointLogVersion(long version) { + this.checkpointLogVersion.set(version); + } + + @Override + public long incrementAndGetCheckpointLogVersion() { + return this.checkpointLogVersion.incrementAndGet(); + } + + @Override + public long getCurrentLogVersion() { + return this.logVersion.get(); + } + + @Override + public long getCheckpointLogVersion() { + return this.checkpointLogVersion.get(); + } +} diff --git a/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory512.java b/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory512.java new file mode 100644 index 0000000000..1e59fae310 --- /dev/null +++ b/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory512.java @@ -0,0 +1,43 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.internal.recordstorage; + +import org.neo4j.kernel.KernelVersion; +import org.neo4j.storageengine.api.CommandReader; +import org.neo4j.storageengine.api.CommandReaderFactory; + +public class InMemoryStorageCommandReaderFactory512 implements CommandReaderFactory { + + public static final CommandReaderFactory INSTANCE = new InMemoryStorageCommandReaderFactory512(); + + @Override + public CommandReader get(KernelVersion kernelVersion) { + switch (kernelVersion) { + case V4_2: + return LogCommandSerializationV4_2.INSTANCE; + case V4_3_D4: + return LogCommandSerializationV4_3_D3.INSTANCE; + case V5_0: + return LogCommandSerializationV5_0.INSTANCE; + default: + throw new IllegalArgumentException("Unsupported kernel version " + kernelVersion); + } + } +} diff --git a/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageReader512.java b/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageReader512.java new file mode 100644 index 0000000000..9a67420e60 --- /dev/null +++ b/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageReader512.java @@ -0,0 +1,333 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.internal.recordstorage; + +import org.eclipse.collections.api.set.primitive.IntSet; +import org.eclipse.collections.impl.set.immutable.primitive.ImmutableIntSetFactoryImpl; +import org.neo4j.common.EntityType; +import org.neo4j.common.TokenNameLookup; +import org.neo4j.counts.CountsStore; +import org.neo4j.gds.compat._512.InMemoryNodeCursor; +import org.neo4j.gds.compat._512.InMemoryPropertyCursor; +import org.neo4j.gds.compat._512.InMemoryRelationshipScanCursor; +import org.neo4j.gds.compat._512.InMemoryRelationshipTraversalCursor; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.internal.schema.ConstraintDescriptor; +import org.neo4j.internal.schema.IndexDescriptor; +import org.neo4j.internal.schema.IndexType; +import org.neo4j.internal.schema.SchemaDescriptor; +import org.neo4j.internal.schema.constraints.IndexBackedConstraintDescriptor; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.memory.MemoryTracker; +import org.neo4j.storageengine.api.AllNodeScan; +import org.neo4j.storageengine.api.AllRelationshipsScan; +import org.neo4j.storageengine.api.StorageNodeCursor; +import org.neo4j.storageengine.api.StoragePropertyCursor; +import org.neo4j.storageengine.api.StorageReader; +import org.neo4j.storageengine.api.StorageRelationshipScanCursor; +import org.neo4j.storageengine.api.StorageRelationshipTraversalCursor; +import org.neo4j.storageengine.api.StorageSchemaReader; +import org.neo4j.storageengine.api.cursor.StoreCursors; +import org.neo4j.token.TokenHolders; + +import java.util.Collection; +import java.util.Collections; +import java.util.Iterator; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Function; + +public class InMemoryStorageReader512 implements StorageReader { + + protected final CypherGraphStore graphStore; + protected final TokenHolders tokenHolders; + protected final CountsStore counts; + private final Map, Object> dependantState; + private boolean closed; + + public InMemoryStorageReader512( + CypherGraphStore graphStore, + TokenHolders tokenHolders, + CountsStore counts + ) { + this.graphStore = graphStore; + + this.tokenHolders = tokenHolders; + this.counts = counts; + this.dependantState = new ConcurrentHashMap<>(); + } + + @Override + public Collection uniquenessConstraintsGetRelated( + long[] changedLabels, + long[] unchangedLabels, + int[] propertyKeyIds, + boolean propertyKeyListIsComplete, + EntityType entityType + ) { + return Collections.emptyList(); + } + + @Override + public long relationshipsGetCount(CursorContext cursorTracer) { + return graphStore.relationshipCount(); + } + + @Override + public boolean nodeExists(long id, StoreCursors storeCursors) { + var originalId = graphStore.nodes().toOriginalNodeId(id); + return graphStore.nodes().containsOriginalId(originalId); + } + + @Override + public boolean relationshipExists(long id, StoreCursors storeCursors) { + return true; + } + + @Override + public StorageNodeCursor allocateNodeCursor( + CursorContext cursorContext, StoreCursors storeCursors + ) { + return new InMemoryNodeCursor(graphStore, tokenHolders); + } + + @Override + public StoragePropertyCursor allocatePropertyCursor( + CursorContext cursorContext, StoreCursors storeCursors, MemoryTracker memoryTracker + ) { + return new InMemoryPropertyCursor(graphStore, tokenHolders); + } + + @Override + public StorageRelationshipTraversalCursor allocateRelationshipTraversalCursor( + CursorContext cursorContext, StoreCursors storeCursors + ) { + return new InMemoryRelationshipTraversalCursor(graphStore, tokenHolders); + } + + @Override + public StorageRelationshipScanCursor allocateRelationshipScanCursor( + CursorContext cursorContext, StoreCursors storeCursors + ) { + return new InMemoryRelationshipScanCursor(graphStore, tokenHolders); + } + + @Override + public IndexDescriptor indexGetForSchemaAndType( + SchemaDescriptor descriptor, IndexType type + ) { + return null; + } + + @Override + public AllRelationshipsScan allRelationshipScan() { + return new AbstractInMemoryAllRelationshipScan() { + @Override + boolean scanRange(AbstractInMemoryRelationshipScanCursor cursor, long start, long stopInclusive) { + return cursor.scanRange(start, stopInclusive); + } + + @Override + public boolean scanBatch(long sizeHint, AbstractInMemoryRelationshipScanCursor cursor) { + return super.scanBatch(sizeHint, cursor); + } + }; + } + + @Override + public Iterator indexGetForSchema(SchemaDescriptor descriptor) { + return Collections.emptyIterator(); + } + + @Override + public Iterator indexesGetForLabel(int labelId) { + return Collections.emptyIterator(); + } + + @Override + public Iterator indexesGetForRelationshipType(int relationshipType) { + return Collections.emptyIterator(); + } + + @Override + public IndexDescriptor indexGetForName(String name) { + return null; + } + + @Override + public ConstraintDescriptor constraintGetForName(String name) { + return null; + } + + @Override + public boolean indexExists(IndexDescriptor index) { + return false; + } + + @Override + public Iterator indexesGetAll() { + return Collections.emptyIterator(); + } + + @Override + public Collection valueIndexesGetRelated( + long[] tokens, int propertyKeyId, EntityType entityType + ) { + return valueIndexesGetRelated(tokens, new int[]{propertyKeyId}, entityType); + } + + @Override + public Collection valueIndexesGetRelated( + long[] tokens, int[] propertyKeyIds, EntityType entityType + ) { + return Collections.emptyList(); + } + + @Override + public Collection uniquenessConstraintsGetRelated( + long[] labels, + int propertyKeyId, + EntityType entityType + ) { + return Collections.emptyList(); + } + + @Override + public Collection uniquenessConstraintsGetRelated( + long[] tokens, + int[] propertyKeyIds, + EntityType entityType + ) { + return Collections.emptyList(); + } + + @Override + public boolean hasRelatedSchema(long[] labels, int propertyKey, EntityType entityType) { + return false; + } + + @Override + public boolean hasRelatedSchema(int label, EntityType entityType) { + return false; + } + + @Override + public Iterator constraintsGetForSchema(SchemaDescriptor descriptor) { + return Collections.emptyIterator(); + } + + @Override + public boolean constraintExists(ConstraintDescriptor descriptor) { + return false; + } + + @Override + public Iterator constraintsGetForLabel(int labelId) { + return Collections.emptyIterator(); + } + + @Override + public Iterator constraintsGetForRelationshipType(int typeId) { + return Collections.emptyIterator(); + } + + @Override + public Iterator constraintsGetAll() { + return Collections.emptyIterator(); + } + + @Override + public IntSet constraintsGetPropertyTokensForLogicalKey(int token, EntityType entityType) { + return ImmutableIntSetFactoryImpl.INSTANCE.empty(); + } + + @Override + public Long indexGetOwningUniquenessConstraintId(IndexDescriptor index) { + return null; + } + + @Override + public long countsForNode(int labelId, CursorContext cursorContext) { + return counts.nodeCount(labelId, cursorContext); + } + + @Override + public long countsForRelationship(int startLabelId, int typeId, int endLabelId, CursorContext cursorContext) { + return counts.relationshipCount(startLabelId, typeId, endLabelId, cursorContext); + } + + @Override + public long nodesGetCount(CursorContext cursorContext) { + return graphStore.nodeCount(); + } + + @Override + public int labelCount() { + return graphStore.nodes().availableNodeLabels().size(); + } + + @Override + public int propertyKeyCount() { + int nodePropertyCount = graphStore + .schema() + .nodeSchema() + .unionProperties() + .size(); + int relPropertyCount = graphStore + .schema() + .relationshipSchema() + .unionProperties() + .size(); + + return nodePropertyCount + relPropertyCount; + } + + @Override + public int relationshipTypeCount() { + return graphStore.schema().relationshipSchema().entries().size(); + } + + @Override + public T getOrCreateSchemaDependantState(Class type, Function factory) { + return type.cast(dependantState.computeIfAbsent(type, key -> factory.apply(this))); + } + + @Override + public AllNodeScan allNodeScan() { + return new InMemoryNodeScan(); + } + + @Override + public void close() { + assert !closed; + closed = true; + } + + @Override + public StorageSchemaReader schemaSnapshot() { + return this; + } + + @Override + public TokenNameLookup tokenNameLookup() { + return tokenHolders; + } + +} diff --git a/gradle/dependencies.gradle b/gradle/dependencies.gradle index ca19058f2a..c8ae007e83 100644 --- a/gradle/dependencies.gradle +++ b/gradle/dependencies.gradle @@ -13,6 +13,7 @@ ext { '5.10': properties.getOrDefault('neo4jVersion510', '5.10.0'), '5.11': properties.getOrDefault('neo4jVersion511', '5.11.0'), '5.12': properties.getOrDefault('neo4jVersion511', '5.12.0'), + '5.12': properties.getOrDefault('neo4jVersion512', '5.12.0'), ] neo4jDefault = neos.'4.4' From 4841e185d348e77482ac6f914c24da92d324424a Mon Sep 17 00:00:00 2001 From: yuval Date: Thu, 7 Sep 2023 21:07:00 +0300 Subject: [PATCH 221/273] updated licenses --- .../gds/compat/_512/Neo4jProxyFactoryImpl.java | 17 +++++++++++++++-- .../compat/_512/SettingProxyFactoryImpl.java | 17 +++++++++++++++-- .../_512/StorageEngineProxyFactoryImpl.java | 17 +++++++++++++++-- 3 files changed, 45 insertions(+), 6 deletions(-) diff --git a/compatibility/5.12/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_512/Neo4jProxyFactoryImpl.java b/compatibility/5.12/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_512/Neo4jProxyFactoryImpl.java index 664ac89c8e..3c34c8cd8d 100644 --- a/compatibility/5.12/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_512/Neo4jProxyFactoryImpl.java +++ b/compatibility/5.12/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_512/Neo4jProxyFactoryImpl.java @@ -1,8 +1,21 @@ /* * Copyright (c) "Neo4j" * Neo4j Sweden AB [http://neo4j.com] - * This file contains proprietary code that is only available via a commercial license from Neo4j. - * For more information, see https://neo4j.com/contact-us/ + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package org.neo4j.gds.compat._512; diff --git a/compatibility/5.12/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_512/SettingProxyFactoryImpl.java b/compatibility/5.12/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_512/SettingProxyFactoryImpl.java index 06032e87fa..2c165303d6 100644 --- a/compatibility/5.12/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_512/SettingProxyFactoryImpl.java +++ b/compatibility/5.12/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_512/SettingProxyFactoryImpl.java @@ -1,8 +1,21 @@ /* * Copyright (c) "Neo4j" * Neo4j Sweden AB [http://neo4j.com] - * This file contains proprietary code that is only available via a commercial license from Neo4j. - * For more information, see https://neo4j.com/contact-us/ + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package org.neo4j.gds.compat._512; diff --git a/compatibility/5.12/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_512/StorageEngineProxyFactoryImpl.java b/compatibility/5.12/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_512/StorageEngineProxyFactoryImpl.java index dbdff6ea60..9ac73ba4f3 100644 --- a/compatibility/5.12/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_512/StorageEngineProxyFactoryImpl.java +++ b/compatibility/5.12/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_512/StorageEngineProxyFactoryImpl.java @@ -1,8 +1,21 @@ /* * Copyright (c) "Neo4j" * Neo4j Sweden AB [http://neo4j.com] - * This file contains proprietary code that is only available via a commercial license from Neo4j. - * For more information, see https://neo4j.com/contact-us/ + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package org.neo4j.gds.compat._512; From d68a50b0b5b80251bc3d2eeadf8c3a23a07ed5c5 Mon Sep 17 00:00:00 2001 From: yuval Date: Thu, 7 Sep 2023 21:10:59 +0300 Subject: [PATCH 222/273] added 5.12 enum --- .../src/main/java/org/neo4j/gds/compat/Neo4jVersion.java | 5 +++++ .../test/java/org/neo4j/gds/compat/Neo4jVersionTest.java | 3 ++- .../src/test/java/org/neo4j/gds/SysInfoProcTest.java | 8 ++++++++ 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/neo4j-adapter/src/main/java/org/neo4j/gds/compat/Neo4jVersion.java b/neo4j-adapter/src/main/java/org/neo4j/gds/compat/Neo4jVersion.java index 10dfe4b431..298009dce6 100644 --- a/neo4j-adapter/src/main/java/org/neo4j/gds/compat/Neo4jVersion.java +++ b/neo4j-adapter/src/main/java/org/neo4j/gds/compat/Neo4jVersion.java @@ -39,6 +39,7 @@ public enum Neo4jVersion { V_5_9, V_5_10, V_5_11, + V_5_12, V_RC; @Override @@ -68,6 +69,8 @@ public String toString() { return "5.10"; case V_5_11: return "5.11"; + case V_5_12: + return "5.12"; case V_RC: return "rc"; default: @@ -163,6 +166,8 @@ static Neo4jVersion parse(String version) { } else if (minorVersion == 11) { return Neo4jVersion.V_5_11; } else if (minorVersion == 12) { + return Neo4jVersion.V_5_12; + } else if (minorVersion == 13) { return Neo4jVersion.V_RC; } } diff --git a/neo4j-adapter/src/test/java/org/neo4j/gds/compat/Neo4jVersionTest.java b/neo4j-adapter/src/test/java/org/neo4j/gds/compat/Neo4jVersionTest.java index be948ec070..6c78c65129 100644 --- a/neo4j-adapter/src/test/java/org/neo4j/gds/compat/Neo4jVersionTest.java +++ b/neo4j-adapter/src/test/java/org/neo4j/gds/compat/Neo4jVersionTest.java @@ -52,7 +52,8 @@ class Neo4jVersionTest { "5.9.0, V_5_9", "5.10.0, V_5_10", "5.11.0, V_5_11", - "5.12.0, V_RC", + "5.12.0, V_5_12", + "5.13.0, V_RC", }) void testParse(String input, Neo4jVersion expected) { assertEquals(expected.name(), Neo4jVersion.parse(input).name()); diff --git a/proc/sysinfo/src/test/java/org/neo4j/gds/SysInfoProcTest.java b/proc/sysinfo/src/test/java/org/neo4j/gds/SysInfoProcTest.java index 786b0ca3c5..3f50671375 100644 --- a/proc/sysinfo/src/test/java/org/neo4j/gds/SysInfoProcTest.java +++ b/proc/sysinfo/src/test/java/org/neo4j/gds/SysInfoProcTest.java @@ -255,6 +255,14 @@ void testSysInfoProc() throws IOException { "Neo4j 5.11" ); break; + case V_5_12: + expectedCompatibilities = Set.of( + "Neo4j Settings 5.12 (placeholder)", + "Neo4j Settings 5.12", + "Neo4j 5.12 (placeholder)", + "Neo4j 5.12" + ); + break; case V_RC: expectedCompatibilities = Set.of( "Neo4j Settings RC", From 95c9072f759368f90044ef6db4937242804e20fa Mon Sep 17 00:00:00 2001 From: yuval Date: Thu, 7 Sep 2023 22:18:01 +0300 Subject: [PATCH 223/273] fixed test --- .../sysinfo/src/test/java/org/neo4j/gds/SysInfoProcTest.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/proc/sysinfo/src/test/java/org/neo4j/gds/SysInfoProcTest.java b/proc/sysinfo/src/test/java/org/neo4j/gds/SysInfoProcTest.java index 3f50671375..79744260a9 100644 --- a/proc/sysinfo/src/test/java/org/neo4j/gds/SysInfoProcTest.java +++ b/proc/sysinfo/src/test/java/org/neo4j/gds/SysInfoProcTest.java @@ -107,6 +107,11 @@ class SysInfoProcTest extends BaseProcTest { "Neo4j Settings 5.11", "Neo4j Settings 5.11 (placeholder)", + "Neo4j 5.12", + "Neo4j 5.12 (placeholder)", + "Neo4j Settings 5.12", + "Neo4j Settings 5.12 (placeholder)", + "Neo4j DEV", "Neo4j DEV (placeholder)", "Neo4j Settings DEV", From f98836a05999cc411f9bb6a09cb04002892f803e Mon Sep 17 00:00:00 2001 From: yuval Date: Fri, 8 Sep 2023 15:16:55 +0300 Subject: [PATCH 224/273] bump Aura version to 33 --- gradle/version.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle/version.gradle b/gradle/version.gradle index bdf585c5e8..df5d50fd06 100644 --- a/gradle/version.gradle +++ b/gradle/version.gradle @@ -1,6 +1,6 @@ ext { gdsBaseVersion = '2.4.6' - gdsAuraVersion = '32' + gdsAuraVersion = '33' gdsVersion = gdsBaseVersion + (rootProject.hasProperty('aurads') ? "+${gdsAuraVersion}" : "") } From 8ea506f5f9d6094b582a0affad0ebcb9fec568b0 Mon Sep 17 00:00:00 2001 From: Mats Rydberg Date: Mon, 11 Sep 2023 09:35:51 +0200 Subject: [PATCH 225/273] Extract default ExecutorService out from Pools --- .../AbstractCentralityResultBuilder.java | 4 +- .../AbstractCommunityResultBuilder.java | 4 +- .../gds/result/CommunityStatisticsTest.java | 8 ++-- .../org/neo4j/gds/scaling/CenterTest.java | 6 +-- .../org/neo4j/gds/scaling/L1NormTest.java | 8 ++-- .../org/neo4j/gds/scaling/L2NormTest.java | 8 ++-- .../org/neo4j/gds/scaling/LogScalerTest.java | 4 +- .../java/org/neo4j/gds/scaling/MaxTest.java | 8 ++-- .../java/org/neo4j/gds/scaling/MeanTest.java | 8 ++-- .../org/neo4j/gds/scaling/MinMaxTest.java | 8 ++-- .../org/neo4j/gds/scaling/ScalerTest.java | 6 +-- .../org/neo4j/gds/scaling/StdScoreTest.java | 8 ++-- .../approxmaxkcut/ApproxMaxKCutFactory.java | 4 +- .../BetweennessCentralityFactory.java | 4 +- .../closeness/ClosenessCentralityFactory.java | 4 +- .../gds/conductance/ConductanceFactory.java | 4 +- .../gds/degree/DegreeCentralityFactory.java | 4 +- .../neo4j/gds/embeddings/fastrp/FastRP.java | 4 +- .../algo/GraphSageAlgorithmFactory.java | 4 +- .../algo/GraphSageTrainAlgorithmFactory.java | 4 +- .../gds/embeddings/node2vec/Node2Vec.java | 4 +- .../HarmonicCentralityAlgorithmFactory.java | 4 +- .../InverseRelationshipsAlgorithmFactory.java | 4 +- .../CELFAlgorithmFactory.java | 4 +- .../gds/k1coloring/K1ColoringFactory.java | 4 +- .../gds/kmeans/KmeansAlgorithmFactory.java | 4 +- .../org/neo4j/gds/kmeans/KmeansContext.java | 4 +- .../LabelPropagationFactory.java | 4 +- .../java/org/neo4j/gds/leiden/Leiden.java | 4 +- .../gds/louvain/LouvainAlgorithmFactory.java | 4 +- .../ModularityOptimizationFactory.java | 4 +- .../pagerank/PageRankAlgorithmFactory.java | 6 +-- .../gds/paths/delta/DeltaSteppingFactory.java | 4 +- .../org/neo4j/gds/paths/traverse/BFS.java | 4 +- .../java/org/neo4j/gds/paths/yens/Yens.java | 4 +- .../ScalePropertiesFactory.java | 4 +- .../filteredknn/FilteredKnnFactory.java | 4 +- .../FilteredNodeSimilarityFactory.java | 4 +- .../neo4j/gds/similarity/knn/KnnContext.java | 4 +- .../neo4j/gds/similarity/knn/KnnFactory.java | 4 +- .../nodesim/NodeSimilarityFactory.java | 4 +- .../steiner/SteinerTreeAlgorithmFactory.java | 4 +- .../traversal/RandomWalkAlgorithmFactory.java | 4 +- .../IntersectingTriangleCountFactory.java | 4 +- .../gds/triangle/TriangleStreamFactory.java | 4 +- .../ToUndirectedAlgorithmFactory.java | 4 +- .../walking/CollapsePathAlgorithmFactory.java | 4 +- .../neo4j/gds/wcc/WccAlgorithmFactory.java | 4 +- .../MSBFSAllShortestPathsTest.java | 4 +- .../WeightedAllShortestPathsTest.java | 6 +-- .../gds/approxmaxkcut/ApproxMaxKCutTest.java | 6 +-- .../BetweennessCentralityTest.java | 6 +-- .../betweenness/SelectionStrategyTest.java | 24 +++++------ .../WeightedBetweennessCentralityTest.java | 8 ++-- .../ClosenessCentralityDirectedTest.java | 6 +-- .../ClosenessCentralityDiscoTest.java | 4 +- .../closeness/ClosenessCentralityTest.java | 6 +-- .../gds/conductance/ConductanceTest.java | 4 +- .../gds/degree/DegreeCentralityTest.java | 8 ++-- .../gds/embeddings/fastrp/FastRPTest.java | 6 +-- .../GraphSageEmbeddingsGeneratorTest.java | 14 +++---- .../graphsage/GraphSageModelTrainerTest.java | 30 +++++++------- .../embeddings/graphsage/GraphSageTest.java | 8 ++-- .../algo/MultiLabelGraphSageTrainTest.java | 12 +++--- .../gds/embeddings/hashgnn/HashGNNTest.java | 6 +-- .../gds/embeddings/node2vec/Node2VecTest.java | 6 +-- .../gds/harmonic/HarmonicCentralityTest.java | 6 +-- .../InverseRelationshipsTest.java | 6 +-- .../CELFOnConnectedGraphTest.java | 4 +- .../CELFOnTreeGraphTest.java | 4 +- .../gds/influenceMaximization/CelfTest.java | 4 +- .../neo4j/gds/k1coloring/K1ColoringTest.java | 10 ++--- .../LabelPropagationTest.java | 10 ++--- .../NonStabilizingLabelPropagationTest.java | 4 +- .../gds/leiden/GraphWithSelfLoopTest.java | 7 ++-- .../gds/leiden/ModularityComputerTest.java | 8 ++-- .../gds/leiden/RefinementPhaseKarateTest.java | 4 +- .../neo4j/gds/leiden/RefinementPhaseTest.java | 4 +- .../WeightedModularityComputerTest.java | 8 ++-- .../org/neo4j/gds/louvain/LouvainTest.java | 20 +++++----- .../modularityoptimization/FootballTest.java | 4 +- .../ModularityOptimizationTest.java | 4 +- ...ityOptimizationWithoutOrientationTest.java | 4 +- .../MultiSourceBFSAccessMethodsTest.java | 14 +++---- .../gds/paths/delta/DeltaSteppingTest.java | 14 +++---- .../ScalePropertiesMissingPropsTest.java | 6 +-- .../scaleproperties/ScalePropertiesTest.java | 26 ++++++------ .../NodeSimilarityTerminationTest.java | 4 +- .../nodesim/NodeSimilarityTest.java | 40 +++++++++---------- .../nodesim/SimilarityGraphBuilderTest.java | 8 ++-- .../UnionGraphWeightedNodeSimilarityTest.java | 4 +- ...rtestPathSteinerAlgorithmExtendedTest.java | 12 +++--- ...estPathsSteinerAlgorithmReroutingTest.java | 26 ++++++------ .../ShortestPathsSteinerAlgorithmTest.java | 4 +- .../neo4j/gds/traversal/RandomWalkTest.java | 22 +++++----- ...sectingTriangleCountFilteredGraphTest.java | 4 +- .../IntersectingTriangleCountTest.java | 4 +- .../LargeIntersectingTriangleCountTest.java | 6 +-- .../gds/triangle/TriangleStreamTest.java | 6 +-- .../UnionGraphTriangleCountingTest.java | 4 +- .../gds/undirected/ToUndirectedTest.java | 14 +++---- .../gds/walking/CollapseMultiPathsTest.java | 6 +-- .../neo4j/gds/walking/CollapsePathTest.java | 8 ++-- .../org/neo4j/gds/wcc/IncrementalWccTest.java | 4 +- .../test/java/org/neo4j/gds/wcc/WccTest.java | 4 +- .../org/neo4j/gds/wcc/WccThresholdTest.java | 4 +- .../java/org/neo4j/gds/pregel/HitsTest.java | 4 +- .../gds/pregel/SpeakerListenerLPATest.java | 8 ++-- .../NativeRelationshipStreamExporter.java | 4 +- .../write/NativeNodeLabelExporterTest.java | 4 +- .../write/NativeNodePropertyExporterTest.java | 6 +-- .../org/neo4j/gds/api/GraphLoaderContext.java | 4 +- .../core/concurrency/ExecutorServices.java | 29 ++++++++++++++ .../org/neo4j/gds/core/concurrency/Pools.java | 1 - .../core/concurrency/RunWithConcurrency.java | 4 +- .../core/loading/ArrayIdMapBuilderOps.java | 4 +- .../loading/construction/GraphFactory.java | 4 +- .../DoubleArrayNodePropertiesBuilder.java | 4 +- .../DoubleNodePropertiesBuilder.java | 4 +- .../FloatArrayNodePropertiesBuilder.java | 4 +- .../LongArrayNodePropertiesBuilder.java | 4 +- .../LongNodePropertiesBuilder.java | 4 +- .../java/org/neo4j/gds/TerminationTest.java | 4 +- .../java/org/neo4j/gds/core/LoadingTest.java | 4 +- .../varlong/TransientCsrListTest.java | 4 +- .../core/loading/LazyIdMapBuilderTest.java | 4 +- .../ScanningRelationshipsImporterTest.java | 4 +- .../paged/HugeAtomicGrowingBitSetTest.java | 4 +- .../utils/paged/ShardedLongLongMapTest.java | 4 +- .../ExamplePregelComputationAlgoTest.java | 2 +- .../beta/pregel/bfs/BFSPregelAlgoTest.java | 8 ++-- .../cc/ConnectedComponentsPregelAlgoTest.java | 4 +- .../lp/LabelPropagationPregelAlgoTest.java | 6 +-- .../pregel/pr/PageRankPregelAlgoTest.java | 4 +- .../pr/WeightedPageRankPregelAlgoTest.java | 4 +- ...ingleSourceShortestPathPregelAlgoTest.java | 4 +- .../TriangleCountPregelTest.java | 4 +- .../graphsampling/GraphSampleConstructor.java | 4 +- .../io/file/FileToGraphStoreImporter.java | 8 ++-- .../neo4j/gds/ml/splitting/EdgeSplitter.java | 4 +- .../pregel/generator/AlgorithmGenerator.java | 4 +- .../generator/AlgorithmGeneratorTest.java | 2 +- .../BidirectionalComputationAlgorithm.java | 4 +- .../expected/ComputationAlgorithm.java | 4 +- .../org/neo4j/gds/beta/pregel/PregelTest.java | 28 ++++++------- .../neo4j/gds/catalog/GraphProjectProc.java | 4 +- .../gds/catalog/GraphWriteNodeLabelProc.java | 6 +-- .../neo4j/gds/catalog/NodeLabelMutator.java | 4 +- .../gds/catalog/NodePropertiesWriter.java | 4 +- .../gds/catalog/GraphProjectProcTest.java | 4 +- .../org/neo4j/gds/CommunityProcCompanion.java | 4 +- ...dePropertiesComputationResultConsumer.java | 4 +- .../org/neo4j/gds/kmeans/KmeansWriteSpec.java | 4 +- .../predict/ApproximateLinkPrediction.java | 4 +- ...redictionPipelineMutateResultConsumer.java | 4 +- .../paths/all/AllShortestPathsStreamSpec.java | 6 +-- .../randomwalk/RandomWalkStreamProcTest.java | 4 +- .../neo4j/gds/pregel/proc/PregelProcTest.java | 4 +- .../FilteredNodeSimilarityMutateSpec.java | 4 +- .../NodeSimilarityMutateSpecification.java | 4 +- .../org/neo4j/gds/GraphLoaderBuilders.java | 4 +- .../gds/core/idmap/IdMapBuilderTest.java | 4 +- 162 files changed, 529 insertions(+), 502 deletions(-) create mode 100644 core/src/main/java/org/neo4j/gds/core/concurrency/ExecutorServices.java diff --git a/algo-common/src/main/java/org/neo4j/gds/result/AbstractCentralityResultBuilder.java b/algo-common/src/main/java/org/neo4j/gds/result/AbstractCentralityResultBuilder.java index c9c6b94c9d..d8f79fe932 100644 --- a/algo-common/src/main/java/org/neo4j/gds/result/AbstractCentralityResultBuilder.java +++ b/algo-common/src/main/java/org/neo4j/gds/result/AbstractCentralityResultBuilder.java @@ -22,7 +22,7 @@ import org.HdrHistogram.DoubleHistogram; import org.jetbrains.annotations.NotNull; import org.neo4j.gds.api.ProcedureReturnColumns; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.utils.ProgressTimer; import org.neo4j.gds.scaling.LogScaler; import org.neo4j.gds.scaling.ScalerFactory; @@ -95,7 +95,7 @@ private Optional computeCentralityHistogram() { return Optional.of(CentralityStatistics.histogram( nodeCount, centralityFunction, - Pools.DEFAULT, + ExecutorServices.DEFAULT, concurrency )); } catch (ArrayIndexOutOfBoundsException e) { diff --git a/algo-common/src/main/java/org/neo4j/gds/result/AbstractCommunityResultBuilder.java b/algo-common/src/main/java/org/neo4j/gds/result/AbstractCommunityResultBuilder.java index ae78203533..27bca6b6c0 100644 --- a/algo-common/src/main/java/org/neo4j/gds/result/AbstractCommunityResultBuilder.java +++ b/algo-common/src/main/java/org/neo4j/gds/result/AbstractCommunityResultBuilder.java @@ -22,7 +22,7 @@ import org.HdrHistogram.Histogram; import org.jetbrains.annotations.Nullable; import org.neo4j.gds.api.ProcedureReturnColumns; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.utils.ProgressTimer; import java.util.Map; @@ -54,7 +54,7 @@ protected AbstractCommunityResultBuilder( ProcedureReturnColumns returnColumns, int concurrency ) { - this(returnColumns, Pools.DEFAULT, concurrency); + this(returnColumns, ExecutorServices.DEFAULT, concurrency); } protected AbstractCommunityResultBuilder( diff --git a/algo-common/src/test/java/org/neo4j/gds/result/CommunityStatisticsTest.java b/algo-common/src/test/java/org/neo4j/gds/result/CommunityStatisticsTest.java index ff83368200..a5cd2274ba 100644 --- a/algo-common/src/test/java/org/neo4j/gds/result/CommunityStatisticsTest.java +++ b/algo-common/src/test/java/org/neo4j/gds/result/CommunityStatisticsTest.java @@ -24,7 +24,7 @@ import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; import org.neo4j.gds.TestSupport; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import java.util.HashMap; import java.util.Map; @@ -49,7 +49,7 @@ void communitySizes( var communitySizes = CommunityStatistics.communitySizes( nodeCount, communityFunction, - Pools.DEFAULT, + ExecutorServices.DEFAULT, concurrency ); expectedCommunitySizes.forEach((communityId, expectedSize) -> { @@ -72,7 +72,7 @@ void communityCount( CommunityStatistics.communityCount( nodeCount, communityFunction, - Pools.DEFAULT, + ExecutorServices.DEFAULT, concurrency ) ); @@ -91,7 +91,7 @@ void communityCountAndHistogram( var communityCountAndHistogram = CommunityStatistics.communityCountAndHistogram( nodeCount, communityFunction, - Pools.DEFAULT, + ExecutorServices.DEFAULT, concurrency ); diff --git a/algo-common/src/test/java/org/neo4j/gds/scaling/CenterTest.java b/algo-common/src/test/java/org/neo4j/gds/scaling/CenterTest.java index 24159e087e..f45bf10998 100644 --- a/algo-common/src/test/java/org/neo4j/gds/scaling/CenterTest.java +++ b/algo-common/src/test/java/org/neo4j/gds/scaling/CenterTest.java @@ -26,7 +26,7 @@ import org.junit.jupiter.params.provider.MethodSource; import org.neo4j.gds.api.properties.nodes.NodePropertyValues; import org.neo4j.gds.core.CypherMapWrapper; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.nodeproperties.DoubleTestPropertyValues; @@ -63,7 +63,7 @@ void normalizes(NodePropertyValues properties, double avg, double[] expected) { 10, 1, ProgressTracker.NULL_TRACKER, - Pools.DEFAULT + ExecutorServices.DEFAULT ); assertThat(scaler.avg).isEqualTo(avg); @@ -81,7 +81,7 @@ void handlesMissingValue() { 10, 1, ProgressTracker.NULL_TRACKER, - Pools.DEFAULT + ExecutorServices.DEFAULT ); var avg = 4.444; diff --git a/algo-common/src/test/java/org/neo4j/gds/scaling/L1NormTest.java b/algo-common/src/test/java/org/neo4j/gds/scaling/L1NormTest.java index 1acf05847a..38532f316d 100644 --- a/algo-common/src/test/java/org/neo4j/gds/scaling/L1NormTest.java +++ b/algo-common/src/test/java/org/neo4j/gds/scaling/L1NormTest.java @@ -26,7 +26,7 @@ import org.junit.jupiter.params.provider.MethodSource; import org.neo4j.gds.api.properties.nodes.NodePropertyValues; import org.neo4j.gds.core.CypherMapWrapper; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.nodeproperties.DoubleTestPropertyValues; @@ -62,7 +62,7 @@ void scale(int nodeCount, NodePropertyValues properties, double l1norm, double[] nodeCount, 1, ProgressTracker.NULL_TRACKER, - Pools.DEFAULT + ExecutorServices.DEFAULT ); assertThat(scaler.l1Norm).isEqualTo(l1norm); @@ -79,7 +79,7 @@ void avoidsDivByZero() { 10, 1, ProgressTracker.NULL_TRACKER, - Pools.DEFAULT + ExecutorServices.DEFAULT ); for (int i = 0; i < 10; i++) { @@ -95,7 +95,7 @@ void handlesMissingValue() { 10, 1, ProgressTracker.NULL_TRACKER, - Pools.DEFAULT + ExecutorServices.DEFAULT ); var l1norm = 40D; diff --git a/algo-common/src/test/java/org/neo4j/gds/scaling/L2NormTest.java b/algo-common/src/test/java/org/neo4j/gds/scaling/L2NormTest.java index 0808221440..89f0b509df 100644 --- a/algo-common/src/test/java/org/neo4j/gds/scaling/L2NormTest.java +++ b/algo-common/src/test/java/org/neo4j/gds/scaling/L2NormTest.java @@ -26,7 +26,7 @@ import org.junit.jupiter.params.provider.MethodSource; import org.neo4j.gds.api.properties.nodes.NodePropertyValues; import org.neo4j.gds.core.CypherMapWrapper; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.nodeproperties.DoubleTestPropertyValues; @@ -53,7 +53,7 @@ void normalizes(NodePropertyValues properties, double euclideanLength, double[] 10, 1, ProgressTracker.NULL_TRACKER, - Pools.DEFAULT + ExecutorServices.DEFAULT ); assertThat(scaler.euclideanLength).isEqualTo(euclideanLength); @@ -70,7 +70,7 @@ void avoidsDivByZero() { 10, 1, ProgressTracker.NULL_TRACKER, - Pools.DEFAULT + ExecutorServices.DEFAULT ); for (int i = 0; i < 10; i++) { @@ -86,7 +86,7 @@ void handlesMissingValue() { 10, 1, ProgressTracker.NULL_TRACKER, - Pools.DEFAULT + ExecutorServices.DEFAULT ); var euclideanLength = 16.124; diff --git a/algo-common/src/test/java/org/neo4j/gds/scaling/LogScalerTest.java b/algo-common/src/test/java/org/neo4j/gds/scaling/LogScalerTest.java index 220eebde88..390733cb23 100644 --- a/algo-common/src/test/java/org/neo4j/gds/scaling/LogScalerTest.java +++ b/algo-common/src/test/java/org/neo4j/gds/scaling/LogScalerTest.java @@ -26,7 +26,7 @@ import org.junit.jupiter.params.provider.MethodSource; import org.neo4j.gds.api.properties.nodes.NodePropertyValues; import org.neo4j.gds.core.CypherMapWrapper; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.nodeproperties.DoubleTestPropertyValues; import org.neo4j.gds.nodeproperties.LongTestPropertyValues; @@ -72,7 +72,7 @@ void handlesMissingValue() { 10, 1, ProgressTracker.NULL_TRACKER, - Pools.DEFAULT + ExecutorServices.DEFAULT ); for (int i = 0; i < 5; i++) { diff --git a/algo-common/src/test/java/org/neo4j/gds/scaling/MaxTest.java b/algo-common/src/test/java/org/neo4j/gds/scaling/MaxTest.java index ba805420c8..f381e52be2 100644 --- a/algo-common/src/test/java/org/neo4j/gds/scaling/MaxTest.java +++ b/algo-common/src/test/java/org/neo4j/gds/scaling/MaxTest.java @@ -26,7 +26,7 @@ import org.junit.jupiter.params.provider.MethodSource; import org.neo4j.gds.api.properties.nodes.NodePropertyValues; import org.neo4j.gds.core.CypherMapWrapper; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.nodeproperties.DoubleTestPropertyValues; @@ -70,7 +70,7 @@ void scale(int nodeCount, NodePropertyValues properties, double absMax, double[] nodeCount, 1, ProgressTracker.NULL_TRACKER, - Pools.DEFAULT + ExecutorServices.DEFAULT ); assertThat(scaler.maxAbs).isEqualTo(absMax); @@ -88,7 +88,7 @@ void avoidsDivByZero() { 10, 1, ProgressTracker.NULL_TRACKER, - Pools.DEFAULT + ExecutorServices.DEFAULT ); assertThat(scaler.statistics()).containsExactlyEntriesOf(Map.of("absMax", List.of(0D))); @@ -106,7 +106,7 @@ void handlesMissingValue() { 10, 1, ProgressTracker.NULL_TRACKER, - Pools.DEFAULT + ExecutorServices.DEFAULT ); diff --git a/algo-common/src/test/java/org/neo4j/gds/scaling/MeanTest.java b/algo-common/src/test/java/org/neo4j/gds/scaling/MeanTest.java index 80ac7bc12a..f0f6411d3e 100644 --- a/algo-common/src/test/java/org/neo4j/gds/scaling/MeanTest.java +++ b/algo-common/src/test/java/org/neo4j/gds/scaling/MeanTest.java @@ -26,7 +26,7 @@ import org.junit.jupiter.params.provider.MethodSource; import org.neo4j.gds.api.properties.nodes.NodePropertyValues; import org.neo4j.gds.core.CypherMapWrapper; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.nodeproperties.DoubleTestPropertyValues; @@ -57,7 +57,7 @@ void normalizes(NodePropertyValues properties, double avg, double min, double ma 10, 1, ProgressTracker.NULL_TRACKER, - Pools.DEFAULT + ExecutorServices.DEFAULT ); assertThat(scaler.avg).isEqualTo(avg); @@ -81,7 +81,7 @@ void avoidsDivByZero() { 10, 1, ProgressTracker.NULL_TRACKER, - Pools.DEFAULT + ExecutorServices.DEFAULT ); assertThat(scaler.statistics()).containsExactlyInAnyOrderEntriesOf(Map.of( @@ -103,7 +103,7 @@ void handlesMissingValue() { 10, 1, ProgressTracker.NULL_TRACKER, - Pools.DEFAULT + ExecutorServices.DEFAULT ); var avg = 4.444; diff --git a/algo-common/src/test/java/org/neo4j/gds/scaling/MinMaxTest.java b/algo-common/src/test/java/org/neo4j/gds/scaling/MinMaxTest.java index d2f0fe13f9..b0c442413f 100644 --- a/algo-common/src/test/java/org/neo4j/gds/scaling/MinMaxTest.java +++ b/algo-common/src/test/java/org/neo4j/gds/scaling/MinMaxTest.java @@ -26,7 +26,7 @@ import org.junit.jupiter.params.provider.MethodSource; import org.neo4j.gds.api.properties.nodes.NodePropertyValues; import org.neo4j.gds.core.CypherMapWrapper; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.nodeproperties.DoubleTestPropertyValues; @@ -53,7 +53,7 @@ void normalizes(NodePropertyValues properties, double min, double max) { 10, 1, ProgressTracker.NULL_TRACKER, - Pools.DEFAULT + ExecutorServices.DEFAULT ); assertThat(scaler.min).isEqualTo(min); @@ -77,7 +77,7 @@ void avoidsDivByZero() { 10, 1, ProgressTracker.NULL_TRACKER, - Pools.DEFAULT + ExecutorServices.DEFAULT ); assertThat(scaler.statistics()).containsExactlyEntriesOf(Map.of( @@ -98,7 +98,7 @@ void handlesMissingValue() { 10, 1, ProgressTracker.NULL_TRACKER, - Pools.DEFAULT + ExecutorServices.DEFAULT ); diff --git a/algo-common/src/test/java/org/neo4j/gds/scaling/ScalerTest.java b/algo-common/src/test/java/org/neo4j/gds/scaling/ScalerTest.java index c67bc368df..9896bf61aa 100644 --- a/algo-common/src/test/java/org/neo4j/gds/scaling/ScalerTest.java +++ b/algo-common/src/test/java/org/neo4j/gds/scaling/ScalerTest.java @@ -21,7 +21,7 @@ import org.junit.jupiter.api.Test; import org.neo4j.gds.core.CypherMapWrapper; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.nodeproperties.DoubleTestPropertyValues; @@ -39,14 +39,14 @@ void shouldAccumulateStatsCorrectly() { 10, 1, ProgressTracker.NULL_TRACKER, - Pools.DEFAULT + ExecutorServices.DEFAULT ); var meanScaler2 = (Mean) Mean.buildFrom(CypherMapWrapper.empty()).create( new DoubleTestPropertyValues(nodeId -> 2 * nodeId), 10, 1, ProgressTracker.NULL_TRACKER, - Pools.DEFAULT + ExecutorServices.DEFAULT ); var meanScaler1Stats = meanScaler1.statistics(); diff --git a/algo-common/src/test/java/org/neo4j/gds/scaling/StdScoreTest.java b/algo-common/src/test/java/org/neo4j/gds/scaling/StdScoreTest.java index 8495767c73..9f4857704c 100644 --- a/algo-common/src/test/java/org/neo4j/gds/scaling/StdScoreTest.java +++ b/algo-common/src/test/java/org/neo4j/gds/scaling/StdScoreTest.java @@ -26,7 +26,7 @@ import org.junit.jupiter.params.provider.MethodSource; import org.neo4j.gds.api.properties.nodes.NodePropertyValues; import org.neo4j.gds.core.CypherMapWrapper; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.nodeproperties.DoubleTestPropertyValues; @@ -56,7 +56,7 @@ void normalizes(NodePropertyValues properties, double avg, double std, double[] 10, 1, ProgressTracker.NULL_TRACKER, - Pools.DEFAULT + ExecutorServices.DEFAULT ); assertThat(scaler.avg).isEqualTo(avg); @@ -78,7 +78,7 @@ void handlesMissingValue() { 10, 1, ProgressTracker.NULL_TRACKER, - Pools.DEFAULT + ExecutorServices.DEFAULT ); var avg = 4.444; @@ -100,7 +100,7 @@ void avoidsDivByZero() { 10, 1, ProgressTracker.NULL_TRACKER, - Pools.DEFAULT + ExecutorServices.DEFAULT ); for (int i = 0; i < 10; i++) { diff --git a/algo/src/main/java/org/neo4j/gds/approxmaxkcut/ApproxMaxKCutFactory.java b/algo/src/main/java/org/neo4j/gds/approxmaxkcut/ApproxMaxKCutFactory.java index c8e6a41d9b..58d1e86c85 100644 --- a/algo/src/main/java/org/neo4j/gds/approxmaxkcut/ApproxMaxKCutFactory.java +++ b/algo/src/main/java/org/neo4j/gds/approxmaxkcut/ApproxMaxKCutFactory.java @@ -24,7 +24,7 @@ import org.neo4j.gds.approxmaxkcut.config.ApproxMaxKCutConfig; import org.neo4j.gds.collections.haa.HugeAtomicByteArray; import org.neo4j.gds.collections.haa.HugeAtomicDoubleArray; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.utils.mem.MemoryEstimation; import org.neo4j.gds.core.utils.mem.MemoryEstimations; import org.neo4j.gds.core.utils.paged.HugeByteArray; @@ -51,7 +51,7 @@ public ApproxMaxKCut build( CONFIG configuration, ProgressTracker progressTracker ) { - return new ApproxMaxKCut(graph, Pools.DEFAULT, configuration, progressTracker); + return new ApproxMaxKCut(graph, ExecutorServices.DEFAULT, configuration, progressTracker); } @Override diff --git a/algo/src/main/java/org/neo4j/gds/betweenness/BetweennessCentralityFactory.java b/algo/src/main/java/org/neo4j/gds/betweenness/BetweennessCentralityFactory.java index 7925fc0cb2..c0434491c4 100644 --- a/algo/src/main/java/org/neo4j/gds/betweenness/BetweennessCentralityFactory.java +++ b/algo/src/main/java/org/neo4j/gds/betweenness/BetweennessCentralityFactory.java @@ -23,7 +23,7 @@ import org.neo4j.gds.GraphAlgorithmFactory; import org.neo4j.gds.api.Graph; import org.neo4j.gds.collections.haa.HugeAtomicDoubleArray; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.utils.mem.MemoryEstimation; import org.neo4j.gds.core.utils.mem.MemoryEstimations; import org.neo4j.gds.core.utils.paged.HugeDoubleArray; @@ -66,7 +66,7 @@ public BetweennessCentrality build( graph, strategy, traverserFactory, - Pools.DEFAULT, + ExecutorServices.DEFAULT, configuration.concurrency(), progressTracker ); diff --git a/algo/src/main/java/org/neo4j/gds/closeness/ClosenessCentralityFactory.java b/algo/src/main/java/org/neo4j/gds/closeness/ClosenessCentralityFactory.java index c2219971cc..5a07ee9767 100644 --- a/algo/src/main/java/org/neo4j/gds/closeness/ClosenessCentralityFactory.java +++ b/algo/src/main/java/org/neo4j/gds/closeness/ClosenessCentralityFactory.java @@ -22,7 +22,7 @@ import org.neo4j.gds.GraphAlgorithmFactory; import org.neo4j.gds.api.Graph; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.core.utils.progress.tasks.Task; import org.neo4j.gds.core.utils.progress.tasks.Tasks; @@ -37,7 +37,7 @@ public ClosenessCentrality build( return ClosenessCentrality.of( graph, configuration, - Pools.DEFAULT, + ExecutorServices.DEFAULT, progressTracker ); } diff --git a/algo/src/main/java/org/neo4j/gds/conductance/ConductanceFactory.java b/algo/src/main/java/org/neo4j/gds/conductance/ConductanceFactory.java index ace1f7357e..751a132c27 100644 --- a/algo/src/main/java/org/neo4j/gds/conductance/ConductanceFactory.java +++ b/algo/src/main/java/org/neo4j/gds/conductance/ConductanceFactory.java @@ -21,7 +21,7 @@ import org.neo4j.gds.GraphAlgorithmFactory; import org.neo4j.gds.api.Graph; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.core.utils.progress.tasks.Task; import org.neo4j.gds.core.utils.progress.tasks.Tasks; @@ -43,7 +43,7 @@ public Conductance build( CONFIG configuration, ProgressTracker progressTracker ) { - return new Conductance(graph, Pools.DEFAULT, configuration, progressTracker); + return new Conductance(graph, ExecutorServices.DEFAULT, configuration, progressTracker); } @Override diff --git a/algo/src/main/java/org/neo4j/gds/degree/DegreeCentralityFactory.java b/algo/src/main/java/org/neo4j/gds/degree/DegreeCentralityFactory.java index 150061c525..ecf4dd1705 100644 --- a/algo/src/main/java/org/neo4j/gds/degree/DegreeCentralityFactory.java +++ b/algo/src/main/java/org/neo4j/gds/degree/DegreeCentralityFactory.java @@ -22,7 +22,7 @@ import org.jetbrains.annotations.NotNull; import org.neo4j.gds.GraphAlgorithmFactory; import org.neo4j.gds.api.Graph; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.utils.mem.MemoryEstimation; import org.neo4j.gds.core.utils.mem.MemoryEstimations; import org.neo4j.gds.core.utils.paged.HugeDoubleArray; @@ -45,7 +45,7 @@ public DegreeCentrality build( CONFIG configuration, ProgressTracker progressTracker ) { - return new DegreeCentrality(graph, Pools.DEFAULT, configuration, progressTracker); + return new DegreeCentrality(graph, ExecutorServices.DEFAULT, configuration, progressTracker); } @Override diff --git a/algo/src/main/java/org/neo4j/gds/embeddings/fastrp/FastRP.java b/algo/src/main/java/org/neo4j/gds/embeddings/fastrp/FastRP.java index 3423460498..daf27b31cd 100644 --- a/algo/src/main/java/org/neo4j/gds/embeddings/fastrp/FastRP.java +++ b/algo/src/main/java/org/neo4j/gds/embeddings/fastrp/FastRP.java @@ -22,8 +22,8 @@ import org.jetbrains.annotations.TestOnly; import org.neo4j.gds.Algorithm; import org.neo4j.gds.api.Graph; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.concurrency.ParallelUtil; -import org.neo4j.gds.core.concurrency.Pools; import org.neo4j.gds.core.concurrency.RunWithConcurrency; import org.neo4j.gds.core.utils.mem.MemoryEstimation; import org.neo4j.gds.core.utils.mem.MemoryEstimations; @@ -221,7 +221,7 @@ void propagateEmbeddings() { ); ParallelUtil.parallelPartitionsConsume( - RunWithConcurrency.builder().executor(Pools.DEFAULT).concurrency(concurrency), + RunWithConcurrency.builder().executor(ExecutorServices.DEFAULT).concurrency(concurrency), partitions.stream(), taskSupplier ); diff --git a/algo/src/main/java/org/neo4j/gds/embeddings/graphsage/algo/GraphSageAlgorithmFactory.java b/algo/src/main/java/org/neo4j/gds/embeddings/graphsage/algo/GraphSageAlgorithmFactory.java index 8d1736e0f4..d724ec9968 100644 --- a/algo/src/main/java/org/neo4j/gds/embeddings/graphsage/algo/GraphSageAlgorithmFactory.java +++ b/algo/src/main/java/org/neo4j/gds/embeddings/graphsage/algo/GraphSageAlgorithmFactory.java @@ -22,7 +22,7 @@ import org.neo4j.gds.GraphStoreAlgorithmFactory; import org.neo4j.gds.api.GraphStore; import org.neo4j.gds.config.MutateConfig; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.model.ModelCatalog; import org.neo4j.gds.core.utils.mem.MemoryEstimation; import org.neo4j.gds.core.utils.mem.MemoryEstimations; @@ -49,7 +49,7 @@ public GraphSageAlgorithmFactory(ModelCatalog modelCatalog) { @Override public GraphSage build(GraphStore graphStore, CONFIG configuration, ProgressTracker progressTracker) { - var executorService = Pools.DEFAULT; + var executorService = ExecutorServices.DEFAULT; var model = resolveModel( modelCatalog, configuration.modelUser(), diff --git a/algo/src/main/java/org/neo4j/gds/embeddings/graphsage/algo/GraphSageTrainAlgorithmFactory.java b/algo/src/main/java/org/neo4j/gds/embeddings/graphsage/algo/GraphSageTrainAlgorithmFactory.java index d8ef87ba90..b7938da074 100644 --- a/algo/src/main/java/org/neo4j/gds/embeddings/graphsage/algo/GraphSageTrainAlgorithmFactory.java +++ b/algo/src/main/java/org/neo4j/gds/embeddings/graphsage/algo/GraphSageTrainAlgorithmFactory.java @@ -21,7 +21,7 @@ import org.neo4j.gds.GraphAlgorithmFactory; import org.neo4j.gds.api.Graph; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.utils.mem.MemoryEstimation; import org.neo4j.gds.core.utils.mem.MemoryEstimations; import org.neo4j.gds.core.utils.mem.MemoryRange; @@ -58,7 +58,7 @@ public GraphSageTrain build( GraphSageTrainConfig configuration, ProgressTracker progressTracker ) { - var executorService = Pools.DEFAULT; + var executorService = ExecutorServices.DEFAULT; if(configuration.hasRelationshipWeightProperty()) { validateRelationshipWeightPropertyValue(graph, configuration.concurrency(), executorService); } diff --git a/algo/src/main/java/org/neo4j/gds/embeddings/node2vec/Node2Vec.java b/algo/src/main/java/org/neo4j/gds/embeddings/node2vec/Node2Vec.java index 4b78949d26..86de6046d2 100644 --- a/algo/src/main/java/org/neo4j/gds/embeddings/node2vec/Node2Vec.java +++ b/algo/src/main/java/org/neo4j/gds/embeddings/node2vec/Node2Vec.java @@ -21,7 +21,7 @@ import org.neo4j.gds.Algorithm; import org.neo4j.gds.api.Graph; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.utils.mem.MemoryEstimation; import org.neo4j.gds.core.utils.mem.MemoryEstimations; import org.neo4j.gds.core.utils.paged.HugeObjectArray; @@ -60,7 +60,7 @@ public Node2VecModel.Result compute() { graph, config, progressTracker, - Pools.DEFAULT + ExecutorServices.DEFAULT ); var probabilitiesBuilder = new RandomWalkProbabilities.Builder( diff --git a/algo/src/main/java/org/neo4j/gds/harmonic/HarmonicCentralityAlgorithmFactory.java b/algo/src/main/java/org/neo4j/gds/harmonic/HarmonicCentralityAlgorithmFactory.java index 82ed47682a..2393a83afb 100644 --- a/algo/src/main/java/org/neo4j/gds/harmonic/HarmonicCentralityAlgorithmFactory.java +++ b/algo/src/main/java/org/neo4j/gds/harmonic/HarmonicCentralityAlgorithmFactory.java @@ -21,7 +21,7 @@ import org.neo4j.gds.GraphAlgorithmFactory; import org.neo4j.gds.api.Graph; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; public class HarmonicCentralityAlgorithmFactory extends GraphAlgorithmFactory { @@ -39,7 +39,7 @@ public HarmonicCentrality build( return new HarmonicCentrality( graph, configuration.concurrency(), - Pools.DEFAULT, + ExecutorServices.DEFAULT, progressTracker ); } diff --git a/algo/src/main/java/org/neo4j/gds/indexInverse/InverseRelationshipsAlgorithmFactory.java b/algo/src/main/java/org/neo4j/gds/indexInverse/InverseRelationshipsAlgorithmFactory.java index 8430d1e0f9..a4c5912508 100644 --- a/algo/src/main/java/org/neo4j/gds/indexInverse/InverseRelationshipsAlgorithmFactory.java +++ b/algo/src/main/java/org/neo4j/gds/indexInverse/InverseRelationshipsAlgorithmFactory.java @@ -23,7 +23,7 @@ import org.neo4j.gds.GraphStoreAlgorithmFactory; import org.neo4j.gds.RelationshipType; import org.neo4j.gds.api.GraphStore; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.loading.AdjacencyListBehavior; import org.neo4j.gds.core.utils.mem.MemoryEstimation; import org.neo4j.gds.core.utils.mem.MemoryEstimations; @@ -45,7 +45,7 @@ public InverseRelationships build( InverseRelationshipsConfig configuration, ProgressTracker progressTracker ) { - return new InverseRelationships(graphStore, configuration, progressTracker, Pools.DEFAULT); + return new InverseRelationships(graphStore, configuration, progressTracker, ExecutorServices.DEFAULT); } @Override diff --git a/algo/src/main/java/org/neo4j/gds/influenceMaximization/CELFAlgorithmFactory.java b/algo/src/main/java/org/neo4j/gds/influenceMaximization/CELFAlgorithmFactory.java index 69e3692a5b..08b5b6145d 100644 --- a/algo/src/main/java/org/neo4j/gds/influenceMaximization/CELFAlgorithmFactory.java +++ b/algo/src/main/java/org/neo4j/gds/influenceMaximization/CELFAlgorithmFactory.java @@ -21,7 +21,7 @@ import org.neo4j.gds.GraphAlgorithmFactory; import org.neo4j.gds.api.Graph; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.utils.mem.MemoryEstimation; import org.neo4j.gds.core.utils.mem.MemoryEstimations; import org.neo4j.gds.core.utils.mem.MemoryRange; @@ -53,7 +53,7 @@ public CELF build( configuration.seedSetSize(), configuration.propagationProbability(), configuration.monteCarloSimulations(), - Pools.DEFAULT, + ExecutorServices.DEFAULT, configuration.concurrency(), configuration.randomSeed().orElse(0L), DEFAULT_BATCH_SIZE, diff --git a/algo/src/main/java/org/neo4j/gds/k1coloring/K1ColoringFactory.java b/algo/src/main/java/org/neo4j/gds/k1coloring/K1ColoringFactory.java index 1f532f1c71..ad5e6c1a4c 100644 --- a/algo/src/main/java/org/neo4j/gds/k1coloring/K1ColoringFactory.java +++ b/algo/src/main/java/org/neo4j/gds/k1coloring/K1ColoringFactory.java @@ -23,7 +23,7 @@ import org.neo4j.gds.api.Graph; import org.neo4j.gds.config.BaseConfig; import org.neo4j.gds.config.IterationsConfig; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.utils.mem.MemoryEstimation; import org.neo4j.gds.core.utils.mem.MemoryEstimations; import org.neo4j.gds.core.utils.paged.HugeLongArray; @@ -54,7 +54,7 @@ public K1Coloring build( configuration.maxIterations(), configuration.batchSize(), configuration.concurrency(), - Pools.DEFAULT, + ExecutorServices.DEFAULT, progressTracker ); } diff --git a/algo/src/main/java/org/neo4j/gds/kmeans/KmeansAlgorithmFactory.java b/algo/src/main/java/org/neo4j/gds/kmeans/KmeansAlgorithmFactory.java index b1f5d26521..b4a0da33a9 100644 --- a/algo/src/main/java/org/neo4j/gds/kmeans/KmeansAlgorithmFactory.java +++ b/algo/src/main/java/org/neo4j/gds/kmeans/KmeansAlgorithmFactory.java @@ -22,7 +22,7 @@ import org.jetbrains.annotations.NotNull; import org.neo4j.gds.GraphAlgorithmFactory; import org.neo4j.gds.api.Graph; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.utils.mem.MemoryEstimation; import org.neo4j.gds.core.utils.mem.MemoryEstimations; import org.neo4j.gds.core.utils.paged.HugeDoubleArray; @@ -61,7 +61,7 @@ public Kmeans build( return Kmeans.createKmeans(graph, configuration, ImmutableKmeansContext .builder() .progressTracker(progressTracker) - .executor(Pools.DEFAULT) + .executor(ExecutorServices.DEFAULT) .build()); } diff --git a/algo/src/main/java/org/neo4j/gds/kmeans/KmeansContext.java b/algo/src/main/java/org/neo4j/gds/kmeans/KmeansContext.java index d90627dfef..03fbb4a16a 100644 --- a/algo/src/main/java/org/neo4j/gds/kmeans/KmeansContext.java +++ b/algo/src/main/java/org/neo4j/gds/kmeans/KmeansContext.java @@ -21,7 +21,7 @@ import org.immutables.value.Value; import org.neo4j.gds.annotation.ValueClass; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import java.util.concurrent.ExecutorService; @@ -31,7 +31,7 @@ public interface KmeansContext { @Value.Default default ExecutorService executor() { - return Pools.DEFAULT; + return ExecutorServices.DEFAULT; } @Value.Default diff --git a/algo/src/main/java/org/neo4j/gds/labelpropagation/LabelPropagationFactory.java b/algo/src/main/java/org/neo4j/gds/labelpropagation/LabelPropagationFactory.java index 33c1a0db59..0288faf563 100644 --- a/algo/src/main/java/org/neo4j/gds/labelpropagation/LabelPropagationFactory.java +++ b/algo/src/main/java/org/neo4j/gds/labelpropagation/LabelPropagationFactory.java @@ -22,7 +22,7 @@ import com.carrotsearch.hppc.LongDoubleScatterMap; import org.neo4j.gds.GraphAlgorithmFactory; import org.neo4j.gds.api.Graph; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.utils.mem.MemoryEstimation; import org.neo4j.gds.core.utils.mem.MemoryEstimations; import org.neo4j.gds.core.utils.mem.MemoryRange; @@ -53,7 +53,7 @@ public LabelPropagation build( return new LabelPropagation( graph, configuration, - Pools.DEFAULT, + ExecutorServices.DEFAULT, progressTracker ); } diff --git a/algo/src/main/java/org/neo4j/gds/leiden/Leiden.java b/algo/src/main/java/org/neo4j/gds/leiden/Leiden.java index 5fd0d03707..6594e91474 100644 --- a/algo/src/main/java/org/neo4j/gds/leiden/Leiden.java +++ b/algo/src/main/java/org/neo4j/gds/leiden/Leiden.java @@ -25,7 +25,7 @@ import org.neo4j.gds.api.Graph; import org.neo4j.gds.api.properties.nodes.NodePropertyValues; import org.neo4j.gds.api.schema.Direction; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.concurrency.RunWithConcurrency; import org.neo4j.gds.core.utils.paged.HugeDoubleArray; import org.neo4j.gds.core.utils.paged.HugeLongArray; @@ -77,7 +77,7 @@ public Leiden( this.theta = theta; this.randomSeed = randomSeed; // TODO: Pass these two as parameters - this.executorService = Pools.DEFAULT; + this.executorService = ExecutorServices.DEFAULT; this.concurrency = concurrency; this.dendrogramManager = new LeidenDendrogramManager( rootGraph, diff --git a/algo/src/main/java/org/neo4j/gds/louvain/LouvainAlgorithmFactory.java b/algo/src/main/java/org/neo4j/gds/louvain/LouvainAlgorithmFactory.java index 9898c9e3db..cdf36896a5 100644 --- a/algo/src/main/java/org/neo4j/gds/louvain/LouvainAlgorithmFactory.java +++ b/algo/src/main/java/org/neo4j/gds/louvain/LouvainAlgorithmFactory.java @@ -31,7 +31,7 @@ import org.neo4j.gds.core.Aggregation; import org.neo4j.gds.core.GraphDimensions; import org.neo4j.gds.core.ImmutableGraphDimensions; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.loading.NativeFactory; import org.neo4j.gds.core.utils.mem.MemoryEstimation; import org.neo4j.gds.core.utils.mem.MemoryEstimations; @@ -61,7 +61,7 @@ public Louvain build( configuration.tolerance(), configuration.concurrency(), progressTracker, - Pools.DEFAULT + ExecutorServices.DEFAULT ); } diff --git a/algo/src/main/java/org/neo4j/gds/modularityoptimization/ModularityOptimizationFactory.java b/algo/src/main/java/org/neo4j/gds/modularityoptimization/ModularityOptimizationFactory.java index 17fa061e0e..5764bc9db5 100644 --- a/algo/src/main/java/org/neo4j/gds/modularityoptimization/ModularityOptimizationFactory.java +++ b/algo/src/main/java/org/neo4j/gds/modularityoptimization/ModularityOptimizationFactory.java @@ -25,7 +25,7 @@ import org.neo4j.gds.collections.haa.HugeAtomicDoubleArray; import org.neo4j.gds.config.BaseConfig; import org.neo4j.gds.config.IterationsConfig; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.utils.mem.MemoryEstimation; import org.neo4j.gds.core.utils.mem.MemoryEstimations; import org.neo4j.gds.core.utils.mem.MemoryRange; @@ -106,7 +106,7 @@ public ModularityOptimization build( seedProperty, configuration.concurrency(), configuration.batchSize(), - Pools.DEFAULT, + ExecutorServices.DEFAULT, progressTracker ); } diff --git a/algo/src/main/java/org/neo4j/gds/pagerank/PageRankAlgorithmFactory.java b/algo/src/main/java/org/neo4j/gds/pagerank/PageRankAlgorithmFactory.java index e076c0856e..92ba1f8713 100644 --- a/algo/src/main/java/org/neo4j/gds/pagerank/PageRankAlgorithmFactory.java +++ b/algo/src/main/java/org/neo4j/gds/pagerank/PageRankAlgorithmFactory.java @@ -27,8 +27,8 @@ import org.neo4j.gds.beta.pregel.Pregel; import org.neo4j.gds.beta.pregel.PregelComputation; import org.neo4j.gds.beta.pregel.PregelSchema; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.concurrency.ParallelUtil; -import org.neo4j.gds.core.concurrency.Pools; import org.neo4j.gds.core.utils.TerminationFlag; import org.neo4j.gds.core.utils.mem.MemoryEstimation; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; @@ -121,7 +121,7 @@ public PageRankAlgorithm build( configuration, computation, mode, - Pools.DEFAULT, + ExecutorServices.DEFAULT, progressTracker ); } @@ -143,7 +143,7 @@ private LongToDoubleFunction degreeFunction( var degreeCentrality = new DegreeCentrality( graph, - Pools.DEFAULT, + ExecutorServices.DEFAULT, config, ProgressTracker.NULL_TRACKER ); diff --git a/algo/src/main/java/org/neo4j/gds/paths/delta/DeltaSteppingFactory.java b/algo/src/main/java/org/neo4j/gds/paths/delta/DeltaSteppingFactory.java index b7d38aba17..20056af714 100644 --- a/algo/src/main/java/org/neo4j/gds/paths/delta/DeltaSteppingFactory.java +++ b/algo/src/main/java/org/neo4j/gds/paths/delta/DeltaSteppingFactory.java @@ -21,7 +21,7 @@ import org.neo4j.gds.GraphAlgorithmFactory; import org.neo4j.gds.api.Graph; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.utils.mem.MemoryEstimation; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.core.utils.progress.tasks.Task; @@ -38,7 +38,7 @@ public DeltaStepping build( T configuration, ProgressTracker progressTracker ) { - return DeltaStepping.of(graph, configuration, Pools.DEFAULT, progressTracker); + return DeltaStepping.of(graph, configuration, ExecutorServices.DEFAULT, progressTracker); } @Override diff --git a/algo/src/main/java/org/neo4j/gds/paths/traverse/BFS.java b/algo/src/main/java/org/neo4j/gds/paths/traverse/BFS.java index 4f21860c24..63f65b288f 100644 --- a/algo/src/main/java/org/neo4j/gds/paths/traverse/BFS.java +++ b/algo/src/main/java/org/neo4j/gds/paths/traverse/BFS.java @@ -23,8 +23,8 @@ import org.neo4j.gds.Algorithm; import org.neo4j.gds.api.Graph; import org.neo4j.gds.collections.haa.HugeAtomicLongArray; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.concurrency.ParallelUtil; -import org.neo4j.gds.core.concurrency.Pools; import org.neo4j.gds.core.utils.paged.HugeAtomicBitSet; import org.neo4j.gds.core.utils.paged.HugeDoubleArray; import org.neo4j.gds.core.utils.paged.HugeLongArray; @@ -199,7 +199,7 @@ public HugeLongArray compute() { if (currentDepth == maximumDepth) { break; } - ParallelUtil.run(bfsTaskList, Pools.DEFAULT); + ParallelUtil.run(bfsTaskList, ExecutorServices.DEFAULT); if (targetFoundIndex.get() != Long.MAX_VALUE) { break; diff --git a/algo/src/main/java/org/neo4j/gds/paths/yens/Yens.java b/algo/src/main/java/org/neo4j/gds/paths/yens/Yens.java index 7eb0cf73b9..524834fb4f 100644 --- a/algo/src/main/java/org/neo4j/gds/paths/yens/Yens.java +++ b/algo/src/main/java/org/neo4j/gds/paths/yens/Yens.java @@ -21,7 +21,7 @@ import org.neo4j.gds.Algorithm; import org.neo4j.gds.api.Graph; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.concurrency.RunWithConcurrency; import org.neo4j.gds.core.utils.mem.MemoryEstimation; import org.neo4j.gds.core.utils.mem.MemoryEstimations; @@ -113,7 +113,7 @@ public PathFindingResult compute() { RunWithConcurrency.builder() .concurrency(config.concurrency()) .tasks(tasks) - .executor(Pools.DEFAULT) + .executor(ExecutorServices.DEFAULT) .run(); progressTracker.logProgress(); diff --git a/algo/src/main/java/org/neo4j/gds/scaleproperties/ScalePropertiesFactory.java b/algo/src/main/java/org/neo4j/gds/scaleproperties/ScalePropertiesFactory.java index 99f7dbb249..66ffa940ec 100644 --- a/algo/src/main/java/org/neo4j/gds/scaleproperties/ScalePropertiesFactory.java +++ b/algo/src/main/java/org/neo4j/gds/scaleproperties/ScalePropertiesFactory.java @@ -21,7 +21,7 @@ import org.neo4j.gds.GraphAlgorithmFactory; import org.neo4j.gds.api.Graph; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.utils.mem.MemoryEstimation; import org.neo4j.gds.core.utils.mem.MemoryEstimations; import org.neo4j.gds.core.utils.mem.MemoryRange; @@ -69,7 +69,7 @@ public ScaleProperties build( graph, configuration, progressTracker, - Pools.DEFAULT + ExecutorServices.DEFAULT ); } diff --git a/algo/src/main/java/org/neo4j/gds/similarity/filteredknn/FilteredKnnFactory.java b/algo/src/main/java/org/neo4j/gds/similarity/filteredknn/FilteredKnnFactory.java index ef6dbe66a7..f77cf68bbb 100644 --- a/algo/src/main/java/org/neo4j/gds/similarity/filteredknn/FilteredKnnFactory.java +++ b/algo/src/main/java/org/neo4j/gds/similarity/filteredknn/FilteredKnnFactory.java @@ -23,7 +23,7 @@ import org.apache.commons.lang3.function.TriFunction; import org.neo4j.gds.GraphAlgorithmFactory; import org.neo4j.gds.api.Graph; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.utils.mem.MemoryEstimation; import org.neo4j.gds.core.utils.mem.MemoryEstimations; import org.neo4j.gds.core.utils.mem.MemoryRange; @@ -68,7 +68,7 @@ public FilteredKnn build(Graph graph, CONFIG configuration, ProgressTracker prog KnnContext knnContext = ImmutableKnnContext .builder() .progressTracker(progressTracker) - .executor(Pools.DEFAULT) + .executor(ExecutorServices.DEFAULT) .build(); if (configuration.seedTargetNodes()) { diff --git a/algo/src/main/java/org/neo4j/gds/similarity/filterednodesim/FilteredNodeSimilarityFactory.java b/algo/src/main/java/org/neo4j/gds/similarity/filterednodesim/FilteredNodeSimilarityFactory.java index 9e4940c914..13958a7c21 100644 --- a/algo/src/main/java/org/neo4j/gds/similarity/filterednodesim/FilteredNodeSimilarityFactory.java +++ b/algo/src/main/java/org/neo4j/gds/similarity/filterednodesim/FilteredNodeSimilarityFactory.java @@ -22,7 +22,7 @@ import com.carrotsearch.hppc.BitSet; import org.neo4j.gds.GraphAlgorithmFactory; import org.neo4j.gds.api.Graph; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.utils.mem.MemoryEstimation; import org.neo4j.gds.core.utils.mem.MemoryEstimations; import org.neo4j.gds.core.utils.mem.MemoryRange; @@ -59,7 +59,7 @@ public NodeSimilarity build( configuration.sourceNodeFilter().toNodeFilter(graph), configuration.targetNodeFilter().toNodeFilter(graph), configuration.concurrency(), - Pools.DEFAULT, + ExecutorServices.DEFAULT, progressTracker ); } diff --git a/algo/src/main/java/org/neo4j/gds/similarity/knn/KnnContext.java b/algo/src/main/java/org/neo4j/gds/similarity/knn/KnnContext.java index cab6b05a4f..c7e57a3033 100644 --- a/algo/src/main/java/org/neo4j/gds/similarity/knn/KnnContext.java +++ b/algo/src/main/java/org/neo4j/gds/similarity/knn/KnnContext.java @@ -21,7 +21,7 @@ import org.immutables.value.Value; import org.neo4j.gds.annotation.ValueClass; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import java.util.concurrent.ExecutorService; @@ -31,7 +31,7 @@ public interface KnnContext { @Value.Default default ExecutorService executor() { - return Pools.DEFAULT; + return ExecutorServices.DEFAULT; } @Value.Default diff --git a/algo/src/main/java/org/neo4j/gds/similarity/knn/KnnFactory.java b/algo/src/main/java/org/neo4j/gds/similarity/knn/KnnFactory.java index d09bff5787..f7f1a9e6d7 100644 --- a/algo/src/main/java/org/neo4j/gds/similarity/knn/KnnFactory.java +++ b/algo/src/main/java/org/neo4j/gds/similarity/knn/KnnFactory.java @@ -22,7 +22,7 @@ import com.carrotsearch.hppc.LongArrayList; import org.neo4j.gds.GraphAlgorithmFactory; import org.neo4j.gds.api.Graph; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.utils.mem.MemoryEstimation; import org.neo4j.gds.core.utils.mem.MemoryEstimations; import org.neo4j.gds.core.utils.mem.MemoryRange; @@ -59,7 +59,7 @@ public Knn build( ImmutableKnnContext .builder() .progressTracker(progressTracker) - .executor(Pools.DEFAULT) + .executor(ExecutorServices.DEFAULT) .build() ); } diff --git a/algo/src/main/java/org/neo4j/gds/similarity/nodesim/NodeSimilarityFactory.java b/algo/src/main/java/org/neo4j/gds/similarity/nodesim/NodeSimilarityFactory.java index 4da718731e..ffa573b2d7 100644 --- a/algo/src/main/java/org/neo4j/gds/similarity/nodesim/NodeSimilarityFactory.java +++ b/algo/src/main/java/org/neo4j/gds/similarity/nodesim/NodeSimilarityFactory.java @@ -22,7 +22,7 @@ import com.carrotsearch.hppc.BitSet; import org.neo4j.gds.GraphAlgorithmFactory; import org.neo4j.gds.api.Graph; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.utils.mem.MemoryEstimation; import org.neo4j.gds.core.utils.mem.MemoryEstimations; import org.neo4j.gds.core.utils.mem.MemoryRange; @@ -54,7 +54,7 @@ public NodeSimilarity build( configuration, similarityComputer, configuration.concurrency(), - Pools.DEFAULT, + ExecutorServices.DEFAULT, progressTracker ); } diff --git a/algo/src/main/java/org/neo4j/gds/steiner/SteinerTreeAlgorithmFactory.java b/algo/src/main/java/org/neo4j/gds/steiner/SteinerTreeAlgorithmFactory.java index b0fa97c7eb..c2232f37d4 100644 --- a/algo/src/main/java/org/neo4j/gds/steiner/SteinerTreeAlgorithmFactory.java +++ b/algo/src/main/java/org/neo4j/gds/steiner/SteinerTreeAlgorithmFactory.java @@ -21,7 +21,7 @@ import org.neo4j.gds.GraphAlgorithmFactory; import org.neo4j.gds.api.Graph; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.core.utils.progress.tasks.Task; import org.neo4j.gds.core.utils.progress.tasks.Tasks; @@ -51,7 +51,7 @@ public ShortestPathsSteinerAlgorithm build( configuration.delta(), configuration.concurrency(), configuration.applyRerouting(), - Pools.DEFAULT, + ExecutorServices.DEFAULT, progressTracker ); } diff --git a/algo/src/main/java/org/neo4j/gds/traversal/RandomWalkAlgorithmFactory.java b/algo/src/main/java/org/neo4j/gds/traversal/RandomWalkAlgorithmFactory.java index 831f7f3d86..0da5a824fc 100644 --- a/algo/src/main/java/org/neo4j/gds/traversal/RandomWalkAlgorithmFactory.java +++ b/algo/src/main/java/org/neo4j/gds/traversal/RandomWalkAlgorithmFactory.java @@ -21,7 +21,7 @@ import org.neo4j.gds.GraphAlgorithmFactory; import org.neo4j.gds.api.Graph; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.utils.mem.MemoryEstimation; import org.neo4j.gds.core.utils.mem.MemoryEstimations; import org.neo4j.gds.core.utils.mem.MemoryRange; @@ -45,7 +45,7 @@ public RandomWalk build( RandomWalkBaseConfig configuration, ProgressTracker progressTracker ) { - return RandomWalk.create(graph, configuration, progressTracker, Pools.DEFAULT); + return RandomWalk.create(graph, configuration, progressTracker, ExecutorServices.DEFAULT); } @Override diff --git a/algo/src/main/java/org/neo4j/gds/triangle/IntersectingTriangleCountFactory.java b/algo/src/main/java/org/neo4j/gds/triangle/IntersectingTriangleCountFactory.java index f76deba421..14ebd8f047 100644 --- a/algo/src/main/java/org/neo4j/gds/triangle/IntersectingTriangleCountFactory.java +++ b/algo/src/main/java/org/neo4j/gds/triangle/IntersectingTriangleCountFactory.java @@ -23,7 +23,7 @@ import org.neo4j.gds.GraphAlgorithmFactory; import org.neo4j.gds.api.Graph; import org.neo4j.gds.collections.haa.HugeAtomicLongArray; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.utils.mem.MemoryEstimation; import org.neo4j.gds.core.utils.mem.MemoryEstimations; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; @@ -48,7 +48,7 @@ public IntersectingTriangleCount build( return IntersectingTriangleCount.create( graph, configuration, - Pools.DEFAULT, + ExecutorServices.DEFAULT, progressTracker ); } diff --git a/algo/src/main/java/org/neo4j/gds/triangle/TriangleStreamFactory.java b/algo/src/main/java/org/neo4j/gds/triangle/TriangleStreamFactory.java index 83a9692f4c..2156472243 100644 --- a/algo/src/main/java/org/neo4j/gds/triangle/TriangleStreamFactory.java +++ b/algo/src/main/java/org/neo4j/gds/triangle/TriangleStreamFactory.java @@ -21,7 +21,7 @@ import org.neo4j.gds.GraphAlgorithmFactory; import org.neo4j.gds.api.Graph; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; public class TriangleStreamFactory extends GraphAlgorithmFactory { @@ -33,6 +33,6 @@ public String taskName() { @Override public TriangleStream build(Graph graph, TriangleCountBaseConfig configuration, ProgressTracker progressTracker) { - return TriangleStream.create(graph, Pools.DEFAULT, configuration.concurrency()); + return TriangleStream.create(graph, ExecutorServices.DEFAULT, configuration.concurrency()); } } diff --git a/algo/src/main/java/org/neo4j/gds/undirected/ToUndirectedAlgorithmFactory.java b/algo/src/main/java/org/neo4j/gds/undirected/ToUndirectedAlgorithmFactory.java index e3ace5ed1a..f508d88ca9 100644 --- a/algo/src/main/java/org/neo4j/gds/undirected/ToUndirectedAlgorithmFactory.java +++ b/algo/src/main/java/org/neo4j/gds/undirected/ToUndirectedAlgorithmFactory.java @@ -22,7 +22,7 @@ import org.neo4j.gds.GraphStoreAlgorithmFactory; import org.neo4j.gds.RelationshipType; import org.neo4j.gds.api.GraphStore; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.loading.AdjacencyListBehavior; import org.neo4j.gds.core.utils.mem.MemoryEstimation; import org.neo4j.gds.core.utils.mem.MemoryEstimations; @@ -39,7 +39,7 @@ public ToUndirected build( ToUndirectedConfig configuration, ProgressTracker progressTracker ) { - return new ToUndirected(graphStore, configuration, progressTracker, Pools.DEFAULT); + return new ToUndirected(graphStore, configuration, progressTracker, ExecutorServices.DEFAULT); } @Override diff --git a/algo/src/main/java/org/neo4j/gds/walking/CollapsePathAlgorithmFactory.java b/algo/src/main/java/org/neo4j/gds/walking/CollapsePathAlgorithmFactory.java index fbb5471a1f..a017c3254c 100644 --- a/algo/src/main/java/org/neo4j/gds/walking/CollapsePathAlgorithmFactory.java +++ b/algo/src/main/java/org/neo4j/gds/walking/CollapsePathAlgorithmFactory.java @@ -24,7 +24,7 @@ import org.neo4j.gds.RelationshipType; import org.neo4j.gds.api.Graph; import org.neo4j.gds.api.GraphStore; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import java.util.Collection; @@ -66,7 +66,7 @@ public CollapsePath build( config.allowSelfLoops(), RelationshipType.of(config.mutateRelationshipType()), config.concurrency(), - Pools.DEFAULT + ExecutorServices.DEFAULT ); } diff --git a/algo/src/main/java/org/neo4j/gds/wcc/WccAlgorithmFactory.java b/algo/src/main/java/org/neo4j/gds/wcc/WccAlgorithmFactory.java index 12452cfa33..31f17a6ae0 100644 --- a/algo/src/main/java/org/neo4j/gds/wcc/WccAlgorithmFactory.java +++ b/algo/src/main/java/org/neo4j/gds/wcc/WccAlgorithmFactory.java @@ -21,8 +21,8 @@ import org.neo4j.gds.GraphAlgorithmFactory; import org.neo4j.gds.api.Graph; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.concurrency.ParallelUtil; -import org.neo4j.gds.core.concurrency.Pools; import org.neo4j.gds.core.utils.mem.MemoryEstimation; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.core.utils.progress.tasks.Task; @@ -50,7 +50,7 @@ public Wcc build( } return new Wcc( graph, - Pools.DEFAULT, + ExecutorServices.DEFAULT, ParallelUtil.DEFAULT_BATCH_SIZE, configuration, progressTracker diff --git a/algo/src/test/java/org/neo4j/gds/allshortestpaths/MSBFSAllShortestPathsTest.java b/algo/src/test/java/org/neo4j/gds/allshortestpaths/MSBFSAllShortestPathsTest.java index c12834b6ba..53f7b6352f 100644 --- a/algo/src/test/java/org/neo4j/gds/allshortestpaths/MSBFSAllShortestPathsTest.java +++ b/algo/src/test/java/org/neo4j/gds/allshortestpaths/MSBFSAllShortestPathsTest.java @@ -25,7 +25,7 @@ import org.neo4j.gds.StoreLoaderBuilder; import org.neo4j.gds.api.Graph; import org.neo4j.gds.config.ConcurrencyConfig; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.graphbuilder.GraphBuilder; import static org.junit.jupiter.api.Assertions.fail; @@ -76,7 +76,7 @@ void testResults() { .addRelationshipType(RELATIONSHIP) .build() .graph(); - testASP(new MSBFSAllShortestPaths(graph, ConcurrencyConfig.DEFAULT_CONCURRENCY, Pools.DEFAULT)); + testASP(new MSBFSAllShortestPaths(graph, ConcurrencyConfig.DEFAULT_CONCURRENCY, ExecutorServices.DEFAULT)); } private void testASP(final MSBFSASPAlgorithm hugeMSBFSAllShortestPaths) { diff --git a/algo/src/test/java/org/neo4j/gds/allshortestpaths/WeightedAllShortestPathsTest.java b/algo/src/test/java/org/neo4j/gds/allshortestpaths/WeightedAllShortestPathsTest.java index 3a58d13e78..61e92c5655 100644 --- a/algo/src/test/java/org/neo4j/gds/allshortestpaths/WeightedAllShortestPathsTest.java +++ b/algo/src/test/java/org/neo4j/gds/allshortestpaths/WeightedAllShortestPathsTest.java @@ -25,7 +25,7 @@ import org.neo4j.gds.PropertyMapping; import org.neo4j.gds.StoreLoaderBuilder; import org.neo4j.gds.api.Graph; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.graphbuilder.GraphBuilder; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -88,7 +88,7 @@ void testResults() { final ResultConsumer mock = mock(ResultConsumer.class); - new WeightedAllShortestPaths(graph, Pools.DEFAULT, 4) + new WeightedAllShortestPaths(graph, ExecutorServices.DEFAULT, 4) .compute() .forEach(r -> { assertNotEquals(Double.POSITIVE_INFINITY, r.distance); @@ -115,7 +115,7 @@ void shouldThrowIfGraphHasNoRelationshipProperty() { .graph(); UnsupportedOperationException exception = assertThrows(UnsupportedOperationException.class, () -> { - new WeightedAllShortestPaths(graph, Pools.DEFAULT, 4); + new WeightedAllShortestPaths(graph, ExecutorServices.DEFAULT, 4); }); assertTrue(exception.getMessage().contains("not supported")); diff --git a/algo/src/test/java/org/neo4j/gds/approxmaxkcut/ApproxMaxKCutTest.java b/algo/src/test/java/org/neo4j/gds/approxmaxkcut/ApproxMaxKCutTest.java index 397ac21f29..6e81692306 100644 --- a/algo/src/test/java/org/neo4j/gds/approxmaxkcut/ApproxMaxKCutTest.java +++ b/algo/src/test/java/org/neo4j/gds/approxmaxkcut/ApproxMaxKCutTest.java @@ -28,7 +28,7 @@ import org.neo4j.gds.collections.haa.HugeAtomicDoubleArray; import org.neo4j.gds.compat.Neo4jProxy; import org.neo4j.gds.compat.TestLog; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.utils.mem.MemoryRange; import org.neo4j.gds.core.utils.paged.HugeByteArray; import org.neo4j.gds.core.utils.progress.EmptyTaskRegistryFactory; @@ -172,7 +172,7 @@ void computeCorrectResults( var graph = minimize ? minGraph : maxGraph; var approxMaxKCut = new ApproxMaxKCut( graph, - Pools.DEFAULT, + ExecutorServices.DEFAULT, config, ProgressTracker.NULL_TRACKER ); @@ -214,7 +214,7 @@ void respectMinCommunitySizes(int concurrency) { var approxMaxKCut = new ApproxMaxKCut( maxGraph, - Pools.DEFAULT, + ExecutorServices.DEFAULT, config, ProgressTracker.NULL_TRACKER ); diff --git a/algo/src/test/java/org/neo4j/gds/betweenness/BetweennessCentralityTest.java b/algo/src/test/java/org/neo4j/gds/betweenness/BetweennessCentralityTest.java index 161554b0ec..6a8f3777d7 100644 --- a/algo/src/test/java/org/neo4j/gds/betweenness/BetweennessCentralityTest.java +++ b/algo/src/test/java/org/neo4j/gds/betweenness/BetweennessCentralityTest.java @@ -29,7 +29,7 @@ import org.neo4j.gds.compat.Neo4jProxy; import org.neo4j.gds.compat.TestLog; import org.neo4j.gds.core.CypherMapWrapper; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.utils.mem.MemoryRange; import org.neo4j.gds.core.utils.progress.EmptyTaskRegistryFactory; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; @@ -141,7 +141,7 @@ void sampling(int concurrency, TestGraph graph, int samplingSize, Map selectionStrategy.next())).containsExactly( graph.toMappedNodeId("a"), @@ -107,9 +107,9 @@ void neverIncludeZeroDegNodesIfBetterChoicesExist() { TestGraph graph = fromGdl("(), (), (), (), (), (a)-->(), (), (), ()"); SelectionStrategy selectionStrategy = new RandomDegreeSelectionStrategy(1); - selectionStrategy.init(graph, Pools.DEFAULT, 1); + selectionStrategy.init(graph, ExecutorServices.DEFAULT, 1); assertEquals(1, samplingSize(selectionStrategy)); - selectionStrategy.init(graph, Pools.DEFAULT, 1); + selectionStrategy.init(graph, ExecutorServices.DEFAULT, 1); assertEquals(graph.toMappedNodeId("a"), selectionStrategy.next()); } @@ -118,18 +118,18 @@ void not100PercentLikelyUnlessMaxDegNode() { TestGraph graph = fromGdl("(a)-->(b), (b)-->(c), (b)-->(d)"); SelectionStrategy selectionStrategy = new RandomDegreeSelectionStrategy(1, Optional.of(42L)); - selectionStrategy.init(graph, Pools.DEFAULT, 1); + selectionStrategy.init(graph, ExecutorServices.DEFAULT, 1); assertEquals(1, samplingSize(selectionStrategy)); - selectionStrategy.init(graph, Pools.DEFAULT, 1); + selectionStrategy.init(graph, ExecutorServices.DEFAULT, 1); assertEquals(graph.toMappedNodeId("b"), selectionStrategy.next()); } @Test void selectHighDegreeNode() { SelectionStrategy selectionStrategy = new RandomDegreeSelectionStrategy(1); - selectionStrategy.init(graph, Pools.DEFAULT, 1); + selectionStrategy.init(graph, ExecutorServices.DEFAULT, 1); assertEquals(1, samplingSize(selectionStrategy)); - selectionStrategy.init(graph, Pools.DEFAULT, 1); + selectionStrategy.init(graph, ExecutorServices.DEFAULT, 1); var isA = selectionStrategy.next(); var isB = selectionStrategy.next(); assertTrue(isA != SelectionStrategy.NONE_SELECTED || isB != SelectionStrategy.NONE_SELECTED); diff --git a/algo/src/test/java/org/neo4j/gds/betweenness/WeightedBetweennessCentralityTest.java b/algo/src/test/java/org/neo4j/gds/betweenness/WeightedBetweennessCentralityTest.java index f7a7a33f86..88e7ed9034 100644 --- a/algo/src/test/java/org/neo4j/gds/betweenness/WeightedBetweennessCentralityTest.java +++ b/algo/src/test/java/org/neo4j/gds/betweenness/WeightedBetweennessCentralityTest.java @@ -26,7 +26,7 @@ import org.junit.jupiter.params.provider.MethodSource; import org.neo4j.gds.api.Graph; import org.neo4j.gds.core.CypherMapWrapper; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.utils.mem.MemoryRange; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.extension.GdlExtension; @@ -86,7 +86,7 @@ void shouldEqualWithUnweightedWhenWeightsAreEqual() { equallyWeightedGraph, new RandomDegreeSelectionStrategy(7, Optional.of(42L)), ForwardTraverser.Factory.weighted(), - Pools.DEFAULT, + ExecutorServices.DEFAULT, 8, ProgressTracker.NULL_TRACKER ); @@ -94,7 +94,7 @@ void shouldEqualWithUnweightedWhenWeightsAreEqual() { equallyWeightedGraph, new RandomDegreeSelectionStrategy(7, Optional.of(42L)), ForwardTraverser.Factory.unweighted(), - Pools.DEFAULT, + ExecutorServices.DEFAULT, 8, ProgressTracker.NULL_TRACKER ); @@ -119,7 +119,7 @@ void shouldComputeWithWeights() { weightedGraph, new RandomDegreeSelectionStrategy(7, Optional.of(42L)), ForwardTraverser.Factory.weighted(), - Pools.DEFAULT, + ExecutorServices.DEFAULT, 8, ProgressTracker.NULL_TRACKER ); diff --git a/algo/src/test/java/org/neo4j/gds/closeness/ClosenessCentralityDirectedTest.java b/algo/src/test/java/org/neo4j/gds/closeness/ClosenessCentralityDirectedTest.java index 2b24a33c67..fa42fc4a94 100644 --- a/algo/src/test/java/org/neo4j/gds/closeness/ClosenessCentralityDirectedTest.java +++ b/algo/src/test/java/org/neo4j/gds/closeness/ClosenessCentralityDirectedTest.java @@ -20,7 +20,7 @@ package org.neo4j.gds.closeness; import org.junit.jupiter.api.Test; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.extension.GdlExtension; import org.neo4j.gds.extension.GdlGraph; @@ -88,7 +88,7 @@ void testCentrality() { var algo = ClosenessCentrality.of( graph, ImmutableClosenessCentralityStreamConfig.builder().build(), - Pools.DEFAULT, + ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER ); @@ -108,7 +108,7 @@ void testCentralityWithWassermanFaust() { var algo = ClosenessCentrality.of( graph, ImmutableClosenessCentralityStreamConfig.builder().useWassermanFaust(true).build(), - Pools.DEFAULT, + ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER ); diff --git a/algo/src/test/java/org/neo4j/gds/closeness/ClosenessCentralityDiscoTest.java b/algo/src/test/java/org/neo4j/gds/closeness/ClosenessCentralityDiscoTest.java index 2a0fb88d9d..1ea23aa106 100644 --- a/algo/src/test/java/org/neo4j/gds/closeness/ClosenessCentralityDiscoTest.java +++ b/algo/src/test/java/org/neo4j/gds/closeness/ClosenessCentralityDiscoTest.java @@ -22,7 +22,7 @@ import org.assertj.core.data.Offset; import org.junit.jupiter.api.Test; import org.neo4j.gds.Orientation; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.extension.GdlExtension; import org.neo4j.gds.extension.GdlGraph; @@ -67,7 +67,7 @@ void testHuge() { var algo = ClosenessCentrality.of( graph, ImmutableClosenessCentralityStreamConfig.builder().concurrency(2).useWassermanFaust(true).build(), - Pools.DEFAULT, + ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER ); diff --git a/algo/src/test/java/org/neo4j/gds/closeness/ClosenessCentralityTest.java b/algo/src/test/java/org/neo4j/gds/closeness/ClosenessCentralityTest.java index 46702494f6..c66e672ba9 100644 --- a/algo/src/test/java/org/neo4j/gds/closeness/ClosenessCentralityTest.java +++ b/algo/src/test/java/org/neo4j/gds/closeness/ClosenessCentralityTest.java @@ -23,7 +23,7 @@ import org.junit.jupiter.api.Test; import org.neo4j.gds.TestProgressTracker; import org.neo4j.gds.compat.Neo4jProxy; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.utils.progress.EmptyTaskRegistryFactory; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.extension.GdlExtension; @@ -93,7 +93,7 @@ void testGetCentrality() { var algo = ClosenessCentrality.of( graph, ImmutableClosenessCentralityStreamConfig.builder().build(), - Pools.DEFAULT, + ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER ); @@ -116,7 +116,7 @@ void shouldLogProgress() { var algo = ClosenessCentrality.of( graph, config, - Pools.DEFAULT, + ExecutorServices.DEFAULT, progressTracker ); diff --git a/algo/src/test/java/org/neo4j/gds/conductance/ConductanceTest.java b/algo/src/test/java/org/neo4j/gds/conductance/ConductanceTest.java index 3e8ca89510..cefce3db5c 100644 --- a/algo/src/test/java/org/neo4j/gds/conductance/ConductanceTest.java +++ b/algo/src/test/java/org/neo4j/gds/conductance/ConductanceTest.java @@ -28,7 +28,7 @@ import org.neo4j.gds.TestSupport; import org.neo4j.gds.compat.Neo4jProxy; import org.neo4j.gds.compat.TestLog; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.utils.progress.EmptyTaskRegistryFactory; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.core.utils.progress.tasks.TaskProgressTracker; @@ -134,7 +134,7 @@ void computeCorrectResults( var conductance = new Conductance( orientation == Orientation.NATURAL ? naturalGraph : undirectedGraph, - Pools.DEFAULT, + ExecutorServices.DEFAULT, config, ProgressTracker.NULL_TRACKER ); diff --git a/algo/src/test/java/org/neo4j/gds/degree/DegreeCentralityTest.java b/algo/src/test/java/org/neo4j/gds/degree/DegreeCentralityTest.java index bb38b51b23..055d5e580d 100644 --- a/algo/src/test/java/org/neo4j/gds/degree/DegreeCentralityTest.java +++ b/algo/src/test/java/org/neo4j/gds/degree/DegreeCentralityTest.java @@ -28,7 +28,7 @@ import org.neo4j.gds.TestProgressTracker; import org.neo4j.gds.compat.Neo4jProxy; import org.neo4j.gds.compat.TestLog; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.utils.mem.MemoryRange; import org.neo4j.gds.core.utils.paged.HugeDoubleArray; import org.neo4j.gds.core.utils.progress.EmptyTaskRegistryFactory; @@ -148,7 +148,7 @@ void shouldComputeCorrectResults(boolean weighted, Orientation orientation, Map< var degreeCentrality = new DegreeCentrality( graph, - Pools.DEFAULT, + ExecutorServices.DEFAULT, config, ProgressTracker.NULL_TRACKER ); @@ -200,7 +200,7 @@ void testProgressLogging(boolean weighted) { var progressTracker = new TestProgressTracker(progressTask, log, 1, EmptyTaskRegistryFactory.INSTANCE); var degreeCentrality = new DegreeCentrality( graph, - Pools.DEFAULT, + ExecutorServices.DEFAULT, config, progressTracker ); @@ -225,7 +225,7 @@ void shouldSupportAllOrientations(Orientation orientation) { var degreeCentrality = new DegreeCentrality( graph, - Pools.DEFAULT, + ExecutorServices.DEFAULT, config, ProgressTracker.NULL_TRACKER ); diff --git a/algo/src/test/java/org/neo4j/gds/embeddings/fastrp/FastRPTest.java b/algo/src/test/java/org/neo4j/gds/embeddings/fastrp/FastRPTest.java index 451e7ff643..e676d8438f 100644 --- a/algo/src/test/java/org/neo4j/gds/embeddings/fastrp/FastRPTest.java +++ b/algo/src/test/java/org/neo4j/gds/embeddings/fastrp/FastRPTest.java @@ -31,7 +31,7 @@ import org.neo4j.gds.compat.Neo4jProxy; import org.neo4j.gds.compat.TestLog; import org.neo4j.gds.core.ImmutableGraphDimensions; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.loading.ArrayIdMap; import org.neo4j.gds.core.loading.LabelInformationBuilders; import org.neo4j.gds.core.loading.construction.GraphFactory; @@ -626,7 +626,7 @@ void shouldBeDeterministicGivenSameOriginalIds() { .nodes(firstIdMap) .relationshipType(RelationshipType.of("REL")) .orientation(Orientation.UNDIRECTED) - .executorService(Pools.DEFAULT) + .executorService(ExecutorServices.DEFAULT) .build(); var secondMappedToOriginal = HugeLongArray.newArray(nodeCount); @@ -650,7 +650,7 @@ void shouldBeDeterministicGivenSameOriginalIds() { .nodes(secondIdMap) .relationshipType(RelationshipType.of("REL")) .orientation(Orientation.UNDIRECTED) - .executorService(Pools.DEFAULT) + .executorService(ExecutorServices.DEFAULT) .build(); var random = new SplittableRandom(42); diff --git a/algo/src/test/java/org/neo4j/gds/embeddings/graphsage/GraphSageEmbeddingsGeneratorTest.java b/algo/src/test/java/org/neo4j/gds/embeddings/graphsage/GraphSageEmbeddingsGeneratorTest.java index 5cfed40932..e56f0e592b 100644 --- a/algo/src/test/java/org/neo4j/gds/embeddings/graphsage/GraphSageEmbeddingsGeneratorTest.java +++ b/algo/src/test/java/org/neo4j/gds/embeddings/graphsage/GraphSageEmbeddingsGeneratorTest.java @@ -24,7 +24,7 @@ import org.junit.jupiter.params.provider.EnumSource; import org.neo4j.gds.RelationshipType; import org.neo4j.gds.api.Graph; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.loading.CSRGraphStore; import org.neo4j.gds.core.utils.paged.HugeObjectArray; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; @@ -74,7 +74,7 @@ void makesEmbeddings(Aggregator.AggregatorType aggregatorType) { var features = GraphSageHelper.initializeSingleLabelFeatures(weightedGraph, config); - var trainModel = new GraphSageModelTrainer(config, Pools.DEFAULT, ProgressTracker.NULL_TRACKER); + var trainModel = new GraphSageModelTrainer(config, ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER); GraphSageModelTrainer.ModelTrainResult result = trainModel.train(weightedGraph, features); @@ -84,7 +84,7 @@ void makesEmbeddings(Aggregator.AggregatorType aggregatorType) { config.concurrency(), new SingleLabelFeatureFunction(), config.randomSeed(), - Pools.DEFAULT, + ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER ); @@ -112,7 +112,7 @@ void makesEmbeddingsFromMultiLabelModel(Aggregator.AggregatorType aggregatorType var trainer = new MultiLabelGraphSageTrain( weightedGraph, config, - Pools.DEFAULT, + ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER, testGdsVersion ); @@ -125,7 +125,7 @@ void makesEmbeddingsFromMultiLabelModel(Aggregator.AggregatorType aggregatorType config.concurrency(), model.data().featureFunction(), model.trainConfig().randomSeed(), - Pools.DEFAULT, + ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER ); @@ -167,7 +167,7 @@ void embeddingsForNodeFilteredGraph() { var trainer = new SingleLabelGraphSageTrain( filteredGraph, config, - Pools.DEFAULT, + ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER, testGdsVersion ); @@ -180,7 +180,7 @@ void embeddingsForNodeFilteredGraph() { config.concurrency(), model.data().featureFunction(), model.trainConfig().randomSeed(), - Pools.DEFAULT, + ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER ); diff --git a/algo/src/test/java/org/neo4j/gds/embeddings/graphsage/GraphSageModelTrainerTest.java b/algo/src/test/java/org/neo4j/gds/embeddings/graphsage/GraphSageModelTrainerTest.java index 71fb4e5276..d1953f7d0d 100644 --- a/algo/src/test/java/org/neo4j/gds/embeddings/graphsage/GraphSageModelTrainerTest.java +++ b/algo/src/test/java/org/neo4j/gds/embeddings/graphsage/GraphSageModelTrainerTest.java @@ -34,7 +34,7 @@ import org.neo4j.gds.beta.generator.PropertyProducer; import org.neo4j.gds.beta.generator.RandomGraphGenerator; import org.neo4j.gds.beta.generator.RelationshipDistribution; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.utils.paged.HugeObjectArray; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.embeddings.graphsage.algo.GraphSageTrainConfigImpl; @@ -120,7 +120,7 @@ void trainsWithRelationshipWeight(AggregatorType aggregatorType) { .modelUser("") .build(); - var trainModel = new GraphSageModelTrainer(config, Pools.DEFAULT, ProgressTracker.NULL_TRACKER); + var trainModel = new GraphSageModelTrainer(config, ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER); int nodeCount = 5_000; var bigGraph = RandomGraphGenerator @@ -160,7 +160,7 @@ void trainsWithMeanAggregator(boolean useRelationshipWeight) { .modelName(MODEL_NAME) .build(); - var trainModel = new GraphSageModelTrainer(config, Pools.DEFAULT, ProgressTracker.NULL_TRACKER); + var trainModel = new GraphSageModelTrainer(config, ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER); var maybeWeights = useRelationshipWeight ? Optional.of("times") : Optional.empty(); @@ -201,7 +201,7 @@ void trainsWithPoolAggregator(boolean useRelationshipWeight) { .modelName(MODEL_NAME) .build(); - var trainModel = new GraphSageModelTrainer(config, Pools.DEFAULT, ProgressTracker.NULL_TRACKER); + var trainModel = new GraphSageModelTrainer(config, ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER); GraphSageModelTrainer.ModelTrainResult result = trainModel.train(graph, features); Layer[] layers = result.layers(); @@ -251,7 +251,7 @@ void shouldTrainModelWithArrayProperties() { .modelUser("") .build(); - var trainer = new GraphSageModelTrainer(config, Pools.DEFAULT, ProgressTracker.NULL_TRACKER); + var trainer = new GraphSageModelTrainer(config, ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER); var result = trainer.train(arrayGraph, arrayFeatures); @@ -276,7 +276,7 @@ void testLosses() { var trainer = new GraphSageModelTrainer( config, - Pools.DEFAULT, + ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER ); @@ -309,7 +309,7 @@ void testLossesWithPoolAggregator() { var trainer = new GraphSageModelTrainer( config, - Pools.DEFAULT, + ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER ); @@ -329,7 +329,7 @@ void testLossesWithPoolAggregator() { void testConvergence() { var trainer = new GraphSageModelTrainer( configBuilder.modelName("convergingModel:)").tolerance(100.0).epochs(10).build(), - Pools.DEFAULT, + ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER ); @@ -355,13 +355,13 @@ void batchesPerIteration() { var trainResultWithoutSampling = new GraphSageModelTrainer( configBuilder.maybeBatchSamplingRatio(1.0).build(), - Pools.DEFAULT, + ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER ).train(unweightedGraph, features); var trainResultWithSampling = new GraphSageModelTrainer( configBuilder.maybeBatchSamplingRatio(0.01).build(), - Pools.DEFAULT, + ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER ).train(unweightedGraph, features); @@ -386,7 +386,7 @@ void l2Penalty(double penalty, double expectedLoss) { var result = new GraphSageModelTrainer( config, - Pools.DEFAULT, + ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER ).train(unweightedGraph, features); @@ -404,8 +404,8 @@ void seededSingleBatch(long seed) { .concurrency(1) .build(); - var trainer = new GraphSageModelTrainer(config, Pools.DEFAULT, ProgressTracker.NULL_TRACKER); - var otherTrainer = new GraphSageModelTrainer(config, Pools.DEFAULT, ProgressTracker.NULL_TRACKER); + var trainer = new GraphSageModelTrainer(config, ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER); + var otherTrainer = new GraphSageModelTrainer(config, ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER); var result = trainer.train(unweightedGraph, features); var otherResult = otherTrainer.train(unweightedGraph, features); @@ -424,8 +424,8 @@ void seededMultiBatch(long seed) { .batchSize(5) .build(); - var trainer = new GraphSageModelTrainer(config, Pools.DEFAULT, ProgressTracker.NULL_TRACKER); - var otherTrainer = new GraphSageModelTrainer(config, Pools.DEFAULT, ProgressTracker.NULL_TRACKER); + var trainer = new GraphSageModelTrainer(config, ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER); + var otherTrainer = new GraphSageModelTrainer(config, ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER); var result = trainer.train(unweightedGraph, features); var otherResult = otherTrainer.train(unweightedGraph, features); diff --git a/algo/src/test/java/org/neo4j/gds/embeddings/graphsage/GraphSageTest.java b/algo/src/test/java/org/neo4j/gds/embeddings/graphsage/GraphSageTest.java index a201bb9f11..1baa9222d7 100644 --- a/algo/src/test/java/org/neo4j/gds/embeddings/graphsage/GraphSageTest.java +++ b/algo/src/test/java/org/neo4j/gds/embeddings/graphsage/GraphSageTest.java @@ -35,7 +35,7 @@ import org.neo4j.gds.compat.Neo4jProxy; import org.neo4j.gds.config.RandomGraphGeneratorConfig; import org.neo4j.gds.core.Aggregation; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.huge.HugeGraph; import org.neo4j.gds.core.loading.CSRGraphStoreUtil; import org.neo4j.gds.core.loading.construction.NodeLabelTokens; @@ -146,7 +146,7 @@ void shouldNotMakeNanEmbeddings(Aggregator.AggregatorType aggregator) { var trainAlgo = new SingleLabelGraphSageTrain( orphanGraph, trainConfig, - Pools.DEFAULT, + ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER, testGdsVersion ); @@ -180,7 +180,7 @@ void differentTrainAndPredictionGraph() { .concurrency(1) .build(); - var graphSageTrain = new SingleLabelGraphSageTrain(graph, trainConfig, Pools.DEFAULT, ProgressTracker.NULL_TRACKER, testGdsVersion); + var graphSageTrain = new SingleLabelGraphSageTrain(graph, trainConfig, ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER, testGdsVersion); var model = graphSageTrain.compute(); @@ -205,7 +205,7 @@ void differentTrainAndPredictionGraph() { .batchSize(2) .build(); - var graphSage = new GraphSage(trainGraph, model, streamConfig, Pools.DEFAULT, ProgressTracker.NULL_TRACKER); + var graphSage = new GraphSage(trainGraph, model, streamConfig, ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER); assertThat(graphSage.compute().embeddings().size()).isEqualTo(predictNodeCount); } diff --git a/algo/src/test/java/org/neo4j/gds/embeddings/graphsage/algo/MultiLabelGraphSageTrainTest.java b/algo/src/test/java/org/neo4j/gds/embeddings/graphsage/algo/MultiLabelGraphSageTrainTest.java index b4c98ffb21..26463e7d63 100644 --- a/algo/src/test/java/org/neo4j/gds/embeddings/graphsage/algo/MultiLabelGraphSageTrainTest.java +++ b/algo/src/test/java/org/neo4j/gds/embeddings/graphsage/algo/MultiLabelGraphSageTrainTest.java @@ -24,7 +24,7 @@ import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; import org.neo4j.gds.api.Graph; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.embeddings.graphsage.ActivationFunction; import org.neo4j.gds.embeddings.graphsage.Aggregator; @@ -98,7 +98,7 @@ void shouldRunWithDifferentProjectedFeatureSizes(String name, GraphSageTrainConf var multiLabelGraphSageTrain = new MultiLabelGraphSageTrain( weightedGraph, config, - Pools.DEFAULT, + ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER, testGdsVersion ); @@ -119,7 +119,7 @@ void shouldStoreMultiLabelFeatureFunctionInModel() { var multiLabelGraphSageTrain = new MultiLabelGraphSageTrain( weightedGraph, config, - Pools.DEFAULT, + ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER, testGdsVersion ); @@ -148,7 +148,7 @@ void runsTrainingOnMultiLabelGraph() { var graphSageTrain = new MultiLabelGraphSageTrain( weightedGraph, graphSageTrainConfig, - Pools.DEFAULT, + ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER, testGdsVersion ); @@ -191,7 +191,7 @@ void shouldFailUnequalLengthArrays() { var multiLabelGraphSageTrain = new MultiLabelGraphSageTrain( unequalGraph, config, - Pools.DEFAULT, + ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER, testGdsVersion ); @@ -216,7 +216,7 @@ void shouldFailMissingArrayProperty(Graph graph, String property, long missingNo var multiLabelGraphSageTrain = new MultiLabelGraphSageTrain( graph, config, - Pools.DEFAULT, + ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER, testGdsVersion ); diff --git a/algo/src/test/java/org/neo4j/gds/embeddings/hashgnn/HashGNNTest.java b/algo/src/test/java/org/neo4j/gds/embeddings/hashgnn/HashGNNTest.java index a2e414e07b..fc19fc195e 100644 --- a/algo/src/test/java/org/neo4j/gds/embeddings/hashgnn/HashGNNTest.java +++ b/algo/src/test/java/org/neo4j/gds/embeddings/hashgnn/HashGNNTest.java @@ -36,7 +36,7 @@ import org.neo4j.gds.compat.Neo4jProxy; import org.neo4j.gds.compat.TestLog; import org.neo4j.gds.core.GraphDimensions; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.loading.ArrayIdMap; import org.neo4j.gds.core.loading.LabelInformationBuilders; import org.neo4j.gds.core.loading.construction.GraphFactory; @@ -422,7 +422,7 @@ void shouldBeDeterministicGivenSameOriginalIds() { .nodes(firstIdMap) .relationshipType(RelationshipType.of("REL")) .orientation(Orientation.UNDIRECTED) - .executorService(Pools.DEFAULT) + .executorService(ExecutorServices.DEFAULT) .build(); var secondMappedToOriginal = HugeLongArray.newArray(nodeCount); @@ -446,7 +446,7 @@ void shouldBeDeterministicGivenSameOriginalIds() { .nodes(secondIdMap) .relationshipType(RelationshipType.of("REL")) .orientation(Orientation.UNDIRECTED) - .executorService(Pools.DEFAULT) + .executorService(ExecutorServices.DEFAULT) .build(); var random = new SplittableRandom(42); diff --git a/algo/src/test/java/org/neo4j/gds/embeddings/node2vec/Node2VecTest.java b/algo/src/test/java/org/neo4j/gds/embeddings/node2vec/Node2VecTest.java index 06712e2221..2a61c80eee 100644 --- a/algo/src/test/java/org/neo4j/gds/embeddings/node2vec/Node2VecTest.java +++ b/algo/src/test/java/org/neo4j/gds/embeddings/node2vec/Node2VecTest.java @@ -44,7 +44,7 @@ import org.neo4j.gds.compat.Neo4jProxy; import org.neo4j.gds.compat.TestLog; import org.neo4j.gds.core.GraphDimensions; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.loading.ArrayIdMap; import org.neo4j.gds.core.loading.LabelInformationBuilders; import org.neo4j.gds.core.loading.construction.GraphFactory; @@ -280,7 +280,7 @@ void shouldBeFairlyConsistentUnderOriginalIds(EmbeddingInitializer embeddingInit .nodes(firstIdMap) .relationshipType(RelationshipType.of("REL")) .orientation(Orientation.UNDIRECTED) - .executorService(Pools.DEFAULT) + .executorService(ExecutorServices.DEFAULT) .build(); var secondMappedToOriginal = HugeLongArray.newArray(nodeCount); @@ -302,7 +302,7 @@ void shouldBeFairlyConsistentUnderOriginalIds(EmbeddingInitializer embeddingInit .nodes(secondIdMap) .relationshipType(RelationshipType.of("REL")) .orientation(Orientation.UNDIRECTED) - .executorService(Pools.DEFAULT) + .executorService(ExecutorServices.DEFAULT) .build(); var random = new SplittableRandom(42); diff --git a/algo/src/test/java/org/neo4j/gds/harmonic/HarmonicCentralityTest.java b/algo/src/test/java/org/neo4j/gds/harmonic/HarmonicCentralityTest.java index d1129e9da9..aaa01e6a40 100644 --- a/algo/src/test/java/org/neo4j/gds/harmonic/HarmonicCentralityTest.java +++ b/algo/src/test/java/org/neo4j/gds/harmonic/HarmonicCentralityTest.java @@ -24,7 +24,7 @@ import org.neo4j.gds.TestProgressTracker; import org.neo4j.gds.api.Graph; import org.neo4j.gds.compat.Neo4jProxy; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.utils.progress.EmptyTaskRegistryFactory; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.core.utils.progress.tasks.Tasks; @@ -67,7 +67,7 @@ void shouldComputeHarmonicCentrality() { var harmonicCentrality = new HarmonicCentrality( graph, 1, - Pools.DEFAULT, + ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER ); @@ -89,7 +89,7 @@ void testLogging() { var algo = new HarmonicCentrality( graph, 1, - Pools.DEFAULT, + ExecutorServices.DEFAULT, progressTracker ); diff --git a/algo/src/test/java/org/neo4j/gds/indexInverse/InverseRelationshipsTest.java b/algo/src/test/java/org/neo4j/gds/indexInverse/InverseRelationshipsTest.java index a5f46a2c53..92dbf2e8f3 100644 --- a/algo/src/test/java/org/neo4j/gds/indexInverse/InverseRelationshipsTest.java +++ b/algo/src/test/java/org/neo4j/gds/indexInverse/InverseRelationshipsTest.java @@ -27,7 +27,7 @@ import org.neo4j.gds.RelationshipType; import org.neo4j.gds.api.GraphStore; import org.neo4j.gds.compat.Neo4jProxy; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.utils.progress.EmptyTaskRegistryFactory; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.extension.GdlExtension; @@ -79,7 +79,7 @@ void shouldCreateIndexedRelationships(int concurrency) { graphStore, config, ProgressTracker.NULL_TRACKER, - Pools.DEFAULT + ExecutorServices.DEFAULT ).compute(); assertThat(inverseRelationshipsPerType).hasSize(1); @@ -110,7 +110,7 @@ void shouldIndexMultipleTypes(Object relTypes) { graphStore, config, ProgressTracker.NULL_TRACKER, - Pools.DEFAULT + ExecutorServices.DEFAULT ).compute(); assertThat(inverseRelationshipsPerType).hasSize(internalTypes.size()); diff --git a/algo/src/test/java/org/neo4j/gds/influenceMaximization/CELFOnConnectedGraphTest.java b/algo/src/test/java/org/neo4j/gds/influenceMaximization/CELFOnConnectedGraphTest.java index e406cadefd..ba87582c9f 100644 --- a/algo/src/test/java/org/neo4j/gds/influenceMaximization/CELFOnConnectedGraphTest.java +++ b/algo/src/test/java/org/neo4j/gds/influenceMaximization/CELFOnConnectedGraphTest.java @@ -25,7 +25,7 @@ import org.neo4j.gds.Orientation; import org.neo4j.gds.compat.Neo4jProxy; import org.neo4j.gds.compat.TestLog; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.utils.progress.EmptyTaskRegistryFactory; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.core.utils.progress.tasks.TaskProgressTracker; @@ -99,7 +99,7 @@ void testSpreadWithSeed1() { // gain[d|a,b,d,e] : 0 {a already activates d} 1(d) 1(d) = 2/3 =0.667 IdFunction idFunction = variable -> graph.toMappedNodeId(variable); - CELF celf = new CELF(graph, 5, 0.2, 3, Pools.DEFAULT, 2, 0, DEFAULT_BATCH_SIZE, + CELF celf = new CELF(graph, 5, 0.2, 3, ExecutorServices.DEFAULT, 2, 0, DEFAULT_BATCH_SIZE, ProgressTracker.EmptyProgressTracker.NULL_TRACKER ); var celfResult = celf.compute(); diff --git a/algo/src/test/java/org/neo4j/gds/influenceMaximization/CELFOnTreeGraphTest.java b/algo/src/test/java/org/neo4j/gds/influenceMaximization/CELFOnTreeGraphTest.java index 6831373860..41677f1d67 100644 --- a/algo/src/test/java/org/neo4j/gds/influenceMaximization/CELFOnTreeGraphTest.java +++ b/algo/src/test/java/org/neo4j/gds/influenceMaximization/CELFOnTreeGraphTest.java @@ -24,7 +24,7 @@ import org.junit.jupiter.api.Test; import org.neo4j.gds.Orientation; import org.neo4j.gds.api.Graph; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.extension.GdlExtension; import org.neo4j.gds.extension.GdlGraph; @@ -136,7 +136,7 @@ void testSpreadWithSeed1() { 5, 0.51, 3, - Pools.DEFAULT, + ExecutorServices.DEFAULT, 1, 10, DEFAULT_BATCH_SIZE, diff --git a/algo/src/test/java/org/neo4j/gds/influenceMaximization/CelfTest.java b/algo/src/test/java/org/neo4j/gds/influenceMaximization/CelfTest.java index 282613cd08..cb8c5f3770 100644 --- a/algo/src/test/java/org/neo4j/gds/influenceMaximization/CelfTest.java +++ b/algo/src/test/java/org/neo4j/gds/influenceMaximization/CelfTest.java @@ -24,7 +24,7 @@ import org.neo4j.gds.api.schema.Direction; import org.neo4j.gds.beta.generator.RandomGraphGenerator; import org.neo4j.gds.beta.generator.RelationshipDistribution; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import static org.assertj.core.api.Assertions.assertThat; @@ -44,7 +44,7 @@ void shouldNotReturnNegativeGains(int seedSize) { .build() .generate(); - var celf = new CELF(graph, seedSize, 0.1, 3, Pools.DEFAULT, 1, 10, 5, ProgressTracker.NULL_TRACKER).compute(); + var celf = new CELF(graph, seedSize, 0.1, 3, ExecutorServices.DEFAULT, 1, 10, 5, ProgressTracker.NULL_TRACKER).compute(); for (var a : celf) { assertThat(a.value).isNotNegative(); } diff --git a/algo/src/test/java/org/neo4j/gds/k1coloring/K1ColoringTest.java b/algo/src/test/java/org/neo4j/gds/k1coloring/K1ColoringTest.java index 252cdff3b0..c0b4469c8e 100644 --- a/algo/src/test/java/org/neo4j/gds/k1coloring/K1ColoringTest.java +++ b/algo/src/test/java/org/neo4j/gds/k1coloring/K1ColoringTest.java @@ -32,7 +32,7 @@ import org.neo4j.gds.core.Aggregation; import org.neo4j.gds.core.GraphDimensions; import org.neo4j.gds.core.ImmutableGraphDimensions; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.utils.mem.MemoryRange; import org.neo4j.gds.core.utils.progress.EmptyTaskRegistryFactory; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; @@ -69,7 +69,7 @@ void testK1Coloring() { 1000, DEFAULT_BATCH_SIZE, 1, - Pools.DEFAULT, + ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER ); @@ -110,7 +110,7 @@ void testParallelK1Coloring() { 100, DEFAULT_BATCH_SIZE, 8, - Pools.DEFAULT, + ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER ); @@ -177,7 +177,7 @@ void everyNodeShouldHaveBeenColored() { 100, DEFAULT_BATCH_SIZE, 8, - Pools.DEFAULT, + ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER ); @@ -214,7 +214,7 @@ void shouldLogProgress(){ config.maxIterations(), DEFAULT_BATCH_SIZE, config.concurrency(), - Pools.DEFAULT, + ExecutorServices.DEFAULT, progressTracker ); diff --git a/algo/src/test/java/org/neo4j/gds/labelpropagation/LabelPropagationTest.java b/algo/src/test/java/org/neo4j/gds/labelpropagation/LabelPropagationTest.java index 4c2ead6bfc..ebcbe09627 100644 --- a/algo/src/test/java/org/neo4j/gds/labelpropagation/LabelPropagationTest.java +++ b/algo/src/test/java/org/neo4j/gds/labelpropagation/LabelPropagationTest.java @@ -29,7 +29,7 @@ import org.neo4j.gds.compat.Neo4jProxy; import org.neo4j.gds.compat.TestLog; import org.neo4j.gds.core.CypherMapWrapper; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.utils.paged.HugeLongArray; import org.neo4j.gds.core.utils.progress.EmptyTaskRegistryFactory; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; @@ -83,7 +83,7 @@ void shouldUseOriginalNodeIdWhenSeedPropertyIsMissing() { LabelPropagation lp = new LabelPropagation( graph, ImmutableLabelPropagationStreamConfig.builder().maxIterations(1).build(), - Pools.DEFAULT, + ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER ); assertArrayEquals( @@ -109,7 +109,7 @@ void shouldUseSeedProperty() { .seedProperty("seedId") .maxIterations(1) .build(), - Pools.DEFAULT, + ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER ); @@ -146,7 +146,7 @@ private void testLPClustering(Graph graph, int batchSize) { LabelPropagation lp = new LabelPropagation( graph, DEFAULT_CONFIG, - Pools.DEFAULT, + ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER ); lp.withBatchSize(batchSize); @@ -202,7 +202,7 @@ void shouldLogProgress() { var lp = new LabelPropagation( graph, DEFAULT_CONFIG, - Pools.DEFAULT, + ExecutorServices.DEFAULT, testTracker ); diff --git a/algo/src/test/java/org/neo4j/gds/labelpropagation/NonStabilizingLabelPropagationTest.java b/algo/src/test/java/org/neo4j/gds/labelpropagation/NonStabilizingLabelPropagationTest.java index acfea1e94f..bd303c9624 100644 --- a/algo/src/test/java/org/neo4j/gds/labelpropagation/NonStabilizingLabelPropagationTest.java +++ b/algo/src/test/java/org/neo4j/gds/labelpropagation/NonStabilizingLabelPropagationTest.java @@ -21,7 +21,7 @@ import org.junit.jupiter.api.Test; import org.neo4j.gds.api.Graph; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.extension.GdlExtension; import org.neo4j.gds.extension.GdlGraph; @@ -62,7 +62,7 @@ void testLabelPropagationDoesStabilize() { LabelPropagation labelPropagation = new LabelPropagation( graph, ImmutableLabelPropagationStreamConfig.builder().build(), - Pools.DEFAULT, + ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER ); LabelPropagationResult compute = labelPropagation.compute(); diff --git a/algo/src/test/java/org/neo4j/gds/leiden/GraphWithSelfLoopTest.java b/algo/src/test/java/org/neo4j/gds/leiden/GraphWithSelfLoopTest.java index a82da2cf43..7b81c0d4b9 100644 --- a/algo/src/test/java/org/neo4j/gds/leiden/GraphWithSelfLoopTest.java +++ b/algo/src/test/java/org/neo4j/gds/leiden/GraphWithSelfLoopTest.java @@ -23,6 +23,7 @@ import org.junit.jupiter.api.Test; import org.neo4j.gds.Orientation; import org.neo4j.gds.api.schema.Direction; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.concurrency.Pools; import org.neo4j.gds.core.utils.TerminationFlag; import org.neo4j.gds.core.utils.paged.HugeDoubleArray; @@ -70,7 +71,7 @@ void shouldCalculateModularityCorrectly() { 1.0 / graph.relationshipCount(), 1.0 / graph.relationshipCount(), 4, - Pools.DEFAULT, + ExecutorServices.DEFAULT, ProgressTracker.EmptyProgressTracker.NULL_TRACKER ); @@ -164,7 +165,7 @@ void shouldCalculateModularityInSummaryGraph() { Direction.UNDIRECTED, localCommunities, 2, - Pools.DEFAULT, + ExecutorServices.DEFAULT, 1, TerminationFlag.RUNNING_TRUE, ProgressTracker.NULL_TRACKER @@ -180,7 +181,7 @@ void shouldCalculateModularityInSummaryGraph() { 1.0 / graph.relationshipCount(), 1.0 / graph.relationshipCount(), 4, - Pools.DEFAULT, + ExecutorServices.DEFAULT, ProgressTracker.EmptyProgressTracker.NULL_TRACKER ); diff --git a/algo/src/test/java/org/neo4j/gds/leiden/ModularityComputerTest.java b/algo/src/test/java/org/neo4j/gds/leiden/ModularityComputerTest.java index 9264951ada..2af5ce22db 100644 --- a/algo/src/test/java/org/neo4j/gds/leiden/ModularityComputerTest.java +++ b/algo/src/test/java/org/neo4j/gds/leiden/ModularityComputerTest.java @@ -23,7 +23,7 @@ import org.junit.jupiter.api.Test; import org.neo4j.gds.Orientation; import org.neo4j.gds.api.schema.Direction; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.utils.TerminationFlag; import org.neo4j.gds.core.utils.paged.HugeDoubleArray; import org.neo4j.gds.core.utils.paged.HugeLongArray; @@ -83,7 +83,7 @@ void shouldCalculateModularityCorrectly() { 1.0 / graph.relationshipCount(), 1.0 / graph.relationshipCount(), 4, - Pools.DEFAULT, + ExecutorServices.DEFAULT, ProgressTracker.EmptyProgressTracker.NULL_TRACKER ); assertThat(modularity).isCloseTo(0.4230, Offset.offset(1e-3)); @@ -102,7 +102,7 @@ void shouldCalculateModularityInSummaryGraph() { Direction.UNDIRECTED, localCommunities, 1, - Pools.DEFAULT, + ExecutorServices.DEFAULT, 1, TerminationFlag.RUNNING_TRUE, ProgressTracker.NULL_TRACKER @@ -116,7 +116,7 @@ void shouldCalculateModularityInSummaryGraph() { 1.0 / graph.relationshipCount(), 1.0 / graph.relationshipCount(), 4, - Pools.DEFAULT, + ExecutorServices.DEFAULT, ProgressTracker.EmptyProgressTracker.NULL_TRACKER ); assertThat(modularity).isCloseTo(0.4230, Offset.offset(1e-3)); diff --git a/algo/src/test/java/org/neo4j/gds/leiden/RefinementPhaseKarateTest.java b/algo/src/test/java/org/neo4j/gds/leiden/RefinementPhaseKarateTest.java index 89a8ebc17d..1907eae189 100644 --- a/algo/src/test/java/org/neo4j/gds/leiden/RefinementPhaseKarateTest.java +++ b/algo/src/test/java/org/neo4j/gds/leiden/RefinementPhaseKarateTest.java @@ -21,7 +21,7 @@ import org.junit.jupiter.api.Test; import org.neo4j.gds.Orientation; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.utils.paged.HugeDoubleArray; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.extension.GdlExtension; @@ -73,7 +73,7 @@ void testRefinementPhase() { 0.01, 19L, 1, - Pools.DEFAULT, + ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER ); diff --git a/algo/src/test/java/org/neo4j/gds/leiden/RefinementPhaseTest.java b/algo/src/test/java/org/neo4j/gds/leiden/RefinementPhaseTest.java index 2321b3638c..fb17dc35b8 100644 --- a/algo/src/test/java/org/neo4j/gds/leiden/RefinementPhaseTest.java +++ b/algo/src/test/java/org/neo4j/gds/leiden/RefinementPhaseTest.java @@ -21,7 +21,7 @@ import org.junit.jupiter.api.Test; import org.neo4j.gds.Orientation; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.utils.paged.HugeDoubleArray; import org.neo4j.gds.core.utils.paged.HugeLongArray; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; @@ -80,7 +80,7 @@ void shouldRefine() { 0.01, 19L, 1, - Pools.DEFAULT, + ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER ); var refinementResult = refinement.run(); diff --git a/algo/src/test/java/org/neo4j/gds/leiden/WeightedModularityComputerTest.java b/algo/src/test/java/org/neo4j/gds/leiden/WeightedModularityComputerTest.java index 65f3fbd70e..c33c42f68e 100644 --- a/algo/src/test/java/org/neo4j/gds/leiden/WeightedModularityComputerTest.java +++ b/algo/src/test/java/org/neo4j/gds/leiden/WeightedModularityComputerTest.java @@ -23,7 +23,7 @@ import org.junit.jupiter.api.Test; import org.neo4j.gds.Orientation; import org.neo4j.gds.api.schema.Direction; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.utils.TerminationFlag; import org.neo4j.gds.core.utils.paged.HugeDoubleArray; import org.neo4j.gds.core.utils.paged.HugeLongArray; @@ -84,7 +84,7 @@ void shouldCalculateModularityCorrectly() { 1.0 / (4.0 * graph.relationshipCount()), 1.0 / (4.0 * graph.relationshipCount()), 4, - Pools.DEFAULT, + ExecutorServices.DEFAULT, ProgressTracker.EmptyProgressTracker.NULL_TRACKER ); assertThat(modularity).isCloseTo(0.4230, Offset.offset(1e-3)); @@ -103,7 +103,7 @@ void shouldCalculateModularityInSummaryGraph() { Direction.UNDIRECTED, localCommunities, 1, - Pools.DEFAULT, + ExecutorServices.DEFAULT, 1, TerminationFlag.RUNNING_TRUE, ProgressTracker.NULL_TRACKER @@ -117,7 +117,7 @@ void shouldCalculateModularityInSummaryGraph() { 1.0 / (4 * graph.relationshipCount()), 1.0 / (4 * graph.relationshipCount()), 4, - Pools.DEFAULT, + ExecutorServices.DEFAULT, ProgressTracker.EmptyProgressTracker.NULL_TRACKER ); assertThat(modularity).isCloseTo(0.4230, Offset.offset(1e-3)); diff --git a/algo/src/test/java/org/neo4j/gds/louvain/LouvainTest.java b/algo/src/test/java/org/neo4j/gds/louvain/LouvainTest.java index 65f52d8f25..21c85058bd 100644 --- a/algo/src/test/java/org/neo4j/gds/louvain/LouvainTest.java +++ b/algo/src/test/java/org/neo4j/gds/louvain/LouvainTest.java @@ -36,7 +36,7 @@ import org.neo4j.gds.core.Aggregation; import org.neo4j.gds.core.GraphDimensions; import org.neo4j.gds.core.ImmutableGraphDimensions; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.huge.HugeGraph; import org.neo4j.gds.core.utils.TerminationFlag; import org.neo4j.gds.core.utils.mem.MemoryRange; @@ -155,7 +155,7 @@ void testUnweighted() { config.tolerance(), config.concurrency(), ProgressTracker.NULL_TRACKER, - Pools.DEFAULT + ExecutorServices.DEFAULT ); algorithm.setTerminationFlag(TerminationFlag.RUNNING_TRUE); @@ -204,7 +204,7 @@ void testWeighted() { config.tolerance(), config.concurrency(), ProgressTracker.NULL_TRACKER, - Pools.DEFAULT + ExecutorServices.DEFAULT ); algorithm.setTerminationFlag(TerminationFlag.RUNNING_TRUE); @@ -253,7 +253,7 @@ void testSeeded() { config.tolerance(), config.concurrency(), ProgressTracker.NULL_TRACKER, - Pools.DEFAULT + ExecutorServices.DEFAULT ); algorithm.setTerminationFlag(TerminationFlag.RUNNING_TRUE); @@ -304,7 +304,7 @@ void testTolerance() { config.tolerance(), config.concurrency(), ProgressTracker.NULL_TRACKER, - Pools.DEFAULT + ExecutorServices.DEFAULT ); algorithm.setTerminationFlag(TerminationFlag.RUNNING_TRUE); @@ -340,7 +340,7 @@ void testMaxLevels() { config.tolerance(), config.concurrency(), ProgressTracker.NULL_TRACKER, - Pools.DEFAULT + ExecutorServices.DEFAULT ); algorithm.setTerminationFlag(TerminationFlag.RUNNING_TRUE); @@ -457,7 +457,7 @@ void testCanBeInterruptedByTxCancellation() { config.tolerance(), config.concurrency(), ProgressTracker.NULL_TRACKER, - Pools.DEFAULT + ExecutorServices.DEFAULT ); louvain.setTerminationFlag(terminationFlag); @@ -494,7 +494,7 @@ void testLogging() { config.tolerance(), config.concurrency(), progressTracker, - Pools.DEFAULT + ExecutorServices.DEFAULT ); @@ -522,7 +522,7 @@ void shouldThrowOnNegativeSeed() { config.tolerance(), config.concurrency(), ProgressTracker.NULL_TRACKER, - Pools.DEFAULT + ExecutorServices.DEFAULT ); algorithm.setTerminationFlag(TerminationFlag.RUNNING_TRUE); @@ -554,7 +554,7 @@ void shouldGiveSameResultWithCalculator() { TOLERANCE_DEFAULT, 4, ProgressTracker.NULL_TRACKER, - Pools.DEFAULT + ExecutorServices.DEFAULT ); var result = louvain.compute(); diff --git a/algo/src/test/java/org/neo4j/gds/modularityoptimization/FootballTest.java b/algo/src/test/java/org/neo4j/gds/modularityoptimization/FootballTest.java index dfb8f26d47..148b0b40d3 100644 --- a/algo/src/test/java/org/neo4j/gds/modularityoptimization/FootballTest.java +++ b/algo/src/test/java/org/neo4j/gds/modularityoptimization/FootballTest.java @@ -28,7 +28,7 @@ import org.neo4j.gds.api.Graph; import org.neo4j.gds.api.properties.nodes.NodePropertyValues; import org.neo4j.gds.compat.Neo4jProxy; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.utils.paged.HugeDoubleArray; import org.neo4j.gds.core.utils.paged.HugeLongArray; import org.neo4j.gds.core.utils.progress.EmptyTaskRegistryFactory; @@ -127,7 +127,7 @@ private ModularityOptimizationResult compute( properties, concurrency, minBatchSize, - Pools.DEFAULT, + ExecutorServices.DEFAULT, progressTracker ).compute(); } diff --git a/algo/src/test/java/org/neo4j/gds/modularityoptimization/ModularityOptimizationTest.java b/algo/src/test/java/org/neo4j/gds/modularityoptimization/ModularityOptimizationTest.java index 482a520b22..bed182c563 100644 --- a/algo/src/test/java/org/neo4j/gds/modularityoptimization/ModularityOptimizationTest.java +++ b/algo/src/test/java/org/neo4j/gds/modularityoptimization/ModularityOptimizationTest.java @@ -35,7 +35,7 @@ import org.neo4j.gds.compat.Neo4jProxy; import org.neo4j.gds.core.GraphDimensions; import org.neo4j.gds.core.ImmutableGraphDimensions; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.utils.mem.MemoryTree; import org.neo4j.gds.core.utils.progress.EmptyTaskRegistryFactory; import org.neo4j.gds.extension.GdlExtension; @@ -243,7 +243,7 @@ private ModularityOptimizationResult compute( properties, concurrency, minBatchSize, - Pools.DEFAULT, + ExecutorServices.DEFAULT, progressTracker ).compute(); } diff --git a/algo/src/test/java/org/neo4j/gds/modularityoptimization/ModularityOptimizationWithoutOrientationTest.java b/algo/src/test/java/org/neo4j/gds/modularityoptimization/ModularityOptimizationWithoutOrientationTest.java index 312d3ce0d2..29ce1f04d5 100644 --- a/algo/src/test/java/org/neo4j/gds/modularityoptimization/ModularityOptimizationWithoutOrientationTest.java +++ b/algo/src/test/java/org/neo4j/gds/modularityoptimization/ModularityOptimizationWithoutOrientationTest.java @@ -34,7 +34,7 @@ import org.neo4j.gds.compat.Neo4jProxy; import org.neo4j.gds.core.GraphDimensions; import org.neo4j.gds.core.ImmutableGraphDimensions; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.utils.mem.MemoryTree; import org.neo4j.gds.core.utils.progress.EmptyTaskRegistryFactory; import org.neo4j.gds.extension.GdlExtension; @@ -252,7 +252,7 @@ private ModularityOptimizationResult compute( properties, concurrency, minBatchSize, - Pools.DEFAULT, + ExecutorServices.DEFAULT, progressTracker ).compute(); } diff --git a/algo/src/test/java/org/neo4j/gds/msbfs/MultiSourceBFSAccessMethodsTest.java b/algo/src/test/java/org/neo4j/gds/msbfs/MultiSourceBFSAccessMethodsTest.java index 88a176bbf8..29e3c58c31 100644 --- a/algo/src/test/java/org/neo4j/gds/msbfs/MultiSourceBFSAccessMethodsTest.java +++ b/algo/src/test/java/org/neo4j/gds/msbfs/MultiSourceBFSAccessMethodsTest.java @@ -31,7 +31,7 @@ import org.neo4j.gds.api.RelationshipIterator; import org.neo4j.gds.api.RelationshipWithPropertyConsumer; import org.neo4j.gds.config.ConcurrencyConfig; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.graphbuilder.DefaultBuilder; import org.neo4j.gds.graphbuilder.GraphBuilder; @@ -85,7 +85,7 @@ void testWithPredecessor() { new long[]{0, 1} ); - msbfs.run(ConcurrencyConfig.DEFAULT_CONCURRENCY, Pools.DEFAULT); + msbfs.run(ConcurrencyConfig.DEFAULT_CONCURRENCY, ExecutorServices.DEFAULT); verify(bfsConsumerMock).accept(1, 0, toList(1)); verify(bfsConsumerMock).accept(2, 0, toList(2)); @@ -126,7 +126,7 @@ void testWithANP() { new long[]{0, 1} ); - msbfs.run(ConcurrencyConfig.DEFAULT_CONCURRENCY, Pools.DEFAULT); + msbfs.run(ConcurrencyConfig.DEFAULT_CONCURRENCY, ExecutorServices.DEFAULT); verify(mock).accept(3, 1, toList(1, 2)); verify(mock).accept(4, 1, toList(1, 2)); @@ -148,7 +148,7 @@ void testPredecessorWithAllSources() { (i, p, d, s) -> mock.accept(i + 1, p + 1, d, toList(s, x -> x + 1)) ); - msbfs.run(ConcurrencyConfig.DEFAULT_CONCURRENCY, Pools.DEFAULT); + msbfs.run(ConcurrencyConfig.DEFAULT_CONCURRENCY, ExecutorServices.DEFAULT); verify(mock).accept(1, 3, 1, toList(3)); verify(mock).accept(1, 4, 1, toList(4)); @@ -199,7 +199,7 @@ void testANPWithAllSources() { (i, d, s) -> mock.accept(i + 1, d, toList(s, x -> x + 1)) ); - msbfs.run(ConcurrencyConfig.DEFAULT_CONCURRENCY, Pools.DEFAULT); + msbfs.run(ConcurrencyConfig.DEFAULT_CONCURRENCY, ExecutorServices.DEFAULT); verify(mock).accept(1, 1, toList(3, 4)); verify(mock).accept(2, 1, toList(3, 4)); @@ -270,7 +270,7 @@ void testParallel() { } } ); - msbfs.run(ConcurrencyConfig.DEFAULT_CONCURRENCY, Pools.DEFAULT); + msbfs.run(ConcurrencyConfig.DEFAULT_CONCURRENCY, ExecutorServices.DEFAULT); }); for (int i = 0; i < maxNodes; i++) { @@ -382,7 +382,7 @@ public Stream streamRelationships(long nodeId, double fallba } }, sources); - msbfs.run(ConcurrencyConfig.DEFAULT_CONCURRENCY, Pools.DEFAULT); + msbfs.run(ConcurrencyConfig.DEFAULT_CONCURRENCY, ExecutorServices.DEFAULT); for (int i = 0; i < seen.length; i++) { final int[] nodeSeen = seen[i]; diff --git a/algo/src/test/java/org/neo4j/gds/paths/delta/DeltaSteppingTest.java b/algo/src/test/java/org/neo4j/gds/paths/delta/DeltaSteppingTest.java index e20bd89e0d..0a3016030f 100644 --- a/algo/src/test/java/org/neo4j/gds/paths/delta/DeltaSteppingTest.java +++ b/algo/src/test/java/org/neo4j/gds/paths/delta/DeltaSteppingTest.java @@ -36,7 +36,7 @@ import org.neo4j.gds.beta.generator.RelationshipDistribution; import org.neo4j.gds.compat.Neo4jProxy; import org.neo4j.gds.core.GraphDimensions; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.utils.progress.EmptyTaskRegistryFactory; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.extension.GdlExtension; @@ -142,7 +142,7 @@ void singleSource(double delta, int concurrency, long idOffset) { .build(); var paths = DeltaStepping - .of(graph, config, Pools.DEFAULT, ProgressTracker.NULL_TRACKER) + .of(graph, config, ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER) .compute() .pathSet(); @@ -172,7 +172,7 @@ void singleSourceFromDisconnectedNode(double delta, int concurrency, long idOffs .build(); var paths = DeltaStepping - .of(graph, config, Pools.DEFAULT, ProgressTracker.NULL_TRACKER) + .of(graph, config, ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER) .compute() .pathSet(); @@ -191,7 +191,7 @@ void shouldLogProgress() { var testLog = Neo4jProxy.testLog(); var progressTracker = new TestProgressTracker(progressTask, testLog, 1, EmptyTaskRegistryFactory.INSTANCE); - DeltaStepping.of(graph, config, Pools.DEFAULT, progressTracker) + DeltaStepping.of(graph, config, ExecutorServices.DEFAULT, progressTracker) .compute() .pathSet(); @@ -279,7 +279,7 @@ void singleSource(double delta, int concurrency, long idOffset) { .build(); var paths = DeltaStepping - .of(graph, config, Pools.DEFAULT, ProgressTracker.NULL_TRACKER) + .of(graph, config, ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER) .compute() .pathSet(); @@ -335,7 +335,7 @@ void singleSource(double delta, int concurrency, long idOffset) { .build(); var paths = DeltaStepping - .of(graph, config, Pools.DEFAULT, ProgressTracker.NULL_TRACKER) + .of(graph, config, ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER) .compute() .pathSet(); @@ -369,7 +369,7 @@ void shouldGiveSameResultsAsDijkstra() { var deltaStepping = DeltaStepping.of( newGraph, config, - Pools.DEFAULT, + ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER ).compute(); diff --git a/algo/src/test/java/org/neo4j/gds/scaleproperties/ScalePropertiesMissingPropsTest.java b/algo/src/test/java/org/neo4j/gds/scaleproperties/ScalePropertiesMissingPropsTest.java index 0ff42febad..8419afb110 100644 --- a/algo/src/test/java/org/neo4j/gds/scaleproperties/ScalePropertiesMissingPropsTest.java +++ b/algo/src/test/java/org/neo4j/gds/scaleproperties/ScalePropertiesMissingPropsTest.java @@ -21,7 +21,7 @@ import org.junit.jupiter.api.Test; import org.neo4j.gds.core.CypherMapWrapper; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.extension.GdlExtension; import org.neo4j.gds.extension.GdlGraph; @@ -55,7 +55,7 @@ void partialArrays() { .nodeProperties(List.of("arrayOn4", "arrayOn1")) .scaler(Max.buildFrom(CypherMapWrapper.empty())) .build(); - var algo = new ScaleProperties(graph, config, ProgressTracker.NULL_TRACKER, Pools.DEFAULT); + var algo = new ScaleProperties(graph, config, ProgressTracker.NULL_TRACKER, ExecutorServices.DEFAULT); var result = algo.compute(); var resultProperties = result.scaledProperties().toArray(); @@ -73,7 +73,7 @@ void testMissingScalar() { .nodeProperties(List.of("a", "b", "c")) .scaler(StdScore.buildFrom(CypherMapWrapper.empty())) .build(); - var algo = new ScaleProperties(graph, config, ProgressTracker.NULL_TRACKER, Pools.DEFAULT); + var algo = new ScaleProperties(graph, config, ProgressTracker.NULL_TRACKER, ExecutorServices.DEFAULT); var result = algo.compute(); var resultProperties = result.scaledProperties().toArray(); diff --git a/algo/src/test/java/org/neo4j/gds/scaleproperties/ScalePropertiesTest.java b/algo/src/test/java/org/neo4j/gds/scaleproperties/ScalePropertiesTest.java index 057477e127..f2fbfd9e1f 100644 --- a/algo/src/test/java/org/neo4j/gds/scaleproperties/ScalePropertiesTest.java +++ b/algo/src/test/java/org/neo4j/gds/scaleproperties/ScalePropertiesTest.java @@ -31,7 +31,7 @@ import org.neo4j.gds.compat.Neo4jProxy; import org.neo4j.gds.core.CypherMapWrapper; import org.neo4j.gds.core.GraphDimensions; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.utils.mem.MemoryRange; import org.neo4j.gds.core.utils.progress.EmptyTaskRegistryFactory; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; @@ -79,7 +79,7 @@ void scaleSingleProperty() { .scaler(MinMax.buildFrom(CypherMapWrapper.empty())) .concurrency(1) .build(); - var algo = new ScaleProperties(graph, config, ProgressTracker.NULL_TRACKER, Pools.DEFAULT); + var algo = new ScaleProperties(graph, config, ProgressTracker.NULL_TRACKER, ExecutorServices.DEFAULT); var result = algo.compute(); var resultProperties = result.scaledProperties().toArray(); @@ -98,7 +98,7 @@ void scaleMultipleProperties() { .scaler(MinMax.buildFrom(CypherMapWrapper.empty())) .concurrency(1) .build(); - var algo = new ScaleProperties(graph, config, ProgressTracker.NULL_TRACKER, Pools.DEFAULT); + var algo = new ScaleProperties(graph, config, ProgressTracker.NULL_TRACKER, ExecutorServices.DEFAULT); var result = algo.compute(); var resultProperties = result.scaledProperties().toArray(); @@ -130,14 +130,14 @@ void parallelScale() { bigGraph, config.concurrency(4).build(), ProgressTracker.NULL_TRACKER, - Pools.DEFAULT + ExecutorServices.DEFAULT ).compute().scaledProperties(); var expected = new ScaleProperties( bigGraph, config.concurrency(1).build(), ProgressTracker.NULL_TRACKER, - Pools.DEFAULT + ExecutorServices.DEFAULT ).compute().scaledProperties(); IntStream.range(0, nodeCount).forEach(id -> assertEquals(expected.get(id)[0], parallelResult.get(id)[0])); @@ -150,7 +150,7 @@ void scaleArrayProperty() { .scaler(MinMax.buildFrom(CypherMapWrapper.empty())) .build(); - var actual = new ScaleProperties(graph, arrayConfig, ProgressTracker.NULL_TRACKER, Pools.DEFAULT) + var actual = new ScaleProperties(graph, arrayConfig, ProgressTracker.NULL_TRACKER, ExecutorServices.DEFAULT) .compute() .scaledProperties(); @@ -159,7 +159,7 @@ void scaleArrayProperty() { .scaler(MinMax.buildFrom(CypherMapWrapper.empty())) .build(); - var expected = new ScaleProperties(graph, singlePropConfig, ProgressTracker.NULL_TRACKER, Pools.DEFAULT) + var expected = new ScaleProperties(graph, singlePropConfig, ProgressTracker.NULL_TRACKER, ExecutorServices.DEFAULT) .compute() .scaledProperties(); @@ -175,13 +175,13 @@ void supportLongAndFloatArrays(String scaler) { var longArrayBConfig = baseConfigBuilder.nodeProperties(List.of("longArrayB")).build(); var doubleArrayBConfig = baseConfigBuilder.nodeProperties(List.of("floatArrayB")).build(); - var expected = new ScaleProperties(graph, bConfig, ProgressTracker.NULL_TRACKER, Pools.DEFAULT) + var expected = new ScaleProperties(graph, bConfig, ProgressTracker.NULL_TRACKER, ExecutorServices.DEFAULT) .compute() .scaledProperties(); - var actualLong = new ScaleProperties(graph, longArrayBConfig, ProgressTracker.NULL_TRACKER, Pools.DEFAULT) + var actualLong = new ScaleProperties(graph, longArrayBConfig, ProgressTracker.NULL_TRACKER, ExecutorServices.DEFAULT) .compute() .scaledProperties(); - var actualDouble = new ScaleProperties(graph, doubleArrayBConfig, ProgressTracker.NULL_TRACKER, Pools.DEFAULT) + var actualDouble = new ScaleProperties(graph, doubleArrayBConfig, ProgressTracker.NULL_TRACKER, ExecutorServices.DEFAULT) .compute() .scaledProperties(); @@ -197,7 +197,7 @@ void supportDoubleArrays() { var config = baseConfigBuilder.nodeProperties(List.of("doubleArray")).build(); var expected = new double[][]{new double[]{0.0}, new double[]{0.2499999722444236}, new double[]{.5}, new double[]{0.7500000277555764}, new double[]{1.0}}; - var actual = new ScaleProperties(graph, config, ProgressTracker.NULL_TRACKER, Pools.DEFAULT) + var actual = new ScaleProperties(graph, config, ProgressTracker.NULL_TRACKER, ExecutorServices.DEFAULT) .compute() .scaledProperties(); @@ -211,7 +211,7 @@ void failOnArrayPropertyWithUnequalLength() { .scaler(MinMax.buildFrom(CypherMapWrapper.empty())) .build(); - var algo = new ScaleProperties(graph, config, ProgressTracker.NULL_TRACKER, Pools.DEFAULT); + var algo = new ScaleProperties(graph, config, ProgressTracker.NULL_TRACKER, ExecutorServices.DEFAULT); var error = assertThrows(IllegalArgumentException.class, algo::compute); assertThat(error.getMessage(), containsString( @@ -226,7 +226,7 @@ void failOnNonExistentProperty() { .scaler(MinMax.buildFrom(CypherMapWrapper.empty())) .build(); - var algo = new ScaleProperties(graph, config, ProgressTracker.NULL_TRACKER, Pools.DEFAULT); + var algo = new ScaleProperties(graph, config, ProgressTracker.NULL_TRACKER, ExecutorServices.DEFAULT); var error = assertThrows(IllegalArgumentException.class, algo::compute); assertThat(error.getMessage(), containsString("Node property `IMAGINARY_PROP` not found in graph")); diff --git a/algo/src/test/java/org/neo4j/gds/similarity/nodesim/NodeSimilarityTerminationTest.java b/algo/src/test/java/org/neo4j/gds/similarity/nodesim/NodeSimilarityTerminationTest.java index d3ae7902e0..ef215ef923 100644 --- a/algo/src/test/java/org/neo4j/gds/similarity/nodesim/NodeSimilarityTerminationTest.java +++ b/algo/src/test/java/org/neo4j/gds/similarity/nodesim/NodeSimilarityTerminationTest.java @@ -23,7 +23,7 @@ import org.neo4j.gds.BaseTest; import org.neo4j.gds.beta.generator.RandomGraphGenerator; import org.neo4j.gds.beta.generator.RelationshipDistribution; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import static org.neo4j.gds.graphbuilder.TransactionTerminationTestUtils.assertTerminates; @@ -44,7 +44,7 @@ void shouldTerminate() { var nodeSimilarity = NodeSimilarity.create( graph, NodeSimilarityTest.configBuilder().concurrency(1).build(), - Pools.DEFAULT, + ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER ); nodeSimilarity.setTerminationFlag(terminationFlag); diff --git a/algo/src/test/java/org/neo4j/gds/similarity/nodesim/NodeSimilarityTest.java b/algo/src/test/java/org/neo4j/gds/similarity/nodesim/NodeSimilarityTest.java index 17d10622ca..2ee72f75b8 100644 --- a/algo/src/test/java/org/neo4j/gds/similarity/nodesim/NodeSimilarityTest.java +++ b/algo/src/test/java/org/neo4j/gds/similarity/nodesim/NodeSimilarityTest.java @@ -32,7 +32,7 @@ import org.neo4j.gds.compat.TestLog; import org.neo4j.gds.core.GraphDimensions; import org.neo4j.gds.core.ImmutableGraphDimensions; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.utils.mem.MemoryEstimations; import org.neo4j.gds.core.utils.mem.MemoryRange; import org.neo4j.gds.core.utils.mem.MemoryTree; @@ -291,7 +291,7 @@ void shouldComputeWeightedForSupportedDirections(Orientation orientation, int co NodeSimilarity nodeSimilarity = NodeSimilarity.create( graph, configBuilder().relationshipWeightProperty("prop").concurrency(concurrency).build(), - Pools.DEFAULT, + ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER ); @@ -311,7 +311,7 @@ void shouldComputeForSupportedDirections(Orientation orientation, int concurrenc NodeSimilarity nodeSimilarity = NodeSimilarity.create( graph, configBuilder().concurrency(concurrency).build(), - Pools.DEFAULT, + ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER ); @@ -331,7 +331,7 @@ void shouldComputeTopNForSupportedDirections(Orientation orientation, int concur NodeSimilarity nodeSimilarity = NodeSimilarity.create( graph, configBuilder().concurrency(concurrency).topN(1).build(), - Pools.DEFAULT, + ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER ); @@ -351,7 +351,7 @@ void shouldComputeNegativeTopNForSupportedDirections(Orientation orientation, in NodeSimilarity nodeSimilarity = NodeSimilarity.create( graph, configBuilder().concurrency(concurrency).bottomN(1).build(), - Pools.DEFAULT, + ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER ); @@ -373,7 +373,7 @@ void shouldComputeTopKForSupportedDirections(Orientation orientation, int concur NodeSimilarity nodeSimilarity = NodeSimilarity.create( graph, configBuilder().topK(1).concurrency(concurrency).build(), - Pools.DEFAULT, + ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER ); @@ -397,7 +397,7 @@ void shouldComputeNegativeTopKForSupportedDirections(Orientation orientation, in .topK(10) .bottomK(1) .build(), - Pools.DEFAULT, + ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER ); @@ -427,7 +427,7 @@ void shouldComputeWithSimilarityCutoffForSupportedDirections(Orientation orienta NodeSimilarity nodeSimilarity = NodeSimilarity.create( graph, configBuilder().concurrency(concurrency).similarityCutoff(0.1).build(), - Pools.DEFAULT, + ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER ); @@ -450,7 +450,7 @@ void shouldComputeWithDegreeCutoffForSupportedDirections(Orientation orientation NodeSimilarity nodeSimilarity = NodeSimilarity.create( graph, configBuilder().degreeCutoff(2).concurrency(concurrency).build(), - Pools.DEFAULT, + ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER ); @@ -471,7 +471,7 @@ void shouldComputeForUndirectedGraphs(int concurrency) { NodeSimilarity nodeSimilarity = NodeSimilarity.create( undirectedGraph, configBuilder().concurrency(concurrency).build(), - Pools.DEFAULT, + ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER ); Set result = nodeSimilarity.computeToStream().collect(Collectors.toSet()); @@ -483,7 +483,7 @@ void shouldComputeForUnionGraphs() { NodeSimilarity nodeSimilarity = NodeSimilarity.create( naturalGraph, configBuilder().concurrency(1).build(), - Pools.DEFAULT, + ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER ); var result1 = nodeSimilarity.computeToStream().collect(Collectors.toSet()); @@ -491,7 +491,7 @@ void shouldComputeForUnionGraphs() { nodeSimilarity = NodeSimilarity.create( naturalUnionGraph, configBuilder().concurrency(1).build(), - Pools.DEFAULT, + ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER ); var result2 = nodeSimilarity.computeToStream().collect(Collectors.toSet()); @@ -507,7 +507,7 @@ void shouldComputeSimilarityGraphInAllSupportedDirections(Orientation orientatio NodeSimilarity nodeSimilarity = NodeSimilarity.create( graph, configBuilder().concurrency(concurrency).build(), - Pools.DEFAULT, + ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER ); @@ -558,7 +558,7 @@ void shouldComputeToGraphWithUnusedNodesInInputGraph(Orientation orientation, in .topK(100) .topN(1) .build(), - Pools.DEFAULT, + ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER ); @@ -596,7 +596,7 @@ void shouldIgnoreLoops(Orientation orientation, int concurrency) { NodeSimilarity nodeSimilarity = NodeSimilarity.create( graph, configBuilder().concurrency(concurrency).topN(1).build(), - Pools.DEFAULT, + ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER ); @@ -623,7 +623,7 @@ void shouldIgnoreParallelEdges(Orientation orientation, int concurrency) { NodeSimilarity nodeSimilarity = NodeSimilarity.create( graph, configBuilder().concurrency(concurrency).build(), - Pools.DEFAULT, + ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER ); @@ -840,7 +840,7 @@ void shouldLogProgress(int concurrency) { NodeSimilarity.create( graph, config, - Pools.DEFAULT, + ExecutorServices.DEFAULT, progressTracker ).compute().streamResult().count(); @@ -880,7 +880,7 @@ void shouldGiveCorrectResultsWithOverlap() { NodeSimilarity nodeSimilarity = NodeSimilarity.create( graph, configBuilder().concurrency(1).similarityMetric(MetricSimilarityComputer.parse("ovErLaP")).build(), - Pools.DEFAULT, + ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER ); @@ -898,7 +898,7 @@ void shouldGiveCorrectResultsWithOverlap() { .concurrency(1) .similarityMetric(MetricSimilarityComputer.parse("ovErLaP")) .build(), - Pools.DEFAULT, + ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER ); @@ -934,7 +934,7 @@ void shouldWorkForAllDegreeBoundsCombinations(int lowBound, int upperBound, Stri NodeSimilarity nodeSimilarity = NodeSimilarity.create( naturalGraph, configBuilder().upperDegreeCutoff(upperBound).degreeCutoff(lowBound).build(), - Pools.DEFAULT, + ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER ); diff --git a/algo/src/test/java/org/neo4j/gds/similarity/nodesim/SimilarityGraphBuilderTest.java b/algo/src/test/java/org/neo4j/gds/similarity/nodesim/SimilarityGraphBuilderTest.java index d71a5bb42b..da7e30d0c8 100644 --- a/algo/src/test/java/org/neo4j/gds/similarity/nodesim/SimilarityGraphBuilderTest.java +++ b/algo/src/test/java/org/neo4j/gds/similarity/nodesim/SimilarityGraphBuilderTest.java @@ -22,7 +22,7 @@ import org.junit.jupiter.api.Test; import org.neo4j.gds.NodeLabel; import org.neo4j.gds.api.Graph; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.huge.HugeGraph; import org.neo4j.gds.core.huge.UnionGraph; import org.neo4j.gds.core.loading.construction.GraphFactory; @@ -73,7 +73,7 @@ void testConstructionFromHugeGraph() { SimilarityGraphBuilder similarityGraphBuilder = new SimilarityGraphBuilder( unlabelledGraph, 1, - Pools.DEFAULT, + ExecutorServices.DEFAULT, TerminationFlag.RUNNING_TRUE ); @@ -96,7 +96,7 @@ void testConstructionFromUnionGraph() { SimilarityGraphBuilder similarityGraphBuilder = new SimilarityGraphBuilder( graph, 1, - Pools.DEFAULT, + ExecutorServices.DEFAULT, TerminationFlag.RUNNING_TRUE ); @@ -131,7 +131,7 @@ void testConstructFromFilteredGraph() { SimilarityGraphBuilder similarityGraphBuilder = new SimilarityGraphBuilder( filteredIdMap, 1, - Pools.DEFAULT, + ExecutorServices.DEFAULT, TerminationFlag.RUNNING_TRUE ); diff --git a/algo/src/test/java/org/neo4j/gds/similarity/nodesim/UnionGraphWeightedNodeSimilarityTest.java b/algo/src/test/java/org/neo4j/gds/similarity/nodesim/UnionGraphWeightedNodeSimilarityTest.java index 0c4ab250ca..a76509d231 100644 --- a/algo/src/test/java/org/neo4j/gds/similarity/nodesim/UnionGraphWeightedNodeSimilarityTest.java +++ b/algo/src/test/java/org/neo4j/gds/similarity/nodesim/UnionGraphWeightedNodeSimilarityTest.java @@ -22,7 +22,7 @@ import org.assertj.core.data.Offset; import org.junit.jupiter.api.Test; import org.neo4j.gds.api.Graph; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.extension.GdlExtension; import org.neo4j.gds.extension.GdlGraph; @@ -58,7 +58,7 @@ void shouldWorkWithUnionGraph(){ .topN(1) .build(); - var nodeSimilarity = NodeSimilarity.create(graph, config, Pools.DEFAULT, ProgressTracker.NULL_TRACKER); + var nodeSimilarity = NodeSimilarity.create(graph, config, ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER); var result = nodeSimilarity.compute().streamResult().findFirst().get(); //input should be (0 + 10 + 0)/ (5 + 10 + 8) = 10/23 diff --git a/algo/src/test/java/org/neo4j/gds/steiner/ShortestPathSteinerAlgorithmExtendedTest.java b/algo/src/test/java/org/neo4j/gds/steiner/ShortestPathSteinerAlgorithmExtendedTest.java index 5d21acddb4..a144be9c70 100644 --- a/algo/src/test/java/org/neo4j/gds/steiner/ShortestPathSteinerAlgorithmExtendedTest.java +++ b/algo/src/test/java/org/neo4j/gds/steiner/ShortestPathSteinerAlgorithmExtendedTest.java @@ -25,7 +25,7 @@ import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; import org.neo4j.gds.Orientation; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.extension.GdlExtension; import org.neo4j.gds.extension.GdlGraph; @@ -155,7 +155,7 @@ void shouldWorkCorrectly(double delta, int binSizeThreshold) { false, binSizeThreshold, //setting custom threshold for such a small graph allows to not examine everything in a single iteration - Pools.DEFAULT, + ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER ).compute(); @@ -175,7 +175,7 @@ void shouldWorkCorrectlyWithLineGraph() { 2.0, 1, false, - Pools.DEFAULT, + ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER ) .compute(); @@ -203,7 +203,7 @@ void deltaSteppingShouldWorkCorrectly() { isTerminal, 1, SteinerBasedDeltaStepping.BIN_SIZE_THRESHOLD, - Pools.DEFAULT, + ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER ); var result = deltaSteiner.compute().pathSet(); @@ -229,7 +229,7 @@ void shouldWorkIfRevisitsVertices() { 2.0, 1, false, - Pools.DEFAULT, + ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER ).compute(); @@ -258,7 +258,7 @@ void shouldWorkOnTriangle() { 2.0, 1, false, - Pools.DEFAULT, + ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER ).compute(); diff --git a/algo/src/test/java/org/neo4j/gds/steiner/ShortestPathsSteinerAlgorithmReroutingTest.java b/algo/src/test/java/org/neo4j/gds/steiner/ShortestPathsSteinerAlgorithmReroutingTest.java index 4e97db509a..cb3a58b8d9 100644 --- a/algo/src/test/java/org/neo4j/gds/steiner/ShortestPathsSteinerAlgorithmReroutingTest.java +++ b/algo/src/test/java/org/neo4j/gds/steiner/ShortestPathsSteinerAlgorithmReroutingTest.java @@ -25,7 +25,7 @@ import org.neo4j.gds.TestProgressTracker; import org.neo4j.gds.compat.Neo4jProxy; import org.neo4j.gds.compat.TestLog; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.utils.progress.EmptyTaskRegistryFactory; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.core.utils.progress.tasks.Task; @@ -189,7 +189,7 @@ void shouldPruneUnusedIfRerouting() { 2.0, 1, false, - Pools.DEFAULT, + ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER ).compute(); assertThat(steinerResult.totalCost()).isEqualTo(7.0); @@ -203,7 +203,7 @@ void shouldPruneUnusedIfRerouting() { 2.0, 1, true, - Pools.DEFAULT, + ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER ).compute(); assertThat(steinerResultWithReroute.totalCost()).isEqualTo(4.0); @@ -223,7 +223,7 @@ void shouldPruneUnusedIfReroutingOnInvertedIndex() { 2.0, 1, true, - Pools.DEFAULT, + ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER ).compute(); assertThat(steinerResultWithReroute.totalCost()).isEqualTo(4.0); @@ -242,7 +242,7 @@ void rerouteShouldNotCreateLoops() { 2.0, 1, true, - Pools.DEFAULT, + ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER ).compute(); var parent = steinerResult.parentArray().toArray(); @@ -267,7 +267,7 @@ void rerouteShouldNotCreateLoopsOnInvertedIndex() { 2.0, 1, true, - Pools.DEFAULT, + ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER ).compute(); var parent = steinerResult.parentArray().toArray(); @@ -294,7 +294,7 @@ void shouldWorkForUnreachableAndReachableTerminals() { 2.0, 1, true, - Pools.DEFAULT, + ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER ).compute(); assertThat(steinerTreeResult.effectiveTargetNodesCount()).isEqualTo(2); @@ -315,7 +315,7 @@ void shouldWorkIfNoReachableTerminals() { 2.0, 1, true, - Pools.DEFAULT, + ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER ).compute(); assertThat(steinerTreeResult.effectiveTargetNodesCount()).isEqualTo(0); @@ -473,7 +473,7 @@ void shouldNotGetOptimalWithoutBetterRerouting() { 2.0, 1, true, - Pools.DEFAULT, + ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER ).compute(); assertThat(steinerResultWithReroute.totalCost()).isEqualTo(25.0); @@ -498,7 +498,7 @@ void shouldHandleMultiplePruningsOnSameTreeAndGetBetter() { 2.0, 1, true, - Pools.DEFAULT, + ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER ).compute(); assertThat(steinerResultWithReroute.totalCost()).isEqualTo(22.0); @@ -522,7 +522,7 @@ void shouldNotPruneUnprunableNodes() { 2.0, 1, true, - Pools.DEFAULT, + ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER ).compute(); assertThat(steinerResultWithReroute.totalCost()).isEqualTo(170.0 - 19); @@ -546,7 +546,7 @@ void shouldTakeAdvantageOfNewSingleParents() { 2.0, 1, true, - Pools.DEFAULT, + ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER ).compute(); assertThat(steinerResultWithReroute.totalCost()).isEqualTo(20); @@ -554,5 +554,3 @@ void shouldTakeAdvantageOfNewSingleParents() { } } - - diff --git a/algo/src/test/java/org/neo4j/gds/steiner/ShortestPathsSteinerAlgorithmTest.java b/algo/src/test/java/org/neo4j/gds/steiner/ShortestPathsSteinerAlgorithmTest.java index 9abd7b58c0..a736cac2dd 100644 --- a/algo/src/test/java/org/neo4j/gds/steiner/ShortestPathsSteinerAlgorithmTest.java +++ b/algo/src/test/java/org/neo4j/gds/steiner/ShortestPathsSteinerAlgorithmTest.java @@ -21,7 +21,7 @@ import org.junit.jupiter.api.Test; import org.neo4j.gds.Orientation; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.extension.GdlExtension; import org.neo4j.gds.extension.GdlGraph; @@ -75,7 +75,7 @@ void shouldWorkCorrectly() { 2.0, 1, false, - Pools.DEFAULT, + ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER ).compute(); var pruned = ShortestPathsSteinerAlgorithm.PRUNED; diff --git a/algo/src/test/java/org/neo4j/gds/traversal/RandomWalkTest.java b/algo/src/test/java/org/neo4j/gds/traversal/RandomWalkTest.java index f1dde921af..7ae46a3750 100644 --- a/algo/src/test/java/org/neo4j/gds/traversal/RandomWalkTest.java +++ b/algo/src/test/java/org/neo4j/gds/traversal/RandomWalkTest.java @@ -34,7 +34,7 @@ import org.neo4j.gds.beta.generator.RelationshipDistribution; import org.neo4j.gds.compat.Neo4jProxy; import org.neo4j.gds.compat.TestLog; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.utils.TerminationFlag; import org.neo4j.gds.core.utils.progress.GlobalTaskStore; import org.neo4j.gds.core.utils.progress.TaskRegistryFactory; @@ -95,7 +95,7 @@ void testWithDefaultConfig() { graph, config, ProgressTracker.NULL_TRACKER, - Pools.DEFAULT + ExecutorServices.DEFAULT ); List result = randomWalk.compute().collect(Collectors.toList()); @@ -136,7 +136,7 @@ private List runRandomWalkSeeded(Node2VecStreamConfig config, Graph grap graph, config, ProgressTracker.NULL_TRACKER, - Pools.DEFAULT + ExecutorServices.DEFAULT ); return randomWalk.compute().collect(Collectors.toList()); @@ -149,7 +149,7 @@ void testSampleFromMultipleRelationshipTypes() { graph, config, ProgressTracker.NULL_TRACKER, - Pools.DEFAULT + ExecutorServices.DEFAULT ); int expectedNumberOfWalks = config.walksPerNode() * 3; @@ -190,7 +190,7 @@ void returnFactorShouldMakeWalksIncludeStartNodeMoreOften() { graph, config, ProgressTracker.NULL_TRACKER, - Pools.DEFAULT + ExecutorServices.DEFAULT ); var nodeCounter = new HashMap(); @@ -250,7 +250,7 @@ void largeInOutFactorShouldMakeTheWalkKeepTheSameDistance() { graph, config, ProgressTracker.NULL_TRACKER, - Pools.DEFAULT + ExecutorServices.DEFAULT ); var nodeCounter = new HashMap(); @@ -299,7 +299,7 @@ void shouldRespectRelationshipWeights() { graph, config, ProgressTracker.NULL_TRACKER, - Pools.DEFAULT + ExecutorServices.DEFAULT ); var nodeCounter = new HashMap(); @@ -336,7 +336,7 @@ void failOnInvalidRelationshipWeights(double invalidWeight) { graph, config, ProgressTracker.NULL_TRACKER, - Pools.DEFAULT + ExecutorServices.DEFAULT ) ).isInstanceOf(RuntimeException.class) .hasMessage( @@ -371,7 +371,7 @@ void parallelWeighted() { graph, config, ProgressTracker.NULL_TRACKER, - Pools.DEFAULT + ExecutorServices.DEFAULT ); assertThat(randomWalk.compute().collect(Collectors.toList())) @@ -400,7 +400,7 @@ void testWithConfiguredOffsetStartNodes() { graph, config, ProgressTracker.NULL_TRACKER, - Pools.DEFAULT + ExecutorServices.DEFAULT ); assertThat(randomWalk.compute().collect(Collectors.toList())) @@ -423,7 +423,7 @@ void testSetTerminationFlagAndMultipleRuns() { graph, config, ProgressTracker.NULL_TRACKER, - Pools.DEFAULT + ExecutorServices.DEFAULT ); var stream = randomWalk.compute(); diff --git a/algo/src/test/java/org/neo4j/gds/triangle/IntersectingTriangleCountFilteredGraphTest.java b/algo/src/test/java/org/neo4j/gds/triangle/IntersectingTriangleCountFilteredGraphTest.java index 9df195441e..6f4fac3772 100644 --- a/algo/src/test/java/org/neo4j/gds/triangle/IntersectingTriangleCountFilteredGraphTest.java +++ b/algo/src/test/java/org/neo4j/gds/triangle/IntersectingTriangleCountFilteredGraphTest.java @@ -29,7 +29,7 @@ import org.neo4j.gds.StoreLoaderBuilder; import org.neo4j.gds.api.GraphStore; import org.neo4j.gds.core.Aggregation; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.extension.Neo4jGraph; import java.util.Collections; @@ -78,7 +78,7 @@ void testUnionGraphWithNodeFilter() { Optional.empty() ); var config = ImmutableTriangleCountBaseConfig.builder().build(); - var triangleCount = IntersectingTriangleCount.create(graph, config, Pools.DEFAULT); + var triangleCount = IntersectingTriangleCount.create(graph, config, ExecutorServices.DEFAULT); var triangleCountResult = triangleCount.compute(); assertThat(triangleCountResult.globalTriangles()).isEqualTo(1); var triangles = triangleCountResult.localTriangles(); diff --git a/algo/src/test/java/org/neo4j/gds/triangle/IntersectingTriangleCountTest.java b/algo/src/test/java/org/neo4j/gds/triangle/IntersectingTriangleCountTest.java index 5d189682de..6b2d4ad294 100644 --- a/algo/src/test/java/org/neo4j/gds/triangle/IntersectingTriangleCountTest.java +++ b/algo/src/test/java/org/neo4j/gds/triangle/IntersectingTriangleCountTest.java @@ -27,7 +27,7 @@ import org.neo4j.gds.Orientation; import org.neo4j.gds.TestSupport; import org.neo4j.gds.api.Graph; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import java.util.stream.Stream; @@ -531,7 +531,7 @@ private TriangleCountResult compute(Graph graph) { } private TriangleCountResult compute(Graph graph, TriangleCountBaseConfig config) { - return IntersectingTriangleCount.create(graph, config, Pools.DEFAULT).compute(); + return IntersectingTriangleCount.create(graph, config, ExecutorServices.DEFAULT).compute(); } private static Graph fromGdl(String gdl) { diff --git a/algo/src/test/java/org/neo4j/gds/triangle/LargeIntersectingTriangleCountTest.java b/algo/src/test/java/org/neo4j/gds/triangle/LargeIntersectingTriangleCountTest.java index 3e7722bb1c..de321fbf88 100644 --- a/algo/src/test/java/org/neo4j/gds/triangle/LargeIntersectingTriangleCountTest.java +++ b/algo/src/test/java/org/neo4j/gds/triangle/LargeIntersectingTriangleCountTest.java @@ -25,7 +25,7 @@ import org.neo4j.gds.Orientation; import org.neo4j.gds.StoreLoaderBuilder; import org.neo4j.gds.api.Graph; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.utils.paged.PagedAtomicIntegerArray; import org.neo4j.gds.graphbuilder.DefaultBuilder; import org.neo4j.gds.graphbuilder.GraphBuilder; @@ -71,7 +71,7 @@ void testQueue() { var result = IntersectingTriangleCount.create( graph, defaultConfigBuilder().build(), - Pools.DEFAULT + ExecutorServices.DEFAULT ).compute(); assertEquals(TRIANGLE_COUNT, result.globalTriangles()); assertTriangles(result.globalTriangles()); @@ -83,7 +83,7 @@ void testQueueParallel() { var result = IntersectingTriangleCount.create( graph, defaultConfigBuilder().concurrency(4).build(), - Pools.DEFAULT + ExecutorServices.DEFAULT ).compute(); assertEquals(TRIANGLE_COUNT, result.globalTriangles()); assertTriangles(result.globalTriangles()); diff --git a/algo/src/test/java/org/neo4j/gds/triangle/TriangleStreamTest.java b/algo/src/test/java/org/neo4j/gds/triangle/TriangleStreamTest.java index 665b2b1598..0c7d5faec3 100644 --- a/algo/src/test/java/org/neo4j/gds/triangle/TriangleStreamTest.java +++ b/algo/src/test/java/org/neo4j/gds/triangle/TriangleStreamTest.java @@ -25,7 +25,7 @@ import org.neo4j.gds.Orientation; import org.neo4j.gds.StoreLoaderBuilder; import org.neo4j.gds.api.Graph; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.graphbuilder.DefaultBuilder; import org.neo4j.gds.graphbuilder.GraphBuilder; import org.neo4j.graphdb.Node; @@ -73,7 +73,7 @@ void setupGraphDb() { void testSequential() { TripleConsumer mock = mock(TripleConsumer.class); - TriangleStream.create(graph, Pools.DEFAULT, 1) + TriangleStream.create(graph, ExecutorServices.DEFAULT, 1) .compute() .forEach(r -> mock.consume(r.nodeA, r.nodeB, r.nodeC)); @@ -84,7 +84,7 @@ void testSequential() { void testParallel() { TripleConsumer mock = mock(TripleConsumer.class); - TriangleStream.create(graph, Pools.DEFAULT, 8) + TriangleStream.create(graph, ExecutorServices.DEFAULT, 8) .compute() .forEach(r -> mock.consume(r.nodeA, r.nodeB, r.nodeC)); diff --git a/algo/src/test/java/org/neo4j/gds/triangle/UnionGraphTriangleCountingTest.java b/algo/src/test/java/org/neo4j/gds/triangle/UnionGraphTriangleCountingTest.java index 06ce5d9233..0f201d7bd1 100644 --- a/algo/src/test/java/org/neo4j/gds/triangle/UnionGraphTriangleCountingTest.java +++ b/algo/src/test/java/org/neo4j/gds/triangle/UnionGraphTriangleCountingTest.java @@ -22,7 +22,7 @@ import org.junit.jupiter.api.Test; import org.neo4j.gds.Orientation; import org.neo4j.gds.api.Graph; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.extension.GdlExtension; import org.neo4j.gds.extension.GdlGraph; import org.neo4j.gds.extension.Inject; @@ -96,7 +96,7 @@ class UnionGraphTriangleCountingTest { @Test void shouldWorkWithUnionGraphs() { var config=TriangleCountStreamConfigImpl.builder().concurrency(1).build(); - var a=IntersectingTriangleCount.create(graph,config, Pools.DEFAULT); + var a=IntersectingTriangleCount.create(graph,config, ExecutorServices.DEFAULT); var result=a.compute(); assertThat(result.globalTriangles()).isEqualTo(0); } diff --git a/algo/src/test/java/org/neo4j/gds/undirected/ToUndirectedTest.java b/algo/src/test/java/org/neo4j/gds/undirected/ToUndirectedTest.java index b09420ea94..c476c0e753 100644 --- a/algo/src/test/java/org/neo4j/gds/undirected/ToUndirectedTest.java +++ b/algo/src/test/java/org/neo4j/gds/undirected/ToUndirectedTest.java @@ -28,7 +28,7 @@ import org.neo4j.gds.api.GraphStore; import org.neo4j.gds.compat.Neo4jProxy; import org.neo4j.gds.core.Aggregation; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.loading.SingleTypeRelationships; import org.neo4j.gds.core.utils.progress.EmptyTaskRegistryFactory; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; @@ -82,7 +82,7 @@ void shouldCreateUndirectedRelationships(int concurrency) { directedGraphStore, config, ProgressTracker.NULL_TRACKER, - Pools.DEFAULT + ExecutorServices.DEFAULT ).compute(); directedGraphStore.addRelationshipType(undirectedRelationships); @@ -129,7 +129,7 @@ void shouldCreateUndirectedRelationshipsWithSingleRelationshipProperty(int concu singleDirectedGraphStore, config, ProgressTracker.NULL_TRACKER, - Pools.DEFAULT + ExecutorServices.DEFAULT ).compute(); singleDirectedGraphStore.addRelationshipType(undirectedRelationships); @@ -175,7 +175,7 @@ void shouldCreateUndirectedRelationshipsWithNoRelationshipProperty(int concurren noPropertyDirectedGraphStore, config, ProgressTracker.NULL_TRACKER, - Pools.DEFAULT + ExecutorServices.DEFAULT ).compute(); noPropertyDirectedGraphStore.addRelationshipType(undirectedRelationships); @@ -207,7 +207,7 @@ void shouldAggregateWithoutProperties() { inputGraphStore, config, ProgressTracker.NULL_TRACKER, - Pools.DEFAULT + ExecutorServices.DEFAULT ).compute(); inputGraphStore.addRelationshipType(undirectedRels); @@ -240,7 +240,7 @@ void shouldAggregateWithPropertiesAndGlobalAggregation() { input, config, ProgressTracker.NULL_TRACKER, - Pools.DEFAULT + ExecutorServices.DEFAULT ).compute(); input.addRelationshipType(aggregatedUndirectedRelationships); @@ -281,7 +281,7 @@ void shouldAggregateWithPropertiesAndLocalAggregation() { input, config, ProgressTracker.NULL_TRACKER, - Pools.DEFAULT + ExecutorServices.DEFAULT ).compute(); input.addRelationshipType(aggregatedUndirectedRelationships); diff --git a/algo/src/test/java/org/neo4j/gds/walking/CollapseMultiPathsTest.java b/algo/src/test/java/org/neo4j/gds/walking/CollapseMultiPathsTest.java index d5d3dfc364..edd2f73afc 100644 --- a/algo/src/test/java/org/neo4j/gds/walking/CollapseMultiPathsTest.java +++ b/algo/src/test/java/org/neo4j/gds/walking/CollapseMultiPathsTest.java @@ -26,7 +26,7 @@ import org.neo4j.gds.api.AdjacencyList; import org.neo4j.gds.api.Graph; import org.neo4j.gds.api.GraphStore; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.extension.GdlExtension; import org.neo4j.gds.extension.GdlGraph; import org.neo4j.gds.extension.IdFunction; @@ -112,7 +112,7 @@ void shouldFollowRoutesFromMovies() { false, RelationshipType.of("REL"), 2, - Pools.DEFAULT + ExecutorServices.DEFAULT ).compute(); @@ -174,7 +174,7 @@ void shouldTurnEveryRouteIntoRelationship() { false, RelationshipType.of("REL"), 2, - Pools.DEFAULT + ExecutorServices.DEFAULT ).compute(); diff --git a/algo/src/test/java/org/neo4j/gds/walking/CollapsePathTest.java b/algo/src/test/java/org/neo4j/gds/walking/CollapsePathTest.java index 933e43901d..d3c02d9864 100644 --- a/algo/src/test/java/org/neo4j/gds/walking/CollapsePathTest.java +++ b/algo/src/test/java/org/neo4j/gds/walking/CollapsePathTest.java @@ -25,7 +25,7 @@ import org.neo4j.gds.RelationshipType; import org.neo4j.gds.api.Graph; import org.neo4j.gds.api.GraphStore; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.loading.SingleTypeRelationships; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.extension.GdlExtension; @@ -123,7 +123,7 @@ void testCreatingRelationships() { false, RelationshipType.of("SAME_DRUG"), 2, - Pools.DEFAULT + ExecutorServices.DEFAULT ).compute(); @@ -139,7 +139,7 @@ void testAllowCreatingSelfLoops() { true, RelationshipType.of("SAME_DRUG"), 2, - Pools.DEFAULT + ExecutorServices.DEFAULT ).compute(); @@ -153,7 +153,7 @@ void runWithDifferentRelationshipTypes() { false, RelationshipType.of("SAME_DRUG"), 2, - Pools.DEFAULT + ExecutorServices.DEFAULT ).compute(); assertResultGraph(tookGraphStore, relationships, EXPECTED_WITHOUT_LOOPS); diff --git a/algo/src/test/java/org/neo4j/gds/wcc/IncrementalWccTest.java b/algo/src/test/java/org/neo4j/gds/wcc/IncrementalWccTest.java index 9ced3d34b6..13e7f111ec 100644 --- a/algo/src/test/java/org/neo4j/gds/wcc/IncrementalWccTest.java +++ b/algo/src/test/java/org/neo4j/gds/wcc/IncrementalWccTest.java @@ -24,7 +24,7 @@ import org.neo4j.gds.api.Graph; import org.neo4j.gds.api.IdMap; import org.neo4j.gds.config.ConcurrencyConfig; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.utils.paged.dss.DisjointSetStruct; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; @@ -125,7 +125,7 @@ void shouldAssignMinimumCommunityIdOnMerge() { private DisjointSetStruct run(Graph graph, WccBaseConfig config) { return new Wcc( graph, - Pools.DEFAULT, + ExecutorServices.DEFAULT, COMMUNITY_SIZE / ConcurrencyConfig.DEFAULT_CONCURRENCY, config, ProgressTracker.NULL_TRACKER diff --git a/algo/src/test/java/org/neo4j/gds/wcc/WccTest.java b/algo/src/test/java/org/neo4j/gds/wcc/WccTest.java index 88cbde8ba0..eac79b958a 100644 --- a/algo/src/test/java/org/neo4j/gds/wcc/WccTest.java +++ b/algo/src/test/java/org/neo4j/gds/wcc/WccTest.java @@ -35,7 +35,7 @@ import org.neo4j.gds.compat.Neo4jProxy; import org.neo4j.gds.core.GraphDimensions; import org.neo4j.gds.core.ImmutableGraphDimensions; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.utils.paged.dss.DisjointSetStruct; import org.neo4j.gds.core.utils.progress.EmptyTaskRegistryFactory; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; @@ -280,7 +280,7 @@ DisjointSetStruct run(Graph graph, WccBaseConfig config) { DisjointSetStruct run(Graph graph, WccBaseConfig config, int concurrency) { return new Wcc( graph, - Pools.DEFAULT, + ExecutorServices.DEFAULT, communitySize() / concurrency, config, ProgressTracker.NULL_TRACKER diff --git a/algo/src/test/java/org/neo4j/gds/wcc/WccThresholdTest.java b/algo/src/test/java/org/neo4j/gds/wcc/WccThresholdTest.java index 228b1ab852..c700da19c7 100644 --- a/algo/src/test/java/org/neo4j/gds/wcc/WccThresholdTest.java +++ b/algo/src/test/java/org/neo4j/gds/wcc/WccThresholdTest.java @@ -23,7 +23,7 @@ import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; import org.neo4j.gds.CommunityHelper; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.utils.paged.dss.DisjointSetStruct; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.extension.GdlExtension; @@ -110,7 +110,7 @@ private void assertResults(double threshold, TestGraph graph, String[][] expecte DisjointSetStruct dss = new Wcc( graph, - Pools.DEFAULT, + ExecutorServices.DEFAULT, DEFAULT_BATCH_SIZE, wccConfig, ProgressTracker.NULL_TRACKER diff --git a/alpha/alpha-proc/src/test/java/org/neo4j/gds/pregel/HitsTest.java b/alpha/alpha-proc/src/test/java/org/neo4j/gds/pregel/HitsTest.java index 5dcd105759..6380b82b2a 100644 --- a/alpha/alpha-proc/src/test/java/org/neo4j/gds/pregel/HitsTest.java +++ b/alpha/alpha-proc/src/test/java/org/neo4j/gds/pregel/HitsTest.java @@ -21,7 +21,7 @@ import org.junit.jupiter.api.Test; import org.neo4j.gds.beta.pregel.Pregel; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.extension.GdlExtension; import org.neo4j.gds.extension.GdlGraph; @@ -65,7 +65,7 @@ void testHits() { graph, config, new Hits(), - Pools.DEFAULT, + ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER ); diff --git a/alpha/alpha-proc/src/test/java/org/neo4j/gds/pregel/SpeakerListenerLPATest.java b/alpha/alpha-proc/src/test/java/org/neo4j/gds/pregel/SpeakerListenerLPATest.java index 30b5e70d34..eae5372092 100644 --- a/alpha/alpha-proc/src/test/java/org/neo4j/gds/pregel/SpeakerListenerLPATest.java +++ b/alpha/alpha-proc/src/test/java/org/neo4j/gds/pregel/SpeakerListenerLPATest.java @@ -21,7 +21,7 @@ import org.junit.jupiter.api.Test; import org.neo4j.gds.beta.pregel.Pregel; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.extension.GdlExtension; import org.neo4j.gds.extension.GdlGraph; @@ -75,7 +75,7 @@ void testWithoutPruning() { graph, config, new SpeakerListenerLPA(42), - Pools.DEFAULT, + ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER ); @@ -122,7 +122,7 @@ void prunesAwayAfterManyIterations() { graph, config, new SpeakerListenerLPA(42), - Pools.DEFAULT, + ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER ); @@ -164,7 +164,7 @@ void closesThreadLocal() { graph, config, computation, - Pools.DEFAULT, + ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER ).run(); diff --git a/core-write/src/main/java/org/neo4j/gds/core/write/NativeRelationshipStreamExporter.java b/core-write/src/main/java/org/neo4j/gds/core/write/NativeRelationshipStreamExporter.java index 45b1454342..200af92bc5 100644 --- a/core-write/src/main/java/org/neo4j/gds/core/write/NativeRelationshipStreamExporter.java +++ b/core-write/src/main/java/org/neo4j/gds/core/write/NativeRelationshipStreamExporter.java @@ -21,7 +21,7 @@ import org.neo4j.gds.api.IdMap; import org.neo4j.gds.api.nodeproperties.ValueType; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.utils.TerminationFlag; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.transaction.TransactionContext; @@ -97,7 +97,7 @@ public long write(String relationshipType, List propertyKeys, List(bufferPool.poll()); diff --git a/core-write/src/test/java/org/neo4j/gds/core/write/NativeNodeLabelExporterTest.java b/core-write/src/test/java/org/neo4j/gds/core/write/NativeNodeLabelExporterTest.java index 21eeaa4848..899a2a77d3 100644 --- a/core-write/src/test/java/org/neo4j/gds/core/write/NativeNodeLabelExporterTest.java +++ b/core-write/src/test/java/org/neo4j/gds/core/write/NativeNodeLabelExporterTest.java @@ -25,7 +25,7 @@ import org.neo4j.gds.StoreLoaderBuilder; import org.neo4j.gds.TestSupport; import org.neo4j.gds.api.Graph; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.utils.TerminationFlag; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.extension.Neo4jGraph; @@ -58,7 +58,7 @@ void exportLabel(int concurrency) { TerminationFlag.RUNNING_TRUE, ProgressTracker.NULL_TRACKER, concurrency, - Pools.DEFAULT + ExecutorServices.DEFAULT ); exporter.write("GeneratedLabel"); diff --git a/core-write/src/test/java/org/neo4j/gds/core/write/NativeNodePropertyExporterTest.java b/core-write/src/test/java/org/neo4j/gds/core/write/NativeNodePropertyExporterTest.java index 0adb0f7c5d..7a157af237 100644 --- a/core-write/src/test/java/org/neo4j/gds/core/write/NativeNodePropertyExporterTest.java +++ b/core-write/src/test/java/org/neo4j/gds/core/write/NativeNodePropertyExporterTest.java @@ -31,7 +31,7 @@ import org.neo4j.gds.compat.Neo4jProxy; import org.neo4j.gds.compat.TestLog; import org.neo4j.gds.core.Aggregation; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.huge.DirectIdMap; import org.neo4j.gds.core.utils.TerminationFlag; import org.neo4j.gds.core.utils.progress.EmptyTaskRegistryFactory; @@ -149,7 +149,7 @@ void stopsExportingWhenTransactionHasBeenTerminated() { @Test void stopsParallelExportingWhenTransactionHasBeenTerminated() { - transactionTerminationTest(Pools.DEFAULT); + transactionTerminationTest(ExecutorServices.DEFAULT); } @ParameterizedTest @@ -172,7 +172,7 @@ void progressLogging(boolean parallel) { .builder(TestSupport.fullAccessTransaction(db), graph, TerminationFlag.RUNNING_TRUE) .withProgressTracker(progressTracker); if (parallel) { - exporterBuilder = exporterBuilder.parallel(Pools.DEFAULT, writeConcurrency); + exporterBuilder = exporterBuilder.parallel(ExecutorServices.DEFAULT, writeConcurrency); } var exporter = exporterBuilder.build(); diff --git a/core/src/main/java/org/neo4j/gds/api/GraphLoaderContext.java b/core/src/main/java/org/neo4j/gds/api/GraphLoaderContext.java index 020496d3ca..7f249c8407 100644 --- a/core/src/main/java/org/neo4j/gds/api/GraphLoaderContext.java +++ b/core/src/main/java/org/neo4j/gds/api/GraphLoaderContext.java @@ -22,7 +22,7 @@ import org.immutables.value.Value; import org.neo4j.common.DependencyResolver; import org.neo4j.gds.annotation.ValueClass; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.utils.TerminationFlag; import org.neo4j.gds.core.utils.progress.EmptyTaskRegistryFactory; import org.neo4j.gds.core.utils.progress.TaskRegistryFactory; @@ -47,7 +47,7 @@ public interface GraphLoaderContext { @Value.Default default ExecutorService executor() { - return Pools.DEFAULT; + return ExecutorServices.DEFAULT; } @Value.Default diff --git a/core/src/main/java/org/neo4j/gds/core/concurrency/ExecutorServices.java b/core/src/main/java/org/neo4j/gds/core/concurrency/ExecutorServices.java new file mode 100644 index 0000000000..977f8d467a --- /dev/null +++ b/core/src/main/java/org/neo4j/gds/core/concurrency/ExecutorServices.java @@ -0,0 +1,29 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.core.concurrency; + +import java.util.concurrent.ExecutorService; + +public final class ExecutorServices { + + public static final ExecutorService DEFAULT = Pools.createDefaultPool(); + + private ExecutorServices() {} +} diff --git a/core/src/main/java/org/neo4j/gds/core/concurrency/Pools.java b/core/src/main/java/org/neo4j/gds/core/concurrency/Pools.java index a08703dfb7..f259230bd2 100644 --- a/core/src/main/java/org/neo4j/gds/core/concurrency/Pools.java +++ b/core/src/main/java/org/neo4j/gds/core/concurrency/Pools.java @@ -40,7 +40,6 @@ public final class Pools { private static final String THREAD_NAME_PREFIX = "gds"; public static final ThreadFactory DEFAULT_THREAD_FACTORY = NamedThreadFactory.daemon(THREAD_NAME_PREFIX); - public static final ExecutorService DEFAULT = createDefaultPool(); public static final ExecutorService DEFAULT_SINGLE_THREAD_POOL = createSingleThreadPool("algo"); private Pools() { diff --git a/core/src/main/java/org/neo4j/gds/core/concurrency/RunWithConcurrency.java b/core/src/main/java/org/neo4j/gds/core/concurrency/RunWithConcurrency.java index 5d4506fc42..eafa85008e 100644 --- a/core/src/main/java/org/neo4j/gds/core/concurrency/RunWithConcurrency.java +++ b/core/src/main/java/org/neo4j/gds/core/concurrency/RunWithConcurrency.java @@ -186,11 +186,11 @@ default boolean mayInterruptIfRunning() { * If {@link #forceUsageOfExecutor()} is {@code true} however, an {@link java.lang.IllegalArgumentException} * is thrown when this object is constructed. *

- * The default executor is {@link org.neo4j.gds.core.concurrency.Pools#DEFAULT}. + * The default executor is {@link ExecutorServices#DEFAULT}. */ @Value.Default default @Nullable ExecutorService executor() { - return Pools.DEFAULT; + return ExecutorServices.DEFAULT; } /** diff --git a/core/src/main/java/org/neo4j/gds/core/loading/ArrayIdMapBuilderOps.java b/core/src/main/java/org/neo4j/gds/core/loading/ArrayIdMapBuilderOps.java index d414d5938e..f0ed70dcfd 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/ArrayIdMapBuilderOps.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/ArrayIdMapBuilderOps.java @@ -23,8 +23,8 @@ import org.neo4j.gds.api.IdMap; import org.neo4j.gds.collections.HugeSparseLongArray; import org.neo4j.gds.collections.cursor.HugeCursor; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.concurrency.ParallelUtil; -import org.neo4j.gds.core.concurrency.Pools; import org.neo4j.gds.core.loading.construction.NodesBuilder; import org.neo4j.gds.core.utils.paged.HugeLongArray; @@ -75,7 +75,7 @@ static HugeSparseLongArray buildSparseIdMap( ParallelUtil.readParallel( concurrency, nodeCount, - Pools.DEFAULT, + ExecutorServices.DEFAULT, (start, end) -> addNodes(graphIds, idMapBuilder, start, end) ); return idMapBuilder.build(); diff --git a/core/src/main/java/org/neo4j/gds/core/loading/construction/GraphFactory.java b/core/src/main/java/org/neo4j/gds/core/loading/construction/GraphFactory.java index 35bf2d06aa..78609ea190 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/construction/GraphFactory.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/construction/GraphFactory.java @@ -40,7 +40,7 @@ import org.neo4j.gds.api.schema.NodeSchema; import org.neo4j.gds.core.Aggregation; import org.neo4j.gds.core.IdMapBehaviorServiceProvider; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.huge.HugeGraph; import org.neo4j.gds.core.huge.HugeGraphBuilder; import org.neo4j.gds.core.loading.HighLimitIdMap; @@ -295,7 +295,7 @@ static RelationshipsBuilder relationshipsBuilder( .isMultiGraph(isMultiGraph) .loadRelationshipProperty(loadRelationshipProperties) .direction(Direction.fromOrientation(actualOrientation)) - .executorService(executorService.orElse(Pools.DEFAULT)) + .executorService(executorService.orElse(ExecutorServices.DEFAULT)) .concurrency(finalConcurrency); if (indexInverse.orElse(false)) { diff --git a/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/DoubleArrayNodePropertiesBuilder.java b/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/DoubleArrayNodePropertiesBuilder.java index 1bb7ac3fbb..47c7bd14c2 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/DoubleArrayNodePropertiesBuilder.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/DoubleArrayNodePropertiesBuilder.java @@ -24,8 +24,8 @@ import org.neo4j.gds.api.PartialIdMap; import org.neo4j.gds.api.properties.nodes.DoubleArrayNodePropertyValues; import org.neo4j.gds.collections.HugeSparseDoubleArrayArray; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.concurrency.ParallelUtil; -import org.neo4j.gds.core.concurrency.Pools; import org.neo4j.gds.utils.Neo4jValueConversion; import org.neo4j.values.storable.Value; @@ -91,7 +91,7 @@ public DoubleArrayNodePropertyValues build(long size, PartialIdMap idMap, long h } } }).collect(Collectors.toList()); - ParallelUtil.run(tasks, Pools.DEFAULT); + ParallelUtil.run(tasks, ExecutorServices.DEFAULT); var propertyValues = propertiesByMappedIdsBuilder.build(); diff --git a/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/DoubleNodePropertiesBuilder.java b/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/DoubleNodePropertiesBuilder.java index e5b45a1e48..81b589c23b 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/DoubleNodePropertiesBuilder.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/DoubleNodePropertiesBuilder.java @@ -24,8 +24,8 @@ import org.neo4j.gds.api.PartialIdMap; import org.neo4j.gds.api.properties.nodes.DoubleNodePropertyValues; import org.neo4j.gds.collections.HugeSparseDoubleArray; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.concurrency.ParallelUtil; -import org.neo4j.gds.core.concurrency.Pools; import org.neo4j.gds.utils.Neo4jValueConversion; import org.neo4j.values.storable.Value; @@ -116,7 +116,7 @@ public DoubleNodePropertyValues build(long size, PartialIdMap idMap, long highes } }).collect(Collectors.toList()); - ParallelUtil.run(tasks, Pools.DEFAULT); + ParallelUtil.run(tasks, ExecutorServices.DEFAULT); var propertyValues = propertiesByMappedIdsBuilder.build(); diff --git a/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/FloatArrayNodePropertiesBuilder.java b/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/FloatArrayNodePropertiesBuilder.java index 86633552e9..5147519e7d 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/FloatArrayNodePropertiesBuilder.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/FloatArrayNodePropertiesBuilder.java @@ -24,8 +24,8 @@ import org.neo4j.gds.api.PartialIdMap; import org.neo4j.gds.api.properties.nodes.FloatArrayNodePropertyValues; import org.neo4j.gds.collections.HugeSparseFloatArrayArray; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.concurrency.ParallelUtil; -import org.neo4j.gds.core.concurrency.Pools; import org.neo4j.gds.utils.Neo4jValueConversion; import org.neo4j.values.storable.Value; @@ -90,7 +90,7 @@ public FloatArrayNodePropertyValues build(long size, PartialIdMap idMap, long hi } }).collect(Collectors.toList()); - ParallelUtil.run(tasks, Pools.DEFAULT); + ParallelUtil.run(tasks, ExecutorServices.DEFAULT); var propertyValues = propertiesByMappedIdsBuilder.build(); return new FloatArrayStoreNodePropertyValues(propertyValues, size); diff --git a/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/LongArrayNodePropertiesBuilder.java b/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/LongArrayNodePropertiesBuilder.java index 6c3616b641..aa55c63226 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/LongArrayNodePropertiesBuilder.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/LongArrayNodePropertiesBuilder.java @@ -24,8 +24,8 @@ import org.neo4j.gds.api.PartialIdMap; import org.neo4j.gds.api.properties.nodes.LongArrayNodePropertyValues; import org.neo4j.gds.collections.HugeSparseLongArrayArray; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.concurrency.ParallelUtil; -import org.neo4j.gds.core.concurrency.Pools; import org.neo4j.gds.utils.Neo4jValueConversion; import org.neo4j.values.storable.Value; @@ -94,7 +94,7 @@ public LongArrayNodePropertyValues build(long size, PartialIdMap idMap, long hig } }).collect(Collectors.toList()); - ParallelUtil.run(tasks, Pools.DEFAULT); + ParallelUtil.run(tasks, ExecutorServices.DEFAULT); var propertyValues = propertiesByMappedIdsBuilder.build(); diff --git a/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/LongNodePropertiesBuilder.java b/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/LongNodePropertiesBuilder.java index e743d718d1..77bef59af3 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/LongNodePropertiesBuilder.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/LongNodePropertiesBuilder.java @@ -25,8 +25,8 @@ import org.neo4j.gds.api.properties.nodes.LongNodePropertyValues; import org.neo4j.gds.api.properties.nodes.NodePropertyValues; import org.neo4j.gds.collections.HugeSparseLongArray; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.concurrency.ParallelUtil; -import org.neo4j.gds.core.concurrency.Pools; import org.neo4j.gds.utils.Neo4jValueConversion; import org.neo4j.values.storable.Value; @@ -124,7 +124,7 @@ public NodePropertyValues build(long size, PartialIdMap idMap, long highestOrigi } }).collect(Collectors.toList()); - ParallelUtil.run(tasks, Pools.DEFAULT); + ParallelUtil.run(tasks, ExecutorServices.DEFAULT); var propertyValues = propertiesByMappedIdsBuilder.build(); diff --git a/core/src/test/java/org/neo4j/gds/TerminationTest.java b/core/src/test/java/org/neo4j/gds/TerminationTest.java index 0a176c4714..256caf37f5 100644 --- a/core/src/test/java/org/neo4j/gds/TerminationTest.java +++ b/core/src/test/java/org/neo4j/gds/TerminationTest.java @@ -22,8 +22,8 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.neo4j.gds.compat.Neo4jProxy; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.concurrency.ParallelUtil; -import org.neo4j.gds.core.concurrency.Pools; import org.neo4j.graphdb.TransactionFailureException; import org.neo4j.kernel.api.exceptions.Status; import org.neo4j.kernel.api.query.ExecutingQuery; @@ -103,7 +103,7 @@ private void executeAndKill() { }); // submit - ParallelUtil.run(runnables, Pools.DEFAULT); + ParallelUtil.run(runnables, ExecutorServices.DEFAULT); } @Test diff --git a/core/src/test/java/org/neo4j/gds/core/LoadingTest.java b/core/src/test/java/org/neo4j/gds/core/LoadingTest.java index ab9df441eb..2694f4f6b9 100644 --- a/core/src/test/java/org/neo4j/gds/core/LoadingTest.java +++ b/core/src/test/java/org/neo4j/gds/core/LoadingTest.java @@ -25,7 +25,7 @@ import org.neo4j.gds.BaseTest; import org.neo4j.gds.StoreLoaderBuilder; import org.neo4j.gds.api.Graph; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import java.util.Arrays; @@ -59,7 +59,7 @@ void setupGraphDb() { void testBasicLoading() { Graph graph = new StoreLoaderBuilder() .databaseService(db) - .executorService(Pools.DEFAULT) + .executorService(ExecutorServices.DEFAULT) .addNodeLabel("Node") .addRelationshipType("TYPE") .build() diff --git a/core/src/test/java/org/neo4j/gds/core/compression/varlong/TransientCsrListTest.java b/core/src/test/java/org/neo4j/gds/core/compression/varlong/TransientCsrListTest.java index 70d0248b2d..fc3c39a721 100644 --- a/core/src/test/java/org/neo4j/gds/core/compression/varlong/TransientCsrListTest.java +++ b/core/src/test/java/org/neo4j/gds/core/compression/varlong/TransientCsrListTest.java @@ -31,7 +31,7 @@ import org.neo4j.gds.api.IdMap; import org.neo4j.gds.core.TestMethodRunner; import org.neo4j.gds.core.compression.packed.PackedAdjacencyList; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.loading.construction.GraphFactory; import java.util.Arrays; @@ -438,7 +438,7 @@ static AdjacencyList adjacencyListFromTargets(IdMap idMap, long[] targets) { .nodes(idMap) .relationshipType(RelationshipType.of("REL")) .concurrency(1) - .executorService(Pools.DEFAULT) + .executorService(ExecutorServices.DEFAULT) .build(); Arrays.stream(targets).forEach(target -> relationshipsBuilder.add(sourceNodeId, target)); diff --git a/core/src/test/java/org/neo4j/gds/core/loading/LazyIdMapBuilderTest.java b/core/src/test/java/org/neo4j/gds/core/loading/LazyIdMapBuilderTest.java index 2aa3941260..4303f5ebc7 100644 --- a/core/src/test/java/org/neo4j/gds/core/loading/LazyIdMapBuilderTest.java +++ b/core/src/test/java/org/neo4j/gds/core/loading/LazyIdMapBuilderTest.java @@ -22,8 +22,8 @@ import org.assertj.core.api.Assertions; import org.junit.jupiter.api.Test; import org.neo4j.gds.api.PropertyState; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.concurrency.ParallelUtil; -import org.neo4j.gds.core.concurrency.Pools; import org.neo4j.gds.core.loading.construction.NodeLabelTokens; import org.neo4j.gds.core.utils.partition.PartitionUtils; @@ -66,7 +66,7 @@ void parallelAddDuplicateNodes() { }, Optional.empty()); - ParallelUtil.run(tasks, Pools.DEFAULT); + ParallelUtil.run(tasks, ExecutorServices.DEFAULT); var highLimitIdMap = lazyIdMapBuilder.build().idMap(); diff --git a/core/src/test/java/org/neo4j/gds/core/loading/ScanningRelationshipsImporterTest.java b/core/src/test/java/org/neo4j/gds/core/loading/ScanningRelationshipsImporterTest.java index a310f2078a..3b195713ce 100644 --- a/core/src/test/java/org/neo4j/gds/core/loading/ScanningRelationshipsImporterTest.java +++ b/core/src/test/java/org/neo4j/gds/core/loading/ScanningRelationshipsImporterTest.java @@ -37,7 +37,7 @@ import org.neo4j.gds.config.ImmutableGraphProjectFromStoreConfig; import org.neo4j.gds.core.GraphDimensions; import org.neo4j.gds.core.GraphDimensionsStoreReader; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.huge.DirectIdMap; import org.neo4j.gds.core.utils.TerminationFlag; import org.neo4j.gds.core.utils.progress.TaskRegistryFactory; @@ -152,7 +152,7 @@ private long[] nodeIds(String... nodeVariables) { private GraphLoaderContext graphLoaderContext() { return ImmutableGraphLoaderContext.builder() - .executor(Pools.DEFAULT) + .executor(ExecutorServices.DEFAULT) .log(NullLog.getInstance()) .terminationFlag(TerminationFlag.RUNNING_TRUE) .transactionContext(DatabaseTransactionContext.of(db, db.beginTx())) diff --git a/core/src/test/java/org/neo4j/gds/core/utils/paged/HugeAtomicGrowingBitSetTest.java b/core/src/test/java/org/neo4j/gds/core/utils/paged/HugeAtomicGrowingBitSetTest.java index faa8cc0e28..dc489dbb82 100644 --- a/core/src/test/java/org/neo4j/gds/core/utils/paged/HugeAtomicGrowingBitSetTest.java +++ b/core/src/test/java/org/neo4j/gds/core/utils/paged/HugeAtomicGrowingBitSetTest.java @@ -22,8 +22,8 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.concurrency.ParallelUtil; -import org.neo4j.gds.core.concurrency.Pools; import org.neo4j.gds.core.utils.partition.PartitionUtils; import java.util.HashSet; @@ -142,7 +142,7 @@ void testSetParallel(HugeAtomicGrowingBitSet bitSet) { } }, Optional.empty()); - ParallelUtil.run(tasks, Pools.DEFAULT); + ParallelUtil.run(tasks, ExecutorServices.DEFAULT); assertThat(bitSet.cardinality()).isEqualTo(nodeCount); } diff --git a/core/src/test/java/org/neo4j/gds/core/utils/paged/ShardedLongLongMapTest.java b/core/src/test/java/org/neo4j/gds/core/utils/paged/ShardedLongLongMapTest.java index 2393f1a7a2..ae758c0086 100644 --- a/core/src/test/java/org/neo4j/gds/core/utils/paged/ShardedLongLongMapTest.java +++ b/core/src/test/java/org/neo4j/gds/core/utils/paged/ShardedLongLongMapTest.java @@ -27,8 +27,8 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.concurrency.ParallelUtil; -import org.neo4j.gds.core.concurrency.Pools; import org.neo4j.gds.core.utils.partition.PartitionUtils; import java.util.Arrays; @@ -184,7 +184,7 @@ void testAddingMultipleNodesInParallel(@ForAll("fixedSizeIds") long[] originalId Optional.of(100) ); - ParallelUtil.run(tasks, Pools.DEFAULT); + ParallelUtil.run(tasks, ExecutorServices.DEFAULT); var map = builder.build(); diff --git a/examples/pregel-bootstrap/src/test/java/gds/example/ExamplePregelComputationAlgoTest.java b/examples/pregel-bootstrap/src/test/java/gds/example/ExamplePregelComputationAlgoTest.java index 26f51bb395..7ce2e5b753 100644 --- a/examples/pregel-bootstrap/src/test/java/gds/example/ExamplePregelComputationAlgoTest.java +++ b/examples/pregel-bootstrap/src/test/java/gds/example/ExamplePregelComputationAlgoTest.java @@ -64,7 +64,7 @@ void runExamplePregelComputation() { graph, config, new ExamplePregelComputation(), - Pools.DEFAULT, + ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER ); diff --git a/examples/pregel-example/src/test/java/org/neo4j/gds/beta/pregel/bfs/BFSPregelAlgoTest.java b/examples/pregel-example/src/test/java/org/neo4j/gds/beta/pregel/bfs/BFSPregelAlgoTest.java index 5acceeec46..5b9e181562 100644 --- a/examples/pregel-example/src/test/java/org/neo4j/gds/beta/pregel/bfs/BFSPregelAlgoTest.java +++ b/examples/pregel-example/src/test/java/org/neo4j/gds/beta/pregel/bfs/BFSPregelAlgoTest.java @@ -22,7 +22,7 @@ import org.junit.jupiter.api.Test; import org.neo4j.gds.TestSupport; import org.neo4j.gds.beta.pregel.Pregel; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.extension.GdlExtension; import org.neo4j.gds.extension.GdlGraph; @@ -111,7 +111,7 @@ void levelBfs() { graph, config, new BFSLevelPregel(), - Pools.DEFAULT, + ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER ); @@ -147,7 +147,7 @@ void parentBfs() { graph, config, new BFSParentPregel(), - Pools.DEFAULT, + ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER ); @@ -183,7 +183,7 @@ void parentBugTest() { parentGraph, config, new BFSParentPregel(), - Pools.DEFAULT, + ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER ); diff --git a/examples/pregel-example/src/test/java/org/neo4j/gds/beta/pregel/cc/ConnectedComponentsPregelAlgoTest.java b/examples/pregel-example/src/test/java/org/neo4j/gds/beta/pregel/cc/ConnectedComponentsPregelAlgoTest.java index c8f0c96109..b8a554617f 100644 --- a/examples/pregel-example/src/test/java/org/neo4j/gds/beta/pregel/cc/ConnectedComponentsPregelAlgoTest.java +++ b/examples/pregel-example/src/test/java/org/neo4j/gds/beta/pregel/cc/ConnectedComponentsPregelAlgoTest.java @@ -23,7 +23,7 @@ import org.neo4j.gds.Orientation; import org.neo4j.gds.TestSupport; import org.neo4j.gds.beta.pregel.Pregel; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.extension.GdlExtension; import org.neo4j.gds.extension.GdlGraph; @@ -85,7 +85,7 @@ void wcc() { graph, config, new ConnectedComponentsPregel(), - Pools.DEFAULT, + ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER ); diff --git a/examples/pregel-example/src/test/java/org/neo4j/gds/beta/pregel/lp/LabelPropagationPregelAlgoTest.java b/examples/pregel-example/src/test/java/org/neo4j/gds/beta/pregel/lp/LabelPropagationPregelAlgoTest.java index e9ebb5a069..d7c3c325c1 100644 --- a/examples/pregel-example/src/test/java/org/neo4j/gds/beta/pregel/lp/LabelPropagationPregelAlgoTest.java +++ b/examples/pregel-example/src/test/java/org/neo4j/gds/beta/pregel/lp/LabelPropagationPregelAlgoTest.java @@ -22,7 +22,7 @@ import org.junit.jupiter.api.Test; import org.neo4j.gds.Orientation; import org.neo4j.gds.beta.pregel.Pregel; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.extension.GdlExtension; import org.neo4j.gds.extension.GdlGraph; @@ -73,7 +73,7 @@ void runLP() { graph, config, new LabelPropagationPregel(), - Pools.DEFAULT, + ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER ); @@ -111,7 +111,7 @@ public double applyRelationshipWeight(double nodeValue, double relationshipWeigh graph, config, weightedLabelPropagation, - Pools.DEFAULT, + ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER ); diff --git a/examples/pregel-example/src/test/java/org/neo4j/gds/beta/pregel/pr/PageRankPregelAlgoTest.java b/examples/pregel-example/src/test/java/org/neo4j/gds/beta/pregel/pr/PageRankPregelAlgoTest.java index febe951e38..26e5619eb9 100644 --- a/examples/pregel-example/src/test/java/org/neo4j/gds/beta/pregel/pr/PageRankPregelAlgoTest.java +++ b/examples/pregel-example/src/test/java/org/neo4j/gds/beta/pregel/pr/PageRankPregelAlgoTest.java @@ -21,7 +21,7 @@ import org.junit.jupiter.api.Test; import org.neo4j.gds.beta.pregel.Pregel; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.utils.paged.HugeDoubleArray; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.extension.GdlExtension; @@ -88,7 +88,7 @@ void runPR() { graph, config, new PageRankPregel(), - Pools.DEFAULT, + ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER ); diff --git a/examples/pregel-example/src/test/java/org/neo4j/gds/beta/pregel/pr/WeightedPageRankPregelAlgoTest.java b/examples/pregel-example/src/test/java/org/neo4j/gds/beta/pregel/pr/WeightedPageRankPregelAlgoTest.java index 42fa414694..99d64878ce 100644 --- a/examples/pregel-example/src/test/java/org/neo4j/gds/beta/pregel/pr/WeightedPageRankPregelAlgoTest.java +++ b/examples/pregel-example/src/test/java/org/neo4j/gds/beta/pregel/pr/WeightedPageRankPregelAlgoTest.java @@ -21,7 +21,7 @@ import org.junit.jupiter.api.Test; import org.neo4j.gds.beta.pregel.Pregel; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.utils.paged.HugeDoubleArray; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.extension.GdlExtension; @@ -130,7 +130,7 @@ static void assertResult(TestGraph graph, Map expected) { graph, config, new PageRankPregel(), - Pools.DEFAULT, + ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER ); diff --git a/examples/pregel-example/src/test/java/org/neo4j/gds/beta/pregel/sssp/SingleSourceShortestPathPregelAlgoTest.java b/examples/pregel-example/src/test/java/org/neo4j/gds/beta/pregel/sssp/SingleSourceShortestPathPregelAlgoTest.java index da597122be..1a4c141c71 100644 --- a/examples/pregel-example/src/test/java/org/neo4j/gds/beta/pregel/sssp/SingleSourceShortestPathPregelAlgoTest.java +++ b/examples/pregel-example/src/test/java/org/neo4j/gds/beta/pregel/sssp/SingleSourceShortestPathPregelAlgoTest.java @@ -21,7 +21,7 @@ import org.junit.jupiter.api.Test; import org.neo4j.gds.beta.pregel.Pregel; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.utils.paged.HugeLongArray; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.extension.GdlExtension; @@ -83,7 +83,7 @@ void runSSSP() { graph, config, new SingleSourceShortestPathPregel(), - Pools.DEFAULT, + ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER ); diff --git a/examples/pregel-example/src/test/java/org/neo4j/gds/beta/pregel/triangleCount/TriangleCountPregelTest.java b/examples/pregel-example/src/test/java/org/neo4j/gds/beta/pregel/triangleCount/TriangleCountPregelTest.java index fd3e4d3098..f447ac6774 100644 --- a/examples/pregel-example/src/test/java/org/neo4j/gds/beta/pregel/triangleCount/TriangleCountPregelTest.java +++ b/examples/pregel-example/src/test/java/org/neo4j/gds/beta/pregel/triangleCount/TriangleCountPregelTest.java @@ -25,7 +25,7 @@ import org.junit.jupiter.params.provider.MethodSource; import org.neo4j.gds.api.Graph; import org.neo4j.gds.beta.pregel.Pregel; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.utils.paged.HugeLongArray; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; @@ -332,7 +332,7 @@ HugeLongArray nodeWiseTriangles(Graph graph) { graph, config, new TriangleCountPregel(), - Pools.DEFAULT, + ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER ); diff --git a/graph-sampling/src/main/java/org/neo4j/gds/graphsampling/GraphSampleConstructor.java b/graph-sampling/src/main/java/org/neo4j/gds/graphsampling/GraphSampleConstructor.java index 00ed2f9cc6..733f2e899c 100644 --- a/graph-sampling/src/main/java/org/neo4j/gds/graphsampling/GraphSampleConstructor.java +++ b/graph-sampling/src/main/java/org/neo4j/gds/graphsampling/GraphSampleConstructor.java @@ -30,7 +30,7 @@ import org.neo4j.gds.beta.filter.expression.EvaluationContext; import org.neo4j.gds.beta.filter.expression.Expression; import org.neo4j.gds.config.GraphSampleAlgoConfig; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.concurrency.RunWithConcurrency; import org.neo4j.gds.core.loading.GraphStoreBuilder; import org.neo4j.gds.core.loading.ImmutableNodes; @@ -108,7 +108,7 @@ public double evaluate(EvaluationContext context) { idMap, config.concurrency(), Map.of(), - Pools.DEFAULT, + ExecutorServices.DEFAULT, progressTracker ); diff --git a/io/core/src/main/java/org/neo4j/gds/core/io/file/FileToGraphStoreImporter.java b/io/core/src/main/java/org/neo4j/gds/core/io/file/FileToGraphStoreImporter.java index 961d36b1d6..0571b464cb 100644 --- a/io/core/src/main/java/org/neo4j/gds/core/io/file/FileToGraphStoreImporter.java +++ b/io/core/src/main/java/org/neo4j/gds/core/io/file/FileToGraphStoreImporter.java @@ -28,8 +28,8 @@ import org.neo4j.gds.api.Topology; import org.neo4j.gds.api.schema.ImmutableMutableGraphSchema; import org.neo4j.gds.api.schema.MutableNodeSchema; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.concurrency.ParallelUtil; -import org.neo4j.gds.core.concurrency.Pools; import org.neo4j.gds.core.io.GraphStoreGraphPropertyVisitor; import org.neo4j.gds.core.io.GraphStoreRelationshipVisitor; import org.neo4j.gds.core.loading.Capabilities.WriteMode; @@ -175,7 +175,7 @@ private Nodes importNodes(FileInput fileInput) { (index) -> new ElementImportRunner<>(nodeVisitorBuilder.build(), nodesIterator, progressTracker) ); - ParallelUtil.run(tasks, Pools.DEFAULT); + ParallelUtil.run(tasks, ExecutorServices.DEFAULT); var nodes = nodesBuilder.build(); @@ -206,7 +206,7 @@ private void importRelationships(FileInput fileInput, IdMap nodes) { (index) -> new ElementImportRunner<>(relationshipVisitorBuilder.build(), relationshipsIterator, progressTracker) ); - ParallelUtil.run(tasks, Pools.DEFAULT); + ParallelUtil.run(tasks, ExecutorServices.DEFAULT); var relationshipImportResult = relationshipImportResult(relationshipBuildersByType); @@ -235,7 +235,7 @@ private void importGraphProperties(FileInput fileInput) { progressTracker ) ); - ParallelUtil.run(tasks, Pools.DEFAULT); + ParallelUtil.run(tasks, ExecutorServices.DEFAULT); graphStoreGraphPropertyVisitor.close(); graphStoreBuilder.graphProperties(GraphPropertyStoreFromVisitorHelper.fromGraphPropertyVisitor( diff --git a/ml/ml-algo/src/main/java/org/neo4j/gds/ml/splitting/EdgeSplitter.java b/ml/ml-algo/src/main/java/org/neo4j/gds/ml/splitting/EdgeSplitter.java index 769b63caf6..1e89f42706 100644 --- a/ml/ml-algo/src/main/java/org/neo4j/gds/ml/splitting/EdgeSplitter.java +++ b/ml/ml-algo/src/main/java/org/neo4j/gds/ml/splitting/EdgeSplitter.java @@ -30,7 +30,7 @@ import org.neo4j.gds.api.RelationshipWithPropertyConsumer; import org.neo4j.gds.api.schema.Direction; import org.neo4j.gds.core.Aggregation; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.loading.construction.GraphFactory; import org.neo4j.gds.core.loading.construction.RelationshipsBuilder; @@ -170,7 +170,7 @@ private static RelationshipsBuilder newRelationshipsBuilder( .map(key -> List.of(GraphFactory.PropertyConfig.of(key, Aggregation.SINGLE, DefaultValue.forDouble()))) .orElse(List.of())) .concurrency(1) - .executorService(Pools.DEFAULT) + .executorService(ExecutorServices.DEFAULT) .build(); } diff --git a/pregel-proc-generator/src/main/java/org/neo4j/gds/pregel/generator/AlgorithmGenerator.java b/pregel-proc-generator/src/main/java/org/neo4j/gds/pregel/generator/AlgorithmGenerator.java index 1b767603d9..fc1952df54 100644 --- a/pregel-proc-generator/src/main/java/org/neo4j/gds/pregel/generator/AlgorithmGenerator.java +++ b/pregel-proc-generator/src/main/java/org/neo4j/gds/pregel/generator/AlgorithmGenerator.java @@ -29,7 +29,7 @@ import org.neo4j.gds.api.Graph; import org.neo4j.gds.beta.pregel.Pregel; import org.neo4j.gds.beta.pregel.PregelResult; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.utils.TerminationFlag; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; @@ -90,7 +90,7 @@ MethodSpec constructor() { .addStatement( "this.pregelJob = $T.create(graph, configuration, computation, $T.DEFAULT, progressTracker)", Pregel.class, - Pools.class + ExecutorServices.class ).build(); } diff --git a/pregel-proc-generator/src/test/java/org/neo4j/gds/pregel/generator/AlgorithmGeneratorTest.java b/pregel-proc-generator/src/test/java/org/neo4j/gds/pregel/generator/AlgorithmGeneratorTest.java index 381a9936b0..a69779c2e2 100644 --- a/pregel-proc-generator/src/test/java/org/neo4j/gds/pregel/generator/AlgorithmGeneratorTest.java +++ b/pregel-proc-generator/src/test/java/org/neo4j/gds/pregel/generator/AlgorithmGeneratorTest.java @@ -61,7 +61,7 @@ void shouldGenerateConstructor() { " org.neo4j.gds.core.utils.progress.tasks.ProgressTracker progressTracker) {" + NL + " super(progressTracker);" + NL + " var computation = new a.b.C();" + NL + - " this.pregelJob = org.neo4j.gds.beta.pregel.Pregel.create(graph, configuration, computation, org.neo4j.gds.core.concurrency.Pools.DEFAULT, progressTracker);" + NL + + " this.pregelJob = org.neo4j.gds.beta.pregel.Pregel.create(graph, configuration, computation, org.neo4j.gds.core.concurrency.ExecutorServices.DEFAULT, progressTracker);" + NL + "}" + NL ); } diff --git a/pregel-proc-generator/src/test/resources/expected/BidirectionalComputationAlgorithm.java b/pregel-proc-generator/src/test/resources/expected/BidirectionalComputationAlgorithm.java index a42fa3e1a1..d1619bf3d4 100644 --- a/pregel-proc-generator/src/test/resources/expected/BidirectionalComputationAlgorithm.java +++ b/pregel-proc-generator/src/test/resources/expected/BidirectionalComputationAlgorithm.java @@ -25,7 +25,7 @@ import org.neo4j.gds.beta.pregel.Pregel; import org.neo4j.gds.beta.pregel.PregelProcedureConfig; import org.neo4j.gds.beta.pregel.PregelResult; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.utils.TerminationFlag; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; @@ -37,7 +37,7 @@ public final class BidirectionalComputationAlgorithm extends Algorithm { ProgressTracker progressTracker) { super(progressTracker); var computation = new Computation(); - this.pregelJob = Pregel.create(graph, configuration, computation, Pools.DEFAULT, progressTracker); + this.pregelJob = Pregel.create(graph, configuration, computation, ExecutorServices.DEFAULT, progressTracker); } @Override diff --git a/pregel/src/test/java/org/neo4j/gds/beta/pregel/PregelTest.java b/pregel/src/test/java/org/neo4j/gds/beta/pregel/PregelTest.java index 8dc1deac7c..08aa876046 100644 --- a/pregel/src/test/java/org/neo4j/gds/beta/pregel/PregelTest.java +++ b/pregel/src/test/java/org/neo4j/gds/beta/pregel/PregelTest.java @@ -44,7 +44,7 @@ import org.neo4j.gds.compat.Neo4jProxy; import org.neo4j.gds.compat.TestLog; import org.neo4j.gds.core.ImmutableGraphDimensions; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.utils.TerminationFlag; import org.neo4j.gds.core.utils.mem.MemoryRange; import org.neo4j.gds.core.utils.paged.HugeDoubleArray; @@ -109,7 +109,7 @@ void sendsMessages( graph, config, computation, - Pools.DEFAULT, + ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER ); @@ -127,7 +127,7 @@ void stopsEarlyWhenTransactionHasBeenTerminated() { graph, config, new TestPregelComputation(), - Pools.DEFAULT, + ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER ); pregelJob.setTerminationFlag(terminationFlag); @@ -163,7 +163,7 @@ void logProgress(Partitioning partitioning) { graph, config, computation, - Pools.DEFAULT, + ExecutorServices.DEFAULT, progressTracker ).run(); @@ -225,7 +225,7 @@ void cleanupProgressLogging() { graph, config, computation, - Pools.DEFAULT, + ExecutorServices.DEFAULT, progressTracker ); @@ -280,7 +280,7 @@ private HugeDoubleArray run(Graph graph, PregelConfig config, PregelComputation< graph, config, computation, - Pools.DEFAULT, + ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER ); @@ -300,7 +300,7 @@ void sendMessageToSpecificTarget(Partitioning partitioning) { graph, config, new TestSendTo(), - Pools.DEFAULT, + ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER ); @@ -325,7 +325,7 @@ void compositeNodeValueTest(Partitioning partitioning) { graph, config, new CompositeTestComputation(), - Pools.DEFAULT, + ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER ); @@ -363,7 +363,7 @@ void testMasterComputeStep(Partitioning partitioning) { graph, ImmutablePregelConfig.builder().maxIterations(4).partitioning(partitioning).build(), new TestMasterCompute(), - Pools.DEFAULT, + ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER ); @@ -378,7 +378,7 @@ void testMasterComputeStepWithConvergence(Partitioning partitioning) { graph, ImmutablePregelConfig.builder().maxIterations(4).partitioning(partitioning).build(), new TestMasterCompute(2), - Pools.DEFAULT, + ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER ); @@ -515,7 +515,7 @@ public boolean masterCompute(MasterComputeContext context) { return true; } }, - Pools.DEFAULT, + ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER ).run(); @@ -603,7 +603,7 @@ void preventIllegalConcurrencyConfiguration(Partitioning partitioning) { graph, config, new TestSendTo(), - Pools.DEFAULT, + ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER )); } @@ -626,7 +626,7 @@ void messagesInInitialSuperStepShouldBeEmpty(Partitioning partitioning, boolean graph, config, new TestEmptyMessageInInitialSuperstep(), - Pools.DEFAULT, + ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER ); @@ -825,7 +825,7 @@ void throwIfBidirectionalWithoutInverseIndex() { graph, ImmutablePregelConfig.builder().maxIterations(4).build(), new Bidirectional(), - Pools.DEFAULT, + ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER ); diff --git a/proc/catalog/src/main/java/org/neo4j/gds/catalog/GraphProjectProc.java b/proc/catalog/src/main/java/org/neo4j/gds/catalog/GraphProjectProc.java index c73c97037d..2b61152686 100644 --- a/proc/catalog/src/main/java/org/neo4j/gds/catalog/GraphProjectProc.java +++ b/proc/catalog/src/main/java/org/neo4j/gds/catalog/GraphProjectProc.java @@ -30,7 +30,7 @@ import org.neo4j.gds.config.GraphProjectFromGraphConfig; import org.neo4j.gds.config.GraphProjectFromStoreConfig; import org.neo4j.gds.core.CypherMapWrapper; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.loading.GraphStoreCatalog; import org.neo4j.gds.core.utils.ProgressTimer; import org.neo4j.gds.core.utils.mem.MemoryTree; @@ -232,7 +232,7 @@ private GraphProjectSubgraphResult projectGraphFromGraphStore( var graphStore = GraphStoreFilter.filter( fromGraphStore, config, - Pools.DEFAULT, + ExecutorServices.DEFAULT, progressTracker ); diff --git a/proc/catalog/src/main/java/org/neo4j/gds/catalog/GraphWriteNodeLabelProc.java b/proc/catalog/src/main/java/org/neo4j/gds/catalog/GraphWriteNodeLabelProc.java index 7cf025dab3..b66550ad97 100644 --- a/proc/catalog/src/main/java/org/neo4j/gds/catalog/GraphWriteNodeLabelProc.java +++ b/proc/catalog/src/main/java/org/neo4j/gds/catalog/GraphWriteNodeLabelProc.java @@ -21,7 +21,7 @@ import org.neo4j.gds.beta.filter.NodesFilter; import org.neo4j.gds.config.WriteLabelConfig; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.utils.ProgressTimer; import org.neo4j.gds.core.utils.TerminationFlag; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; @@ -66,7 +66,7 @@ public Stream write( nodeFilter, procedureConfig.concurrency(), Map.of(), - Pools.DEFAULT, + ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER ); @@ -74,7 +74,7 @@ public Stream write( .withIdMap(filteredNodes.idMap()) .withTerminationFlag(TerminationFlag.wrap(executionContext().terminationMonitor())) .withArrowConnectionInfo(procedureConfig.arrowConnectionInfo(), graphStore.databaseId().databaseName()) - .parallel(Pools.DEFAULT, procedureConfig.concurrency()) + .parallel(ExecutorServices.DEFAULT, procedureConfig.concurrency()) .build(); runWithExceptionLogging( diff --git a/proc/catalog/src/main/java/org/neo4j/gds/catalog/NodeLabelMutator.java b/proc/catalog/src/main/java/org/neo4j/gds/catalog/NodeLabelMutator.java index 7e6de0142c..0cff4280a0 100644 --- a/proc/catalog/src/main/java/org/neo4j/gds/catalog/NodeLabelMutator.java +++ b/proc/catalog/src/main/java/org/neo4j/gds/catalog/NodeLabelMutator.java @@ -22,7 +22,7 @@ import org.neo4j.gds.NodeLabel; import org.neo4j.gds.beta.filter.NodesFilter; import org.neo4j.gds.config.MutateLabelConfig; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.loading.GraphStoreCatalog; import org.neo4j.gds.core.loading.GraphStoreWithConfig; import org.neo4j.gds.core.loading.ImmutableCatalogRequest; @@ -52,7 +52,7 @@ static Stream mutateNodeLabel(String graphName, String nodeLa nodeFilter, procedureConfig.concurrency(), Map.of(), - Pools.DEFAULT, + ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER ); diff --git a/proc/catalog/src/main/java/org/neo4j/gds/catalog/NodePropertiesWriter.java b/proc/catalog/src/main/java/org/neo4j/gds/catalog/NodePropertiesWriter.java index 4f23f0e456..03a5010496 100644 --- a/proc/catalog/src/main/java/org/neo4j/gds/catalog/NodePropertiesWriter.java +++ b/proc/catalog/src/main/java/org/neo4j/gds/catalog/NodePropertiesWriter.java @@ -27,7 +27,7 @@ import org.neo4j.gds.config.GraphWriteNodePropertiesConfig; import org.neo4j.gds.core.CypherMapAccess; import org.neo4j.gds.core.CypherMapWrapper; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.loading.GraphStoreWithConfig; import org.neo4j.gds.core.utils.ProgressTimer; import org.neo4j.gds.core.utils.TerminationFlag; @@ -147,7 +147,7 @@ private static long writeNodeProperties( var exporter = nodePropertyExporterBuilder .withIdMap(subGraph) .withTerminationFlag(TerminationFlag.wrap(executionContext.terminationMonitor())) - .parallel(Pools.DEFAULT, config.writeConcurrency()) + .parallel(ExecutorServices.DEFAULT, config.writeConcurrency()) .withProgressTracker(progressTracker) .withArrowConnectionInfo(config.arrowConnectionInfo(), graphStore.databaseId().databaseName()) .build(); diff --git a/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphProjectProcTest.java b/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphProjectProcTest.java index 040467e68e..28af113cf8 100644 --- a/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphProjectProcTest.java +++ b/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphProjectProcTest.java @@ -47,9 +47,9 @@ import org.neo4j.gds.config.GraphProjectFromCypherConfig; import org.neo4j.gds.config.GraphProjectFromStoreConfig; import org.neo4j.gds.core.Aggregation; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.CypherMapWrapper; import org.neo4j.gds.core.concurrency.ParallelUtil; -import org.neo4j.gds.core.concurrency.Pools; import org.neo4j.gds.core.loading.GraphStoreCatalog; import org.neo4j.gds.core.utils.progress.GlobalTaskStore; import org.neo4j.gds.core.utils.progress.TaskRegistry; @@ -895,7 +895,7 @@ void loadGraphWithSaturatedThreadPool() { // block all available threads for (int i = 0; i < ConcurrencyConfig.DEFAULT_CONCURRENCY; i++) { futures.add( - Pools.DEFAULT.submit(() -> LockSupport.parkNanos(Duration.ofSeconds(1).toNanos())) + ExecutorServices.DEFAULT.submit(() -> LockSupport.parkNanos(Duration.ofSeconds(1).toNanos())) ); } diff --git a/proc/common/src/main/java/org/neo4j/gds/CommunityProcCompanion.java b/proc/common/src/main/java/org/neo4j/gds/CommunityProcCompanion.java index 66587161d4..0f02b4a905 100644 --- a/proc/common/src/main/java/org/neo4j/gds/CommunityProcCompanion.java +++ b/proc/common/src/main/java/org/neo4j/gds/CommunityProcCompanion.java @@ -27,7 +27,7 @@ import org.neo4j.gds.config.ConcurrencyConfig; import org.neo4j.gds.config.ConsecutiveIdsConfig; import org.neo4j.gds.config.SeedConfig; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.nodeproperties.ConsecutiveLongNodePropertyValues; import org.neo4j.gds.nodeproperties.LongIfChangedNodePropertyValues; import org.neo4j.gds.result.CommunityStatistics; @@ -122,7 +122,7 @@ private static LongNodePropertyValues applySizeFilter( var communitySizes = CommunityStatistics.communitySizes( nodeProperties.nodeCount(), nodeProperties::longValue, - Pools.DEFAULT, + ExecutorServices.DEFAULT, concurrency ); return new CommunitySizeFilter(nodeProperties, communitySizes, size); diff --git a/proc/common/src/main/java/org/neo4j/gds/WriteNodePropertiesComputationResultConsumer.java b/proc/common/src/main/java/org/neo4j/gds/WriteNodePropertiesComputationResultConsumer.java index ffcb462c93..b42a17e944 100644 --- a/proc/common/src/main/java/org/neo4j/gds/WriteNodePropertiesComputationResultConsumer.java +++ b/proc/common/src/main/java/org/neo4j/gds/WriteNodePropertiesComputationResultConsumer.java @@ -23,7 +23,7 @@ import org.neo4j.gds.api.schema.PropertySchema; import org.neo4j.gds.config.AlgoBaseConfig; import org.neo4j.gds.config.WritePropertyConfig; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.loading.Capabilities.WriteMode; import org.neo4j.gds.core.utils.ProgressTimer; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; @@ -111,7 +111,7 @@ void writeToNeo( .withTerminationFlag(computationResult.algorithm().terminationFlag) .withProgressTracker(progressTracker) .withArrowConnectionInfo(config.arrowConnectionInfo(), computationResult.graphStore().databaseId().databaseName()) - .parallel(Pools.DEFAULT, config.writeConcurrency()) + .parallel(ExecutorServices.DEFAULT, config.writeConcurrency()) .build(); try { diff --git a/proc/community/src/main/java/org/neo4j/gds/kmeans/KmeansWriteSpec.java b/proc/community/src/main/java/org/neo4j/gds/kmeans/KmeansWriteSpec.java index aeaa8bcb39..70ac76c2f2 100644 --- a/proc/community/src/main/java/org/neo4j/gds/kmeans/KmeansWriteSpec.java +++ b/proc/community/src/main/java/org/neo4j/gds/kmeans/KmeansWriteSpec.java @@ -21,7 +21,7 @@ import org.neo4j.gds.api.properties.nodes.EmptyLongNodePropertyValues; import org.neo4j.gds.api.properties.nodes.LongNodePropertyValues; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.utils.ProgressTimer; import org.neo4j.gds.core.write.NodePropertyExporter; import org.neo4j.gds.executor.AlgorithmSpec; @@ -101,7 +101,7 @@ public ComputationResultConsumer context, Messages mess context.setNodeValue(LONG_ARRAY_KEY, new long[]{1, 3, 3, 7}); context.setNodeValue(DOUBLE_ARRAY_KEY, new double[]{1, 9, 8, 4}); } - }, Pools.DEFAULT, progressTracker); + }, ExecutorServices.DEFAULT, progressTracker); } @Override diff --git a/proc/similarity/src/main/java/org/neo4j/gds/similarity/filterednodesim/FilteredNodeSimilarityMutateSpec.java b/proc/similarity/src/main/java/org/neo4j/gds/similarity/filterednodesim/FilteredNodeSimilarityMutateSpec.java index 607df985cf..244333ffe6 100644 --- a/proc/similarity/src/main/java/org/neo4j/gds/similarity/filterednodesim/FilteredNodeSimilarityMutateSpec.java +++ b/proc/similarity/src/main/java/org/neo4j/gds/similarity/filterednodesim/FilteredNodeSimilarityMutateSpec.java @@ -27,7 +27,7 @@ import org.neo4j.gds.api.ProcedureReturnColumns; import org.neo4j.gds.api.nodeproperties.ValueType; import org.neo4j.gds.api.schema.RelationshipPropertySchema; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.huge.HugeGraph; import org.neo4j.gds.core.loading.SingleTypeRelationships; import org.neo4j.gds.core.loading.construction.GraphFactory; @@ -145,7 +145,7 @@ private SingleTypeRelationships getRelationships( .orientation(Orientation.NATURAL) .addPropertyConfig(GraphFactory.PropertyConfig.of(relationshipPropertyKey)) .concurrency(1) - .executorService(Pools.DEFAULT) + .executorService(ExecutorServices.DEFAULT) .build(); IdMap idMap = computationResult.graph(); diff --git a/proc/similarity/src/main/java/org/neo4j/gds/similarity/nodesim/NodeSimilarityMutateSpecification.java b/proc/similarity/src/main/java/org/neo4j/gds/similarity/nodesim/NodeSimilarityMutateSpecification.java index 752d0e2224..90aad2a29b 100644 --- a/proc/similarity/src/main/java/org/neo4j/gds/similarity/nodesim/NodeSimilarityMutateSpecification.java +++ b/proc/similarity/src/main/java/org/neo4j/gds/similarity/nodesim/NodeSimilarityMutateSpecification.java @@ -26,7 +26,7 @@ import org.neo4j.gds.api.ProcedureReturnColumns; import org.neo4j.gds.api.nodeproperties.ValueType; import org.neo4j.gds.api.schema.RelationshipPropertySchema; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.huge.HugeGraph; import org.neo4j.gds.core.loading.SingleTypeRelationships; import org.neo4j.gds.core.loading.construction.GraphFactory; @@ -134,7 +134,7 @@ private SingleTypeRelationships getRelationships( .orientation(Orientation.NATURAL) .addPropertyConfig(GraphFactory.PropertyConfig.of(relationshipPropertyKey)) .concurrency(1) - .executorService(Pools.DEFAULT) + .executorService(ExecutorServices.DEFAULT) .build(); IdMap idMap = computationResult.graph(); diff --git a/test-utils/src/main/java/org/neo4j/gds/GraphLoaderBuilders.java b/test-utils/src/main/java/org/neo4j/gds/GraphLoaderBuilders.java index a9484fc980..408a70486b 100644 --- a/test-utils/src/main/java/org/neo4j/gds/GraphLoaderBuilders.java +++ b/test-utils/src/main/java/org/neo4j/gds/GraphLoaderBuilders.java @@ -31,7 +31,7 @@ import org.neo4j.gds.core.Aggregation; import org.neo4j.gds.core.GraphLoader; import org.neo4j.gds.core.ImmutableGraphLoader; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.utils.TerminationFlag; import org.neo4j.gds.core.utils.progress.EmptyTaskRegistryFactory; import org.neo4j.gds.core.utils.progress.JobId; @@ -194,7 +194,7 @@ public static GraphLoader createGraphLoader( .databaseId(DatabaseId.of(databaseService)) .dependencyResolver(GraphDatabaseApiProxy.dependencyResolver(databaseService)) .transactionContext(transactionContext.orElseGet(() -> TestSupport.fullAccessTransaction(databaseService))) - .executor(executorService.orElse(Pools.DEFAULT)) + .executor(executorService.orElse(ExecutorServices.DEFAULT)) .terminationFlag(terminationFlag.orElse(TerminationFlag.RUNNING_TRUE)) .taskRegistryFactory(EmptyTaskRegistryFactory.INSTANCE) .userLogRegistryFactory(EmptyUserLogRegistryFactory.INSTANCE) diff --git a/test-utils/src/main/java/org/neo4j/gds/core/idmap/IdMapBuilderTest.java b/test-utils/src/main/java/org/neo4j/gds/core/idmap/IdMapBuilderTest.java index a7b5742127..7f7bba47a1 100644 --- a/test-utils/src/main/java/org/neo4j/gds/core/idmap/IdMapBuilderTest.java +++ b/test-utils/src/main/java/org/neo4j/gds/core/idmap/IdMapBuilderTest.java @@ -28,8 +28,8 @@ import org.neo4j.gds.NodeLabel; import org.neo4j.gds.annotation.ValueClass; import org.neo4j.gds.api.IdMap; +import org.neo4j.gds.core.concurrency.ExecutorServices; import org.neo4j.gds.core.concurrency.ParallelUtil; -import org.neo4j.gds.core.concurrency.Pools; import org.neo4j.gds.core.loading.IdMapBuilder; import org.neo4j.gds.core.loading.LabelInformationBuilders; import org.neo4j.gds.core.utils.partition.PartitionUtils; @@ -241,7 +241,7 @@ void testBuildParallel( } }, Optional.empty()); - ParallelUtil.run(tasks, Pools.DEFAULT); + ParallelUtil.run(tasks, ExecutorServices.DEFAULT); var idMap = idMapBuilder.build(LabelInformationBuilders.allNodes(), highestOriginalId, concurrency); From 0d4d36cc10dda23cbe9c9360b8b38216521e68d5 Mon Sep 17 00:00:00 2001 From: Mats Rydberg Date: Mon, 11 Sep 2023 09:39:46 +0200 Subject: [PATCH 226/273] Initialise PoolSizes as null MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We always set the pool sizes explicitly, regardless of license state. Init to null will expose errors where we use the PoolSizes before they were set up correctly by the lifecycle adapter Co-authored-by: Jonatan Jäderberg Co-authored-by: Paul Horn --- .../main/java/org/neo4j/gds/concurrency/PoolSizesService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/concurrency-validation-api/src/main/java/org/neo4j/gds/concurrency/PoolSizesService.java b/concurrency-validation-api/src/main/java/org/neo4j/gds/concurrency/PoolSizesService.java index 4f731fbd55..73e11c12fd 100644 --- a/concurrency-validation-api/src/main/java/org/neo4j/gds/concurrency/PoolSizesService.java +++ b/concurrency-validation-api/src/main/java/org/neo4j/gds/concurrency/PoolSizesService.java @@ -21,7 +21,7 @@ public final class PoolSizesService { - private static PoolSizes instance = new OpenGdsPoolSizes(); + private static PoolSizes instance = null; private PoolSizesService() { } From 65f45f211ef5228376ca49a2953aa1d518730eac Mon Sep 17 00:00:00 2001 From: Mats Rydberg Date: Mon, 11 Sep 2023 10:19:44 +0200 Subject: [PATCH 227/273] Rewrite PoolsTest as PoolSizesServiceTest Co-authored-by: Paul Horn --- .../neo4j/gds/concurrency/PoolSizesService.java | 4 +++- .../{PoolsTest.java => PoolSizesServiceTest.java} | 14 +++++--------- 2 files changed, 8 insertions(+), 10 deletions(-) rename core/src/test/java/org/neo4j/gds/core/concurrency/{PoolsTest.java => PoolSizesServiceTest.java} (69%) diff --git a/concurrency-validation-api/src/main/java/org/neo4j/gds/concurrency/PoolSizesService.java b/concurrency-validation-api/src/main/java/org/neo4j/gds/concurrency/PoolSizesService.java index 73e11c12fd..77714b6277 100644 --- a/concurrency-validation-api/src/main/java/org/neo4j/gds/concurrency/PoolSizesService.java +++ b/concurrency-validation-api/src/main/java/org/neo4j/gds/concurrency/PoolSizesService.java @@ -19,6 +19,8 @@ */ package org.neo4j.gds.concurrency; +import java.util.Objects; + public final class PoolSizesService { private static PoolSizes instance = null; @@ -31,6 +33,6 @@ public static void poolSizes(PoolSizes poolSizes) { } public static PoolSizes poolSizes() { - return instance; + return Objects.requireNonNull(instance); } } diff --git a/core/src/test/java/org/neo4j/gds/core/concurrency/PoolsTest.java b/core/src/test/java/org/neo4j/gds/core/concurrency/PoolSizesServiceTest.java similarity index 69% rename from core/src/test/java/org/neo4j/gds/core/concurrency/PoolsTest.java rename to core/src/test/java/org/neo4j/gds/core/concurrency/PoolSizesServiceTest.java index 3e88180c7a..6189572cf5 100644 --- a/core/src/test/java/org/neo4j/gds/core/concurrency/PoolsTest.java +++ b/core/src/test/java/org/neo4j/gds/core/concurrency/PoolSizesServiceTest.java @@ -20,19 +20,15 @@ package org.neo4j.gds.core.concurrency; import org.junit.jupiter.api.Test; +import org.neo4j.gds.concurrency.PoolSizesService; -import java.util.concurrent.ThreadPoolExecutor; +import static org.assertj.core.api.AssertionsForClassTypes.assertThatThrownBy; -import static org.junit.jupiter.api.Assertions.assertEquals; - -class PoolsTest { +class PoolSizesServiceTest { @Test - void shouldGetLimitedPool() { - ThreadPoolExecutor defaultPool = (ThreadPoolExecutor) Pools.createDefaultPool(); - - assertEquals(4, defaultPool.getCorePoolSize()); - assertEquals(4, defaultPool.getMaximumPoolSize()); + void throwOnAccessBeforeProperInitialisation() { + assertThatThrownBy(PoolSizesService::poolSizes).isInstanceOf(NullPointerException.class); } } From b7adb6ad932f547fd8f0f0e540d05f3609287a3c Mon Sep 17 00:00:00 2001 From: Mats Rydberg Date: Mon, 11 Sep 2023 10:23:58 +0200 Subject: [PATCH 228/273] Move createDefaultPool method over to ExecutorServices Co-authored-by: Paul Horn --- .../core/concurrency/ExecutorServices.java | 20 ++++++++++++++++++- .../org/neo4j/gds/core/concurrency/Pools.java | 15 -------------- 2 files changed, 19 insertions(+), 16 deletions(-) diff --git a/core/src/main/java/org/neo4j/gds/core/concurrency/ExecutorServices.java b/core/src/main/java/org/neo4j/gds/core/concurrency/ExecutorServices.java index 977f8d467a..2fa07691c4 100644 --- a/core/src/main/java/org/neo4j/gds/core/concurrency/ExecutorServices.java +++ b/core/src/main/java/org/neo4j/gds/core/concurrency/ExecutorServices.java @@ -19,11 +19,29 @@ */ package org.neo4j.gds.core.concurrency; +import org.neo4j.gds.concurrency.PoolSizes; +import org.neo4j.gds.concurrency.PoolSizesService; + +import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.ExecutorService; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; public final class ExecutorServices { - public static final ExecutorService DEFAULT = Pools.createDefaultPool(); + public static final ExecutorService DEFAULT = createDefaultPool(PoolSizesService.poolSizes()); + + private static ExecutorService createDefaultPool(PoolSizes poolSizes) { + return new ThreadPoolExecutor( + poolSizes.corePoolSize(), + poolSizes.maxPoolSize(), + 30L, + TimeUnit.SECONDS, + new ArrayBlockingQueue<>(poolSizes.corePoolSize() * 50), + Pools.DEFAULT_THREAD_FACTORY, + new Pools.CallerBlocksPolicy() + ); + } private ExecutorServices() {} } diff --git a/core/src/main/java/org/neo4j/gds/core/concurrency/Pools.java b/core/src/main/java/org/neo4j/gds/core/concurrency/Pools.java index f259230bd2..f25a3e2e84 100644 --- a/core/src/main/java/org/neo4j/gds/core/concurrency/Pools.java +++ b/core/src/main/java/org/neo4j/gds/core/concurrency/Pools.java @@ -19,10 +19,8 @@ */ package org.neo4j.gds.core.concurrency; -import org.neo4j.gds.concurrency.PoolSizesService; import org.neo4j.internal.helpers.NamedThreadFactory; -import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.BlockingQueue; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; @@ -46,19 +44,6 @@ private Pools() { throw new UnsupportedOperationException(); } - public static ExecutorService createDefaultPool() { - var poolSizes = PoolSizesService.poolSizes(); - return new ThreadPoolExecutor( - poolSizes.corePoolSize(), - poolSizes.maxPoolSize(), - 30L, - TimeUnit.SECONDS, - new ArrayBlockingQueue<>(poolSizes.corePoolSize() * 50), - DEFAULT_THREAD_FACTORY, - new CallerBlocksPolicy() - ); - } - public static ExecutorService createSingleThreadPool(String threadPrefix) { return Executors.newSingleThreadExecutor(NamedThreadFactory.daemon(threadPrefix)); } From 4bc7930b856c029630c21766f958623203cac765 Mon Sep 17 00:00:00 2001 From: Mats Rydberg Date: Mon, 11 Sep 2023 10:29:42 +0200 Subject: [PATCH 229/273] Rename to `DefaultPool.INSTANCE` Co-authored-by: Paul Horn --- .../AbstractCentralityResultBuilder.java | 4 +- .../AbstractCommunityResultBuilder.java | 4 +- .../gds/result/CommunityStatisticsTest.java | 8 ++-- .../org/neo4j/gds/scaling/CenterTest.java | 6 +-- .../org/neo4j/gds/scaling/L1NormTest.java | 8 ++-- .../org/neo4j/gds/scaling/L2NormTest.java | 8 ++-- .../org/neo4j/gds/scaling/LogScalerTest.java | 4 +- .../java/org/neo4j/gds/scaling/MaxTest.java | 8 ++-- .../java/org/neo4j/gds/scaling/MeanTest.java | 8 ++-- .../org/neo4j/gds/scaling/MinMaxTest.java | 8 ++-- .../org/neo4j/gds/scaling/ScalerTest.java | 6 +-- .../org/neo4j/gds/scaling/StdScoreTest.java | 8 ++-- .../approxmaxkcut/ApproxMaxKCutFactory.java | 4 +- .../BetweennessCentralityFactory.java | 4 +- .../closeness/ClosenessCentralityFactory.java | 4 +- .../gds/conductance/ConductanceFactory.java | 4 +- .../gds/degree/DegreeCentralityFactory.java | 4 +- .../neo4j/gds/embeddings/fastrp/FastRP.java | 4 +- .../algo/GraphSageAlgorithmFactory.java | 4 +- .../algo/GraphSageTrainAlgorithmFactory.java | 4 +- .../gds/embeddings/node2vec/Node2Vec.java | 4 +- .../HarmonicCentralityAlgorithmFactory.java | 4 +- .../InverseRelationshipsAlgorithmFactory.java | 4 +- .../CELFAlgorithmFactory.java | 4 +- .../gds/k1coloring/K1ColoringFactory.java | 4 +- .../gds/kmeans/KmeansAlgorithmFactory.java | 4 +- .../org/neo4j/gds/kmeans/KmeansContext.java | 4 +- .../LabelPropagationFactory.java | 4 +- .../java/org/neo4j/gds/leiden/Leiden.java | 4 +- .../gds/louvain/LouvainAlgorithmFactory.java | 4 +- .../ModularityOptimizationFactory.java | 4 +- .../pagerank/PageRankAlgorithmFactory.java | 6 +-- .../gds/paths/delta/DeltaSteppingFactory.java | 4 +- .../org/neo4j/gds/paths/traverse/BFS.java | 4 +- .../java/org/neo4j/gds/paths/yens/Yens.java | 4 +- .../ScalePropertiesFactory.java | 4 +- .../filteredknn/FilteredKnnFactory.java | 4 +- .../FilteredNodeSimilarityFactory.java | 4 +- .../neo4j/gds/similarity/knn/KnnContext.java | 4 +- .../neo4j/gds/similarity/knn/KnnFactory.java | 4 +- .../nodesim/NodeSimilarityFactory.java | 4 +- .../steiner/SteinerTreeAlgorithmFactory.java | 4 +- .../traversal/RandomWalkAlgorithmFactory.java | 4 +- .../IntersectingTriangleCountFactory.java | 4 +- .../gds/triangle/TriangleStreamFactory.java | 4 +- .../ToUndirectedAlgorithmFactory.java | 4 +- .../walking/CollapsePathAlgorithmFactory.java | 4 +- .../neo4j/gds/wcc/WccAlgorithmFactory.java | 4 +- .../MSBFSAllShortestPathsTest.java | 4 +- .../WeightedAllShortestPathsTest.java | 6 +-- .../gds/approxmaxkcut/ApproxMaxKCutTest.java | 6 +-- .../BetweennessCentralityTest.java | 6 +-- .../betweenness/SelectionStrategyTest.java | 24 +++++------ .../WeightedBetweennessCentralityTest.java | 8 ++-- .../ClosenessCentralityDirectedTest.java | 6 +-- .../ClosenessCentralityDiscoTest.java | 4 +- .../closeness/ClosenessCentralityTest.java | 6 +-- .../gds/conductance/ConductanceTest.java | 4 +- .../gds/degree/DegreeCentralityTest.java | 8 ++-- .../gds/embeddings/fastrp/FastRPTest.java | 6 +-- .../GraphSageEmbeddingsGeneratorTest.java | 14 +++---- .../graphsage/GraphSageModelTrainerTest.java | 30 +++++++------- .../embeddings/graphsage/GraphSageTest.java | 8 ++-- .../algo/MultiLabelGraphSageTrainTest.java | 12 +++--- .../gds/embeddings/hashgnn/HashGNNTest.java | 6 +-- .../gds/embeddings/node2vec/Node2VecTest.java | 6 +-- .../gds/harmonic/HarmonicCentralityTest.java | 6 +-- .../InverseRelationshipsTest.java | 6 +-- .../CELFOnConnectedGraphTest.java | 4 +- .../CELFOnTreeGraphTest.java | 4 +- .../gds/influenceMaximization/CelfTest.java | 4 +- .../neo4j/gds/k1coloring/K1ColoringTest.java | 10 ++--- .../LabelPropagationTest.java | 10 ++--- .../NonStabilizingLabelPropagationTest.java | 4 +- .../gds/leiden/GraphWithSelfLoopTest.java | 8 ++-- .../gds/leiden/ModularityComputerTest.java | 8 ++-- .../gds/leiden/RefinementPhaseKarateTest.java | 4 +- .../neo4j/gds/leiden/RefinementPhaseTest.java | 4 +- .../WeightedModularityComputerTest.java | 8 ++-- .../org/neo4j/gds/louvain/LouvainTest.java | 20 +++++----- .../modularityoptimization/FootballTest.java | 4 +- .../ModularityOptimizationTest.java | 4 +- ...ityOptimizationWithoutOrientationTest.java | 4 +- .../MultiSourceBFSAccessMethodsTest.java | 14 +++---- .../gds/paths/delta/DeltaSteppingTest.java | 14 +++---- .../ScalePropertiesMissingPropsTest.java | 6 +-- .../scaleproperties/ScalePropertiesTest.java | 26 ++++++------ .../NodeSimilarityTerminationTest.java | 4 +- .../nodesim/NodeSimilarityTest.java | 40 +++++++++---------- .../nodesim/SimilarityGraphBuilderTest.java | 8 ++-- .../UnionGraphWeightedNodeSimilarityTest.java | 4 +- ...rtestPathSteinerAlgorithmExtendedTest.java | 12 +++--- ...estPathsSteinerAlgorithmReroutingTest.java | 24 +++++------ .../ShortestPathsSteinerAlgorithmTest.java | 4 +- .../neo4j/gds/traversal/RandomWalkTest.java | 22 +++++----- ...sectingTriangleCountFilteredGraphTest.java | 4 +- .../IntersectingTriangleCountTest.java | 4 +- .../LargeIntersectingTriangleCountTest.java | 6 +-- .../gds/triangle/TriangleStreamTest.java | 6 +-- .../UnionGraphTriangleCountingTest.java | 4 +- .../gds/undirected/ToUndirectedTest.java | 14 +++---- .../gds/walking/CollapseMultiPathsTest.java | 6 +-- .../neo4j/gds/walking/CollapsePathTest.java | 8 ++-- .../org/neo4j/gds/wcc/IncrementalWccTest.java | 4 +- .../test/java/org/neo4j/gds/wcc/WccTest.java | 4 +- .../org/neo4j/gds/wcc/WccThresholdTest.java | 4 +- .../java/org/neo4j/gds/pregel/HitsTest.java | 4 +- .../gds/pregel/SpeakerListenerLPATest.java | 8 ++-- .../NativeRelationshipStreamExporter.java | 4 +- .../write/NativeNodeLabelExporterTest.java | 4 +- .../write/NativeNodePropertyExporterTest.java | 6 +-- .../org/neo4j/gds/api/GraphLoaderContext.java | 4 +- ...ExecutorServices.java => DefaultPool.java} | 6 +-- .../core/concurrency/RunWithConcurrency.java | 4 +- .../core/loading/ArrayIdMapBuilderOps.java | 4 +- .../loading/construction/GraphFactory.java | 4 +- .../DoubleArrayNodePropertiesBuilder.java | 4 +- .../DoubleNodePropertiesBuilder.java | 4 +- .../FloatArrayNodePropertiesBuilder.java | 4 +- .../LongArrayNodePropertiesBuilder.java | 4 +- .../LongNodePropertiesBuilder.java | 4 +- .../java/org/neo4j/gds/TerminationTest.java | 4 +- .../java/org/neo4j/gds/core/LoadingTest.java | 4 +- .../varlong/TransientCsrListTest.java | 4 +- .../core/loading/LazyIdMapBuilderTest.java | 4 +- .../ScanningRelationshipsImporterTest.java | 4 +- .../paged/HugeAtomicGrowingBitSetTest.java | 4 +- .../utils/paged/ShardedLongLongMapTest.java | 4 +- .../beta/pregel/bfs/BFSPregelAlgoTest.java | 8 ++-- .../cc/ConnectedComponentsPregelAlgoTest.java | 4 +- .../lp/LabelPropagationPregelAlgoTest.java | 6 +-- .../pregel/pr/PageRankPregelAlgoTest.java | 4 +- .../pr/WeightedPageRankPregelAlgoTest.java | 4 +- ...ingleSourceShortestPathPregelAlgoTest.java | 4 +- .../TriangleCountPregelTest.java | 4 +- .../graphsampling/GraphSampleConstructor.java | 4 +- .../io/file/FileToGraphStoreImporter.java | 8 ++-- .../neo4j/gds/ml/splitting/EdgeSplitter.java | 4 +- .../pregel/generator/AlgorithmGenerator.java | 6 +-- .../BidirectionalComputationAlgorithm.java | 4 +- .../expected/ComputationAlgorithm.java | 4 +- .../org/neo4j/gds/beta/pregel/PregelTest.java | 28 ++++++------- .../neo4j/gds/catalog/GraphProjectProc.java | 4 +- .../gds/catalog/GraphWriteNodeLabelProc.java | 6 +-- .../neo4j/gds/catalog/NodeLabelMutator.java | 4 +- .../gds/catalog/NodePropertiesWriter.java | 4 +- .../gds/catalog/GraphProjectProcTest.java | 4 +- .../org/neo4j/gds/CommunityProcCompanion.java | 4 +- ...dePropertiesComputationResultConsumer.java | 4 +- .../org/neo4j/gds/kmeans/KmeansWriteSpec.java | 4 +- .../predict/ApproximateLinkPrediction.java | 4 +- ...redictionPipelineMutateResultConsumer.java | 4 +- .../paths/all/AllShortestPathsStreamSpec.java | 6 +-- .../randomwalk/RandomWalkStreamProcTest.java | 4 +- .../neo4j/gds/pregel/proc/PregelProcTest.java | 4 +- .../FilteredNodeSimilarityMutateSpec.java | 4 +- .../NodeSimilarityMutateSpecification.java | 4 +- .../org/neo4j/gds/GraphLoaderBuilders.java | 4 +- .../gds/core/idmap/IdMapBuilderTest.java | 4 +- 159 files changed, 502 insertions(+), 502 deletions(-) rename core/src/main/java/org/neo4j/gds/core/concurrency/{ExecutorServices.java => DefaultPool.java} (89%) diff --git a/algo-common/src/main/java/org/neo4j/gds/result/AbstractCentralityResultBuilder.java b/algo-common/src/main/java/org/neo4j/gds/result/AbstractCentralityResultBuilder.java index d8f79fe932..7fb7529b9c 100644 --- a/algo-common/src/main/java/org/neo4j/gds/result/AbstractCentralityResultBuilder.java +++ b/algo-common/src/main/java/org/neo4j/gds/result/AbstractCentralityResultBuilder.java @@ -22,7 +22,7 @@ import org.HdrHistogram.DoubleHistogram; import org.jetbrains.annotations.NotNull; import org.neo4j.gds.api.ProcedureReturnColumns; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.ProgressTimer; import org.neo4j.gds.scaling.LogScaler; import org.neo4j.gds.scaling.ScalerFactory; @@ -95,7 +95,7 @@ private Optional computeCentralityHistogram() { return Optional.of(CentralityStatistics.histogram( nodeCount, centralityFunction, - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, concurrency )); } catch (ArrayIndexOutOfBoundsException e) { diff --git a/algo-common/src/main/java/org/neo4j/gds/result/AbstractCommunityResultBuilder.java b/algo-common/src/main/java/org/neo4j/gds/result/AbstractCommunityResultBuilder.java index 27bca6b6c0..58ee87adef 100644 --- a/algo-common/src/main/java/org/neo4j/gds/result/AbstractCommunityResultBuilder.java +++ b/algo-common/src/main/java/org/neo4j/gds/result/AbstractCommunityResultBuilder.java @@ -22,7 +22,7 @@ import org.HdrHistogram.Histogram; import org.jetbrains.annotations.Nullable; import org.neo4j.gds.api.ProcedureReturnColumns; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.ProgressTimer; import java.util.Map; @@ -54,7 +54,7 @@ protected AbstractCommunityResultBuilder( ProcedureReturnColumns returnColumns, int concurrency ) { - this(returnColumns, ExecutorServices.DEFAULT, concurrency); + this(returnColumns, DefaultPool.INSTANCE, concurrency); } protected AbstractCommunityResultBuilder( diff --git a/algo-common/src/test/java/org/neo4j/gds/result/CommunityStatisticsTest.java b/algo-common/src/test/java/org/neo4j/gds/result/CommunityStatisticsTest.java index a5cd2274ba..5063bad3c1 100644 --- a/algo-common/src/test/java/org/neo4j/gds/result/CommunityStatisticsTest.java +++ b/algo-common/src/test/java/org/neo4j/gds/result/CommunityStatisticsTest.java @@ -24,7 +24,7 @@ import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; import org.neo4j.gds.TestSupport; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import java.util.HashMap; import java.util.Map; @@ -49,7 +49,7 @@ void communitySizes( var communitySizes = CommunityStatistics.communitySizes( nodeCount, communityFunction, - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, concurrency ); expectedCommunitySizes.forEach((communityId, expectedSize) -> { @@ -72,7 +72,7 @@ void communityCount( CommunityStatistics.communityCount( nodeCount, communityFunction, - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, concurrency ) ); @@ -91,7 +91,7 @@ void communityCountAndHistogram( var communityCountAndHistogram = CommunityStatistics.communityCountAndHistogram( nodeCount, communityFunction, - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, concurrency ); diff --git a/algo-common/src/test/java/org/neo4j/gds/scaling/CenterTest.java b/algo-common/src/test/java/org/neo4j/gds/scaling/CenterTest.java index f45bf10998..20ad760ed7 100644 --- a/algo-common/src/test/java/org/neo4j/gds/scaling/CenterTest.java +++ b/algo-common/src/test/java/org/neo4j/gds/scaling/CenterTest.java @@ -26,7 +26,7 @@ import org.junit.jupiter.params.provider.MethodSource; import org.neo4j.gds.api.properties.nodes.NodePropertyValues; import org.neo4j.gds.core.CypherMapWrapper; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.nodeproperties.DoubleTestPropertyValues; @@ -63,7 +63,7 @@ void normalizes(NodePropertyValues properties, double avg, double[] expected) { 10, 1, ProgressTracker.NULL_TRACKER, - ExecutorServices.DEFAULT + DefaultPool.INSTANCE ); assertThat(scaler.avg).isEqualTo(avg); @@ -81,7 +81,7 @@ void handlesMissingValue() { 10, 1, ProgressTracker.NULL_TRACKER, - ExecutorServices.DEFAULT + DefaultPool.INSTANCE ); var avg = 4.444; diff --git a/algo-common/src/test/java/org/neo4j/gds/scaling/L1NormTest.java b/algo-common/src/test/java/org/neo4j/gds/scaling/L1NormTest.java index 38532f316d..10b7cef741 100644 --- a/algo-common/src/test/java/org/neo4j/gds/scaling/L1NormTest.java +++ b/algo-common/src/test/java/org/neo4j/gds/scaling/L1NormTest.java @@ -26,7 +26,7 @@ import org.junit.jupiter.params.provider.MethodSource; import org.neo4j.gds.api.properties.nodes.NodePropertyValues; import org.neo4j.gds.core.CypherMapWrapper; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.nodeproperties.DoubleTestPropertyValues; @@ -62,7 +62,7 @@ void scale(int nodeCount, NodePropertyValues properties, double l1norm, double[] nodeCount, 1, ProgressTracker.NULL_TRACKER, - ExecutorServices.DEFAULT + DefaultPool.INSTANCE ); assertThat(scaler.l1Norm).isEqualTo(l1norm); @@ -79,7 +79,7 @@ void avoidsDivByZero() { 10, 1, ProgressTracker.NULL_TRACKER, - ExecutorServices.DEFAULT + DefaultPool.INSTANCE ); for (int i = 0; i < 10; i++) { @@ -95,7 +95,7 @@ void handlesMissingValue() { 10, 1, ProgressTracker.NULL_TRACKER, - ExecutorServices.DEFAULT + DefaultPool.INSTANCE ); var l1norm = 40D; diff --git a/algo-common/src/test/java/org/neo4j/gds/scaling/L2NormTest.java b/algo-common/src/test/java/org/neo4j/gds/scaling/L2NormTest.java index 89f0b509df..b8f93a15f2 100644 --- a/algo-common/src/test/java/org/neo4j/gds/scaling/L2NormTest.java +++ b/algo-common/src/test/java/org/neo4j/gds/scaling/L2NormTest.java @@ -26,7 +26,7 @@ import org.junit.jupiter.params.provider.MethodSource; import org.neo4j.gds.api.properties.nodes.NodePropertyValues; import org.neo4j.gds.core.CypherMapWrapper; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.nodeproperties.DoubleTestPropertyValues; @@ -53,7 +53,7 @@ void normalizes(NodePropertyValues properties, double euclideanLength, double[] 10, 1, ProgressTracker.NULL_TRACKER, - ExecutorServices.DEFAULT + DefaultPool.INSTANCE ); assertThat(scaler.euclideanLength).isEqualTo(euclideanLength); @@ -70,7 +70,7 @@ void avoidsDivByZero() { 10, 1, ProgressTracker.NULL_TRACKER, - ExecutorServices.DEFAULT + DefaultPool.INSTANCE ); for (int i = 0; i < 10; i++) { @@ -86,7 +86,7 @@ void handlesMissingValue() { 10, 1, ProgressTracker.NULL_TRACKER, - ExecutorServices.DEFAULT + DefaultPool.INSTANCE ); var euclideanLength = 16.124; diff --git a/algo-common/src/test/java/org/neo4j/gds/scaling/LogScalerTest.java b/algo-common/src/test/java/org/neo4j/gds/scaling/LogScalerTest.java index 390733cb23..680bfd0560 100644 --- a/algo-common/src/test/java/org/neo4j/gds/scaling/LogScalerTest.java +++ b/algo-common/src/test/java/org/neo4j/gds/scaling/LogScalerTest.java @@ -26,7 +26,7 @@ import org.junit.jupiter.params.provider.MethodSource; import org.neo4j.gds.api.properties.nodes.NodePropertyValues; import org.neo4j.gds.core.CypherMapWrapper; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.nodeproperties.DoubleTestPropertyValues; import org.neo4j.gds.nodeproperties.LongTestPropertyValues; @@ -72,7 +72,7 @@ void handlesMissingValue() { 10, 1, ProgressTracker.NULL_TRACKER, - ExecutorServices.DEFAULT + DefaultPool.INSTANCE ); for (int i = 0; i < 5; i++) { diff --git a/algo-common/src/test/java/org/neo4j/gds/scaling/MaxTest.java b/algo-common/src/test/java/org/neo4j/gds/scaling/MaxTest.java index f381e52be2..9d50bbb5af 100644 --- a/algo-common/src/test/java/org/neo4j/gds/scaling/MaxTest.java +++ b/algo-common/src/test/java/org/neo4j/gds/scaling/MaxTest.java @@ -26,7 +26,7 @@ import org.junit.jupiter.params.provider.MethodSource; import org.neo4j.gds.api.properties.nodes.NodePropertyValues; import org.neo4j.gds.core.CypherMapWrapper; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.nodeproperties.DoubleTestPropertyValues; @@ -70,7 +70,7 @@ void scale(int nodeCount, NodePropertyValues properties, double absMax, double[] nodeCount, 1, ProgressTracker.NULL_TRACKER, - ExecutorServices.DEFAULT + DefaultPool.INSTANCE ); assertThat(scaler.maxAbs).isEqualTo(absMax); @@ -88,7 +88,7 @@ void avoidsDivByZero() { 10, 1, ProgressTracker.NULL_TRACKER, - ExecutorServices.DEFAULT + DefaultPool.INSTANCE ); assertThat(scaler.statistics()).containsExactlyEntriesOf(Map.of("absMax", List.of(0D))); @@ -106,7 +106,7 @@ void handlesMissingValue() { 10, 1, ProgressTracker.NULL_TRACKER, - ExecutorServices.DEFAULT + DefaultPool.INSTANCE ); diff --git a/algo-common/src/test/java/org/neo4j/gds/scaling/MeanTest.java b/algo-common/src/test/java/org/neo4j/gds/scaling/MeanTest.java index f0f6411d3e..39b5a4bf82 100644 --- a/algo-common/src/test/java/org/neo4j/gds/scaling/MeanTest.java +++ b/algo-common/src/test/java/org/neo4j/gds/scaling/MeanTest.java @@ -26,7 +26,7 @@ import org.junit.jupiter.params.provider.MethodSource; import org.neo4j.gds.api.properties.nodes.NodePropertyValues; import org.neo4j.gds.core.CypherMapWrapper; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.nodeproperties.DoubleTestPropertyValues; @@ -57,7 +57,7 @@ void normalizes(NodePropertyValues properties, double avg, double min, double ma 10, 1, ProgressTracker.NULL_TRACKER, - ExecutorServices.DEFAULT + DefaultPool.INSTANCE ); assertThat(scaler.avg).isEqualTo(avg); @@ -81,7 +81,7 @@ void avoidsDivByZero() { 10, 1, ProgressTracker.NULL_TRACKER, - ExecutorServices.DEFAULT + DefaultPool.INSTANCE ); assertThat(scaler.statistics()).containsExactlyInAnyOrderEntriesOf(Map.of( @@ -103,7 +103,7 @@ void handlesMissingValue() { 10, 1, ProgressTracker.NULL_TRACKER, - ExecutorServices.DEFAULT + DefaultPool.INSTANCE ); var avg = 4.444; diff --git a/algo-common/src/test/java/org/neo4j/gds/scaling/MinMaxTest.java b/algo-common/src/test/java/org/neo4j/gds/scaling/MinMaxTest.java index b0c442413f..edc881c051 100644 --- a/algo-common/src/test/java/org/neo4j/gds/scaling/MinMaxTest.java +++ b/algo-common/src/test/java/org/neo4j/gds/scaling/MinMaxTest.java @@ -26,7 +26,7 @@ import org.junit.jupiter.params.provider.MethodSource; import org.neo4j.gds.api.properties.nodes.NodePropertyValues; import org.neo4j.gds.core.CypherMapWrapper; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.nodeproperties.DoubleTestPropertyValues; @@ -53,7 +53,7 @@ void normalizes(NodePropertyValues properties, double min, double max) { 10, 1, ProgressTracker.NULL_TRACKER, - ExecutorServices.DEFAULT + DefaultPool.INSTANCE ); assertThat(scaler.min).isEqualTo(min); @@ -77,7 +77,7 @@ void avoidsDivByZero() { 10, 1, ProgressTracker.NULL_TRACKER, - ExecutorServices.DEFAULT + DefaultPool.INSTANCE ); assertThat(scaler.statistics()).containsExactlyEntriesOf(Map.of( @@ -98,7 +98,7 @@ void handlesMissingValue() { 10, 1, ProgressTracker.NULL_TRACKER, - ExecutorServices.DEFAULT + DefaultPool.INSTANCE ); diff --git a/algo-common/src/test/java/org/neo4j/gds/scaling/ScalerTest.java b/algo-common/src/test/java/org/neo4j/gds/scaling/ScalerTest.java index 9896bf61aa..a5f6c2966a 100644 --- a/algo-common/src/test/java/org/neo4j/gds/scaling/ScalerTest.java +++ b/algo-common/src/test/java/org/neo4j/gds/scaling/ScalerTest.java @@ -21,7 +21,7 @@ import org.junit.jupiter.api.Test; import org.neo4j.gds.core.CypherMapWrapper; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.nodeproperties.DoubleTestPropertyValues; @@ -39,14 +39,14 @@ void shouldAccumulateStatsCorrectly() { 10, 1, ProgressTracker.NULL_TRACKER, - ExecutorServices.DEFAULT + DefaultPool.INSTANCE ); var meanScaler2 = (Mean) Mean.buildFrom(CypherMapWrapper.empty()).create( new DoubleTestPropertyValues(nodeId -> 2 * nodeId), 10, 1, ProgressTracker.NULL_TRACKER, - ExecutorServices.DEFAULT + DefaultPool.INSTANCE ); var meanScaler1Stats = meanScaler1.statistics(); diff --git a/algo-common/src/test/java/org/neo4j/gds/scaling/StdScoreTest.java b/algo-common/src/test/java/org/neo4j/gds/scaling/StdScoreTest.java index 9f4857704c..236975d31f 100644 --- a/algo-common/src/test/java/org/neo4j/gds/scaling/StdScoreTest.java +++ b/algo-common/src/test/java/org/neo4j/gds/scaling/StdScoreTest.java @@ -26,7 +26,7 @@ import org.junit.jupiter.params.provider.MethodSource; import org.neo4j.gds.api.properties.nodes.NodePropertyValues; import org.neo4j.gds.core.CypherMapWrapper; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.nodeproperties.DoubleTestPropertyValues; @@ -56,7 +56,7 @@ void normalizes(NodePropertyValues properties, double avg, double std, double[] 10, 1, ProgressTracker.NULL_TRACKER, - ExecutorServices.DEFAULT + DefaultPool.INSTANCE ); assertThat(scaler.avg).isEqualTo(avg); @@ -78,7 +78,7 @@ void handlesMissingValue() { 10, 1, ProgressTracker.NULL_TRACKER, - ExecutorServices.DEFAULT + DefaultPool.INSTANCE ); var avg = 4.444; @@ -100,7 +100,7 @@ void avoidsDivByZero() { 10, 1, ProgressTracker.NULL_TRACKER, - ExecutorServices.DEFAULT + DefaultPool.INSTANCE ); for (int i = 0; i < 10; i++) { diff --git a/algo/src/main/java/org/neo4j/gds/approxmaxkcut/ApproxMaxKCutFactory.java b/algo/src/main/java/org/neo4j/gds/approxmaxkcut/ApproxMaxKCutFactory.java index 58d1e86c85..617846da09 100644 --- a/algo/src/main/java/org/neo4j/gds/approxmaxkcut/ApproxMaxKCutFactory.java +++ b/algo/src/main/java/org/neo4j/gds/approxmaxkcut/ApproxMaxKCutFactory.java @@ -24,7 +24,7 @@ import org.neo4j.gds.approxmaxkcut.config.ApproxMaxKCutConfig; import org.neo4j.gds.collections.haa.HugeAtomicByteArray; import org.neo4j.gds.collections.haa.HugeAtomicDoubleArray; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.mem.MemoryEstimation; import org.neo4j.gds.core.utils.mem.MemoryEstimations; import org.neo4j.gds.core.utils.paged.HugeByteArray; @@ -51,7 +51,7 @@ public ApproxMaxKCut build( CONFIG configuration, ProgressTracker progressTracker ) { - return new ApproxMaxKCut(graph, ExecutorServices.DEFAULT, configuration, progressTracker); + return new ApproxMaxKCut(graph, DefaultPool.INSTANCE, configuration, progressTracker); } @Override diff --git a/algo/src/main/java/org/neo4j/gds/betweenness/BetweennessCentralityFactory.java b/algo/src/main/java/org/neo4j/gds/betweenness/BetweennessCentralityFactory.java index c0434491c4..9bae0ded33 100644 --- a/algo/src/main/java/org/neo4j/gds/betweenness/BetweennessCentralityFactory.java +++ b/algo/src/main/java/org/neo4j/gds/betweenness/BetweennessCentralityFactory.java @@ -23,7 +23,7 @@ import org.neo4j.gds.GraphAlgorithmFactory; import org.neo4j.gds.api.Graph; import org.neo4j.gds.collections.haa.HugeAtomicDoubleArray; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.mem.MemoryEstimation; import org.neo4j.gds.core.utils.mem.MemoryEstimations; import org.neo4j.gds.core.utils.paged.HugeDoubleArray; @@ -66,7 +66,7 @@ public BetweennessCentrality build( graph, strategy, traverserFactory, - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, configuration.concurrency(), progressTracker ); diff --git a/algo/src/main/java/org/neo4j/gds/closeness/ClosenessCentralityFactory.java b/algo/src/main/java/org/neo4j/gds/closeness/ClosenessCentralityFactory.java index 5a07ee9767..df3bd6b035 100644 --- a/algo/src/main/java/org/neo4j/gds/closeness/ClosenessCentralityFactory.java +++ b/algo/src/main/java/org/neo4j/gds/closeness/ClosenessCentralityFactory.java @@ -22,7 +22,7 @@ import org.neo4j.gds.GraphAlgorithmFactory; import org.neo4j.gds.api.Graph; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.core.utils.progress.tasks.Task; import org.neo4j.gds.core.utils.progress.tasks.Tasks; @@ -37,7 +37,7 @@ public ClosenessCentrality build( return ClosenessCentrality.of( graph, configuration, - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, progressTracker ); } diff --git a/algo/src/main/java/org/neo4j/gds/conductance/ConductanceFactory.java b/algo/src/main/java/org/neo4j/gds/conductance/ConductanceFactory.java index 751a132c27..a7d1050f31 100644 --- a/algo/src/main/java/org/neo4j/gds/conductance/ConductanceFactory.java +++ b/algo/src/main/java/org/neo4j/gds/conductance/ConductanceFactory.java @@ -21,7 +21,7 @@ import org.neo4j.gds.GraphAlgorithmFactory; import org.neo4j.gds.api.Graph; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.core.utils.progress.tasks.Task; import org.neo4j.gds.core.utils.progress.tasks.Tasks; @@ -43,7 +43,7 @@ public Conductance build( CONFIG configuration, ProgressTracker progressTracker ) { - return new Conductance(graph, ExecutorServices.DEFAULT, configuration, progressTracker); + return new Conductance(graph, DefaultPool.INSTANCE, configuration, progressTracker); } @Override diff --git a/algo/src/main/java/org/neo4j/gds/degree/DegreeCentralityFactory.java b/algo/src/main/java/org/neo4j/gds/degree/DegreeCentralityFactory.java index ecf4dd1705..c3387586ad 100644 --- a/algo/src/main/java/org/neo4j/gds/degree/DegreeCentralityFactory.java +++ b/algo/src/main/java/org/neo4j/gds/degree/DegreeCentralityFactory.java @@ -22,7 +22,7 @@ import org.jetbrains.annotations.NotNull; import org.neo4j.gds.GraphAlgorithmFactory; import org.neo4j.gds.api.Graph; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.mem.MemoryEstimation; import org.neo4j.gds.core.utils.mem.MemoryEstimations; import org.neo4j.gds.core.utils.paged.HugeDoubleArray; @@ -45,7 +45,7 @@ public DegreeCentrality build( CONFIG configuration, ProgressTracker progressTracker ) { - return new DegreeCentrality(graph, ExecutorServices.DEFAULT, configuration, progressTracker); + return new DegreeCentrality(graph, DefaultPool.INSTANCE, configuration, progressTracker); } @Override diff --git a/algo/src/main/java/org/neo4j/gds/embeddings/fastrp/FastRP.java b/algo/src/main/java/org/neo4j/gds/embeddings/fastrp/FastRP.java index daf27b31cd..14c1e4375e 100644 --- a/algo/src/main/java/org/neo4j/gds/embeddings/fastrp/FastRP.java +++ b/algo/src/main/java/org/neo4j/gds/embeddings/fastrp/FastRP.java @@ -22,7 +22,7 @@ import org.jetbrains.annotations.TestOnly; import org.neo4j.gds.Algorithm; import org.neo4j.gds.api.Graph; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.concurrency.ParallelUtil; import org.neo4j.gds.core.concurrency.RunWithConcurrency; import org.neo4j.gds.core.utils.mem.MemoryEstimation; @@ -221,7 +221,7 @@ void propagateEmbeddings() { ); ParallelUtil.parallelPartitionsConsume( - RunWithConcurrency.builder().executor(ExecutorServices.DEFAULT).concurrency(concurrency), + RunWithConcurrency.builder().executor(DefaultPool.INSTANCE).concurrency(concurrency), partitions.stream(), taskSupplier ); diff --git a/algo/src/main/java/org/neo4j/gds/embeddings/graphsage/algo/GraphSageAlgorithmFactory.java b/algo/src/main/java/org/neo4j/gds/embeddings/graphsage/algo/GraphSageAlgorithmFactory.java index d724ec9968..f47eb2e359 100644 --- a/algo/src/main/java/org/neo4j/gds/embeddings/graphsage/algo/GraphSageAlgorithmFactory.java +++ b/algo/src/main/java/org/neo4j/gds/embeddings/graphsage/algo/GraphSageAlgorithmFactory.java @@ -22,7 +22,7 @@ import org.neo4j.gds.GraphStoreAlgorithmFactory; import org.neo4j.gds.api.GraphStore; import org.neo4j.gds.config.MutateConfig; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.model.ModelCatalog; import org.neo4j.gds.core.utils.mem.MemoryEstimation; import org.neo4j.gds.core.utils.mem.MemoryEstimations; @@ -49,7 +49,7 @@ public GraphSageAlgorithmFactory(ModelCatalog modelCatalog) { @Override public GraphSage build(GraphStore graphStore, CONFIG configuration, ProgressTracker progressTracker) { - var executorService = ExecutorServices.DEFAULT; + var executorService = DefaultPool.INSTANCE; var model = resolveModel( modelCatalog, configuration.modelUser(), diff --git a/algo/src/main/java/org/neo4j/gds/embeddings/graphsage/algo/GraphSageTrainAlgorithmFactory.java b/algo/src/main/java/org/neo4j/gds/embeddings/graphsage/algo/GraphSageTrainAlgorithmFactory.java index b7938da074..358316dbe2 100644 --- a/algo/src/main/java/org/neo4j/gds/embeddings/graphsage/algo/GraphSageTrainAlgorithmFactory.java +++ b/algo/src/main/java/org/neo4j/gds/embeddings/graphsage/algo/GraphSageTrainAlgorithmFactory.java @@ -21,7 +21,7 @@ import org.neo4j.gds.GraphAlgorithmFactory; import org.neo4j.gds.api.Graph; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.mem.MemoryEstimation; import org.neo4j.gds.core.utils.mem.MemoryEstimations; import org.neo4j.gds.core.utils.mem.MemoryRange; @@ -58,7 +58,7 @@ public GraphSageTrain build( GraphSageTrainConfig configuration, ProgressTracker progressTracker ) { - var executorService = ExecutorServices.DEFAULT; + var executorService = DefaultPool.INSTANCE; if(configuration.hasRelationshipWeightProperty()) { validateRelationshipWeightPropertyValue(graph, configuration.concurrency(), executorService); } diff --git a/algo/src/main/java/org/neo4j/gds/embeddings/node2vec/Node2Vec.java b/algo/src/main/java/org/neo4j/gds/embeddings/node2vec/Node2Vec.java index 86de6046d2..48f7d1cba5 100644 --- a/algo/src/main/java/org/neo4j/gds/embeddings/node2vec/Node2Vec.java +++ b/algo/src/main/java/org/neo4j/gds/embeddings/node2vec/Node2Vec.java @@ -21,7 +21,7 @@ import org.neo4j.gds.Algorithm; import org.neo4j.gds.api.Graph; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.mem.MemoryEstimation; import org.neo4j.gds.core.utils.mem.MemoryEstimations; import org.neo4j.gds.core.utils.paged.HugeObjectArray; @@ -60,7 +60,7 @@ public Node2VecModel.Result compute() { graph, config, progressTracker, - ExecutorServices.DEFAULT + DefaultPool.INSTANCE ); var probabilitiesBuilder = new RandomWalkProbabilities.Builder( diff --git a/algo/src/main/java/org/neo4j/gds/harmonic/HarmonicCentralityAlgorithmFactory.java b/algo/src/main/java/org/neo4j/gds/harmonic/HarmonicCentralityAlgorithmFactory.java index 2393a83afb..879f6fa54b 100644 --- a/algo/src/main/java/org/neo4j/gds/harmonic/HarmonicCentralityAlgorithmFactory.java +++ b/algo/src/main/java/org/neo4j/gds/harmonic/HarmonicCentralityAlgorithmFactory.java @@ -21,7 +21,7 @@ import org.neo4j.gds.GraphAlgorithmFactory; import org.neo4j.gds.api.Graph; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; public class HarmonicCentralityAlgorithmFactory extends GraphAlgorithmFactory { @@ -39,7 +39,7 @@ public HarmonicCentrality build( return new HarmonicCentrality( graph, configuration.concurrency(), - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, progressTracker ); } diff --git a/algo/src/main/java/org/neo4j/gds/indexInverse/InverseRelationshipsAlgorithmFactory.java b/algo/src/main/java/org/neo4j/gds/indexInverse/InverseRelationshipsAlgorithmFactory.java index a4c5912508..e1a9eb96a1 100644 --- a/algo/src/main/java/org/neo4j/gds/indexInverse/InverseRelationshipsAlgorithmFactory.java +++ b/algo/src/main/java/org/neo4j/gds/indexInverse/InverseRelationshipsAlgorithmFactory.java @@ -23,7 +23,7 @@ import org.neo4j.gds.GraphStoreAlgorithmFactory; import org.neo4j.gds.RelationshipType; import org.neo4j.gds.api.GraphStore; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.loading.AdjacencyListBehavior; import org.neo4j.gds.core.utils.mem.MemoryEstimation; import org.neo4j.gds.core.utils.mem.MemoryEstimations; @@ -45,7 +45,7 @@ public InverseRelationships build( InverseRelationshipsConfig configuration, ProgressTracker progressTracker ) { - return new InverseRelationships(graphStore, configuration, progressTracker, ExecutorServices.DEFAULT); + return new InverseRelationships(graphStore, configuration, progressTracker, DefaultPool.INSTANCE); } @Override diff --git a/algo/src/main/java/org/neo4j/gds/influenceMaximization/CELFAlgorithmFactory.java b/algo/src/main/java/org/neo4j/gds/influenceMaximization/CELFAlgorithmFactory.java index 08b5b6145d..f479f7d5d6 100644 --- a/algo/src/main/java/org/neo4j/gds/influenceMaximization/CELFAlgorithmFactory.java +++ b/algo/src/main/java/org/neo4j/gds/influenceMaximization/CELFAlgorithmFactory.java @@ -21,7 +21,7 @@ import org.neo4j.gds.GraphAlgorithmFactory; import org.neo4j.gds.api.Graph; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.mem.MemoryEstimation; import org.neo4j.gds.core.utils.mem.MemoryEstimations; import org.neo4j.gds.core.utils.mem.MemoryRange; @@ -53,7 +53,7 @@ public CELF build( configuration.seedSetSize(), configuration.propagationProbability(), configuration.monteCarloSimulations(), - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, configuration.concurrency(), configuration.randomSeed().orElse(0L), DEFAULT_BATCH_SIZE, diff --git a/algo/src/main/java/org/neo4j/gds/k1coloring/K1ColoringFactory.java b/algo/src/main/java/org/neo4j/gds/k1coloring/K1ColoringFactory.java index ad5e6c1a4c..2e4e069485 100644 --- a/algo/src/main/java/org/neo4j/gds/k1coloring/K1ColoringFactory.java +++ b/algo/src/main/java/org/neo4j/gds/k1coloring/K1ColoringFactory.java @@ -23,7 +23,7 @@ import org.neo4j.gds.api.Graph; import org.neo4j.gds.config.BaseConfig; import org.neo4j.gds.config.IterationsConfig; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.mem.MemoryEstimation; import org.neo4j.gds.core.utils.mem.MemoryEstimations; import org.neo4j.gds.core.utils.paged.HugeLongArray; @@ -54,7 +54,7 @@ public K1Coloring build( configuration.maxIterations(), configuration.batchSize(), configuration.concurrency(), - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, progressTracker ); } diff --git a/algo/src/main/java/org/neo4j/gds/kmeans/KmeansAlgorithmFactory.java b/algo/src/main/java/org/neo4j/gds/kmeans/KmeansAlgorithmFactory.java index b4a0da33a9..25aab9a298 100644 --- a/algo/src/main/java/org/neo4j/gds/kmeans/KmeansAlgorithmFactory.java +++ b/algo/src/main/java/org/neo4j/gds/kmeans/KmeansAlgorithmFactory.java @@ -22,7 +22,7 @@ import org.jetbrains.annotations.NotNull; import org.neo4j.gds.GraphAlgorithmFactory; import org.neo4j.gds.api.Graph; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.mem.MemoryEstimation; import org.neo4j.gds.core.utils.mem.MemoryEstimations; import org.neo4j.gds.core.utils.paged.HugeDoubleArray; @@ -61,7 +61,7 @@ public Kmeans build( return Kmeans.createKmeans(graph, configuration, ImmutableKmeansContext .builder() .progressTracker(progressTracker) - .executor(ExecutorServices.DEFAULT) + .executor(DefaultPool.INSTANCE) .build()); } diff --git a/algo/src/main/java/org/neo4j/gds/kmeans/KmeansContext.java b/algo/src/main/java/org/neo4j/gds/kmeans/KmeansContext.java index 03fbb4a16a..b5729e27c4 100644 --- a/algo/src/main/java/org/neo4j/gds/kmeans/KmeansContext.java +++ b/algo/src/main/java/org/neo4j/gds/kmeans/KmeansContext.java @@ -21,7 +21,7 @@ import org.immutables.value.Value; import org.neo4j.gds.annotation.ValueClass; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import java.util.concurrent.ExecutorService; @@ -31,7 +31,7 @@ public interface KmeansContext { @Value.Default default ExecutorService executor() { - return ExecutorServices.DEFAULT; + return DefaultPool.INSTANCE; } @Value.Default diff --git a/algo/src/main/java/org/neo4j/gds/labelpropagation/LabelPropagationFactory.java b/algo/src/main/java/org/neo4j/gds/labelpropagation/LabelPropagationFactory.java index 0288faf563..08232a8ab0 100644 --- a/algo/src/main/java/org/neo4j/gds/labelpropagation/LabelPropagationFactory.java +++ b/algo/src/main/java/org/neo4j/gds/labelpropagation/LabelPropagationFactory.java @@ -22,7 +22,7 @@ import com.carrotsearch.hppc.LongDoubleScatterMap; import org.neo4j.gds.GraphAlgorithmFactory; import org.neo4j.gds.api.Graph; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.mem.MemoryEstimation; import org.neo4j.gds.core.utils.mem.MemoryEstimations; import org.neo4j.gds.core.utils.mem.MemoryRange; @@ -53,7 +53,7 @@ public LabelPropagation build( return new LabelPropagation( graph, configuration, - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, progressTracker ); } diff --git a/algo/src/main/java/org/neo4j/gds/leiden/Leiden.java b/algo/src/main/java/org/neo4j/gds/leiden/Leiden.java index 6594e91474..b9b2779415 100644 --- a/algo/src/main/java/org/neo4j/gds/leiden/Leiden.java +++ b/algo/src/main/java/org/neo4j/gds/leiden/Leiden.java @@ -25,7 +25,7 @@ import org.neo4j.gds.api.Graph; import org.neo4j.gds.api.properties.nodes.NodePropertyValues; import org.neo4j.gds.api.schema.Direction; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.concurrency.RunWithConcurrency; import org.neo4j.gds.core.utils.paged.HugeDoubleArray; import org.neo4j.gds.core.utils.paged.HugeLongArray; @@ -77,7 +77,7 @@ public Leiden( this.theta = theta; this.randomSeed = randomSeed; // TODO: Pass these two as parameters - this.executorService = ExecutorServices.DEFAULT; + this.executorService = DefaultPool.INSTANCE; this.concurrency = concurrency; this.dendrogramManager = new LeidenDendrogramManager( rootGraph, diff --git a/algo/src/main/java/org/neo4j/gds/louvain/LouvainAlgorithmFactory.java b/algo/src/main/java/org/neo4j/gds/louvain/LouvainAlgorithmFactory.java index cdf36896a5..7d8e79dda4 100644 --- a/algo/src/main/java/org/neo4j/gds/louvain/LouvainAlgorithmFactory.java +++ b/algo/src/main/java/org/neo4j/gds/louvain/LouvainAlgorithmFactory.java @@ -31,7 +31,7 @@ import org.neo4j.gds.core.Aggregation; import org.neo4j.gds.core.GraphDimensions; import org.neo4j.gds.core.ImmutableGraphDimensions; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.loading.NativeFactory; import org.neo4j.gds.core.utils.mem.MemoryEstimation; import org.neo4j.gds.core.utils.mem.MemoryEstimations; @@ -61,7 +61,7 @@ public Louvain build( configuration.tolerance(), configuration.concurrency(), progressTracker, - ExecutorServices.DEFAULT + DefaultPool.INSTANCE ); } diff --git a/algo/src/main/java/org/neo4j/gds/modularityoptimization/ModularityOptimizationFactory.java b/algo/src/main/java/org/neo4j/gds/modularityoptimization/ModularityOptimizationFactory.java index 5764bc9db5..e3047b883c 100644 --- a/algo/src/main/java/org/neo4j/gds/modularityoptimization/ModularityOptimizationFactory.java +++ b/algo/src/main/java/org/neo4j/gds/modularityoptimization/ModularityOptimizationFactory.java @@ -25,7 +25,7 @@ import org.neo4j.gds.collections.haa.HugeAtomicDoubleArray; import org.neo4j.gds.config.BaseConfig; import org.neo4j.gds.config.IterationsConfig; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.mem.MemoryEstimation; import org.neo4j.gds.core.utils.mem.MemoryEstimations; import org.neo4j.gds.core.utils.mem.MemoryRange; @@ -106,7 +106,7 @@ public ModularityOptimization build( seedProperty, configuration.concurrency(), configuration.batchSize(), - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, progressTracker ); } diff --git a/algo/src/main/java/org/neo4j/gds/pagerank/PageRankAlgorithmFactory.java b/algo/src/main/java/org/neo4j/gds/pagerank/PageRankAlgorithmFactory.java index 92ba1f8713..d62d9c6d84 100644 --- a/algo/src/main/java/org/neo4j/gds/pagerank/PageRankAlgorithmFactory.java +++ b/algo/src/main/java/org/neo4j/gds/pagerank/PageRankAlgorithmFactory.java @@ -27,7 +27,7 @@ import org.neo4j.gds.beta.pregel.Pregel; import org.neo4j.gds.beta.pregel.PregelComputation; import org.neo4j.gds.beta.pregel.PregelSchema; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.concurrency.ParallelUtil; import org.neo4j.gds.core.utils.TerminationFlag; import org.neo4j.gds.core.utils.mem.MemoryEstimation; @@ -121,7 +121,7 @@ public PageRankAlgorithm build( configuration, computation, mode, - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, progressTracker ); } @@ -143,7 +143,7 @@ private LongToDoubleFunction degreeFunction( var degreeCentrality = new DegreeCentrality( graph, - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, config, ProgressTracker.NULL_TRACKER ); diff --git a/algo/src/main/java/org/neo4j/gds/paths/delta/DeltaSteppingFactory.java b/algo/src/main/java/org/neo4j/gds/paths/delta/DeltaSteppingFactory.java index 20056af714..23456c2dd1 100644 --- a/algo/src/main/java/org/neo4j/gds/paths/delta/DeltaSteppingFactory.java +++ b/algo/src/main/java/org/neo4j/gds/paths/delta/DeltaSteppingFactory.java @@ -21,7 +21,7 @@ import org.neo4j.gds.GraphAlgorithmFactory; import org.neo4j.gds.api.Graph; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.mem.MemoryEstimation; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.core.utils.progress.tasks.Task; @@ -38,7 +38,7 @@ public DeltaStepping build( T configuration, ProgressTracker progressTracker ) { - return DeltaStepping.of(graph, configuration, ExecutorServices.DEFAULT, progressTracker); + return DeltaStepping.of(graph, configuration, DefaultPool.INSTANCE, progressTracker); } @Override diff --git a/algo/src/main/java/org/neo4j/gds/paths/traverse/BFS.java b/algo/src/main/java/org/neo4j/gds/paths/traverse/BFS.java index 63f65b288f..15ea38e5e3 100644 --- a/algo/src/main/java/org/neo4j/gds/paths/traverse/BFS.java +++ b/algo/src/main/java/org/neo4j/gds/paths/traverse/BFS.java @@ -23,7 +23,7 @@ import org.neo4j.gds.Algorithm; import org.neo4j.gds.api.Graph; import org.neo4j.gds.collections.haa.HugeAtomicLongArray; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.concurrency.ParallelUtil; import org.neo4j.gds.core.utils.paged.HugeAtomicBitSet; import org.neo4j.gds.core.utils.paged.HugeDoubleArray; @@ -199,7 +199,7 @@ public HugeLongArray compute() { if (currentDepth == maximumDepth) { break; } - ParallelUtil.run(bfsTaskList, ExecutorServices.DEFAULT); + ParallelUtil.run(bfsTaskList, DefaultPool.INSTANCE); if (targetFoundIndex.get() != Long.MAX_VALUE) { break; diff --git a/algo/src/main/java/org/neo4j/gds/paths/yens/Yens.java b/algo/src/main/java/org/neo4j/gds/paths/yens/Yens.java index 524834fb4f..a07aa9bfce 100644 --- a/algo/src/main/java/org/neo4j/gds/paths/yens/Yens.java +++ b/algo/src/main/java/org/neo4j/gds/paths/yens/Yens.java @@ -21,7 +21,7 @@ import org.neo4j.gds.Algorithm; import org.neo4j.gds.api.Graph; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.concurrency.RunWithConcurrency; import org.neo4j.gds.core.utils.mem.MemoryEstimation; import org.neo4j.gds.core.utils.mem.MemoryEstimations; @@ -113,7 +113,7 @@ public PathFindingResult compute() { RunWithConcurrency.builder() .concurrency(config.concurrency()) .tasks(tasks) - .executor(ExecutorServices.DEFAULT) + .executor(DefaultPool.INSTANCE) .run(); progressTracker.logProgress(); diff --git a/algo/src/main/java/org/neo4j/gds/scaleproperties/ScalePropertiesFactory.java b/algo/src/main/java/org/neo4j/gds/scaleproperties/ScalePropertiesFactory.java index 66ffa940ec..745c32c2a8 100644 --- a/algo/src/main/java/org/neo4j/gds/scaleproperties/ScalePropertiesFactory.java +++ b/algo/src/main/java/org/neo4j/gds/scaleproperties/ScalePropertiesFactory.java @@ -21,7 +21,7 @@ import org.neo4j.gds.GraphAlgorithmFactory; import org.neo4j.gds.api.Graph; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.mem.MemoryEstimation; import org.neo4j.gds.core.utils.mem.MemoryEstimations; import org.neo4j.gds.core.utils.mem.MemoryRange; @@ -69,7 +69,7 @@ public ScaleProperties build( graph, configuration, progressTracker, - ExecutorServices.DEFAULT + DefaultPool.INSTANCE ); } diff --git a/algo/src/main/java/org/neo4j/gds/similarity/filteredknn/FilteredKnnFactory.java b/algo/src/main/java/org/neo4j/gds/similarity/filteredknn/FilteredKnnFactory.java index f77cf68bbb..056fac86cb 100644 --- a/algo/src/main/java/org/neo4j/gds/similarity/filteredknn/FilteredKnnFactory.java +++ b/algo/src/main/java/org/neo4j/gds/similarity/filteredknn/FilteredKnnFactory.java @@ -23,7 +23,7 @@ import org.apache.commons.lang3.function.TriFunction; import org.neo4j.gds.GraphAlgorithmFactory; import org.neo4j.gds.api.Graph; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.mem.MemoryEstimation; import org.neo4j.gds.core.utils.mem.MemoryEstimations; import org.neo4j.gds.core.utils.mem.MemoryRange; @@ -68,7 +68,7 @@ public FilteredKnn build(Graph graph, CONFIG configuration, ProgressTracker prog KnnContext knnContext = ImmutableKnnContext .builder() .progressTracker(progressTracker) - .executor(ExecutorServices.DEFAULT) + .executor(DefaultPool.INSTANCE) .build(); if (configuration.seedTargetNodes()) { diff --git a/algo/src/main/java/org/neo4j/gds/similarity/filterednodesim/FilteredNodeSimilarityFactory.java b/algo/src/main/java/org/neo4j/gds/similarity/filterednodesim/FilteredNodeSimilarityFactory.java index 13958a7c21..42eac124cb 100644 --- a/algo/src/main/java/org/neo4j/gds/similarity/filterednodesim/FilteredNodeSimilarityFactory.java +++ b/algo/src/main/java/org/neo4j/gds/similarity/filterednodesim/FilteredNodeSimilarityFactory.java @@ -22,7 +22,7 @@ import com.carrotsearch.hppc.BitSet; import org.neo4j.gds.GraphAlgorithmFactory; import org.neo4j.gds.api.Graph; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.mem.MemoryEstimation; import org.neo4j.gds.core.utils.mem.MemoryEstimations; import org.neo4j.gds.core.utils.mem.MemoryRange; @@ -59,7 +59,7 @@ public NodeSimilarity build( configuration.sourceNodeFilter().toNodeFilter(graph), configuration.targetNodeFilter().toNodeFilter(graph), configuration.concurrency(), - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, progressTracker ); } diff --git a/algo/src/main/java/org/neo4j/gds/similarity/knn/KnnContext.java b/algo/src/main/java/org/neo4j/gds/similarity/knn/KnnContext.java index c7e57a3033..54acdbc572 100644 --- a/algo/src/main/java/org/neo4j/gds/similarity/knn/KnnContext.java +++ b/algo/src/main/java/org/neo4j/gds/similarity/knn/KnnContext.java @@ -21,7 +21,7 @@ import org.immutables.value.Value; import org.neo4j.gds.annotation.ValueClass; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import java.util.concurrent.ExecutorService; @@ -31,7 +31,7 @@ public interface KnnContext { @Value.Default default ExecutorService executor() { - return ExecutorServices.DEFAULT; + return DefaultPool.INSTANCE; } @Value.Default diff --git a/algo/src/main/java/org/neo4j/gds/similarity/knn/KnnFactory.java b/algo/src/main/java/org/neo4j/gds/similarity/knn/KnnFactory.java index f7f1a9e6d7..8dded7f2ec 100644 --- a/algo/src/main/java/org/neo4j/gds/similarity/knn/KnnFactory.java +++ b/algo/src/main/java/org/neo4j/gds/similarity/knn/KnnFactory.java @@ -22,7 +22,7 @@ import com.carrotsearch.hppc.LongArrayList; import org.neo4j.gds.GraphAlgorithmFactory; import org.neo4j.gds.api.Graph; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.mem.MemoryEstimation; import org.neo4j.gds.core.utils.mem.MemoryEstimations; import org.neo4j.gds.core.utils.mem.MemoryRange; @@ -59,7 +59,7 @@ public Knn build( ImmutableKnnContext .builder() .progressTracker(progressTracker) - .executor(ExecutorServices.DEFAULT) + .executor(DefaultPool.INSTANCE) .build() ); } diff --git a/algo/src/main/java/org/neo4j/gds/similarity/nodesim/NodeSimilarityFactory.java b/algo/src/main/java/org/neo4j/gds/similarity/nodesim/NodeSimilarityFactory.java index ffa573b2d7..b1eb71e31f 100644 --- a/algo/src/main/java/org/neo4j/gds/similarity/nodesim/NodeSimilarityFactory.java +++ b/algo/src/main/java/org/neo4j/gds/similarity/nodesim/NodeSimilarityFactory.java @@ -22,7 +22,7 @@ import com.carrotsearch.hppc.BitSet; import org.neo4j.gds.GraphAlgorithmFactory; import org.neo4j.gds.api.Graph; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.mem.MemoryEstimation; import org.neo4j.gds.core.utils.mem.MemoryEstimations; import org.neo4j.gds.core.utils.mem.MemoryRange; @@ -54,7 +54,7 @@ public NodeSimilarity build( configuration, similarityComputer, configuration.concurrency(), - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, progressTracker ); } diff --git a/algo/src/main/java/org/neo4j/gds/steiner/SteinerTreeAlgorithmFactory.java b/algo/src/main/java/org/neo4j/gds/steiner/SteinerTreeAlgorithmFactory.java index c2232f37d4..60e0761b4b 100644 --- a/algo/src/main/java/org/neo4j/gds/steiner/SteinerTreeAlgorithmFactory.java +++ b/algo/src/main/java/org/neo4j/gds/steiner/SteinerTreeAlgorithmFactory.java @@ -21,7 +21,7 @@ import org.neo4j.gds.GraphAlgorithmFactory; import org.neo4j.gds.api.Graph; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.core.utils.progress.tasks.Task; import org.neo4j.gds.core.utils.progress.tasks.Tasks; @@ -51,7 +51,7 @@ public ShortestPathsSteinerAlgorithm build( configuration.delta(), configuration.concurrency(), configuration.applyRerouting(), - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, progressTracker ); } diff --git a/algo/src/main/java/org/neo4j/gds/traversal/RandomWalkAlgorithmFactory.java b/algo/src/main/java/org/neo4j/gds/traversal/RandomWalkAlgorithmFactory.java index 0da5a824fc..39637f4ee0 100644 --- a/algo/src/main/java/org/neo4j/gds/traversal/RandomWalkAlgorithmFactory.java +++ b/algo/src/main/java/org/neo4j/gds/traversal/RandomWalkAlgorithmFactory.java @@ -21,7 +21,7 @@ import org.neo4j.gds.GraphAlgorithmFactory; import org.neo4j.gds.api.Graph; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.mem.MemoryEstimation; import org.neo4j.gds.core.utils.mem.MemoryEstimations; import org.neo4j.gds.core.utils.mem.MemoryRange; @@ -45,7 +45,7 @@ public RandomWalk build( RandomWalkBaseConfig configuration, ProgressTracker progressTracker ) { - return RandomWalk.create(graph, configuration, progressTracker, ExecutorServices.DEFAULT); + return RandomWalk.create(graph, configuration, progressTracker, DefaultPool.INSTANCE); } @Override diff --git a/algo/src/main/java/org/neo4j/gds/triangle/IntersectingTriangleCountFactory.java b/algo/src/main/java/org/neo4j/gds/triangle/IntersectingTriangleCountFactory.java index 14ebd8f047..4622d95cc8 100644 --- a/algo/src/main/java/org/neo4j/gds/triangle/IntersectingTriangleCountFactory.java +++ b/algo/src/main/java/org/neo4j/gds/triangle/IntersectingTriangleCountFactory.java @@ -23,7 +23,7 @@ import org.neo4j.gds.GraphAlgorithmFactory; import org.neo4j.gds.api.Graph; import org.neo4j.gds.collections.haa.HugeAtomicLongArray; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.mem.MemoryEstimation; import org.neo4j.gds.core.utils.mem.MemoryEstimations; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; @@ -48,7 +48,7 @@ public IntersectingTriangleCount build( return IntersectingTriangleCount.create( graph, configuration, - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, progressTracker ); } diff --git a/algo/src/main/java/org/neo4j/gds/triangle/TriangleStreamFactory.java b/algo/src/main/java/org/neo4j/gds/triangle/TriangleStreamFactory.java index 2156472243..1c07b8b17c 100644 --- a/algo/src/main/java/org/neo4j/gds/triangle/TriangleStreamFactory.java +++ b/algo/src/main/java/org/neo4j/gds/triangle/TriangleStreamFactory.java @@ -21,7 +21,7 @@ import org.neo4j.gds.GraphAlgorithmFactory; import org.neo4j.gds.api.Graph; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; public class TriangleStreamFactory extends GraphAlgorithmFactory { @@ -33,6 +33,6 @@ public String taskName() { @Override public TriangleStream build(Graph graph, TriangleCountBaseConfig configuration, ProgressTracker progressTracker) { - return TriangleStream.create(graph, ExecutorServices.DEFAULT, configuration.concurrency()); + return TriangleStream.create(graph, DefaultPool.INSTANCE, configuration.concurrency()); } } diff --git a/algo/src/main/java/org/neo4j/gds/undirected/ToUndirectedAlgorithmFactory.java b/algo/src/main/java/org/neo4j/gds/undirected/ToUndirectedAlgorithmFactory.java index f508d88ca9..ce858c0903 100644 --- a/algo/src/main/java/org/neo4j/gds/undirected/ToUndirectedAlgorithmFactory.java +++ b/algo/src/main/java/org/neo4j/gds/undirected/ToUndirectedAlgorithmFactory.java @@ -22,7 +22,7 @@ import org.neo4j.gds.GraphStoreAlgorithmFactory; import org.neo4j.gds.RelationshipType; import org.neo4j.gds.api.GraphStore; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.loading.AdjacencyListBehavior; import org.neo4j.gds.core.utils.mem.MemoryEstimation; import org.neo4j.gds.core.utils.mem.MemoryEstimations; @@ -39,7 +39,7 @@ public ToUndirected build( ToUndirectedConfig configuration, ProgressTracker progressTracker ) { - return new ToUndirected(graphStore, configuration, progressTracker, ExecutorServices.DEFAULT); + return new ToUndirected(graphStore, configuration, progressTracker, DefaultPool.INSTANCE); } @Override diff --git a/algo/src/main/java/org/neo4j/gds/walking/CollapsePathAlgorithmFactory.java b/algo/src/main/java/org/neo4j/gds/walking/CollapsePathAlgorithmFactory.java index a017c3254c..da8284beb3 100644 --- a/algo/src/main/java/org/neo4j/gds/walking/CollapsePathAlgorithmFactory.java +++ b/algo/src/main/java/org/neo4j/gds/walking/CollapsePathAlgorithmFactory.java @@ -24,7 +24,7 @@ import org.neo4j.gds.RelationshipType; import org.neo4j.gds.api.Graph; import org.neo4j.gds.api.GraphStore; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import java.util.Collection; @@ -66,7 +66,7 @@ public CollapsePath build( config.allowSelfLoops(), RelationshipType.of(config.mutateRelationshipType()), config.concurrency(), - ExecutorServices.DEFAULT + DefaultPool.INSTANCE ); } diff --git a/algo/src/main/java/org/neo4j/gds/wcc/WccAlgorithmFactory.java b/algo/src/main/java/org/neo4j/gds/wcc/WccAlgorithmFactory.java index 31f17a6ae0..130ae57ecc 100644 --- a/algo/src/main/java/org/neo4j/gds/wcc/WccAlgorithmFactory.java +++ b/algo/src/main/java/org/neo4j/gds/wcc/WccAlgorithmFactory.java @@ -21,7 +21,7 @@ import org.neo4j.gds.GraphAlgorithmFactory; import org.neo4j.gds.api.Graph; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.concurrency.ParallelUtil; import org.neo4j.gds.core.utils.mem.MemoryEstimation; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; @@ -50,7 +50,7 @@ public Wcc build( } return new Wcc( graph, - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, ParallelUtil.DEFAULT_BATCH_SIZE, configuration, progressTracker diff --git a/algo/src/test/java/org/neo4j/gds/allshortestpaths/MSBFSAllShortestPathsTest.java b/algo/src/test/java/org/neo4j/gds/allshortestpaths/MSBFSAllShortestPathsTest.java index 53f7b6352f..4a817d7be8 100644 --- a/algo/src/test/java/org/neo4j/gds/allshortestpaths/MSBFSAllShortestPathsTest.java +++ b/algo/src/test/java/org/neo4j/gds/allshortestpaths/MSBFSAllShortestPathsTest.java @@ -25,7 +25,7 @@ import org.neo4j.gds.StoreLoaderBuilder; import org.neo4j.gds.api.Graph; import org.neo4j.gds.config.ConcurrencyConfig; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.graphbuilder.GraphBuilder; import static org.junit.jupiter.api.Assertions.fail; @@ -76,7 +76,7 @@ void testResults() { .addRelationshipType(RELATIONSHIP) .build() .graph(); - testASP(new MSBFSAllShortestPaths(graph, ConcurrencyConfig.DEFAULT_CONCURRENCY, ExecutorServices.DEFAULT)); + testASP(new MSBFSAllShortestPaths(graph, ConcurrencyConfig.DEFAULT_CONCURRENCY, DefaultPool.INSTANCE)); } private void testASP(final MSBFSASPAlgorithm hugeMSBFSAllShortestPaths) { diff --git a/algo/src/test/java/org/neo4j/gds/allshortestpaths/WeightedAllShortestPathsTest.java b/algo/src/test/java/org/neo4j/gds/allshortestpaths/WeightedAllShortestPathsTest.java index 61e92c5655..53defe79ee 100644 --- a/algo/src/test/java/org/neo4j/gds/allshortestpaths/WeightedAllShortestPathsTest.java +++ b/algo/src/test/java/org/neo4j/gds/allshortestpaths/WeightedAllShortestPathsTest.java @@ -25,7 +25,7 @@ import org.neo4j.gds.PropertyMapping; import org.neo4j.gds.StoreLoaderBuilder; import org.neo4j.gds.api.Graph; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.graphbuilder.GraphBuilder; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -88,7 +88,7 @@ void testResults() { final ResultConsumer mock = mock(ResultConsumer.class); - new WeightedAllShortestPaths(graph, ExecutorServices.DEFAULT, 4) + new WeightedAllShortestPaths(graph, DefaultPool.INSTANCE, 4) .compute() .forEach(r -> { assertNotEquals(Double.POSITIVE_INFINITY, r.distance); @@ -115,7 +115,7 @@ void shouldThrowIfGraphHasNoRelationshipProperty() { .graph(); UnsupportedOperationException exception = assertThrows(UnsupportedOperationException.class, () -> { - new WeightedAllShortestPaths(graph, ExecutorServices.DEFAULT, 4); + new WeightedAllShortestPaths(graph, DefaultPool.INSTANCE, 4); }); assertTrue(exception.getMessage().contains("not supported")); diff --git a/algo/src/test/java/org/neo4j/gds/approxmaxkcut/ApproxMaxKCutTest.java b/algo/src/test/java/org/neo4j/gds/approxmaxkcut/ApproxMaxKCutTest.java index 6e81692306..8a9b33b6b2 100644 --- a/algo/src/test/java/org/neo4j/gds/approxmaxkcut/ApproxMaxKCutTest.java +++ b/algo/src/test/java/org/neo4j/gds/approxmaxkcut/ApproxMaxKCutTest.java @@ -28,7 +28,7 @@ import org.neo4j.gds.collections.haa.HugeAtomicDoubleArray; import org.neo4j.gds.compat.Neo4jProxy; import org.neo4j.gds.compat.TestLog; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.mem.MemoryRange; import org.neo4j.gds.core.utils.paged.HugeByteArray; import org.neo4j.gds.core.utils.progress.EmptyTaskRegistryFactory; @@ -172,7 +172,7 @@ void computeCorrectResults( var graph = minimize ? minGraph : maxGraph; var approxMaxKCut = new ApproxMaxKCut( graph, - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, config, ProgressTracker.NULL_TRACKER ); @@ -214,7 +214,7 @@ void respectMinCommunitySizes(int concurrency) { var approxMaxKCut = new ApproxMaxKCut( maxGraph, - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, config, ProgressTracker.NULL_TRACKER ); diff --git a/algo/src/test/java/org/neo4j/gds/betweenness/BetweennessCentralityTest.java b/algo/src/test/java/org/neo4j/gds/betweenness/BetweennessCentralityTest.java index 6a8f3777d7..dee1368f0c 100644 --- a/algo/src/test/java/org/neo4j/gds/betweenness/BetweennessCentralityTest.java +++ b/algo/src/test/java/org/neo4j/gds/betweenness/BetweennessCentralityTest.java @@ -29,7 +29,7 @@ import org.neo4j.gds.compat.Neo4jProxy; import org.neo4j.gds.compat.TestLog; import org.neo4j.gds.core.CypherMapWrapper; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.mem.MemoryRange; import org.neo4j.gds.core.utils.progress.EmptyTaskRegistryFactory; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; @@ -141,7 +141,7 @@ void sampling(int concurrency, TestGraph graph, int samplingSize, Map selectionStrategy.next())).containsExactly( graph.toMappedNodeId("a"), @@ -107,9 +107,9 @@ void neverIncludeZeroDegNodesIfBetterChoicesExist() { TestGraph graph = fromGdl("(), (), (), (), (), (a)-->(), (), (), ()"); SelectionStrategy selectionStrategy = new RandomDegreeSelectionStrategy(1); - selectionStrategy.init(graph, ExecutorServices.DEFAULT, 1); + selectionStrategy.init(graph, DefaultPool.INSTANCE, 1); assertEquals(1, samplingSize(selectionStrategy)); - selectionStrategy.init(graph, ExecutorServices.DEFAULT, 1); + selectionStrategy.init(graph, DefaultPool.INSTANCE, 1); assertEquals(graph.toMappedNodeId("a"), selectionStrategy.next()); } @@ -118,18 +118,18 @@ void not100PercentLikelyUnlessMaxDegNode() { TestGraph graph = fromGdl("(a)-->(b), (b)-->(c), (b)-->(d)"); SelectionStrategy selectionStrategy = new RandomDegreeSelectionStrategy(1, Optional.of(42L)); - selectionStrategy.init(graph, ExecutorServices.DEFAULT, 1); + selectionStrategy.init(graph, DefaultPool.INSTANCE, 1); assertEquals(1, samplingSize(selectionStrategy)); - selectionStrategy.init(graph, ExecutorServices.DEFAULT, 1); + selectionStrategy.init(graph, DefaultPool.INSTANCE, 1); assertEquals(graph.toMappedNodeId("b"), selectionStrategy.next()); } @Test void selectHighDegreeNode() { SelectionStrategy selectionStrategy = new RandomDegreeSelectionStrategy(1); - selectionStrategy.init(graph, ExecutorServices.DEFAULT, 1); + selectionStrategy.init(graph, DefaultPool.INSTANCE, 1); assertEquals(1, samplingSize(selectionStrategy)); - selectionStrategy.init(graph, ExecutorServices.DEFAULT, 1); + selectionStrategy.init(graph, DefaultPool.INSTANCE, 1); var isA = selectionStrategy.next(); var isB = selectionStrategy.next(); assertTrue(isA != SelectionStrategy.NONE_SELECTED || isB != SelectionStrategy.NONE_SELECTED); diff --git a/algo/src/test/java/org/neo4j/gds/betweenness/WeightedBetweennessCentralityTest.java b/algo/src/test/java/org/neo4j/gds/betweenness/WeightedBetweennessCentralityTest.java index 88e7ed9034..6b1d8db152 100644 --- a/algo/src/test/java/org/neo4j/gds/betweenness/WeightedBetweennessCentralityTest.java +++ b/algo/src/test/java/org/neo4j/gds/betweenness/WeightedBetweennessCentralityTest.java @@ -26,7 +26,7 @@ import org.junit.jupiter.params.provider.MethodSource; import org.neo4j.gds.api.Graph; import org.neo4j.gds.core.CypherMapWrapper; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.mem.MemoryRange; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.extension.GdlExtension; @@ -86,7 +86,7 @@ void shouldEqualWithUnweightedWhenWeightsAreEqual() { equallyWeightedGraph, new RandomDegreeSelectionStrategy(7, Optional.of(42L)), ForwardTraverser.Factory.weighted(), - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, 8, ProgressTracker.NULL_TRACKER ); @@ -94,7 +94,7 @@ void shouldEqualWithUnweightedWhenWeightsAreEqual() { equallyWeightedGraph, new RandomDegreeSelectionStrategy(7, Optional.of(42L)), ForwardTraverser.Factory.unweighted(), - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, 8, ProgressTracker.NULL_TRACKER ); @@ -119,7 +119,7 @@ void shouldComputeWithWeights() { weightedGraph, new RandomDegreeSelectionStrategy(7, Optional.of(42L)), ForwardTraverser.Factory.weighted(), - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, 8, ProgressTracker.NULL_TRACKER ); diff --git a/algo/src/test/java/org/neo4j/gds/closeness/ClosenessCentralityDirectedTest.java b/algo/src/test/java/org/neo4j/gds/closeness/ClosenessCentralityDirectedTest.java index fa42fc4a94..df826b3294 100644 --- a/algo/src/test/java/org/neo4j/gds/closeness/ClosenessCentralityDirectedTest.java +++ b/algo/src/test/java/org/neo4j/gds/closeness/ClosenessCentralityDirectedTest.java @@ -20,7 +20,7 @@ package org.neo4j.gds.closeness; import org.junit.jupiter.api.Test; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.extension.GdlExtension; import org.neo4j.gds.extension.GdlGraph; @@ -88,7 +88,7 @@ void testCentrality() { var algo = ClosenessCentrality.of( graph, ImmutableClosenessCentralityStreamConfig.builder().build(), - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); @@ -108,7 +108,7 @@ void testCentralityWithWassermanFaust() { var algo = ClosenessCentrality.of( graph, ImmutableClosenessCentralityStreamConfig.builder().useWassermanFaust(true).build(), - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); diff --git a/algo/src/test/java/org/neo4j/gds/closeness/ClosenessCentralityDiscoTest.java b/algo/src/test/java/org/neo4j/gds/closeness/ClosenessCentralityDiscoTest.java index 1ea23aa106..c225605215 100644 --- a/algo/src/test/java/org/neo4j/gds/closeness/ClosenessCentralityDiscoTest.java +++ b/algo/src/test/java/org/neo4j/gds/closeness/ClosenessCentralityDiscoTest.java @@ -22,7 +22,7 @@ import org.assertj.core.data.Offset; import org.junit.jupiter.api.Test; import org.neo4j.gds.Orientation; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.extension.GdlExtension; import org.neo4j.gds.extension.GdlGraph; @@ -67,7 +67,7 @@ void testHuge() { var algo = ClosenessCentrality.of( graph, ImmutableClosenessCentralityStreamConfig.builder().concurrency(2).useWassermanFaust(true).build(), - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); diff --git a/algo/src/test/java/org/neo4j/gds/closeness/ClosenessCentralityTest.java b/algo/src/test/java/org/neo4j/gds/closeness/ClosenessCentralityTest.java index c66e672ba9..eaa49008d1 100644 --- a/algo/src/test/java/org/neo4j/gds/closeness/ClosenessCentralityTest.java +++ b/algo/src/test/java/org/neo4j/gds/closeness/ClosenessCentralityTest.java @@ -23,7 +23,7 @@ import org.junit.jupiter.api.Test; import org.neo4j.gds.TestProgressTracker; import org.neo4j.gds.compat.Neo4jProxy; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.progress.EmptyTaskRegistryFactory; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.extension.GdlExtension; @@ -93,7 +93,7 @@ void testGetCentrality() { var algo = ClosenessCentrality.of( graph, ImmutableClosenessCentralityStreamConfig.builder().build(), - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); @@ -116,7 +116,7 @@ void shouldLogProgress() { var algo = ClosenessCentrality.of( graph, config, - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, progressTracker ); diff --git a/algo/src/test/java/org/neo4j/gds/conductance/ConductanceTest.java b/algo/src/test/java/org/neo4j/gds/conductance/ConductanceTest.java index cefce3db5c..a5045d4cd2 100644 --- a/algo/src/test/java/org/neo4j/gds/conductance/ConductanceTest.java +++ b/algo/src/test/java/org/neo4j/gds/conductance/ConductanceTest.java @@ -28,7 +28,7 @@ import org.neo4j.gds.TestSupport; import org.neo4j.gds.compat.Neo4jProxy; import org.neo4j.gds.compat.TestLog; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.progress.EmptyTaskRegistryFactory; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.core.utils.progress.tasks.TaskProgressTracker; @@ -134,7 +134,7 @@ void computeCorrectResults( var conductance = new Conductance( orientation == Orientation.NATURAL ? naturalGraph : undirectedGraph, - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, config, ProgressTracker.NULL_TRACKER ); diff --git a/algo/src/test/java/org/neo4j/gds/degree/DegreeCentralityTest.java b/algo/src/test/java/org/neo4j/gds/degree/DegreeCentralityTest.java index 055d5e580d..e3f795a479 100644 --- a/algo/src/test/java/org/neo4j/gds/degree/DegreeCentralityTest.java +++ b/algo/src/test/java/org/neo4j/gds/degree/DegreeCentralityTest.java @@ -28,7 +28,7 @@ import org.neo4j.gds.TestProgressTracker; import org.neo4j.gds.compat.Neo4jProxy; import org.neo4j.gds.compat.TestLog; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.mem.MemoryRange; import org.neo4j.gds.core.utils.paged.HugeDoubleArray; import org.neo4j.gds.core.utils.progress.EmptyTaskRegistryFactory; @@ -148,7 +148,7 @@ void shouldComputeCorrectResults(boolean weighted, Orientation orientation, Map< var degreeCentrality = new DegreeCentrality( graph, - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, config, ProgressTracker.NULL_TRACKER ); @@ -200,7 +200,7 @@ void testProgressLogging(boolean weighted) { var progressTracker = new TestProgressTracker(progressTask, log, 1, EmptyTaskRegistryFactory.INSTANCE); var degreeCentrality = new DegreeCentrality( graph, - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, config, progressTracker ); @@ -225,7 +225,7 @@ void shouldSupportAllOrientations(Orientation orientation) { var degreeCentrality = new DegreeCentrality( graph, - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, config, ProgressTracker.NULL_TRACKER ); diff --git a/algo/src/test/java/org/neo4j/gds/embeddings/fastrp/FastRPTest.java b/algo/src/test/java/org/neo4j/gds/embeddings/fastrp/FastRPTest.java index e676d8438f..2d549b3b57 100644 --- a/algo/src/test/java/org/neo4j/gds/embeddings/fastrp/FastRPTest.java +++ b/algo/src/test/java/org/neo4j/gds/embeddings/fastrp/FastRPTest.java @@ -31,7 +31,7 @@ import org.neo4j.gds.compat.Neo4jProxy; import org.neo4j.gds.compat.TestLog; import org.neo4j.gds.core.ImmutableGraphDimensions; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.loading.ArrayIdMap; import org.neo4j.gds.core.loading.LabelInformationBuilders; import org.neo4j.gds.core.loading.construction.GraphFactory; @@ -626,7 +626,7 @@ void shouldBeDeterministicGivenSameOriginalIds() { .nodes(firstIdMap) .relationshipType(RelationshipType.of("REL")) .orientation(Orientation.UNDIRECTED) - .executorService(ExecutorServices.DEFAULT) + .executorService(DefaultPool.INSTANCE) .build(); var secondMappedToOriginal = HugeLongArray.newArray(nodeCount); @@ -650,7 +650,7 @@ void shouldBeDeterministicGivenSameOriginalIds() { .nodes(secondIdMap) .relationshipType(RelationshipType.of("REL")) .orientation(Orientation.UNDIRECTED) - .executorService(ExecutorServices.DEFAULT) + .executorService(DefaultPool.INSTANCE) .build(); var random = new SplittableRandom(42); diff --git a/algo/src/test/java/org/neo4j/gds/embeddings/graphsage/GraphSageEmbeddingsGeneratorTest.java b/algo/src/test/java/org/neo4j/gds/embeddings/graphsage/GraphSageEmbeddingsGeneratorTest.java index e56f0e592b..a1b4fe0032 100644 --- a/algo/src/test/java/org/neo4j/gds/embeddings/graphsage/GraphSageEmbeddingsGeneratorTest.java +++ b/algo/src/test/java/org/neo4j/gds/embeddings/graphsage/GraphSageEmbeddingsGeneratorTest.java @@ -24,7 +24,7 @@ import org.junit.jupiter.params.provider.EnumSource; import org.neo4j.gds.RelationshipType; import org.neo4j.gds.api.Graph; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.loading.CSRGraphStore; import org.neo4j.gds.core.utils.paged.HugeObjectArray; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; @@ -74,7 +74,7 @@ void makesEmbeddings(Aggregator.AggregatorType aggregatorType) { var features = GraphSageHelper.initializeSingleLabelFeatures(weightedGraph, config); - var trainModel = new GraphSageModelTrainer(config, ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER); + var trainModel = new GraphSageModelTrainer(config, DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER); GraphSageModelTrainer.ModelTrainResult result = trainModel.train(weightedGraph, features); @@ -84,7 +84,7 @@ void makesEmbeddings(Aggregator.AggregatorType aggregatorType) { config.concurrency(), new SingleLabelFeatureFunction(), config.randomSeed(), - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); @@ -112,7 +112,7 @@ void makesEmbeddingsFromMultiLabelModel(Aggregator.AggregatorType aggregatorType var trainer = new MultiLabelGraphSageTrain( weightedGraph, config, - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER, testGdsVersion ); @@ -125,7 +125,7 @@ void makesEmbeddingsFromMultiLabelModel(Aggregator.AggregatorType aggregatorType config.concurrency(), model.data().featureFunction(), model.trainConfig().randomSeed(), - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); @@ -167,7 +167,7 @@ void embeddingsForNodeFilteredGraph() { var trainer = new SingleLabelGraphSageTrain( filteredGraph, config, - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER, testGdsVersion ); @@ -180,7 +180,7 @@ void embeddingsForNodeFilteredGraph() { config.concurrency(), model.data().featureFunction(), model.trainConfig().randomSeed(), - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); diff --git a/algo/src/test/java/org/neo4j/gds/embeddings/graphsage/GraphSageModelTrainerTest.java b/algo/src/test/java/org/neo4j/gds/embeddings/graphsage/GraphSageModelTrainerTest.java index d1953f7d0d..93c3cdc022 100644 --- a/algo/src/test/java/org/neo4j/gds/embeddings/graphsage/GraphSageModelTrainerTest.java +++ b/algo/src/test/java/org/neo4j/gds/embeddings/graphsage/GraphSageModelTrainerTest.java @@ -34,7 +34,7 @@ import org.neo4j.gds.beta.generator.PropertyProducer; import org.neo4j.gds.beta.generator.RandomGraphGenerator; import org.neo4j.gds.beta.generator.RelationshipDistribution; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.paged.HugeObjectArray; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.embeddings.graphsage.algo.GraphSageTrainConfigImpl; @@ -120,7 +120,7 @@ void trainsWithRelationshipWeight(AggregatorType aggregatorType) { .modelUser("") .build(); - var trainModel = new GraphSageModelTrainer(config, ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER); + var trainModel = new GraphSageModelTrainer(config, DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER); int nodeCount = 5_000; var bigGraph = RandomGraphGenerator @@ -160,7 +160,7 @@ void trainsWithMeanAggregator(boolean useRelationshipWeight) { .modelName(MODEL_NAME) .build(); - var trainModel = new GraphSageModelTrainer(config, ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER); + var trainModel = new GraphSageModelTrainer(config, DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER); var maybeWeights = useRelationshipWeight ? Optional.of("times") : Optional.empty(); @@ -201,7 +201,7 @@ void trainsWithPoolAggregator(boolean useRelationshipWeight) { .modelName(MODEL_NAME) .build(); - var trainModel = new GraphSageModelTrainer(config, ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER); + var trainModel = new GraphSageModelTrainer(config, DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER); GraphSageModelTrainer.ModelTrainResult result = trainModel.train(graph, features); Layer[] layers = result.layers(); @@ -251,7 +251,7 @@ void shouldTrainModelWithArrayProperties() { .modelUser("") .build(); - var trainer = new GraphSageModelTrainer(config, ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER); + var trainer = new GraphSageModelTrainer(config, DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER); var result = trainer.train(arrayGraph, arrayFeatures); @@ -276,7 +276,7 @@ void testLosses() { var trainer = new GraphSageModelTrainer( config, - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); @@ -309,7 +309,7 @@ void testLossesWithPoolAggregator() { var trainer = new GraphSageModelTrainer( config, - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); @@ -329,7 +329,7 @@ void testLossesWithPoolAggregator() { void testConvergence() { var trainer = new GraphSageModelTrainer( configBuilder.modelName("convergingModel:)").tolerance(100.0).epochs(10).build(), - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); @@ -355,13 +355,13 @@ void batchesPerIteration() { var trainResultWithoutSampling = new GraphSageModelTrainer( configBuilder.maybeBatchSamplingRatio(1.0).build(), - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ).train(unweightedGraph, features); var trainResultWithSampling = new GraphSageModelTrainer( configBuilder.maybeBatchSamplingRatio(0.01).build(), - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ).train(unweightedGraph, features); @@ -386,7 +386,7 @@ void l2Penalty(double penalty, double expectedLoss) { var result = new GraphSageModelTrainer( config, - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ).train(unweightedGraph, features); @@ -404,8 +404,8 @@ void seededSingleBatch(long seed) { .concurrency(1) .build(); - var trainer = new GraphSageModelTrainer(config, ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER); - var otherTrainer = new GraphSageModelTrainer(config, ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER); + var trainer = new GraphSageModelTrainer(config, DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER); + var otherTrainer = new GraphSageModelTrainer(config, DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER); var result = trainer.train(unweightedGraph, features); var otherResult = otherTrainer.train(unweightedGraph, features); @@ -424,8 +424,8 @@ void seededMultiBatch(long seed) { .batchSize(5) .build(); - var trainer = new GraphSageModelTrainer(config, ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER); - var otherTrainer = new GraphSageModelTrainer(config, ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER); + var trainer = new GraphSageModelTrainer(config, DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER); + var otherTrainer = new GraphSageModelTrainer(config, DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER); var result = trainer.train(unweightedGraph, features); var otherResult = otherTrainer.train(unweightedGraph, features); diff --git a/algo/src/test/java/org/neo4j/gds/embeddings/graphsage/GraphSageTest.java b/algo/src/test/java/org/neo4j/gds/embeddings/graphsage/GraphSageTest.java index 1baa9222d7..eb6f3cd9b8 100644 --- a/algo/src/test/java/org/neo4j/gds/embeddings/graphsage/GraphSageTest.java +++ b/algo/src/test/java/org/neo4j/gds/embeddings/graphsage/GraphSageTest.java @@ -35,7 +35,7 @@ import org.neo4j.gds.compat.Neo4jProxy; import org.neo4j.gds.config.RandomGraphGeneratorConfig; import org.neo4j.gds.core.Aggregation; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.huge.HugeGraph; import org.neo4j.gds.core.loading.CSRGraphStoreUtil; import org.neo4j.gds.core.loading.construction.NodeLabelTokens; @@ -146,7 +146,7 @@ void shouldNotMakeNanEmbeddings(Aggregator.AggregatorType aggregator) { var trainAlgo = new SingleLabelGraphSageTrain( orphanGraph, trainConfig, - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER, testGdsVersion ); @@ -180,7 +180,7 @@ void differentTrainAndPredictionGraph() { .concurrency(1) .build(); - var graphSageTrain = new SingleLabelGraphSageTrain(graph, trainConfig, ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER, testGdsVersion); + var graphSageTrain = new SingleLabelGraphSageTrain(graph, trainConfig, DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER, testGdsVersion); var model = graphSageTrain.compute(); @@ -205,7 +205,7 @@ void differentTrainAndPredictionGraph() { .batchSize(2) .build(); - var graphSage = new GraphSage(trainGraph, model, streamConfig, ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER); + var graphSage = new GraphSage(trainGraph, model, streamConfig, DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER); assertThat(graphSage.compute().embeddings().size()).isEqualTo(predictNodeCount); } diff --git a/algo/src/test/java/org/neo4j/gds/embeddings/graphsage/algo/MultiLabelGraphSageTrainTest.java b/algo/src/test/java/org/neo4j/gds/embeddings/graphsage/algo/MultiLabelGraphSageTrainTest.java index 26463e7d63..58aac3b66b 100644 --- a/algo/src/test/java/org/neo4j/gds/embeddings/graphsage/algo/MultiLabelGraphSageTrainTest.java +++ b/algo/src/test/java/org/neo4j/gds/embeddings/graphsage/algo/MultiLabelGraphSageTrainTest.java @@ -24,7 +24,7 @@ import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; import org.neo4j.gds.api.Graph; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.embeddings.graphsage.ActivationFunction; import org.neo4j.gds.embeddings.graphsage.Aggregator; @@ -98,7 +98,7 @@ void shouldRunWithDifferentProjectedFeatureSizes(String name, GraphSageTrainConf var multiLabelGraphSageTrain = new MultiLabelGraphSageTrain( weightedGraph, config, - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER, testGdsVersion ); @@ -119,7 +119,7 @@ void shouldStoreMultiLabelFeatureFunctionInModel() { var multiLabelGraphSageTrain = new MultiLabelGraphSageTrain( weightedGraph, config, - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER, testGdsVersion ); @@ -148,7 +148,7 @@ void runsTrainingOnMultiLabelGraph() { var graphSageTrain = new MultiLabelGraphSageTrain( weightedGraph, graphSageTrainConfig, - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER, testGdsVersion ); @@ -191,7 +191,7 @@ void shouldFailUnequalLengthArrays() { var multiLabelGraphSageTrain = new MultiLabelGraphSageTrain( unequalGraph, config, - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER, testGdsVersion ); @@ -216,7 +216,7 @@ void shouldFailMissingArrayProperty(Graph graph, String property, long missingNo var multiLabelGraphSageTrain = new MultiLabelGraphSageTrain( graph, config, - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER, testGdsVersion ); diff --git a/algo/src/test/java/org/neo4j/gds/embeddings/hashgnn/HashGNNTest.java b/algo/src/test/java/org/neo4j/gds/embeddings/hashgnn/HashGNNTest.java index fc19fc195e..e830ae4465 100644 --- a/algo/src/test/java/org/neo4j/gds/embeddings/hashgnn/HashGNNTest.java +++ b/algo/src/test/java/org/neo4j/gds/embeddings/hashgnn/HashGNNTest.java @@ -36,7 +36,7 @@ import org.neo4j.gds.compat.Neo4jProxy; import org.neo4j.gds.compat.TestLog; import org.neo4j.gds.core.GraphDimensions; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.loading.ArrayIdMap; import org.neo4j.gds.core.loading.LabelInformationBuilders; import org.neo4j.gds.core.loading.construction.GraphFactory; @@ -422,7 +422,7 @@ void shouldBeDeterministicGivenSameOriginalIds() { .nodes(firstIdMap) .relationshipType(RelationshipType.of("REL")) .orientation(Orientation.UNDIRECTED) - .executorService(ExecutorServices.DEFAULT) + .executorService(DefaultPool.INSTANCE) .build(); var secondMappedToOriginal = HugeLongArray.newArray(nodeCount); @@ -446,7 +446,7 @@ void shouldBeDeterministicGivenSameOriginalIds() { .nodes(secondIdMap) .relationshipType(RelationshipType.of("REL")) .orientation(Orientation.UNDIRECTED) - .executorService(ExecutorServices.DEFAULT) + .executorService(DefaultPool.INSTANCE) .build(); var random = new SplittableRandom(42); diff --git a/algo/src/test/java/org/neo4j/gds/embeddings/node2vec/Node2VecTest.java b/algo/src/test/java/org/neo4j/gds/embeddings/node2vec/Node2VecTest.java index 2a61c80eee..8b4ede616d 100644 --- a/algo/src/test/java/org/neo4j/gds/embeddings/node2vec/Node2VecTest.java +++ b/algo/src/test/java/org/neo4j/gds/embeddings/node2vec/Node2VecTest.java @@ -44,7 +44,7 @@ import org.neo4j.gds.compat.Neo4jProxy; import org.neo4j.gds.compat.TestLog; import org.neo4j.gds.core.GraphDimensions; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.loading.ArrayIdMap; import org.neo4j.gds.core.loading.LabelInformationBuilders; import org.neo4j.gds.core.loading.construction.GraphFactory; @@ -280,7 +280,7 @@ void shouldBeFairlyConsistentUnderOriginalIds(EmbeddingInitializer embeddingInit .nodes(firstIdMap) .relationshipType(RelationshipType.of("REL")) .orientation(Orientation.UNDIRECTED) - .executorService(ExecutorServices.DEFAULT) + .executorService(DefaultPool.INSTANCE) .build(); var secondMappedToOriginal = HugeLongArray.newArray(nodeCount); @@ -302,7 +302,7 @@ void shouldBeFairlyConsistentUnderOriginalIds(EmbeddingInitializer embeddingInit .nodes(secondIdMap) .relationshipType(RelationshipType.of("REL")) .orientation(Orientation.UNDIRECTED) - .executorService(ExecutorServices.DEFAULT) + .executorService(DefaultPool.INSTANCE) .build(); var random = new SplittableRandom(42); diff --git a/algo/src/test/java/org/neo4j/gds/harmonic/HarmonicCentralityTest.java b/algo/src/test/java/org/neo4j/gds/harmonic/HarmonicCentralityTest.java index aaa01e6a40..99009e5bb6 100644 --- a/algo/src/test/java/org/neo4j/gds/harmonic/HarmonicCentralityTest.java +++ b/algo/src/test/java/org/neo4j/gds/harmonic/HarmonicCentralityTest.java @@ -24,7 +24,7 @@ import org.neo4j.gds.TestProgressTracker; import org.neo4j.gds.api.Graph; import org.neo4j.gds.compat.Neo4jProxy; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.progress.EmptyTaskRegistryFactory; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.core.utils.progress.tasks.Tasks; @@ -67,7 +67,7 @@ void shouldComputeHarmonicCentrality() { var harmonicCentrality = new HarmonicCentrality( graph, 1, - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); @@ -89,7 +89,7 @@ void testLogging() { var algo = new HarmonicCentrality( graph, 1, - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, progressTracker ); diff --git a/algo/src/test/java/org/neo4j/gds/indexInverse/InverseRelationshipsTest.java b/algo/src/test/java/org/neo4j/gds/indexInverse/InverseRelationshipsTest.java index 92dbf2e8f3..9e17be857d 100644 --- a/algo/src/test/java/org/neo4j/gds/indexInverse/InverseRelationshipsTest.java +++ b/algo/src/test/java/org/neo4j/gds/indexInverse/InverseRelationshipsTest.java @@ -27,7 +27,7 @@ import org.neo4j.gds.RelationshipType; import org.neo4j.gds.api.GraphStore; import org.neo4j.gds.compat.Neo4jProxy; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.progress.EmptyTaskRegistryFactory; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.extension.GdlExtension; @@ -79,7 +79,7 @@ void shouldCreateIndexedRelationships(int concurrency) { graphStore, config, ProgressTracker.NULL_TRACKER, - ExecutorServices.DEFAULT + DefaultPool.INSTANCE ).compute(); assertThat(inverseRelationshipsPerType).hasSize(1); @@ -110,7 +110,7 @@ void shouldIndexMultipleTypes(Object relTypes) { graphStore, config, ProgressTracker.NULL_TRACKER, - ExecutorServices.DEFAULT + DefaultPool.INSTANCE ).compute(); assertThat(inverseRelationshipsPerType).hasSize(internalTypes.size()); diff --git a/algo/src/test/java/org/neo4j/gds/influenceMaximization/CELFOnConnectedGraphTest.java b/algo/src/test/java/org/neo4j/gds/influenceMaximization/CELFOnConnectedGraphTest.java index ba87582c9f..7125941382 100644 --- a/algo/src/test/java/org/neo4j/gds/influenceMaximization/CELFOnConnectedGraphTest.java +++ b/algo/src/test/java/org/neo4j/gds/influenceMaximization/CELFOnConnectedGraphTest.java @@ -25,7 +25,7 @@ import org.neo4j.gds.Orientation; import org.neo4j.gds.compat.Neo4jProxy; import org.neo4j.gds.compat.TestLog; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.progress.EmptyTaskRegistryFactory; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.core.utils.progress.tasks.TaskProgressTracker; @@ -99,7 +99,7 @@ void testSpreadWithSeed1() { // gain[d|a,b,d,e] : 0 {a already activates d} 1(d) 1(d) = 2/3 =0.667 IdFunction idFunction = variable -> graph.toMappedNodeId(variable); - CELF celf = new CELF(graph, 5, 0.2, 3, ExecutorServices.DEFAULT, 2, 0, DEFAULT_BATCH_SIZE, + CELF celf = new CELF(graph, 5, 0.2, 3, DefaultPool.INSTANCE, 2, 0, DEFAULT_BATCH_SIZE, ProgressTracker.EmptyProgressTracker.NULL_TRACKER ); var celfResult = celf.compute(); diff --git a/algo/src/test/java/org/neo4j/gds/influenceMaximization/CELFOnTreeGraphTest.java b/algo/src/test/java/org/neo4j/gds/influenceMaximization/CELFOnTreeGraphTest.java index 41677f1d67..43198fb627 100644 --- a/algo/src/test/java/org/neo4j/gds/influenceMaximization/CELFOnTreeGraphTest.java +++ b/algo/src/test/java/org/neo4j/gds/influenceMaximization/CELFOnTreeGraphTest.java @@ -24,7 +24,7 @@ import org.junit.jupiter.api.Test; import org.neo4j.gds.Orientation; import org.neo4j.gds.api.Graph; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.extension.GdlExtension; import org.neo4j.gds.extension.GdlGraph; @@ -136,7 +136,7 @@ void testSpreadWithSeed1() { 5, 0.51, 3, - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, 1, 10, DEFAULT_BATCH_SIZE, diff --git a/algo/src/test/java/org/neo4j/gds/influenceMaximization/CelfTest.java b/algo/src/test/java/org/neo4j/gds/influenceMaximization/CelfTest.java index cb8c5f3770..80e7254bdc 100644 --- a/algo/src/test/java/org/neo4j/gds/influenceMaximization/CelfTest.java +++ b/algo/src/test/java/org/neo4j/gds/influenceMaximization/CelfTest.java @@ -24,7 +24,7 @@ import org.neo4j.gds.api.schema.Direction; import org.neo4j.gds.beta.generator.RandomGraphGenerator; import org.neo4j.gds.beta.generator.RelationshipDistribution; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import static org.assertj.core.api.Assertions.assertThat; @@ -44,7 +44,7 @@ void shouldNotReturnNegativeGains(int seedSize) { .build() .generate(); - var celf = new CELF(graph, seedSize, 0.1, 3, ExecutorServices.DEFAULT, 1, 10, 5, ProgressTracker.NULL_TRACKER).compute(); + var celf = new CELF(graph, seedSize, 0.1, 3, DefaultPool.INSTANCE, 1, 10, 5, ProgressTracker.NULL_TRACKER).compute(); for (var a : celf) { assertThat(a.value).isNotNegative(); } diff --git a/algo/src/test/java/org/neo4j/gds/k1coloring/K1ColoringTest.java b/algo/src/test/java/org/neo4j/gds/k1coloring/K1ColoringTest.java index c0b4469c8e..121bd5ca2a 100644 --- a/algo/src/test/java/org/neo4j/gds/k1coloring/K1ColoringTest.java +++ b/algo/src/test/java/org/neo4j/gds/k1coloring/K1ColoringTest.java @@ -32,7 +32,7 @@ import org.neo4j.gds.core.Aggregation; import org.neo4j.gds.core.GraphDimensions; import org.neo4j.gds.core.ImmutableGraphDimensions; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.mem.MemoryRange; import org.neo4j.gds.core.utils.progress.EmptyTaskRegistryFactory; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; @@ -69,7 +69,7 @@ void testK1Coloring() { 1000, DEFAULT_BATCH_SIZE, 1, - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); @@ -110,7 +110,7 @@ void testParallelK1Coloring() { 100, DEFAULT_BATCH_SIZE, 8, - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); @@ -177,7 +177,7 @@ void everyNodeShouldHaveBeenColored() { 100, DEFAULT_BATCH_SIZE, 8, - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); @@ -214,7 +214,7 @@ void shouldLogProgress(){ config.maxIterations(), DEFAULT_BATCH_SIZE, config.concurrency(), - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, progressTracker ); diff --git a/algo/src/test/java/org/neo4j/gds/labelpropagation/LabelPropagationTest.java b/algo/src/test/java/org/neo4j/gds/labelpropagation/LabelPropagationTest.java index ebcbe09627..2f60b5a398 100644 --- a/algo/src/test/java/org/neo4j/gds/labelpropagation/LabelPropagationTest.java +++ b/algo/src/test/java/org/neo4j/gds/labelpropagation/LabelPropagationTest.java @@ -29,7 +29,7 @@ import org.neo4j.gds.compat.Neo4jProxy; import org.neo4j.gds.compat.TestLog; import org.neo4j.gds.core.CypherMapWrapper; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.paged.HugeLongArray; import org.neo4j.gds.core.utils.progress.EmptyTaskRegistryFactory; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; @@ -83,7 +83,7 @@ void shouldUseOriginalNodeIdWhenSeedPropertyIsMissing() { LabelPropagation lp = new LabelPropagation( graph, ImmutableLabelPropagationStreamConfig.builder().maxIterations(1).build(), - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); assertArrayEquals( @@ -109,7 +109,7 @@ void shouldUseSeedProperty() { .seedProperty("seedId") .maxIterations(1) .build(), - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); @@ -146,7 +146,7 @@ private void testLPClustering(Graph graph, int batchSize) { LabelPropagation lp = new LabelPropagation( graph, DEFAULT_CONFIG, - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); lp.withBatchSize(batchSize); @@ -202,7 +202,7 @@ void shouldLogProgress() { var lp = new LabelPropagation( graph, DEFAULT_CONFIG, - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, testTracker ); diff --git a/algo/src/test/java/org/neo4j/gds/labelpropagation/NonStabilizingLabelPropagationTest.java b/algo/src/test/java/org/neo4j/gds/labelpropagation/NonStabilizingLabelPropagationTest.java index bd303c9624..aa25dc610c 100644 --- a/algo/src/test/java/org/neo4j/gds/labelpropagation/NonStabilizingLabelPropagationTest.java +++ b/algo/src/test/java/org/neo4j/gds/labelpropagation/NonStabilizingLabelPropagationTest.java @@ -21,7 +21,7 @@ import org.junit.jupiter.api.Test; import org.neo4j.gds.api.Graph; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.extension.GdlExtension; import org.neo4j.gds.extension.GdlGraph; @@ -62,7 +62,7 @@ void testLabelPropagationDoesStabilize() { LabelPropagation labelPropagation = new LabelPropagation( graph, ImmutableLabelPropagationStreamConfig.builder().build(), - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); LabelPropagationResult compute = labelPropagation.compute(); diff --git a/algo/src/test/java/org/neo4j/gds/leiden/GraphWithSelfLoopTest.java b/algo/src/test/java/org/neo4j/gds/leiden/GraphWithSelfLoopTest.java index 7b81c0d4b9..a6e8baf301 100644 --- a/algo/src/test/java/org/neo4j/gds/leiden/GraphWithSelfLoopTest.java +++ b/algo/src/test/java/org/neo4j/gds/leiden/GraphWithSelfLoopTest.java @@ -23,7 +23,7 @@ import org.junit.jupiter.api.Test; import org.neo4j.gds.Orientation; import org.neo4j.gds.api.schema.Direction; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.concurrency.Pools; import org.neo4j.gds.core.utils.TerminationFlag; import org.neo4j.gds.core.utils.paged.HugeDoubleArray; @@ -71,7 +71,7 @@ void shouldCalculateModularityCorrectly() { 1.0 / graph.relationshipCount(), 1.0 / graph.relationshipCount(), 4, - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.EmptyProgressTracker.NULL_TRACKER ); @@ -165,7 +165,7 @@ void shouldCalculateModularityInSummaryGraph() { Direction.UNDIRECTED, localCommunities, 2, - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, 1, TerminationFlag.RUNNING_TRUE, ProgressTracker.NULL_TRACKER @@ -181,7 +181,7 @@ void shouldCalculateModularityInSummaryGraph() { 1.0 / graph.relationshipCount(), 1.0 / graph.relationshipCount(), 4, - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.EmptyProgressTracker.NULL_TRACKER ); diff --git a/algo/src/test/java/org/neo4j/gds/leiden/ModularityComputerTest.java b/algo/src/test/java/org/neo4j/gds/leiden/ModularityComputerTest.java index 2af5ce22db..5d11a2e780 100644 --- a/algo/src/test/java/org/neo4j/gds/leiden/ModularityComputerTest.java +++ b/algo/src/test/java/org/neo4j/gds/leiden/ModularityComputerTest.java @@ -23,7 +23,7 @@ import org.junit.jupiter.api.Test; import org.neo4j.gds.Orientation; import org.neo4j.gds.api.schema.Direction; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.TerminationFlag; import org.neo4j.gds.core.utils.paged.HugeDoubleArray; import org.neo4j.gds.core.utils.paged.HugeLongArray; @@ -83,7 +83,7 @@ void shouldCalculateModularityCorrectly() { 1.0 / graph.relationshipCount(), 1.0 / graph.relationshipCount(), 4, - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.EmptyProgressTracker.NULL_TRACKER ); assertThat(modularity).isCloseTo(0.4230, Offset.offset(1e-3)); @@ -102,7 +102,7 @@ void shouldCalculateModularityInSummaryGraph() { Direction.UNDIRECTED, localCommunities, 1, - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, 1, TerminationFlag.RUNNING_TRUE, ProgressTracker.NULL_TRACKER @@ -116,7 +116,7 @@ void shouldCalculateModularityInSummaryGraph() { 1.0 / graph.relationshipCount(), 1.0 / graph.relationshipCount(), 4, - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.EmptyProgressTracker.NULL_TRACKER ); assertThat(modularity).isCloseTo(0.4230, Offset.offset(1e-3)); diff --git a/algo/src/test/java/org/neo4j/gds/leiden/RefinementPhaseKarateTest.java b/algo/src/test/java/org/neo4j/gds/leiden/RefinementPhaseKarateTest.java index 1907eae189..cab2f6b6dc 100644 --- a/algo/src/test/java/org/neo4j/gds/leiden/RefinementPhaseKarateTest.java +++ b/algo/src/test/java/org/neo4j/gds/leiden/RefinementPhaseKarateTest.java @@ -21,7 +21,7 @@ import org.junit.jupiter.api.Test; import org.neo4j.gds.Orientation; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.paged.HugeDoubleArray; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.extension.GdlExtension; @@ -73,7 +73,7 @@ void testRefinementPhase() { 0.01, 19L, 1, - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); diff --git a/algo/src/test/java/org/neo4j/gds/leiden/RefinementPhaseTest.java b/algo/src/test/java/org/neo4j/gds/leiden/RefinementPhaseTest.java index fb17dc35b8..f9597a50a1 100644 --- a/algo/src/test/java/org/neo4j/gds/leiden/RefinementPhaseTest.java +++ b/algo/src/test/java/org/neo4j/gds/leiden/RefinementPhaseTest.java @@ -21,7 +21,7 @@ import org.junit.jupiter.api.Test; import org.neo4j.gds.Orientation; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.paged.HugeDoubleArray; import org.neo4j.gds.core.utils.paged.HugeLongArray; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; @@ -80,7 +80,7 @@ void shouldRefine() { 0.01, 19L, 1, - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); var refinementResult = refinement.run(); diff --git a/algo/src/test/java/org/neo4j/gds/leiden/WeightedModularityComputerTest.java b/algo/src/test/java/org/neo4j/gds/leiden/WeightedModularityComputerTest.java index c33c42f68e..92de248242 100644 --- a/algo/src/test/java/org/neo4j/gds/leiden/WeightedModularityComputerTest.java +++ b/algo/src/test/java/org/neo4j/gds/leiden/WeightedModularityComputerTest.java @@ -23,7 +23,7 @@ import org.junit.jupiter.api.Test; import org.neo4j.gds.Orientation; import org.neo4j.gds.api.schema.Direction; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.TerminationFlag; import org.neo4j.gds.core.utils.paged.HugeDoubleArray; import org.neo4j.gds.core.utils.paged.HugeLongArray; @@ -84,7 +84,7 @@ void shouldCalculateModularityCorrectly() { 1.0 / (4.0 * graph.relationshipCount()), 1.0 / (4.0 * graph.relationshipCount()), 4, - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.EmptyProgressTracker.NULL_TRACKER ); assertThat(modularity).isCloseTo(0.4230, Offset.offset(1e-3)); @@ -103,7 +103,7 @@ void shouldCalculateModularityInSummaryGraph() { Direction.UNDIRECTED, localCommunities, 1, - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, 1, TerminationFlag.RUNNING_TRUE, ProgressTracker.NULL_TRACKER @@ -117,7 +117,7 @@ void shouldCalculateModularityInSummaryGraph() { 1.0 / (4 * graph.relationshipCount()), 1.0 / (4 * graph.relationshipCount()), 4, - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.EmptyProgressTracker.NULL_TRACKER ); assertThat(modularity).isCloseTo(0.4230, Offset.offset(1e-3)); diff --git a/algo/src/test/java/org/neo4j/gds/louvain/LouvainTest.java b/algo/src/test/java/org/neo4j/gds/louvain/LouvainTest.java index 21c85058bd..db34cff670 100644 --- a/algo/src/test/java/org/neo4j/gds/louvain/LouvainTest.java +++ b/algo/src/test/java/org/neo4j/gds/louvain/LouvainTest.java @@ -36,7 +36,7 @@ import org.neo4j.gds.core.Aggregation; import org.neo4j.gds.core.GraphDimensions; import org.neo4j.gds.core.ImmutableGraphDimensions; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.huge.HugeGraph; import org.neo4j.gds.core.utils.TerminationFlag; import org.neo4j.gds.core.utils.mem.MemoryRange; @@ -155,7 +155,7 @@ void testUnweighted() { config.tolerance(), config.concurrency(), ProgressTracker.NULL_TRACKER, - ExecutorServices.DEFAULT + DefaultPool.INSTANCE ); algorithm.setTerminationFlag(TerminationFlag.RUNNING_TRUE); @@ -204,7 +204,7 @@ void testWeighted() { config.tolerance(), config.concurrency(), ProgressTracker.NULL_TRACKER, - ExecutorServices.DEFAULT + DefaultPool.INSTANCE ); algorithm.setTerminationFlag(TerminationFlag.RUNNING_TRUE); @@ -253,7 +253,7 @@ void testSeeded() { config.tolerance(), config.concurrency(), ProgressTracker.NULL_TRACKER, - ExecutorServices.DEFAULT + DefaultPool.INSTANCE ); algorithm.setTerminationFlag(TerminationFlag.RUNNING_TRUE); @@ -304,7 +304,7 @@ void testTolerance() { config.tolerance(), config.concurrency(), ProgressTracker.NULL_TRACKER, - ExecutorServices.DEFAULT + DefaultPool.INSTANCE ); algorithm.setTerminationFlag(TerminationFlag.RUNNING_TRUE); @@ -340,7 +340,7 @@ void testMaxLevels() { config.tolerance(), config.concurrency(), ProgressTracker.NULL_TRACKER, - ExecutorServices.DEFAULT + DefaultPool.INSTANCE ); algorithm.setTerminationFlag(TerminationFlag.RUNNING_TRUE); @@ -457,7 +457,7 @@ void testCanBeInterruptedByTxCancellation() { config.tolerance(), config.concurrency(), ProgressTracker.NULL_TRACKER, - ExecutorServices.DEFAULT + DefaultPool.INSTANCE ); louvain.setTerminationFlag(terminationFlag); @@ -494,7 +494,7 @@ void testLogging() { config.tolerance(), config.concurrency(), progressTracker, - ExecutorServices.DEFAULT + DefaultPool.INSTANCE ); @@ -522,7 +522,7 @@ void shouldThrowOnNegativeSeed() { config.tolerance(), config.concurrency(), ProgressTracker.NULL_TRACKER, - ExecutorServices.DEFAULT + DefaultPool.INSTANCE ); algorithm.setTerminationFlag(TerminationFlag.RUNNING_TRUE); @@ -554,7 +554,7 @@ void shouldGiveSameResultWithCalculator() { TOLERANCE_DEFAULT, 4, ProgressTracker.NULL_TRACKER, - ExecutorServices.DEFAULT + DefaultPool.INSTANCE ); var result = louvain.compute(); diff --git a/algo/src/test/java/org/neo4j/gds/modularityoptimization/FootballTest.java b/algo/src/test/java/org/neo4j/gds/modularityoptimization/FootballTest.java index 148b0b40d3..458b1b3bd0 100644 --- a/algo/src/test/java/org/neo4j/gds/modularityoptimization/FootballTest.java +++ b/algo/src/test/java/org/neo4j/gds/modularityoptimization/FootballTest.java @@ -28,7 +28,7 @@ import org.neo4j.gds.api.Graph; import org.neo4j.gds.api.properties.nodes.NodePropertyValues; import org.neo4j.gds.compat.Neo4jProxy; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.paged.HugeDoubleArray; import org.neo4j.gds.core.utils.paged.HugeLongArray; import org.neo4j.gds.core.utils.progress.EmptyTaskRegistryFactory; @@ -127,7 +127,7 @@ private ModularityOptimizationResult compute( properties, concurrency, minBatchSize, - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, progressTracker ).compute(); } diff --git a/algo/src/test/java/org/neo4j/gds/modularityoptimization/ModularityOptimizationTest.java b/algo/src/test/java/org/neo4j/gds/modularityoptimization/ModularityOptimizationTest.java index bed182c563..96f2f5c004 100644 --- a/algo/src/test/java/org/neo4j/gds/modularityoptimization/ModularityOptimizationTest.java +++ b/algo/src/test/java/org/neo4j/gds/modularityoptimization/ModularityOptimizationTest.java @@ -35,7 +35,7 @@ import org.neo4j.gds.compat.Neo4jProxy; import org.neo4j.gds.core.GraphDimensions; import org.neo4j.gds.core.ImmutableGraphDimensions; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.mem.MemoryTree; import org.neo4j.gds.core.utils.progress.EmptyTaskRegistryFactory; import org.neo4j.gds.extension.GdlExtension; @@ -243,7 +243,7 @@ private ModularityOptimizationResult compute( properties, concurrency, minBatchSize, - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, progressTracker ).compute(); } diff --git a/algo/src/test/java/org/neo4j/gds/modularityoptimization/ModularityOptimizationWithoutOrientationTest.java b/algo/src/test/java/org/neo4j/gds/modularityoptimization/ModularityOptimizationWithoutOrientationTest.java index 29ce1f04d5..db0ef84da0 100644 --- a/algo/src/test/java/org/neo4j/gds/modularityoptimization/ModularityOptimizationWithoutOrientationTest.java +++ b/algo/src/test/java/org/neo4j/gds/modularityoptimization/ModularityOptimizationWithoutOrientationTest.java @@ -34,7 +34,7 @@ import org.neo4j.gds.compat.Neo4jProxy; import org.neo4j.gds.core.GraphDimensions; import org.neo4j.gds.core.ImmutableGraphDimensions; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.mem.MemoryTree; import org.neo4j.gds.core.utils.progress.EmptyTaskRegistryFactory; import org.neo4j.gds.extension.GdlExtension; @@ -252,7 +252,7 @@ private ModularityOptimizationResult compute( properties, concurrency, minBatchSize, - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, progressTracker ).compute(); } diff --git a/algo/src/test/java/org/neo4j/gds/msbfs/MultiSourceBFSAccessMethodsTest.java b/algo/src/test/java/org/neo4j/gds/msbfs/MultiSourceBFSAccessMethodsTest.java index 29e3c58c31..f0b4fd9a4a 100644 --- a/algo/src/test/java/org/neo4j/gds/msbfs/MultiSourceBFSAccessMethodsTest.java +++ b/algo/src/test/java/org/neo4j/gds/msbfs/MultiSourceBFSAccessMethodsTest.java @@ -31,7 +31,7 @@ import org.neo4j.gds.api.RelationshipIterator; import org.neo4j.gds.api.RelationshipWithPropertyConsumer; import org.neo4j.gds.config.ConcurrencyConfig; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.graphbuilder.DefaultBuilder; import org.neo4j.gds.graphbuilder.GraphBuilder; @@ -85,7 +85,7 @@ void testWithPredecessor() { new long[]{0, 1} ); - msbfs.run(ConcurrencyConfig.DEFAULT_CONCURRENCY, ExecutorServices.DEFAULT); + msbfs.run(ConcurrencyConfig.DEFAULT_CONCURRENCY, DefaultPool.INSTANCE); verify(bfsConsumerMock).accept(1, 0, toList(1)); verify(bfsConsumerMock).accept(2, 0, toList(2)); @@ -126,7 +126,7 @@ void testWithANP() { new long[]{0, 1} ); - msbfs.run(ConcurrencyConfig.DEFAULT_CONCURRENCY, ExecutorServices.DEFAULT); + msbfs.run(ConcurrencyConfig.DEFAULT_CONCURRENCY, DefaultPool.INSTANCE); verify(mock).accept(3, 1, toList(1, 2)); verify(mock).accept(4, 1, toList(1, 2)); @@ -148,7 +148,7 @@ void testPredecessorWithAllSources() { (i, p, d, s) -> mock.accept(i + 1, p + 1, d, toList(s, x -> x + 1)) ); - msbfs.run(ConcurrencyConfig.DEFAULT_CONCURRENCY, ExecutorServices.DEFAULT); + msbfs.run(ConcurrencyConfig.DEFAULT_CONCURRENCY, DefaultPool.INSTANCE); verify(mock).accept(1, 3, 1, toList(3)); verify(mock).accept(1, 4, 1, toList(4)); @@ -199,7 +199,7 @@ void testANPWithAllSources() { (i, d, s) -> mock.accept(i + 1, d, toList(s, x -> x + 1)) ); - msbfs.run(ConcurrencyConfig.DEFAULT_CONCURRENCY, ExecutorServices.DEFAULT); + msbfs.run(ConcurrencyConfig.DEFAULT_CONCURRENCY, DefaultPool.INSTANCE); verify(mock).accept(1, 1, toList(3, 4)); verify(mock).accept(2, 1, toList(3, 4)); @@ -270,7 +270,7 @@ void testParallel() { } } ); - msbfs.run(ConcurrencyConfig.DEFAULT_CONCURRENCY, ExecutorServices.DEFAULT); + msbfs.run(ConcurrencyConfig.DEFAULT_CONCURRENCY, DefaultPool.INSTANCE); }); for (int i = 0; i < maxNodes; i++) { @@ -382,7 +382,7 @@ public Stream streamRelationships(long nodeId, double fallba } }, sources); - msbfs.run(ConcurrencyConfig.DEFAULT_CONCURRENCY, ExecutorServices.DEFAULT); + msbfs.run(ConcurrencyConfig.DEFAULT_CONCURRENCY, DefaultPool.INSTANCE); for (int i = 0; i < seen.length; i++) { final int[] nodeSeen = seen[i]; diff --git a/algo/src/test/java/org/neo4j/gds/paths/delta/DeltaSteppingTest.java b/algo/src/test/java/org/neo4j/gds/paths/delta/DeltaSteppingTest.java index 0a3016030f..bf6470e364 100644 --- a/algo/src/test/java/org/neo4j/gds/paths/delta/DeltaSteppingTest.java +++ b/algo/src/test/java/org/neo4j/gds/paths/delta/DeltaSteppingTest.java @@ -36,7 +36,7 @@ import org.neo4j.gds.beta.generator.RelationshipDistribution; import org.neo4j.gds.compat.Neo4jProxy; import org.neo4j.gds.core.GraphDimensions; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.progress.EmptyTaskRegistryFactory; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.extension.GdlExtension; @@ -142,7 +142,7 @@ void singleSource(double delta, int concurrency, long idOffset) { .build(); var paths = DeltaStepping - .of(graph, config, ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER) + .of(graph, config, DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER) .compute() .pathSet(); @@ -172,7 +172,7 @@ void singleSourceFromDisconnectedNode(double delta, int concurrency, long idOffs .build(); var paths = DeltaStepping - .of(graph, config, ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER) + .of(graph, config, DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER) .compute() .pathSet(); @@ -191,7 +191,7 @@ void shouldLogProgress() { var testLog = Neo4jProxy.testLog(); var progressTracker = new TestProgressTracker(progressTask, testLog, 1, EmptyTaskRegistryFactory.INSTANCE); - DeltaStepping.of(graph, config, ExecutorServices.DEFAULT, progressTracker) + DeltaStepping.of(graph, config, DefaultPool.INSTANCE, progressTracker) .compute() .pathSet(); @@ -279,7 +279,7 @@ void singleSource(double delta, int concurrency, long idOffset) { .build(); var paths = DeltaStepping - .of(graph, config, ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER) + .of(graph, config, DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER) .compute() .pathSet(); @@ -335,7 +335,7 @@ void singleSource(double delta, int concurrency, long idOffset) { .build(); var paths = DeltaStepping - .of(graph, config, ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER) + .of(graph, config, DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER) .compute() .pathSet(); @@ -369,7 +369,7 @@ void shouldGiveSameResultsAsDijkstra() { var deltaStepping = DeltaStepping.of( newGraph, config, - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ).compute(); diff --git a/algo/src/test/java/org/neo4j/gds/scaleproperties/ScalePropertiesMissingPropsTest.java b/algo/src/test/java/org/neo4j/gds/scaleproperties/ScalePropertiesMissingPropsTest.java index 8419afb110..222f31e5be 100644 --- a/algo/src/test/java/org/neo4j/gds/scaleproperties/ScalePropertiesMissingPropsTest.java +++ b/algo/src/test/java/org/neo4j/gds/scaleproperties/ScalePropertiesMissingPropsTest.java @@ -21,7 +21,7 @@ import org.junit.jupiter.api.Test; import org.neo4j.gds.core.CypherMapWrapper; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.extension.GdlExtension; import org.neo4j.gds.extension.GdlGraph; @@ -55,7 +55,7 @@ void partialArrays() { .nodeProperties(List.of("arrayOn4", "arrayOn1")) .scaler(Max.buildFrom(CypherMapWrapper.empty())) .build(); - var algo = new ScaleProperties(graph, config, ProgressTracker.NULL_TRACKER, ExecutorServices.DEFAULT); + var algo = new ScaleProperties(graph, config, ProgressTracker.NULL_TRACKER, DefaultPool.INSTANCE); var result = algo.compute(); var resultProperties = result.scaledProperties().toArray(); @@ -73,7 +73,7 @@ void testMissingScalar() { .nodeProperties(List.of("a", "b", "c")) .scaler(StdScore.buildFrom(CypherMapWrapper.empty())) .build(); - var algo = new ScaleProperties(graph, config, ProgressTracker.NULL_TRACKER, ExecutorServices.DEFAULT); + var algo = new ScaleProperties(graph, config, ProgressTracker.NULL_TRACKER, DefaultPool.INSTANCE); var result = algo.compute(); var resultProperties = result.scaledProperties().toArray(); diff --git a/algo/src/test/java/org/neo4j/gds/scaleproperties/ScalePropertiesTest.java b/algo/src/test/java/org/neo4j/gds/scaleproperties/ScalePropertiesTest.java index f2fbfd9e1f..026fa49575 100644 --- a/algo/src/test/java/org/neo4j/gds/scaleproperties/ScalePropertiesTest.java +++ b/algo/src/test/java/org/neo4j/gds/scaleproperties/ScalePropertiesTest.java @@ -31,7 +31,7 @@ import org.neo4j.gds.compat.Neo4jProxy; import org.neo4j.gds.core.CypherMapWrapper; import org.neo4j.gds.core.GraphDimensions; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.mem.MemoryRange; import org.neo4j.gds.core.utils.progress.EmptyTaskRegistryFactory; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; @@ -79,7 +79,7 @@ void scaleSingleProperty() { .scaler(MinMax.buildFrom(CypherMapWrapper.empty())) .concurrency(1) .build(); - var algo = new ScaleProperties(graph, config, ProgressTracker.NULL_TRACKER, ExecutorServices.DEFAULT); + var algo = new ScaleProperties(graph, config, ProgressTracker.NULL_TRACKER, DefaultPool.INSTANCE); var result = algo.compute(); var resultProperties = result.scaledProperties().toArray(); @@ -98,7 +98,7 @@ void scaleMultipleProperties() { .scaler(MinMax.buildFrom(CypherMapWrapper.empty())) .concurrency(1) .build(); - var algo = new ScaleProperties(graph, config, ProgressTracker.NULL_TRACKER, ExecutorServices.DEFAULT); + var algo = new ScaleProperties(graph, config, ProgressTracker.NULL_TRACKER, DefaultPool.INSTANCE); var result = algo.compute(); var resultProperties = result.scaledProperties().toArray(); @@ -130,14 +130,14 @@ void parallelScale() { bigGraph, config.concurrency(4).build(), ProgressTracker.NULL_TRACKER, - ExecutorServices.DEFAULT + DefaultPool.INSTANCE ).compute().scaledProperties(); var expected = new ScaleProperties( bigGraph, config.concurrency(1).build(), ProgressTracker.NULL_TRACKER, - ExecutorServices.DEFAULT + DefaultPool.INSTANCE ).compute().scaledProperties(); IntStream.range(0, nodeCount).forEach(id -> assertEquals(expected.get(id)[0], parallelResult.get(id)[0])); @@ -150,7 +150,7 @@ void scaleArrayProperty() { .scaler(MinMax.buildFrom(CypherMapWrapper.empty())) .build(); - var actual = new ScaleProperties(graph, arrayConfig, ProgressTracker.NULL_TRACKER, ExecutorServices.DEFAULT) + var actual = new ScaleProperties(graph, arrayConfig, ProgressTracker.NULL_TRACKER, DefaultPool.INSTANCE) .compute() .scaledProperties(); @@ -159,7 +159,7 @@ void scaleArrayProperty() { .scaler(MinMax.buildFrom(CypherMapWrapper.empty())) .build(); - var expected = new ScaleProperties(graph, singlePropConfig, ProgressTracker.NULL_TRACKER, ExecutorServices.DEFAULT) + var expected = new ScaleProperties(graph, singlePropConfig, ProgressTracker.NULL_TRACKER, DefaultPool.INSTANCE) .compute() .scaledProperties(); @@ -175,13 +175,13 @@ void supportLongAndFloatArrays(String scaler) { var longArrayBConfig = baseConfigBuilder.nodeProperties(List.of("longArrayB")).build(); var doubleArrayBConfig = baseConfigBuilder.nodeProperties(List.of("floatArrayB")).build(); - var expected = new ScaleProperties(graph, bConfig, ProgressTracker.NULL_TRACKER, ExecutorServices.DEFAULT) + var expected = new ScaleProperties(graph, bConfig, ProgressTracker.NULL_TRACKER, DefaultPool.INSTANCE) .compute() .scaledProperties(); - var actualLong = new ScaleProperties(graph, longArrayBConfig, ProgressTracker.NULL_TRACKER, ExecutorServices.DEFAULT) + var actualLong = new ScaleProperties(graph, longArrayBConfig, ProgressTracker.NULL_TRACKER, DefaultPool.INSTANCE) .compute() .scaledProperties(); - var actualDouble = new ScaleProperties(graph, doubleArrayBConfig, ProgressTracker.NULL_TRACKER, ExecutorServices.DEFAULT) + var actualDouble = new ScaleProperties(graph, doubleArrayBConfig, ProgressTracker.NULL_TRACKER, DefaultPool.INSTANCE) .compute() .scaledProperties(); @@ -197,7 +197,7 @@ void supportDoubleArrays() { var config = baseConfigBuilder.nodeProperties(List.of("doubleArray")).build(); var expected = new double[][]{new double[]{0.0}, new double[]{0.2499999722444236}, new double[]{.5}, new double[]{0.7500000277555764}, new double[]{1.0}}; - var actual = new ScaleProperties(graph, config, ProgressTracker.NULL_TRACKER, ExecutorServices.DEFAULT) + var actual = new ScaleProperties(graph, config, ProgressTracker.NULL_TRACKER, DefaultPool.INSTANCE) .compute() .scaledProperties(); @@ -211,7 +211,7 @@ void failOnArrayPropertyWithUnequalLength() { .scaler(MinMax.buildFrom(CypherMapWrapper.empty())) .build(); - var algo = new ScaleProperties(graph, config, ProgressTracker.NULL_TRACKER, ExecutorServices.DEFAULT); + var algo = new ScaleProperties(graph, config, ProgressTracker.NULL_TRACKER, DefaultPool.INSTANCE); var error = assertThrows(IllegalArgumentException.class, algo::compute); assertThat(error.getMessage(), containsString( @@ -226,7 +226,7 @@ void failOnNonExistentProperty() { .scaler(MinMax.buildFrom(CypherMapWrapper.empty())) .build(); - var algo = new ScaleProperties(graph, config, ProgressTracker.NULL_TRACKER, ExecutorServices.DEFAULT); + var algo = new ScaleProperties(graph, config, ProgressTracker.NULL_TRACKER, DefaultPool.INSTANCE); var error = assertThrows(IllegalArgumentException.class, algo::compute); assertThat(error.getMessage(), containsString("Node property `IMAGINARY_PROP` not found in graph")); diff --git a/algo/src/test/java/org/neo4j/gds/similarity/nodesim/NodeSimilarityTerminationTest.java b/algo/src/test/java/org/neo4j/gds/similarity/nodesim/NodeSimilarityTerminationTest.java index ef215ef923..599fc2d134 100644 --- a/algo/src/test/java/org/neo4j/gds/similarity/nodesim/NodeSimilarityTerminationTest.java +++ b/algo/src/test/java/org/neo4j/gds/similarity/nodesim/NodeSimilarityTerminationTest.java @@ -23,7 +23,7 @@ import org.neo4j.gds.BaseTest; import org.neo4j.gds.beta.generator.RandomGraphGenerator; import org.neo4j.gds.beta.generator.RelationshipDistribution; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import static org.neo4j.gds.graphbuilder.TransactionTerminationTestUtils.assertTerminates; @@ -44,7 +44,7 @@ void shouldTerminate() { var nodeSimilarity = NodeSimilarity.create( graph, NodeSimilarityTest.configBuilder().concurrency(1).build(), - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); nodeSimilarity.setTerminationFlag(terminationFlag); diff --git a/algo/src/test/java/org/neo4j/gds/similarity/nodesim/NodeSimilarityTest.java b/algo/src/test/java/org/neo4j/gds/similarity/nodesim/NodeSimilarityTest.java index 2ee72f75b8..4e8a244864 100644 --- a/algo/src/test/java/org/neo4j/gds/similarity/nodesim/NodeSimilarityTest.java +++ b/algo/src/test/java/org/neo4j/gds/similarity/nodesim/NodeSimilarityTest.java @@ -32,7 +32,7 @@ import org.neo4j.gds.compat.TestLog; import org.neo4j.gds.core.GraphDimensions; import org.neo4j.gds.core.ImmutableGraphDimensions; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.mem.MemoryEstimations; import org.neo4j.gds.core.utils.mem.MemoryRange; import org.neo4j.gds.core.utils.mem.MemoryTree; @@ -291,7 +291,7 @@ void shouldComputeWeightedForSupportedDirections(Orientation orientation, int co NodeSimilarity nodeSimilarity = NodeSimilarity.create( graph, configBuilder().relationshipWeightProperty("prop").concurrency(concurrency).build(), - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); @@ -311,7 +311,7 @@ void shouldComputeForSupportedDirections(Orientation orientation, int concurrenc NodeSimilarity nodeSimilarity = NodeSimilarity.create( graph, configBuilder().concurrency(concurrency).build(), - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); @@ -331,7 +331,7 @@ void shouldComputeTopNForSupportedDirections(Orientation orientation, int concur NodeSimilarity nodeSimilarity = NodeSimilarity.create( graph, configBuilder().concurrency(concurrency).topN(1).build(), - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); @@ -351,7 +351,7 @@ void shouldComputeNegativeTopNForSupportedDirections(Orientation orientation, in NodeSimilarity nodeSimilarity = NodeSimilarity.create( graph, configBuilder().concurrency(concurrency).bottomN(1).build(), - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); @@ -373,7 +373,7 @@ void shouldComputeTopKForSupportedDirections(Orientation orientation, int concur NodeSimilarity nodeSimilarity = NodeSimilarity.create( graph, configBuilder().topK(1).concurrency(concurrency).build(), - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); @@ -397,7 +397,7 @@ void shouldComputeNegativeTopKForSupportedDirections(Orientation orientation, in .topK(10) .bottomK(1) .build(), - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); @@ -427,7 +427,7 @@ void shouldComputeWithSimilarityCutoffForSupportedDirections(Orientation orienta NodeSimilarity nodeSimilarity = NodeSimilarity.create( graph, configBuilder().concurrency(concurrency).similarityCutoff(0.1).build(), - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); @@ -450,7 +450,7 @@ void shouldComputeWithDegreeCutoffForSupportedDirections(Orientation orientation NodeSimilarity nodeSimilarity = NodeSimilarity.create( graph, configBuilder().degreeCutoff(2).concurrency(concurrency).build(), - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); @@ -471,7 +471,7 @@ void shouldComputeForUndirectedGraphs(int concurrency) { NodeSimilarity nodeSimilarity = NodeSimilarity.create( undirectedGraph, configBuilder().concurrency(concurrency).build(), - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); Set result = nodeSimilarity.computeToStream().collect(Collectors.toSet()); @@ -483,7 +483,7 @@ void shouldComputeForUnionGraphs() { NodeSimilarity nodeSimilarity = NodeSimilarity.create( naturalGraph, configBuilder().concurrency(1).build(), - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); var result1 = nodeSimilarity.computeToStream().collect(Collectors.toSet()); @@ -491,7 +491,7 @@ void shouldComputeForUnionGraphs() { nodeSimilarity = NodeSimilarity.create( naturalUnionGraph, configBuilder().concurrency(1).build(), - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); var result2 = nodeSimilarity.computeToStream().collect(Collectors.toSet()); @@ -507,7 +507,7 @@ void shouldComputeSimilarityGraphInAllSupportedDirections(Orientation orientatio NodeSimilarity nodeSimilarity = NodeSimilarity.create( graph, configBuilder().concurrency(concurrency).build(), - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); @@ -558,7 +558,7 @@ void shouldComputeToGraphWithUnusedNodesInInputGraph(Orientation orientation, in .topK(100) .topN(1) .build(), - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); @@ -596,7 +596,7 @@ void shouldIgnoreLoops(Orientation orientation, int concurrency) { NodeSimilarity nodeSimilarity = NodeSimilarity.create( graph, configBuilder().concurrency(concurrency).topN(1).build(), - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); @@ -623,7 +623,7 @@ void shouldIgnoreParallelEdges(Orientation orientation, int concurrency) { NodeSimilarity nodeSimilarity = NodeSimilarity.create( graph, configBuilder().concurrency(concurrency).build(), - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); @@ -840,7 +840,7 @@ void shouldLogProgress(int concurrency) { NodeSimilarity.create( graph, config, - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, progressTracker ).compute().streamResult().count(); @@ -880,7 +880,7 @@ void shouldGiveCorrectResultsWithOverlap() { NodeSimilarity nodeSimilarity = NodeSimilarity.create( graph, configBuilder().concurrency(1).similarityMetric(MetricSimilarityComputer.parse("ovErLaP")).build(), - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); @@ -898,7 +898,7 @@ void shouldGiveCorrectResultsWithOverlap() { .concurrency(1) .similarityMetric(MetricSimilarityComputer.parse("ovErLaP")) .build(), - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); @@ -934,7 +934,7 @@ void shouldWorkForAllDegreeBoundsCombinations(int lowBound, int upperBound, Stri NodeSimilarity nodeSimilarity = NodeSimilarity.create( naturalGraph, configBuilder().upperDegreeCutoff(upperBound).degreeCutoff(lowBound).build(), - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); diff --git a/algo/src/test/java/org/neo4j/gds/similarity/nodesim/SimilarityGraphBuilderTest.java b/algo/src/test/java/org/neo4j/gds/similarity/nodesim/SimilarityGraphBuilderTest.java index da7e30d0c8..858524a05f 100644 --- a/algo/src/test/java/org/neo4j/gds/similarity/nodesim/SimilarityGraphBuilderTest.java +++ b/algo/src/test/java/org/neo4j/gds/similarity/nodesim/SimilarityGraphBuilderTest.java @@ -22,7 +22,7 @@ import org.junit.jupiter.api.Test; import org.neo4j.gds.NodeLabel; import org.neo4j.gds.api.Graph; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.huge.HugeGraph; import org.neo4j.gds.core.huge.UnionGraph; import org.neo4j.gds.core.loading.construction.GraphFactory; @@ -73,7 +73,7 @@ void testConstructionFromHugeGraph() { SimilarityGraphBuilder similarityGraphBuilder = new SimilarityGraphBuilder( unlabelledGraph, 1, - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, TerminationFlag.RUNNING_TRUE ); @@ -96,7 +96,7 @@ void testConstructionFromUnionGraph() { SimilarityGraphBuilder similarityGraphBuilder = new SimilarityGraphBuilder( graph, 1, - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, TerminationFlag.RUNNING_TRUE ); @@ -131,7 +131,7 @@ void testConstructFromFilteredGraph() { SimilarityGraphBuilder similarityGraphBuilder = new SimilarityGraphBuilder( filteredIdMap, 1, - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, TerminationFlag.RUNNING_TRUE ); diff --git a/algo/src/test/java/org/neo4j/gds/similarity/nodesim/UnionGraphWeightedNodeSimilarityTest.java b/algo/src/test/java/org/neo4j/gds/similarity/nodesim/UnionGraphWeightedNodeSimilarityTest.java index a76509d231..b7c371db0d 100644 --- a/algo/src/test/java/org/neo4j/gds/similarity/nodesim/UnionGraphWeightedNodeSimilarityTest.java +++ b/algo/src/test/java/org/neo4j/gds/similarity/nodesim/UnionGraphWeightedNodeSimilarityTest.java @@ -22,7 +22,7 @@ import org.assertj.core.data.Offset; import org.junit.jupiter.api.Test; import org.neo4j.gds.api.Graph; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.extension.GdlExtension; import org.neo4j.gds.extension.GdlGraph; @@ -58,7 +58,7 @@ void shouldWorkWithUnionGraph(){ .topN(1) .build(); - var nodeSimilarity = NodeSimilarity.create(graph, config, ExecutorServices.DEFAULT, ProgressTracker.NULL_TRACKER); + var nodeSimilarity = NodeSimilarity.create(graph, config, DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER); var result = nodeSimilarity.compute().streamResult().findFirst().get(); //input should be (0 + 10 + 0)/ (5 + 10 + 8) = 10/23 diff --git a/algo/src/test/java/org/neo4j/gds/steiner/ShortestPathSteinerAlgorithmExtendedTest.java b/algo/src/test/java/org/neo4j/gds/steiner/ShortestPathSteinerAlgorithmExtendedTest.java index a144be9c70..b826bccc72 100644 --- a/algo/src/test/java/org/neo4j/gds/steiner/ShortestPathSteinerAlgorithmExtendedTest.java +++ b/algo/src/test/java/org/neo4j/gds/steiner/ShortestPathSteinerAlgorithmExtendedTest.java @@ -25,7 +25,7 @@ import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; import org.neo4j.gds.Orientation; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.extension.GdlExtension; import org.neo4j.gds.extension.GdlGraph; @@ -155,7 +155,7 @@ void shouldWorkCorrectly(double delta, int binSizeThreshold) { false, binSizeThreshold, //setting custom threshold for such a small graph allows to not examine everything in a single iteration - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ).compute(); @@ -175,7 +175,7 @@ void shouldWorkCorrectlyWithLineGraph() { 2.0, 1, false, - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ) .compute(); @@ -203,7 +203,7 @@ void deltaSteppingShouldWorkCorrectly() { isTerminal, 1, SteinerBasedDeltaStepping.BIN_SIZE_THRESHOLD, - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); var result = deltaSteiner.compute().pathSet(); @@ -229,7 +229,7 @@ void shouldWorkIfRevisitsVertices() { 2.0, 1, false, - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ).compute(); @@ -258,7 +258,7 @@ void shouldWorkOnTriangle() { 2.0, 1, false, - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ).compute(); diff --git a/algo/src/test/java/org/neo4j/gds/steiner/ShortestPathsSteinerAlgorithmReroutingTest.java b/algo/src/test/java/org/neo4j/gds/steiner/ShortestPathsSteinerAlgorithmReroutingTest.java index cb3a58b8d9..688070202a 100644 --- a/algo/src/test/java/org/neo4j/gds/steiner/ShortestPathsSteinerAlgorithmReroutingTest.java +++ b/algo/src/test/java/org/neo4j/gds/steiner/ShortestPathsSteinerAlgorithmReroutingTest.java @@ -25,7 +25,7 @@ import org.neo4j.gds.TestProgressTracker; import org.neo4j.gds.compat.Neo4jProxy; import org.neo4j.gds.compat.TestLog; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.progress.EmptyTaskRegistryFactory; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.core.utils.progress.tasks.Task; @@ -189,7 +189,7 @@ void shouldPruneUnusedIfRerouting() { 2.0, 1, false, - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ).compute(); assertThat(steinerResult.totalCost()).isEqualTo(7.0); @@ -203,7 +203,7 @@ void shouldPruneUnusedIfRerouting() { 2.0, 1, true, - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ).compute(); assertThat(steinerResultWithReroute.totalCost()).isEqualTo(4.0); @@ -223,7 +223,7 @@ void shouldPruneUnusedIfReroutingOnInvertedIndex() { 2.0, 1, true, - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ).compute(); assertThat(steinerResultWithReroute.totalCost()).isEqualTo(4.0); @@ -242,7 +242,7 @@ void rerouteShouldNotCreateLoops() { 2.0, 1, true, - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ).compute(); var parent = steinerResult.parentArray().toArray(); @@ -267,7 +267,7 @@ void rerouteShouldNotCreateLoopsOnInvertedIndex() { 2.0, 1, true, - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ).compute(); var parent = steinerResult.parentArray().toArray(); @@ -294,7 +294,7 @@ void shouldWorkForUnreachableAndReachableTerminals() { 2.0, 1, true, - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ).compute(); assertThat(steinerTreeResult.effectiveTargetNodesCount()).isEqualTo(2); @@ -315,7 +315,7 @@ void shouldWorkIfNoReachableTerminals() { 2.0, 1, true, - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ).compute(); assertThat(steinerTreeResult.effectiveTargetNodesCount()).isEqualTo(0); @@ -473,7 +473,7 @@ void shouldNotGetOptimalWithoutBetterRerouting() { 2.0, 1, true, - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ).compute(); assertThat(steinerResultWithReroute.totalCost()).isEqualTo(25.0); @@ -498,7 +498,7 @@ void shouldHandleMultiplePruningsOnSameTreeAndGetBetter() { 2.0, 1, true, - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ).compute(); assertThat(steinerResultWithReroute.totalCost()).isEqualTo(22.0); @@ -522,7 +522,7 @@ void shouldNotPruneUnprunableNodes() { 2.0, 1, true, - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ).compute(); assertThat(steinerResultWithReroute.totalCost()).isEqualTo(170.0 - 19); @@ -546,7 +546,7 @@ void shouldTakeAdvantageOfNewSingleParents() { 2.0, 1, true, - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ).compute(); assertThat(steinerResultWithReroute.totalCost()).isEqualTo(20); diff --git a/algo/src/test/java/org/neo4j/gds/steiner/ShortestPathsSteinerAlgorithmTest.java b/algo/src/test/java/org/neo4j/gds/steiner/ShortestPathsSteinerAlgorithmTest.java index a736cac2dd..d057bcb99e 100644 --- a/algo/src/test/java/org/neo4j/gds/steiner/ShortestPathsSteinerAlgorithmTest.java +++ b/algo/src/test/java/org/neo4j/gds/steiner/ShortestPathsSteinerAlgorithmTest.java @@ -21,7 +21,7 @@ import org.junit.jupiter.api.Test; import org.neo4j.gds.Orientation; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.extension.GdlExtension; import org.neo4j.gds.extension.GdlGraph; @@ -75,7 +75,7 @@ void shouldWorkCorrectly() { 2.0, 1, false, - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ).compute(); var pruned = ShortestPathsSteinerAlgorithm.PRUNED; diff --git a/algo/src/test/java/org/neo4j/gds/traversal/RandomWalkTest.java b/algo/src/test/java/org/neo4j/gds/traversal/RandomWalkTest.java index 7ae46a3750..689a1b8186 100644 --- a/algo/src/test/java/org/neo4j/gds/traversal/RandomWalkTest.java +++ b/algo/src/test/java/org/neo4j/gds/traversal/RandomWalkTest.java @@ -34,7 +34,7 @@ import org.neo4j.gds.beta.generator.RelationshipDistribution; import org.neo4j.gds.compat.Neo4jProxy; import org.neo4j.gds.compat.TestLog; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.TerminationFlag; import org.neo4j.gds.core.utils.progress.GlobalTaskStore; import org.neo4j.gds.core.utils.progress.TaskRegistryFactory; @@ -95,7 +95,7 @@ void testWithDefaultConfig() { graph, config, ProgressTracker.NULL_TRACKER, - ExecutorServices.DEFAULT + DefaultPool.INSTANCE ); List result = randomWalk.compute().collect(Collectors.toList()); @@ -136,7 +136,7 @@ private List runRandomWalkSeeded(Node2VecStreamConfig config, Graph grap graph, config, ProgressTracker.NULL_TRACKER, - ExecutorServices.DEFAULT + DefaultPool.INSTANCE ); return randomWalk.compute().collect(Collectors.toList()); @@ -149,7 +149,7 @@ void testSampleFromMultipleRelationshipTypes() { graph, config, ProgressTracker.NULL_TRACKER, - ExecutorServices.DEFAULT + DefaultPool.INSTANCE ); int expectedNumberOfWalks = config.walksPerNode() * 3; @@ -190,7 +190,7 @@ void returnFactorShouldMakeWalksIncludeStartNodeMoreOften() { graph, config, ProgressTracker.NULL_TRACKER, - ExecutorServices.DEFAULT + DefaultPool.INSTANCE ); var nodeCounter = new HashMap(); @@ -250,7 +250,7 @@ void largeInOutFactorShouldMakeTheWalkKeepTheSameDistance() { graph, config, ProgressTracker.NULL_TRACKER, - ExecutorServices.DEFAULT + DefaultPool.INSTANCE ); var nodeCounter = new HashMap(); @@ -299,7 +299,7 @@ void shouldRespectRelationshipWeights() { graph, config, ProgressTracker.NULL_TRACKER, - ExecutorServices.DEFAULT + DefaultPool.INSTANCE ); var nodeCounter = new HashMap(); @@ -336,7 +336,7 @@ void failOnInvalidRelationshipWeights(double invalidWeight) { graph, config, ProgressTracker.NULL_TRACKER, - ExecutorServices.DEFAULT + DefaultPool.INSTANCE ) ).isInstanceOf(RuntimeException.class) .hasMessage( @@ -371,7 +371,7 @@ void parallelWeighted() { graph, config, ProgressTracker.NULL_TRACKER, - ExecutorServices.DEFAULT + DefaultPool.INSTANCE ); assertThat(randomWalk.compute().collect(Collectors.toList())) @@ -400,7 +400,7 @@ void testWithConfiguredOffsetStartNodes() { graph, config, ProgressTracker.NULL_TRACKER, - ExecutorServices.DEFAULT + DefaultPool.INSTANCE ); assertThat(randomWalk.compute().collect(Collectors.toList())) @@ -423,7 +423,7 @@ void testSetTerminationFlagAndMultipleRuns() { graph, config, ProgressTracker.NULL_TRACKER, - ExecutorServices.DEFAULT + DefaultPool.INSTANCE ); var stream = randomWalk.compute(); diff --git a/algo/src/test/java/org/neo4j/gds/triangle/IntersectingTriangleCountFilteredGraphTest.java b/algo/src/test/java/org/neo4j/gds/triangle/IntersectingTriangleCountFilteredGraphTest.java index 6f4fac3772..7a146f2368 100644 --- a/algo/src/test/java/org/neo4j/gds/triangle/IntersectingTriangleCountFilteredGraphTest.java +++ b/algo/src/test/java/org/neo4j/gds/triangle/IntersectingTriangleCountFilteredGraphTest.java @@ -29,7 +29,7 @@ import org.neo4j.gds.StoreLoaderBuilder; import org.neo4j.gds.api.GraphStore; import org.neo4j.gds.core.Aggregation; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.extension.Neo4jGraph; import java.util.Collections; @@ -78,7 +78,7 @@ void testUnionGraphWithNodeFilter() { Optional.empty() ); var config = ImmutableTriangleCountBaseConfig.builder().build(); - var triangleCount = IntersectingTriangleCount.create(graph, config, ExecutorServices.DEFAULT); + var triangleCount = IntersectingTriangleCount.create(graph, config, DefaultPool.INSTANCE); var triangleCountResult = triangleCount.compute(); assertThat(triangleCountResult.globalTriangles()).isEqualTo(1); var triangles = triangleCountResult.localTriangles(); diff --git a/algo/src/test/java/org/neo4j/gds/triangle/IntersectingTriangleCountTest.java b/algo/src/test/java/org/neo4j/gds/triangle/IntersectingTriangleCountTest.java index 6b2d4ad294..a3403f0582 100644 --- a/algo/src/test/java/org/neo4j/gds/triangle/IntersectingTriangleCountTest.java +++ b/algo/src/test/java/org/neo4j/gds/triangle/IntersectingTriangleCountTest.java @@ -27,7 +27,7 @@ import org.neo4j.gds.Orientation; import org.neo4j.gds.TestSupport; import org.neo4j.gds.api.Graph; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import java.util.stream.Stream; @@ -531,7 +531,7 @@ private TriangleCountResult compute(Graph graph) { } private TriangleCountResult compute(Graph graph, TriangleCountBaseConfig config) { - return IntersectingTriangleCount.create(graph, config, ExecutorServices.DEFAULT).compute(); + return IntersectingTriangleCount.create(graph, config, DefaultPool.INSTANCE).compute(); } private static Graph fromGdl(String gdl) { diff --git a/algo/src/test/java/org/neo4j/gds/triangle/LargeIntersectingTriangleCountTest.java b/algo/src/test/java/org/neo4j/gds/triangle/LargeIntersectingTriangleCountTest.java index de321fbf88..23f5eea35f 100644 --- a/algo/src/test/java/org/neo4j/gds/triangle/LargeIntersectingTriangleCountTest.java +++ b/algo/src/test/java/org/neo4j/gds/triangle/LargeIntersectingTriangleCountTest.java @@ -25,7 +25,7 @@ import org.neo4j.gds.Orientation; import org.neo4j.gds.StoreLoaderBuilder; import org.neo4j.gds.api.Graph; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.paged.PagedAtomicIntegerArray; import org.neo4j.gds.graphbuilder.DefaultBuilder; import org.neo4j.gds.graphbuilder.GraphBuilder; @@ -71,7 +71,7 @@ void testQueue() { var result = IntersectingTriangleCount.create( graph, defaultConfigBuilder().build(), - ExecutorServices.DEFAULT + DefaultPool.INSTANCE ).compute(); assertEquals(TRIANGLE_COUNT, result.globalTriangles()); assertTriangles(result.globalTriangles()); @@ -83,7 +83,7 @@ void testQueueParallel() { var result = IntersectingTriangleCount.create( graph, defaultConfigBuilder().concurrency(4).build(), - ExecutorServices.DEFAULT + DefaultPool.INSTANCE ).compute(); assertEquals(TRIANGLE_COUNT, result.globalTriangles()); assertTriangles(result.globalTriangles()); diff --git a/algo/src/test/java/org/neo4j/gds/triangle/TriangleStreamTest.java b/algo/src/test/java/org/neo4j/gds/triangle/TriangleStreamTest.java index 0c7d5faec3..6a29c4ca06 100644 --- a/algo/src/test/java/org/neo4j/gds/triangle/TriangleStreamTest.java +++ b/algo/src/test/java/org/neo4j/gds/triangle/TriangleStreamTest.java @@ -25,7 +25,7 @@ import org.neo4j.gds.Orientation; import org.neo4j.gds.StoreLoaderBuilder; import org.neo4j.gds.api.Graph; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.graphbuilder.DefaultBuilder; import org.neo4j.gds.graphbuilder.GraphBuilder; import org.neo4j.graphdb.Node; @@ -73,7 +73,7 @@ void setupGraphDb() { void testSequential() { TripleConsumer mock = mock(TripleConsumer.class); - TriangleStream.create(graph, ExecutorServices.DEFAULT, 1) + TriangleStream.create(graph, DefaultPool.INSTANCE, 1) .compute() .forEach(r -> mock.consume(r.nodeA, r.nodeB, r.nodeC)); @@ -84,7 +84,7 @@ void testSequential() { void testParallel() { TripleConsumer mock = mock(TripleConsumer.class); - TriangleStream.create(graph, ExecutorServices.DEFAULT, 8) + TriangleStream.create(graph, DefaultPool.INSTANCE, 8) .compute() .forEach(r -> mock.consume(r.nodeA, r.nodeB, r.nodeC)); diff --git a/algo/src/test/java/org/neo4j/gds/triangle/UnionGraphTriangleCountingTest.java b/algo/src/test/java/org/neo4j/gds/triangle/UnionGraphTriangleCountingTest.java index 0f201d7bd1..1e71af7ac1 100644 --- a/algo/src/test/java/org/neo4j/gds/triangle/UnionGraphTriangleCountingTest.java +++ b/algo/src/test/java/org/neo4j/gds/triangle/UnionGraphTriangleCountingTest.java @@ -22,7 +22,7 @@ import org.junit.jupiter.api.Test; import org.neo4j.gds.Orientation; import org.neo4j.gds.api.Graph; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.extension.GdlExtension; import org.neo4j.gds.extension.GdlGraph; import org.neo4j.gds.extension.Inject; @@ -96,7 +96,7 @@ class UnionGraphTriangleCountingTest { @Test void shouldWorkWithUnionGraphs() { var config=TriangleCountStreamConfigImpl.builder().concurrency(1).build(); - var a=IntersectingTriangleCount.create(graph,config, ExecutorServices.DEFAULT); + var a=IntersectingTriangleCount.create(graph,config, DefaultPool.INSTANCE); var result=a.compute(); assertThat(result.globalTriangles()).isEqualTo(0); } diff --git a/algo/src/test/java/org/neo4j/gds/undirected/ToUndirectedTest.java b/algo/src/test/java/org/neo4j/gds/undirected/ToUndirectedTest.java index c476c0e753..94ecc433db 100644 --- a/algo/src/test/java/org/neo4j/gds/undirected/ToUndirectedTest.java +++ b/algo/src/test/java/org/neo4j/gds/undirected/ToUndirectedTest.java @@ -28,7 +28,7 @@ import org.neo4j.gds.api.GraphStore; import org.neo4j.gds.compat.Neo4jProxy; import org.neo4j.gds.core.Aggregation; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.loading.SingleTypeRelationships; import org.neo4j.gds.core.utils.progress.EmptyTaskRegistryFactory; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; @@ -82,7 +82,7 @@ void shouldCreateUndirectedRelationships(int concurrency) { directedGraphStore, config, ProgressTracker.NULL_TRACKER, - ExecutorServices.DEFAULT + DefaultPool.INSTANCE ).compute(); directedGraphStore.addRelationshipType(undirectedRelationships); @@ -129,7 +129,7 @@ void shouldCreateUndirectedRelationshipsWithSingleRelationshipProperty(int concu singleDirectedGraphStore, config, ProgressTracker.NULL_TRACKER, - ExecutorServices.DEFAULT + DefaultPool.INSTANCE ).compute(); singleDirectedGraphStore.addRelationshipType(undirectedRelationships); @@ -175,7 +175,7 @@ void shouldCreateUndirectedRelationshipsWithNoRelationshipProperty(int concurren noPropertyDirectedGraphStore, config, ProgressTracker.NULL_TRACKER, - ExecutorServices.DEFAULT + DefaultPool.INSTANCE ).compute(); noPropertyDirectedGraphStore.addRelationshipType(undirectedRelationships); @@ -207,7 +207,7 @@ void shouldAggregateWithoutProperties() { inputGraphStore, config, ProgressTracker.NULL_TRACKER, - ExecutorServices.DEFAULT + DefaultPool.INSTANCE ).compute(); inputGraphStore.addRelationshipType(undirectedRels); @@ -240,7 +240,7 @@ void shouldAggregateWithPropertiesAndGlobalAggregation() { input, config, ProgressTracker.NULL_TRACKER, - ExecutorServices.DEFAULT + DefaultPool.INSTANCE ).compute(); input.addRelationshipType(aggregatedUndirectedRelationships); @@ -281,7 +281,7 @@ void shouldAggregateWithPropertiesAndLocalAggregation() { input, config, ProgressTracker.NULL_TRACKER, - ExecutorServices.DEFAULT + DefaultPool.INSTANCE ).compute(); input.addRelationshipType(aggregatedUndirectedRelationships); diff --git a/algo/src/test/java/org/neo4j/gds/walking/CollapseMultiPathsTest.java b/algo/src/test/java/org/neo4j/gds/walking/CollapseMultiPathsTest.java index edd2f73afc..08fb482dda 100644 --- a/algo/src/test/java/org/neo4j/gds/walking/CollapseMultiPathsTest.java +++ b/algo/src/test/java/org/neo4j/gds/walking/CollapseMultiPathsTest.java @@ -26,7 +26,7 @@ import org.neo4j.gds.api.AdjacencyList; import org.neo4j.gds.api.Graph; import org.neo4j.gds.api.GraphStore; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.extension.GdlExtension; import org.neo4j.gds.extension.GdlGraph; import org.neo4j.gds.extension.IdFunction; @@ -112,7 +112,7 @@ void shouldFollowRoutesFromMovies() { false, RelationshipType.of("REL"), 2, - ExecutorServices.DEFAULT + DefaultPool.INSTANCE ).compute(); @@ -174,7 +174,7 @@ void shouldTurnEveryRouteIntoRelationship() { false, RelationshipType.of("REL"), 2, - ExecutorServices.DEFAULT + DefaultPool.INSTANCE ).compute(); diff --git a/algo/src/test/java/org/neo4j/gds/walking/CollapsePathTest.java b/algo/src/test/java/org/neo4j/gds/walking/CollapsePathTest.java index d3c02d9864..02a83910f8 100644 --- a/algo/src/test/java/org/neo4j/gds/walking/CollapsePathTest.java +++ b/algo/src/test/java/org/neo4j/gds/walking/CollapsePathTest.java @@ -25,7 +25,7 @@ import org.neo4j.gds.RelationshipType; import org.neo4j.gds.api.Graph; import org.neo4j.gds.api.GraphStore; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.loading.SingleTypeRelationships; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.extension.GdlExtension; @@ -123,7 +123,7 @@ void testCreatingRelationships() { false, RelationshipType.of("SAME_DRUG"), 2, - ExecutorServices.DEFAULT + DefaultPool.INSTANCE ).compute(); @@ -139,7 +139,7 @@ void testAllowCreatingSelfLoops() { true, RelationshipType.of("SAME_DRUG"), 2, - ExecutorServices.DEFAULT + DefaultPool.INSTANCE ).compute(); @@ -153,7 +153,7 @@ void runWithDifferentRelationshipTypes() { false, RelationshipType.of("SAME_DRUG"), 2, - ExecutorServices.DEFAULT + DefaultPool.INSTANCE ).compute(); assertResultGraph(tookGraphStore, relationships, EXPECTED_WITHOUT_LOOPS); diff --git a/algo/src/test/java/org/neo4j/gds/wcc/IncrementalWccTest.java b/algo/src/test/java/org/neo4j/gds/wcc/IncrementalWccTest.java index 13e7f111ec..65bd6e58e7 100644 --- a/algo/src/test/java/org/neo4j/gds/wcc/IncrementalWccTest.java +++ b/algo/src/test/java/org/neo4j/gds/wcc/IncrementalWccTest.java @@ -24,7 +24,7 @@ import org.neo4j.gds.api.Graph; import org.neo4j.gds.api.IdMap; import org.neo4j.gds.config.ConcurrencyConfig; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.paged.dss.DisjointSetStruct; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; @@ -125,7 +125,7 @@ void shouldAssignMinimumCommunityIdOnMerge() { private DisjointSetStruct run(Graph graph, WccBaseConfig config) { return new Wcc( graph, - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, COMMUNITY_SIZE / ConcurrencyConfig.DEFAULT_CONCURRENCY, config, ProgressTracker.NULL_TRACKER diff --git a/algo/src/test/java/org/neo4j/gds/wcc/WccTest.java b/algo/src/test/java/org/neo4j/gds/wcc/WccTest.java index eac79b958a..9c2402ee09 100644 --- a/algo/src/test/java/org/neo4j/gds/wcc/WccTest.java +++ b/algo/src/test/java/org/neo4j/gds/wcc/WccTest.java @@ -35,7 +35,7 @@ import org.neo4j.gds.compat.Neo4jProxy; import org.neo4j.gds.core.GraphDimensions; import org.neo4j.gds.core.ImmutableGraphDimensions; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.paged.dss.DisjointSetStruct; import org.neo4j.gds.core.utils.progress.EmptyTaskRegistryFactory; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; @@ -280,7 +280,7 @@ DisjointSetStruct run(Graph graph, WccBaseConfig config) { DisjointSetStruct run(Graph graph, WccBaseConfig config, int concurrency) { return new Wcc( graph, - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, communitySize() / concurrency, config, ProgressTracker.NULL_TRACKER diff --git a/algo/src/test/java/org/neo4j/gds/wcc/WccThresholdTest.java b/algo/src/test/java/org/neo4j/gds/wcc/WccThresholdTest.java index c700da19c7..7b05ab9023 100644 --- a/algo/src/test/java/org/neo4j/gds/wcc/WccThresholdTest.java +++ b/algo/src/test/java/org/neo4j/gds/wcc/WccThresholdTest.java @@ -23,7 +23,7 @@ import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; import org.neo4j.gds.CommunityHelper; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.paged.dss.DisjointSetStruct; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.extension.GdlExtension; @@ -110,7 +110,7 @@ private void assertResults(double threshold, TestGraph graph, String[][] expecte DisjointSetStruct dss = new Wcc( graph, - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, DEFAULT_BATCH_SIZE, wccConfig, ProgressTracker.NULL_TRACKER diff --git a/alpha/alpha-proc/src/test/java/org/neo4j/gds/pregel/HitsTest.java b/alpha/alpha-proc/src/test/java/org/neo4j/gds/pregel/HitsTest.java index 6380b82b2a..a68d12abb9 100644 --- a/alpha/alpha-proc/src/test/java/org/neo4j/gds/pregel/HitsTest.java +++ b/alpha/alpha-proc/src/test/java/org/neo4j/gds/pregel/HitsTest.java @@ -21,7 +21,7 @@ import org.junit.jupiter.api.Test; import org.neo4j.gds.beta.pregel.Pregel; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.extension.GdlExtension; import org.neo4j.gds.extension.GdlGraph; @@ -65,7 +65,7 @@ void testHits() { graph, config, new Hits(), - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); diff --git a/alpha/alpha-proc/src/test/java/org/neo4j/gds/pregel/SpeakerListenerLPATest.java b/alpha/alpha-proc/src/test/java/org/neo4j/gds/pregel/SpeakerListenerLPATest.java index eae5372092..4752a25b2a 100644 --- a/alpha/alpha-proc/src/test/java/org/neo4j/gds/pregel/SpeakerListenerLPATest.java +++ b/alpha/alpha-proc/src/test/java/org/neo4j/gds/pregel/SpeakerListenerLPATest.java @@ -21,7 +21,7 @@ import org.junit.jupiter.api.Test; import org.neo4j.gds.beta.pregel.Pregel; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.extension.GdlExtension; import org.neo4j.gds.extension.GdlGraph; @@ -75,7 +75,7 @@ void testWithoutPruning() { graph, config, new SpeakerListenerLPA(42), - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); @@ -122,7 +122,7 @@ void prunesAwayAfterManyIterations() { graph, config, new SpeakerListenerLPA(42), - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); @@ -164,7 +164,7 @@ void closesThreadLocal() { graph, config, computation, - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ).run(); diff --git a/core-write/src/main/java/org/neo4j/gds/core/write/NativeRelationshipStreamExporter.java b/core-write/src/main/java/org/neo4j/gds/core/write/NativeRelationshipStreamExporter.java index 200af92bc5..53cc05e72f 100644 --- a/core-write/src/main/java/org/neo4j/gds/core/write/NativeRelationshipStreamExporter.java +++ b/core-write/src/main/java/org/neo4j/gds/core/write/NativeRelationshipStreamExporter.java @@ -21,7 +21,7 @@ import org.neo4j.gds.api.IdMap; import org.neo4j.gds.api.nodeproperties.ValueType; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.TerminationFlag; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.transaction.TransactionContext; @@ -97,7 +97,7 @@ public long write(String relationshipType, List propertyKeys, List(bufferPool.poll()); diff --git a/core-write/src/test/java/org/neo4j/gds/core/write/NativeNodeLabelExporterTest.java b/core-write/src/test/java/org/neo4j/gds/core/write/NativeNodeLabelExporterTest.java index 899a2a77d3..366743f211 100644 --- a/core-write/src/test/java/org/neo4j/gds/core/write/NativeNodeLabelExporterTest.java +++ b/core-write/src/test/java/org/neo4j/gds/core/write/NativeNodeLabelExporterTest.java @@ -25,7 +25,7 @@ import org.neo4j.gds.StoreLoaderBuilder; import org.neo4j.gds.TestSupport; import org.neo4j.gds.api.Graph; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.TerminationFlag; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.extension.Neo4jGraph; @@ -58,7 +58,7 @@ void exportLabel(int concurrency) { TerminationFlag.RUNNING_TRUE, ProgressTracker.NULL_TRACKER, concurrency, - ExecutorServices.DEFAULT + DefaultPool.INSTANCE ); exporter.write("GeneratedLabel"); diff --git a/core-write/src/test/java/org/neo4j/gds/core/write/NativeNodePropertyExporterTest.java b/core-write/src/test/java/org/neo4j/gds/core/write/NativeNodePropertyExporterTest.java index 7a157af237..740e9d6d2d 100644 --- a/core-write/src/test/java/org/neo4j/gds/core/write/NativeNodePropertyExporterTest.java +++ b/core-write/src/test/java/org/neo4j/gds/core/write/NativeNodePropertyExporterTest.java @@ -31,7 +31,7 @@ import org.neo4j.gds.compat.Neo4jProxy; import org.neo4j.gds.compat.TestLog; import org.neo4j.gds.core.Aggregation; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.huge.DirectIdMap; import org.neo4j.gds.core.utils.TerminationFlag; import org.neo4j.gds.core.utils.progress.EmptyTaskRegistryFactory; @@ -149,7 +149,7 @@ void stopsExportingWhenTransactionHasBeenTerminated() { @Test void stopsParallelExportingWhenTransactionHasBeenTerminated() { - transactionTerminationTest(ExecutorServices.DEFAULT); + transactionTerminationTest(DefaultPool.INSTANCE); } @ParameterizedTest @@ -172,7 +172,7 @@ void progressLogging(boolean parallel) { .builder(TestSupport.fullAccessTransaction(db), graph, TerminationFlag.RUNNING_TRUE) .withProgressTracker(progressTracker); if (parallel) { - exporterBuilder = exporterBuilder.parallel(ExecutorServices.DEFAULT, writeConcurrency); + exporterBuilder = exporterBuilder.parallel(DefaultPool.INSTANCE, writeConcurrency); } var exporter = exporterBuilder.build(); diff --git a/core/src/main/java/org/neo4j/gds/api/GraphLoaderContext.java b/core/src/main/java/org/neo4j/gds/api/GraphLoaderContext.java index 7f249c8407..c0bdda5442 100644 --- a/core/src/main/java/org/neo4j/gds/api/GraphLoaderContext.java +++ b/core/src/main/java/org/neo4j/gds/api/GraphLoaderContext.java @@ -22,7 +22,7 @@ import org.immutables.value.Value; import org.neo4j.common.DependencyResolver; import org.neo4j.gds.annotation.ValueClass; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.TerminationFlag; import org.neo4j.gds.core.utils.progress.EmptyTaskRegistryFactory; import org.neo4j.gds.core.utils.progress.TaskRegistryFactory; @@ -47,7 +47,7 @@ public interface GraphLoaderContext { @Value.Default default ExecutorService executor() { - return ExecutorServices.DEFAULT; + return DefaultPool.INSTANCE; } @Value.Default diff --git a/core/src/main/java/org/neo4j/gds/core/concurrency/ExecutorServices.java b/core/src/main/java/org/neo4j/gds/core/concurrency/DefaultPool.java similarity index 89% rename from core/src/main/java/org/neo4j/gds/core/concurrency/ExecutorServices.java rename to core/src/main/java/org/neo4j/gds/core/concurrency/DefaultPool.java index 2fa07691c4..e1ba5a6bd6 100644 --- a/core/src/main/java/org/neo4j/gds/core/concurrency/ExecutorServices.java +++ b/core/src/main/java/org/neo4j/gds/core/concurrency/DefaultPool.java @@ -27,9 +27,9 @@ import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; -public final class ExecutorServices { +public final class DefaultPool { - public static final ExecutorService DEFAULT = createDefaultPool(PoolSizesService.poolSizes()); + public static final ExecutorService INSTANCE = createDefaultPool(PoolSizesService.poolSizes()); private static ExecutorService createDefaultPool(PoolSizes poolSizes) { return new ThreadPoolExecutor( @@ -43,5 +43,5 @@ private static ExecutorService createDefaultPool(PoolSizes poolSizes) { ); } - private ExecutorServices() {} + private DefaultPool() {} } diff --git a/core/src/main/java/org/neo4j/gds/core/concurrency/RunWithConcurrency.java b/core/src/main/java/org/neo4j/gds/core/concurrency/RunWithConcurrency.java index eafa85008e..0928ce08ec 100644 --- a/core/src/main/java/org/neo4j/gds/core/concurrency/RunWithConcurrency.java +++ b/core/src/main/java/org/neo4j/gds/core/concurrency/RunWithConcurrency.java @@ -186,11 +186,11 @@ default boolean mayInterruptIfRunning() { * If {@link #forceUsageOfExecutor()} is {@code true} however, an {@link java.lang.IllegalArgumentException} * is thrown when this object is constructed. *

- * The default executor is {@link ExecutorServices#DEFAULT}. + * The default executor is {@link DefaultPool#INSTANCE}. */ @Value.Default default @Nullable ExecutorService executor() { - return ExecutorServices.DEFAULT; + return DefaultPool.INSTANCE; } /** diff --git a/core/src/main/java/org/neo4j/gds/core/loading/ArrayIdMapBuilderOps.java b/core/src/main/java/org/neo4j/gds/core/loading/ArrayIdMapBuilderOps.java index f0ed70dcfd..b1723d1731 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/ArrayIdMapBuilderOps.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/ArrayIdMapBuilderOps.java @@ -23,7 +23,7 @@ import org.neo4j.gds.api.IdMap; import org.neo4j.gds.collections.HugeSparseLongArray; import org.neo4j.gds.collections.cursor.HugeCursor; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.concurrency.ParallelUtil; import org.neo4j.gds.core.loading.construction.NodesBuilder; import org.neo4j.gds.core.utils.paged.HugeLongArray; @@ -75,7 +75,7 @@ static HugeSparseLongArray buildSparseIdMap( ParallelUtil.readParallel( concurrency, nodeCount, - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, (start, end) -> addNodes(graphIds, idMapBuilder, start, end) ); return idMapBuilder.build(); diff --git a/core/src/main/java/org/neo4j/gds/core/loading/construction/GraphFactory.java b/core/src/main/java/org/neo4j/gds/core/loading/construction/GraphFactory.java index 78609ea190..3ba8858e93 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/construction/GraphFactory.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/construction/GraphFactory.java @@ -40,7 +40,7 @@ import org.neo4j.gds.api.schema.NodeSchema; import org.neo4j.gds.core.Aggregation; import org.neo4j.gds.core.IdMapBehaviorServiceProvider; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.huge.HugeGraph; import org.neo4j.gds.core.huge.HugeGraphBuilder; import org.neo4j.gds.core.loading.HighLimitIdMap; @@ -295,7 +295,7 @@ static RelationshipsBuilder relationshipsBuilder( .isMultiGraph(isMultiGraph) .loadRelationshipProperty(loadRelationshipProperties) .direction(Direction.fromOrientation(actualOrientation)) - .executorService(executorService.orElse(ExecutorServices.DEFAULT)) + .executorService(executorService.orElse(DefaultPool.INSTANCE)) .concurrency(finalConcurrency); if (indexInverse.orElse(false)) { diff --git a/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/DoubleArrayNodePropertiesBuilder.java b/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/DoubleArrayNodePropertiesBuilder.java index 47c7bd14c2..d7db8d37e1 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/DoubleArrayNodePropertiesBuilder.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/DoubleArrayNodePropertiesBuilder.java @@ -24,7 +24,7 @@ import org.neo4j.gds.api.PartialIdMap; import org.neo4j.gds.api.properties.nodes.DoubleArrayNodePropertyValues; import org.neo4j.gds.collections.HugeSparseDoubleArrayArray; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.concurrency.ParallelUtil; import org.neo4j.gds.utils.Neo4jValueConversion; import org.neo4j.values.storable.Value; @@ -91,7 +91,7 @@ public DoubleArrayNodePropertyValues build(long size, PartialIdMap idMap, long h } } }).collect(Collectors.toList()); - ParallelUtil.run(tasks, ExecutorServices.DEFAULT); + ParallelUtil.run(tasks, DefaultPool.INSTANCE); var propertyValues = propertiesByMappedIdsBuilder.build(); diff --git a/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/DoubleNodePropertiesBuilder.java b/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/DoubleNodePropertiesBuilder.java index 81b589c23b..f7b112e8a9 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/DoubleNodePropertiesBuilder.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/DoubleNodePropertiesBuilder.java @@ -24,7 +24,7 @@ import org.neo4j.gds.api.PartialIdMap; import org.neo4j.gds.api.properties.nodes.DoubleNodePropertyValues; import org.neo4j.gds.collections.HugeSparseDoubleArray; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.concurrency.ParallelUtil; import org.neo4j.gds.utils.Neo4jValueConversion; import org.neo4j.values.storable.Value; @@ -116,7 +116,7 @@ public DoubleNodePropertyValues build(long size, PartialIdMap idMap, long highes } }).collect(Collectors.toList()); - ParallelUtil.run(tasks, ExecutorServices.DEFAULT); + ParallelUtil.run(tasks, DefaultPool.INSTANCE); var propertyValues = propertiesByMappedIdsBuilder.build(); diff --git a/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/FloatArrayNodePropertiesBuilder.java b/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/FloatArrayNodePropertiesBuilder.java index 5147519e7d..8b5563b678 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/FloatArrayNodePropertiesBuilder.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/FloatArrayNodePropertiesBuilder.java @@ -24,7 +24,7 @@ import org.neo4j.gds.api.PartialIdMap; import org.neo4j.gds.api.properties.nodes.FloatArrayNodePropertyValues; import org.neo4j.gds.collections.HugeSparseFloatArrayArray; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.concurrency.ParallelUtil; import org.neo4j.gds.utils.Neo4jValueConversion; import org.neo4j.values.storable.Value; @@ -90,7 +90,7 @@ public FloatArrayNodePropertyValues build(long size, PartialIdMap idMap, long hi } }).collect(Collectors.toList()); - ParallelUtil.run(tasks, ExecutorServices.DEFAULT); + ParallelUtil.run(tasks, DefaultPool.INSTANCE); var propertyValues = propertiesByMappedIdsBuilder.build(); return new FloatArrayStoreNodePropertyValues(propertyValues, size); diff --git a/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/LongArrayNodePropertiesBuilder.java b/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/LongArrayNodePropertiesBuilder.java index aa55c63226..ddfbb31ef2 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/LongArrayNodePropertiesBuilder.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/LongArrayNodePropertiesBuilder.java @@ -24,7 +24,7 @@ import org.neo4j.gds.api.PartialIdMap; import org.neo4j.gds.api.properties.nodes.LongArrayNodePropertyValues; import org.neo4j.gds.collections.HugeSparseLongArrayArray; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.concurrency.ParallelUtil; import org.neo4j.gds.utils.Neo4jValueConversion; import org.neo4j.values.storable.Value; @@ -94,7 +94,7 @@ public LongArrayNodePropertyValues build(long size, PartialIdMap idMap, long hig } }).collect(Collectors.toList()); - ParallelUtil.run(tasks, ExecutorServices.DEFAULT); + ParallelUtil.run(tasks, DefaultPool.INSTANCE); var propertyValues = propertiesByMappedIdsBuilder.build(); diff --git a/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/LongNodePropertiesBuilder.java b/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/LongNodePropertiesBuilder.java index 77bef59af3..7e3752cbcb 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/LongNodePropertiesBuilder.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/LongNodePropertiesBuilder.java @@ -25,7 +25,7 @@ import org.neo4j.gds.api.properties.nodes.LongNodePropertyValues; import org.neo4j.gds.api.properties.nodes.NodePropertyValues; import org.neo4j.gds.collections.HugeSparseLongArray; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.concurrency.ParallelUtil; import org.neo4j.gds.utils.Neo4jValueConversion; import org.neo4j.values.storable.Value; @@ -124,7 +124,7 @@ public NodePropertyValues build(long size, PartialIdMap idMap, long highestOrigi } }).collect(Collectors.toList()); - ParallelUtil.run(tasks, ExecutorServices.DEFAULT); + ParallelUtil.run(tasks, DefaultPool.INSTANCE); var propertyValues = propertiesByMappedIdsBuilder.build(); diff --git a/core/src/test/java/org/neo4j/gds/TerminationTest.java b/core/src/test/java/org/neo4j/gds/TerminationTest.java index 256caf37f5..26ac0b6716 100644 --- a/core/src/test/java/org/neo4j/gds/TerminationTest.java +++ b/core/src/test/java/org/neo4j/gds/TerminationTest.java @@ -22,7 +22,7 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.neo4j.gds.compat.Neo4jProxy; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.concurrency.ParallelUtil; import org.neo4j.graphdb.TransactionFailureException; import org.neo4j.kernel.api.exceptions.Status; @@ -103,7 +103,7 @@ private void executeAndKill() { }); // submit - ParallelUtil.run(runnables, ExecutorServices.DEFAULT); + ParallelUtil.run(runnables, DefaultPool.INSTANCE); } @Test diff --git a/core/src/test/java/org/neo4j/gds/core/LoadingTest.java b/core/src/test/java/org/neo4j/gds/core/LoadingTest.java index 2694f4f6b9..e81e505edb 100644 --- a/core/src/test/java/org/neo4j/gds/core/LoadingTest.java +++ b/core/src/test/java/org/neo4j/gds/core/LoadingTest.java @@ -25,7 +25,7 @@ import org.neo4j.gds.BaseTest; import org.neo4j.gds.StoreLoaderBuilder; import org.neo4j.gds.api.Graph; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import java.util.Arrays; @@ -59,7 +59,7 @@ void setupGraphDb() { void testBasicLoading() { Graph graph = new StoreLoaderBuilder() .databaseService(db) - .executorService(ExecutorServices.DEFAULT) + .executorService(DefaultPool.INSTANCE) .addNodeLabel("Node") .addRelationshipType("TYPE") .build() diff --git a/core/src/test/java/org/neo4j/gds/core/compression/varlong/TransientCsrListTest.java b/core/src/test/java/org/neo4j/gds/core/compression/varlong/TransientCsrListTest.java index fc3c39a721..d7fb8ab120 100644 --- a/core/src/test/java/org/neo4j/gds/core/compression/varlong/TransientCsrListTest.java +++ b/core/src/test/java/org/neo4j/gds/core/compression/varlong/TransientCsrListTest.java @@ -31,7 +31,7 @@ import org.neo4j.gds.api.IdMap; import org.neo4j.gds.core.TestMethodRunner; import org.neo4j.gds.core.compression.packed.PackedAdjacencyList; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.loading.construction.GraphFactory; import java.util.Arrays; @@ -438,7 +438,7 @@ static AdjacencyList adjacencyListFromTargets(IdMap idMap, long[] targets) { .nodes(idMap) .relationshipType(RelationshipType.of("REL")) .concurrency(1) - .executorService(ExecutorServices.DEFAULT) + .executorService(DefaultPool.INSTANCE) .build(); Arrays.stream(targets).forEach(target -> relationshipsBuilder.add(sourceNodeId, target)); diff --git a/core/src/test/java/org/neo4j/gds/core/loading/LazyIdMapBuilderTest.java b/core/src/test/java/org/neo4j/gds/core/loading/LazyIdMapBuilderTest.java index 4303f5ebc7..d5d078c26d 100644 --- a/core/src/test/java/org/neo4j/gds/core/loading/LazyIdMapBuilderTest.java +++ b/core/src/test/java/org/neo4j/gds/core/loading/LazyIdMapBuilderTest.java @@ -22,7 +22,7 @@ import org.assertj.core.api.Assertions; import org.junit.jupiter.api.Test; import org.neo4j.gds.api.PropertyState; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.concurrency.ParallelUtil; import org.neo4j.gds.core.loading.construction.NodeLabelTokens; import org.neo4j.gds.core.utils.partition.PartitionUtils; @@ -66,7 +66,7 @@ void parallelAddDuplicateNodes() { }, Optional.empty()); - ParallelUtil.run(tasks, ExecutorServices.DEFAULT); + ParallelUtil.run(tasks, DefaultPool.INSTANCE); var highLimitIdMap = lazyIdMapBuilder.build().idMap(); diff --git a/core/src/test/java/org/neo4j/gds/core/loading/ScanningRelationshipsImporterTest.java b/core/src/test/java/org/neo4j/gds/core/loading/ScanningRelationshipsImporterTest.java index 3b195713ce..54340a1ddb 100644 --- a/core/src/test/java/org/neo4j/gds/core/loading/ScanningRelationshipsImporterTest.java +++ b/core/src/test/java/org/neo4j/gds/core/loading/ScanningRelationshipsImporterTest.java @@ -37,7 +37,7 @@ import org.neo4j.gds.config.ImmutableGraphProjectFromStoreConfig; import org.neo4j.gds.core.GraphDimensions; import org.neo4j.gds.core.GraphDimensionsStoreReader; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.huge.DirectIdMap; import org.neo4j.gds.core.utils.TerminationFlag; import org.neo4j.gds.core.utils.progress.TaskRegistryFactory; @@ -152,7 +152,7 @@ private long[] nodeIds(String... nodeVariables) { private GraphLoaderContext graphLoaderContext() { return ImmutableGraphLoaderContext.builder() - .executor(ExecutorServices.DEFAULT) + .executor(DefaultPool.INSTANCE) .log(NullLog.getInstance()) .terminationFlag(TerminationFlag.RUNNING_TRUE) .transactionContext(DatabaseTransactionContext.of(db, db.beginTx())) diff --git a/core/src/test/java/org/neo4j/gds/core/utils/paged/HugeAtomicGrowingBitSetTest.java b/core/src/test/java/org/neo4j/gds/core/utils/paged/HugeAtomicGrowingBitSetTest.java index dc489dbb82..5e06652895 100644 --- a/core/src/test/java/org/neo4j/gds/core/utils/paged/HugeAtomicGrowingBitSetTest.java +++ b/core/src/test/java/org/neo4j/gds/core/utils/paged/HugeAtomicGrowingBitSetTest.java @@ -22,7 +22,7 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.concurrency.ParallelUtil; import org.neo4j.gds.core.utils.partition.PartitionUtils; @@ -142,7 +142,7 @@ void testSetParallel(HugeAtomicGrowingBitSet bitSet) { } }, Optional.empty()); - ParallelUtil.run(tasks, ExecutorServices.DEFAULT); + ParallelUtil.run(tasks, DefaultPool.INSTANCE); assertThat(bitSet.cardinality()).isEqualTo(nodeCount); } diff --git a/core/src/test/java/org/neo4j/gds/core/utils/paged/ShardedLongLongMapTest.java b/core/src/test/java/org/neo4j/gds/core/utils/paged/ShardedLongLongMapTest.java index ae758c0086..e6bbed2181 100644 --- a/core/src/test/java/org/neo4j/gds/core/utils/paged/ShardedLongLongMapTest.java +++ b/core/src/test/java/org/neo4j/gds/core/utils/paged/ShardedLongLongMapTest.java @@ -27,7 +27,7 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.concurrency.ParallelUtil; import org.neo4j.gds.core.utils.partition.PartitionUtils; @@ -184,7 +184,7 @@ void testAddingMultipleNodesInParallel(@ForAll("fixedSizeIds") long[] originalId Optional.of(100) ); - ParallelUtil.run(tasks, ExecutorServices.DEFAULT); + ParallelUtil.run(tasks, DefaultPool.INSTANCE); var map = builder.build(); diff --git a/examples/pregel-example/src/test/java/org/neo4j/gds/beta/pregel/bfs/BFSPregelAlgoTest.java b/examples/pregel-example/src/test/java/org/neo4j/gds/beta/pregel/bfs/BFSPregelAlgoTest.java index 5b9e181562..af7d71fbe7 100644 --- a/examples/pregel-example/src/test/java/org/neo4j/gds/beta/pregel/bfs/BFSPregelAlgoTest.java +++ b/examples/pregel-example/src/test/java/org/neo4j/gds/beta/pregel/bfs/BFSPregelAlgoTest.java @@ -22,7 +22,7 @@ import org.junit.jupiter.api.Test; import org.neo4j.gds.TestSupport; import org.neo4j.gds.beta.pregel.Pregel; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.extension.GdlExtension; import org.neo4j.gds.extension.GdlGraph; @@ -111,7 +111,7 @@ void levelBfs() { graph, config, new BFSLevelPregel(), - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); @@ -147,7 +147,7 @@ void parentBfs() { graph, config, new BFSParentPregel(), - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); @@ -183,7 +183,7 @@ void parentBugTest() { parentGraph, config, new BFSParentPregel(), - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); diff --git a/examples/pregel-example/src/test/java/org/neo4j/gds/beta/pregel/cc/ConnectedComponentsPregelAlgoTest.java b/examples/pregel-example/src/test/java/org/neo4j/gds/beta/pregel/cc/ConnectedComponentsPregelAlgoTest.java index b8a554617f..856755c1f9 100644 --- a/examples/pregel-example/src/test/java/org/neo4j/gds/beta/pregel/cc/ConnectedComponentsPregelAlgoTest.java +++ b/examples/pregel-example/src/test/java/org/neo4j/gds/beta/pregel/cc/ConnectedComponentsPregelAlgoTest.java @@ -23,7 +23,7 @@ import org.neo4j.gds.Orientation; import org.neo4j.gds.TestSupport; import org.neo4j.gds.beta.pregel.Pregel; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.extension.GdlExtension; import org.neo4j.gds.extension.GdlGraph; @@ -85,7 +85,7 @@ void wcc() { graph, config, new ConnectedComponentsPregel(), - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); diff --git a/examples/pregel-example/src/test/java/org/neo4j/gds/beta/pregel/lp/LabelPropagationPregelAlgoTest.java b/examples/pregel-example/src/test/java/org/neo4j/gds/beta/pregel/lp/LabelPropagationPregelAlgoTest.java index d7c3c325c1..4a6611a9f4 100644 --- a/examples/pregel-example/src/test/java/org/neo4j/gds/beta/pregel/lp/LabelPropagationPregelAlgoTest.java +++ b/examples/pregel-example/src/test/java/org/neo4j/gds/beta/pregel/lp/LabelPropagationPregelAlgoTest.java @@ -22,7 +22,7 @@ import org.junit.jupiter.api.Test; import org.neo4j.gds.Orientation; import org.neo4j.gds.beta.pregel.Pregel; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.extension.GdlExtension; import org.neo4j.gds.extension.GdlGraph; @@ -73,7 +73,7 @@ void runLP() { graph, config, new LabelPropagationPregel(), - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); @@ -111,7 +111,7 @@ public double applyRelationshipWeight(double nodeValue, double relationshipWeigh graph, config, weightedLabelPropagation, - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); diff --git a/examples/pregel-example/src/test/java/org/neo4j/gds/beta/pregel/pr/PageRankPregelAlgoTest.java b/examples/pregel-example/src/test/java/org/neo4j/gds/beta/pregel/pr/PageRankPregelAlgoTest.java index 26e5619eb9..86b211d519 100644 --- a/examples/pregel-example/src/test/java/org/neo4j/gds/beta/pregel/pr/PageRankPregelAlgoTest.java +++ b/examples/pregel-example/src/test/java/org/neo4j/gds/beta/pregel/pr/PageRankPregelAlgoTest.java @@ -21,7 +21,7 @@ import org.junit.jupiter.api.Test; import org.neo4j.gds.beta.pregel.Pregel; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.paged.HugeDoubleArray; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.extension.GdlExtension; @@ -88,7 +88,7 @@ void runPR() { graph, config, new PageRankPregel(), - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); diff --git a/examples/pregel-example/src/test/java/org/neo4j/gds/beta/pregel/pr/WeightedPageRankPregelAlgoTest.java b/examples/pregel-example/src/test/java/org/neo4j/gds/beta/pregel/pr/WeightedPageRankPregelAlgoTest.java index 99d64878ce..c0e44d4f74 100644 --- a/examples/pregel-example/src/test/java/org/neo4j/gds/beta/pregel/pr/WeightedPageRankPregelAlgoTest.java +++ b/examples/pregel-example/src/test/java/org/neo4j/gds/beta/pregel/pr/WeightedPageRankPregelAlgoTest.java @@ -21,7 +21,7 @@ import org.junit.jupiter.api.Test; import org.neo4j.gds.beta.pregel.Pregel; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.paged.HugeDoubleArray; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.extension.GdlExtension; @@ -130,7 +130,7 @@ static void assertResult(TestGraph graph, Map expected) { graph, config, new PageRankPregel(), - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); diff --git a/examples/pregel-example/src/test/java/org/neo4j/gds/beta/pregel/sssp/SingleSourceShortestPathPregelAlgoTest.java b/examples/pregel-example/src/test/java/org/neo4j/gds/beta/pregel/sssp/SingleSourceShortestPathPregelAlgoTest.java index 1a4c141c71..48566f62d8 100644 --- a/examples/pregel-example/src/test/java/org/neo4j/gds/beta/pregel/sssp/SingleSourceShortestPathPregelAlgoTest.java +++ b/examples/pregel-example/src/test/java/org/neo4j/gds/beta/pregel/sssp/SingleSourceShortestPathPregelAlgoTest.java @@ -21,7 +21,7 @@ import org.junit.jupiter.api.Test; import org.neo4j.gds.beta.pregel.Pregel; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.paged.HugeLongArray; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.extension.GdlExtension; @@ -83,7 +83,7 @@ void runSSSP() { graph, config, new SingleSourceShortestPathPregel(), - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); diff --git a/examples/pregel-example/src/test/java/org/neo4j/gds/beta/pregel/triangleCount/TriangleCountPregelTest.java b/examples/pregel-example/src/test/java/org/neo4j/gds/beta/pregel/triangleCount/TriangleCountPregelTest.java index f447ac6774..5582660c6e 100644 --- a/examples/pregel-example/src/test/java/org/neo4j/gds/beta/pregel/triangleCount/TriangleCountPregelTest.java +++ b/examples/pregel-example/src/test/java/org/neo4j/gds/beta/pregel/triangleCount/TriangleCountPregelTest.java @@ -25,7 +25,7 @@ import org.junit.jupiter.params.provider.MethodSource; import org.neo4j.gds.api.Graph; import org.neo4j.gds.beta.pregel.Pregel; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.paged.HugeLongArray; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; @@ -332,7 +332,7 @@ HugeLongArray nodeWiseTriangles(Graph graph) { graph, config, new TriangleCountPregel(), - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); diff --git a/graph-sampling/src/main/java/org/neo4j/gds/graphsampling/GraphSampleConstructor.java b/graph-sampling/src/main/java/org/neo4j/gds/graphsampling/GraphSampleConstructor.java index 733f2e899c..c7836c479d 100644 --- a/graph-sampling/src/main/java/org/neo4j/gds/graphsampling/GraphSampleConstructor.java +++ b/graph-sampling/src/main/java/org/neo4j/gds/graphsampling/GraphSampleConstructor.java @@ -30,7 +30,7 @@ import org.neo4j.gds.beta.filter.expression.EvaluationContext; import org.neo4j.gds.beta.filter.expression.Expression; import org.neo4j.gds.config.GraphSampleAlgoConfig; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.concurrency.RunWithConcurrency; import org.neo4j.gds.core.loading.GraphStoreBuilder; import org.neo4j.gds.core.loading.ImmutableNodes; @@ -108,7 +108,7 @@ public double evaluate(EvaluationContext context) { idMap, config.concurrency(), Map.of(), - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, progressTracker ); diff --git a/io/core/src/main/java/org/neo4j/gds/core/io/file/FileToGraphStoreImporter.java b/io/core/src/main/java/org/neo4j/gds/core/io/file/FileToGraphStoreImporter.java index 0571b464cb..e33c3119a6 100644 --- a/io/core/src/main/java/org/neo4j/gds/core/io/file/FileToGraphStoreImporter.java +++ b/io/core/src/main/java/org/neo4j/gds/core/io/file/FileToGraphStoreImporter.java @@ -28,7 +28,7 @@ import org.neo4j.gds.api.Topology; import org.neo4j.gds.api.schema.ImmutableMutableGraphSchema; import org.neo4j.gds.api.schema.MutableNodeSchema; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.concurrency.ParallelUtil; import org.neo4j.gds.core.io.GraphStoreGraphPropertyVisitor; import org.neo4j.gds.core.io.GraphStoreRelationshipVisitor; @@ -175,7 +175,7 @@ private Nodes importNodes(FileInput fileInput) { (index) -> new ElementImportRunner<>(nodeVisitorBuilder.build(), nodesIterator, progressTracker) ); - ParallelUtil.run(tasks, ExecutorServices.DEFAULT); + ParallelUtil.run(tasks, DefaultPool.INSTANCE); var nodes = nodesBuilder.build(); @@ -206,7 +206,7 @@ private void importRelationships(FileInput fileInput, IdMap nodes) { (index) -> new ElementImportRunner<>(relationshipVisitorBuilder.build(), relationshipsIterator, progressTracker) ); - ParallelUtil.run(tasks, ExecutorServices.DEFAULT); + ParallelUtil.run(tasks, DefaultPool.INSTANCE); var relationshipImportResult = relationshipImportResult(relationshipBuildersByType); @@ -235,7 +235,7 @@ private void importGraphProperties(FileInput fileInput) { progressTracker ) ); - ParallelUtil.run(tasks, ExecutorServices.DEFAULT); + ParallelUtil.run(tasks, DefaultPool.INSTANCE); graphStoreGraphPropertyVisitor.close(); graphStoreBuilder.graphProperties(GraphPropertyStoreFromVisitorHelper.fromGraphPropertyVisitor( diff --git a/ml/ml-algo/src/main/java/org/neo4j/gds/ml/splitting/EdgeSplitter.java b/ml/ml-algo/src/main/java/org/neo4j/gds/ml/splitting/EdgeSplitter.java index 1e89f42706..d5982ac951 100644 --- a/ml/ml-algo/src/main/java/org/neo4j/gds/ml/splitting/EdgeSplitter.java +++ b/ml/ml-algo/src/main/java/org/neo4j/gds/ml/splitting/EdgeSplitter.java @@ -30,7 +30,7 @@ import org.neo4j.gds.api.RelationshipWithPropertyConsumer; import org.neo4j.gds.api.schema.Direction; import org.neo4j.gds.core.Aggregation; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.loading.construction.GraphFactory; import org.neo4j.gds.core.loading.construction.RelationshipsBuilder; @@ -170,7 +170,7 @@ private static RelationshipsBuilder newRelationshipsBuilder( .map(key -> List.of(GraphFactory.PropertyConfig.of(key, Aggregation.SINGLE, DefaultValue.forDouble()))) .orElse(List.of())) .concurrency(1) - .executorService(ExecutorServices.DEFAULT) + .executorService(DefaultPool.INSTANCE) .build(); } diff --git a/pregel-proc-generator/src/main/java/org/neo4j/gds/pregel/generator/AlgorithmGenerator.java b/pregel-proc-generator/src/main/java/org/neo4j/gds/pregel/generator/AlgorithmGenerator.java index fc1952df54..35e73d4348 100644 --- a/pregel-proc-generator/src/main/java/org/neo4j/gds/pregel/generator/AlgorithmGenerator.java +++ b/pregel-proc-generator/src/main/java/org/neo4j/gds/pregel/generator/AlgorithmGenerator.java @@ -29,7 +29,7 @@ import org.neo4j.gds.api.Graph; import org.neo4j.gds.beta.pregel.Pregel; import org.neo4j.gds.beta.pregel.PregelResult; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.TerminationFlag; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; @@ -88,9 +88,9 @@ MethodSpec constructor() { .addStatement("super(progressTracker)") .addStatement("var computation = new $T()", typeNames.computation()) .addStatement( - "this.pregelJob = $T.create(graph, configuration, computation, $T.DEFAULT, progressTracker)", + "this.pregelJob = $T.create(graph, configuration, computation, $T.INSTANCE, progressTracker)", Pregel.class, - ExecutorServices.class + DefaultPool.class ).build(); } diff --git a/pregel-proc-generator/src/test/resources/expected/BidirectionalComputationAlgorithm.java b/pregel-proc-generator/src/test/resources/expected/BidirectionalComputationAlgorithm.java index d1619bf3d4..d682a26ab7 100644 --- a/pregel-proc-generator/src/test/resources/expected/BidirectionalComputationAlgorithm.java +++ b/pregel-proc-generator/src/test/resources/expected/BidirectionalComputationAlgorithm.java @@ -25,7 +25,7 @@ import org.neo4j.gds.beta.pregel.Pregel; import org.neo4j.gds.beta.pregel.PregelProcedureConfig; import org.neo4j.gds.beta.pregel.PregelResult; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.TerminationFlag; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; @@ -37,7 +37,7 @@ public final class BidirectionalComputationAlgorithm extends Algorithm { ProgressTracker progressTracker) { super(progressTracker); var computation = new Computation(); - this.pregelJob = Pregel.create(graph, configuration, computation, ExecutorServices.DEFAULT, progressTracker); + this.pregelJob = Pregel.create(graph, configuration, computation, DefaultPool.INSTANCE, progressTracker); } @Override diff --git a/pregel/src/test/java/org/neo4j/gds/beta/pregel/PregelTest.java b/pregel/src/test/java/org/neo4j/gds/beta/pregel/PregelTest.java index 08aa876046..d65e7879a0 100644 --- a/pregel/src/test/java/org/neo4j/gds/beta/pregel/PregelTest.java +++ b/pregel/src/test/java/org/neo4j/gds/beta/pregel/PregelTest.java @@ -44,7 +44,7 @@ import org.neo4j.gds.compat.Neo4jProxy; import org.neo4j.gds.compat.TestLog; import org.neo4j.gds.core.ImmutableGraphDimensions; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.TerminationFlag; import org.neo4j.gds.core.utils.mem.MemoryRange; import org.neo4j.gds.core.utils.paged.HugeDoubleArray; @@ -109,7 +109,7 @@ void sendsMessages( graph, config, computation, - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); @@ -127,7 +127,7 @@ void stopsEarlyWhenTransactionHasBeenTerminated() { graph, config, new TestPregelComputation(), - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); pregelJob.setTerminationFlag(terminationFlag); @@ -163,7 +163,7 @@ void logProgress(Partitioning partitioning) { graph, config, computation, - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, progressTracker ).run(); @@ -225,7 +225,7 @@ void cleanupProgressLogging() { graph, config, computation, - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, progressTracker ); @@ -280,7 +280,7 @@ private HugeDoubleArray run(Graph graph, PregelConfig config, PregelComputation< graph, config, computation, - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); @@ -300,7 +300,7 @@ void sendMessageToSpecificTarget(Partitioning partitioning) { graph, config, new TestSendTo(), - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); @@ -325,7 +325,7 @@ void compositeNodeValueTest(Partitioning partitioning) { graph, config, new CompositeTestComputation(), - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); @@ -363,7 +363,7 @@ void testMasterComputeStep(Partitioning partitioning) { graph, ImmutablePregelConfig.builder().maxIterations(4).partitioning(partitioning).build(), new TestMasterCompute(), - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); @@ -378,7 +378,7 @@ void testMasterComputeStepWithConvergence(Partitioning partitioning) { graph, ImmutablePregelConfig.builder().maxIterations(4).partitioning(partitioning).build(), new TestMasterCompute(2), - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); @@ -515,7 +515,7 @@ public boolean masterCompute(MasterComputeContext context) { return true; } }, - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ).run(); @@ -603,7 +603,7 @@ void preventIllegalConcurrencyConfiguration(Partitioning partitioning) { graph, config, new TestSendTo(), - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER )); } @@ -626,7 +626,7 @@ void messagesInInitialSuperStepShouldBeEmpty(Partitioning partitioning, boolean graph, config, new TestEmptyMessageInInitialSuperstep(), - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); @@ -825,7 +825,7 @@ void throwIfBidirectionalWithoutInverseIndex() { graph, ImmutablePregelConfig.builder().maxIterations(4).build(), new Bidirectional(), - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); diff --git a/proc/catalog/src/main/java/org/neo4j/gds/catalog/GraphProjectProc.java b/proc/catalog/src/main/java/org/neo4j/gds/catalog/GraphProjectProc.java index 2b61152686..d3144ba909 100644 --- a/proc/catalog/src/main/java/org/neo4j/gds/catalog/GraphProjectProc.java +++ b/proc/catalog/src/main/java/org/neo4j/gds/catalog/GraphProjectProc.java @@ -30,7 +30,7 @@ import org.neo4j.gds.config.GraphProjectFromGraphConfig; import org.neo4j.gds.config.GraphProjectFromStoreConfig; import org.neo4j.gds.core.CypherMapWrapper; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.loading.GraphStoreCatalog; import org.neo4j.gds.core.utils.ProgressTimer; import org.neo4j.gds.core.utils.mem.MemoryTree; @@ -232,7 +232,7 @@ private GraphProjectSubgraphResult projectGraphFromGraphStore( var graphStore = GraphStoreFilter.filter( fromGraphStore, config, - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, progressTracker ); diff --git a/proc/catalog/src/main/java/org/neo4j/gds/catalog/GraphWriteNodeLabelProc.java b/proc/catalog/src/main/java/org/neo4j/gds/catalog/GraphWriteNodeLabelProc.java index b66550ad97..d05279ca8a 100644 --- a/proc/catalog/src/main/java/org/neo4j/gds/catalog/GraphWriteNodeLabelProc.java +++ b/proc/catalog/src/main/java/org/neo4j/gds/catalog/GraphWriteNodeLabelProc.java @@ -21,7 +21,7 @@ import org.neo4j.gds.beta.filter.NodesFilter; import org.neo4j.gds.config.WriteLabelConfig; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.ProgressTimer; import org.neo4j.gds.core.utils.TerminationFlag; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; @@ -66,7 +66,7 @@ public Stream write( nodeFilter, procedureConfig.concurrency(), Map.of(), - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); @@ -74,7 +74,7 @@ public Stream write( .withIdMap(filteredNodes.idMap()) .withTerminationFlag(TerminationFlag.wrap(executionContext().terminationMonitor())) .withArrowConnectionInfo(procedureConfig.arrowConnectionInfo(), graphStore.databaseId().databaseName()) - .parallel(ExecutorServices.DEFAULT, procedureConfig.concurrency()) + .parallel(DefaultPool.INSTANCE, procedureConfig.concurrency()) .build(); runWithExceptionLogging( diff --git a/proc/catalog/src/main/java/org/neo4j/gds/catalog/NodeLabelMutator.java b/proc/catalog/src/main/java/org/neo4j/gds/catalog/NodeLabelMutator.java index 0cff4280a0..f660e7fa6d 100644 --- a/proc/catalog/src/main/java/org/neo4j/gds/catalog/NodeLabelMutator.java +++ b/proc/catalog/src/main/java/org/neo4j/gds/catalog/NodeLabelMutator.java @@ -22,7 +22,7 @@ import org.neo4j.gds.NodeLabel; import org.neo4j.gds.beta.filter.NodesFilter; import org.neo4j.gds.config.MutateLabelConfig; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.loading.GraphStoreCatalog; import org.neo4j.gds.core.loading.GraphStoreWithConfig; import org.neo4j.gds.core.loading.ImmutableCatalogRequest; @@ -52,7 +52,7 @@ static Stream mutateNodeLabel(String graphName, String nodeLa nodeFilter, procedureConfig.concurrency(), Map.of(), - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); diff --git a/proc/catalog/src/main/java/org/neo4j/gds/catalog/NodePropertiesWriter.java b/proc/catalog/src/main/java/org/neo4j/gds/catalog/NodePropertiesWriter.java index 03a5010496..7a338f2c15 100644 --- a/proc/catalog/src/main/java/org/neo4j/gds/catalog/NodePropertiesWriter.java +++ b/proc/catalog/src/main/java/org/neo4j/gds/catalog/NodePropertiesWriter.java @@ -27,7 +27,7 @@ import org.neo4j.gds.config.GraphWriteNodePropertiesConfig; import org.neo4j.gds.core.CypherMapAccess; import org.neo4j.gds.core.CypherMapWrapper; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.loading.GraphStoreWithConfig; import org.neo4j.gds.core.utils.ProgressTimer; import org.neo4j.gds.core.utils.TerminationFlag; @@ -147,7 +147,7 @@ private static long writeNodeProperties( var exporter = nodePropertyExporterBuilder .withIdMap(subGraph) .withTerminationFlag(TerminationFlag.wrap(executionContext.terminationMonitor())) - .parallel(ExecutorServices.DEFAULT, config.writeConcurrency()) + .parallel(DefaultPool.INSTANCE, config.writeConcurrency()) .withProgressTracker(progressTracker) .withArrowConnectionInfo(config.arrowConnectionInfo(), graphStore.databaseId().databaseName()) .build(); diff --git a/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphProjectProcTest.java b/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphProjectProcTest.java index 28af113cf8..22feadcb18 100644 --- a/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphProjectProcTest.java +++ b/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphProjectProcTest.java @@ -47,7 +47,7 @@ import org.neo4j.gds.config.GraphProjectFromCypherConfig; import org.neo4j.gds.config.GraphProjectFromStoreConfig; import org.neo4j.gds.core.Aggregation; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.CypherMapWrapper; import org.neo4j.gds.core.concurrency.ParallelUtil; import org.neo4j.gds.core.loading.GraphStoreCatalog; @@ -895,7 +895,7 @@ void loadGraphWithSaturatedThreadPool() { // block all available threads for (int i = 0; i < ConcurrencyConfig.DEFAULT_CONCURRENCY; i++) { futures.add( - ExecutorServices.DEFAULT.submit(() -> LockSupport.parkNanos(Duration.ofSeconds(1).toNanos())) + DefaultPool.INSTANCE.submit(() -> LockSupport.parkNanos(Duration.ofSeconds(1).toNanos())) ); } diff --git a/proc/common/src/main/java/org/neo4j/gds/CommunityProcCompanion.java b/proc/common/src/main/java/org/neo4j/gds/CommunityProcCompanion.java index 0f02b4a905..0a5eb7ff16 100644 --- a/proc/common/src/main/java/org/neo4j/gds/CommunityProcCompanion.java +++ b/proc/common/src/main/java/org/neo4j/gds/CommunityProcCompanion.java @@ -27,7 +27,7 @@ import org.neo4j.gds.config.ConcurrencyConfig; import org.neo4j.gds.config.ConsecutiveIdsConfig; import org.neo4j.gds.config.SeedConfig; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.nodeproperties.ConsecutiveLongNodePropertyValues; import org.neo4j.gds.nodeproperties.LongIfChangedNodePropertyValues; import org.neo4j.gds.result.CommunityStatistics; @@ -122,7 +122,7 @@ private static LongNodePropertyValues applySizeFilter( var communitySizes = CommunityStatistics.communitySizes( nodeProperties.nodeCount(), nodeProperties::longValue, - ExecutorServices.DEFAULT, + DefaultPool.INSTANCE, concurrency ); return new CommunitySizeFilter(nodeProperties, communitySizes, size); diff --git a/proc/common/src/main/java/org/neo4j/gds/WriteNodePropertiesComputationResultConsumer.java b/proc/common/src/main/java/org/neo4j/gds/WriteNodePropertiesComputationResultConsumer.java index b42a17e944..1fb2c06ab9 100644 --- a/proc/common/src/main/java/org/neo4j/gds/WriteNodePropertiesComputationResultConsumer.java +++ b/proc/common/src/main/java/org/neo4j/gds/WriteNodePropertiesComputationResultConsumer.java @@ -23,7 +23,7 @@ import org.neo4j.gds.api.schema.PropertySchema; import org.neo4j.gds.config.AlgoBaseConfig; import org.neo4j.gds.config.WritePropertyConfig; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.loading.Capabilities.WriteMode; import org.neo4j.gds.core.utils.ProgressTimer; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; @@ -111,7 +111,7 @@ void writeToNeo( .withTerminationFlag(computationResult.algorithm().terminationFlag) .withProgressTracker(progressTracker) .withArrowConnectionInfo(config.arrowConnectionInfo(), computationResult.graphStore().databaseId().databaseName()) - .parallel(ExecutorServices.DEFAULT, config.writeConcurrency()) + .parallel(DefaultPool.INSTANCE, config.writeConcurrency()) .build(); try { diff --git a/proc/community/src/main/java/org/neo4j/gds/kmeans/KmeansWriteSpec.java b/proc/community/src/main/java/org/neo4j/gds/kmeans/KmeansWriteSpec.java index 70ac76c2f2..c81841e927 100644 --- a/proc/community/src/main/java/org/neo4j/gds/kmeans/KmeansWriteSpec.java +++ b/proc/community/src/main/java/org/neo4j/gds/kmeans/KmeansWriteSpec.java @@ -21,7 +21,7 @@ import org.neo4j.gds.api.properties.nodes.EmptyLongNodePropertyValues; import org.neo4j.gds.api.properties.nodes.LongNodePropertyValues; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.ProgressTimer; import org.neo4j.gds.core.write.NodePropertyExporter; import org.neo4j.gds.executor.AlgorithmSpec; @@ -101,7 +101,7 @@ public ComputationResultConsumer context, Messages mess context.setNodeValue(LONG_ARRAY_KEY, new long[]{1, 3, 3, 7}); context.setNodeValue(DOUBLE_ARRAY_KEY, new double[]{1, 9, 8, 4}); } - }, ExecutorServices.DEFAULT, progressTracker); + }, DefaultPool.INSTANCE, progressTracker); } @Override diff --git a/proc/similarity/src/main/java/org/neo4j/gds/similarity/filterednodesim/FilteredNodeSimilarityMutateSpec.java b/proc/similarity/src/main/java/org/neo4j/gds/similarity/filterednodesim/FilteredNodeSimilarityMutateSpec.java index 244333ffe6..c99b33a85b 100644 --- a/proc/similarity/src/main/java/org/neo4j/gds/similarity/filterednodesim/FilteredNodeSimilarityMutateSpec.java +++ b/proc/similarity/src/main/java/org/neo4j/gds/similarity/filterednodesim/FilteredNodeSimilarityMutateSpec.java @@ -27,7 +27,7 @@ import org.neo4j.gds.api.ProcedureReturnColumns; import org.neo4j.gds.api.nodeproperties.ValueType; import org.neo4j.gds.api.schema.RelationshipPropertySchema; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.huge.HugeGraph; import org.neo4j.gds.core.loading.SingleTypeRelationships; import org.neo4j.gds.core.loading.construction.GraphFactory; @@ -145,7 +145,7 @@ private SingleTypeRelationships getRelationships( .orientation(Orientation.NATURAL) .addPropertyConfig(GraphFactory.PropertyConfig.of(relationshipPropertyKey)) .concurrency(1) - .executorService(ExecutorServices.DEFAULT) + .executorService(DefaultPool.INSTANCE) .build(); IdMap idMap = computationResult.graph(); diff --git a/proc/similarity/src/main/java/org/neo4j/gds/similarity/nodesim/NodeSimilarityMutateSpecification.java b/proc/similarity/src/main/java/org/neo4j/gds/similarity/nodesim/NodeSimilarityMutateSpecification.java index 90aad2a29b..ed48632baf 100644 --- a/proc/similarity/src/main/java/org/neo4j/gds/similarity/nodesim/NodeSimilarityMutateSpecification.java +++ b/proc/similarity/src/main/java/org/neo4j/gds/similarity/nodesim/NodeSimilarityMutateSpecification.java @@ -26,7 +26,7 @@ import org.neo4j.gds.api.ProcedureReturnColumns; import org.neo4j.gds.api.nodeproperties.ValueType; import org.neo4j.gds.api.schema.RelationshipPropertySchema; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.huge.HugeGraph; import org.neo4j.gds.core.loading.SingleTypeRelationships; import org.neo4j.gds.core.loading.construction.GraphFactory; @@ -134,7 +134,7 @@ private SingleTypeRelationships getRelationships( .orientation(Orientation.NATURAL) .addPropertyConfig(GraphFactory.PropertyConfig.of(relationshipPropertyKey)) .concurrency(1) - .executorService(ExecutorServices.DEFAULT) + .executorService(DefaultPool.INSTANCE) .build(); IdMap idMap = computationResult.graph(); diff --git a/test-utils/src/main/java/org/neo4j/gds/GraphLoaderBuilders.java b/test-utils/src/main/java/org/neo4j/gds/GraphLoaderBuilders.java index 408a70486b..b1e151f913 100644 --- a/test-utils/src/main/java/org/neo4j/gds/GraphLoaderBuilders.java +++ b/test-utils/src/main/java/org/neo4j/gds/GraphLoaderBuilders.java @@ -31,7 +31,7 @@ import org.neo4j.gds.core.Aggregation; import org.neo4j.gds.core.GraphLoader; import org.neo4j.gds.core.ImmutableGraphLoader; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.TerminationFlag; import org.neo4j.gds.core.utils.progress.EmptyTaskRegistryFactory; import org.neo4j.gds.core.utils.progress.JobId; @@ -194,7 +194,7 @@ public static GraphLoader createGraphLoader( .databaseId(DatabaseId.of(databaseService)) .dependencyResolver(GraphDatabaseApiProxy.dependencyResolver(databaseService)) .transactionContext(transactionContext.orElseGet(() -> TestSupport.fullAccessTransaction(databaseService))) - .executor(executorService.orElse(ExecutorServices.DEFAULT)) + .executor(executorService.orElse(DefaultPool.INSTANCE)) .terminationFlag(terminationFlag.orElse(TerminationFlag.RUNNING_TRUE)) .taskRegistryFactory(EmptyTaskRegistryFactory.INSTANCE) .userLogRegistryFactory(EmptyUserLogRegistryFactory.INSTANCE) diff --git a/test-utils/src/main/java/org/neo4j/gds/core/idmap/IdMapBuilderTest.java b/test-utils/src/main/java/org/neo4j/gds/core/idmap/IdMapBuilderTest.java index 7f7bba47a1..e0c114266f 100644 --- a/test-utils/src/main/java/org/neo4j/gds/core/idmap/IdMapBuilderTest.java +++ b/test-utils/src/main/java/org/neo4j/gds/core/idmap/IdMapBuilderTest.java @@ -28,7 +28,7 @@ import org.neo4j.gds.NodeLabel; import org.neo4j.gds.annotation.ValueClass; import org.neo4j.gds.api.IdMap; -import org.neo4j.gds.core.concurrency.ExecutorServices; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.concurrency.ParallelUtil; import org.neo4j.gds.core.loading.IdMapBuilder; import org.neo4j.gds.core.loading.LabelInformationBuilders; @@ -241,7 +241,7 @@ void testBuildParallel( } }, Optional.empty()); - ParallelUtil.run(tasks, ExecutorServices.DEFAULT); + ParallelUtil.run(tasks, DefaultPool.INSTANCE); var idMap = idMapBuilder.build(LabelInformationBuilders.allNodes(), highestOriginalId, concurrency); From 3f7af5d14b18f0527a439cc99608f29e1123163a Mon Sep 17 00:00:00 2001 From: Mats Rydberg Date: Mon, 11 Sep 2023 10:31:39 +0200 Subject: [PATCH 230/273] Rename to `ExecutorServiceUtil` Co-authored-by: Paul Horn --- .../java/org/neo4j/gds/topologicalsort/TopologicalSort.java | 4 ++-- algo/src/main/java/org/neo4j/gds/traversal/RandomWalk.java | 4 ++-- .../org/neo4j/gds/leiden/GraphAggregationPhaseTest.java | 4 ++-- .../java/org/neo4j/gds/leiden/GraphWithSelfLoopTest.java | 6 +++--- algo/src/test/java/org/neo4j/gds/leiden/LeidenTest.java | 4 ++-- .../neo4j/gds/core/write/NativeRelationshipExporter.java | 4 ++-- .../write/NativeRelationshipPropertiesExporterBuilder.java | 4 ++-- .../write/NativeRelationshipPropertiesExporterTest.java | 4 ++-- .../java/org/neo4j/gds/core/concurrency/DefaultPool.java | 4 ++-- .../concurrency/{Pools.java => ExecutorServiceUtil.java} | 6 +++--- .../java/org/neo4j/gds/core/concurrency/ParallelUtil.java | 2 +- .../java/org/neo4j/gds/core/utils/paged/HugeMergeSort.java | 4 ++-- .../org/neo4j/gds/core/concurrency/ParallelUtilTest.java | 2 +- etc/forbidden-apis | 4 ++-- pregel/src/main/java/org/neo4j/gds/beta/pregel/Pregel.java | 4 ++-- .../java/org/neo4j/gds/beta/filter/NodesFilterTest.java | 4 ++-- 16 files changed, 32 insertions(+), 32 deletions(-) rename core/src/main/java/org/neo4j/gds/core/concurrency/{Pools.java => ExecutorServiceUtil.java} (95%) diff --git a/algo/src/main/java/org/neo4j/gds/topologicalsort/TopologicalSort.java b/algo/src/main/java/org/neo4j/gds/topologicalsort/TopologicalSort.java index 86f7c36258..5a9f1f3e07 100644 --- a/algo/src/main/java/org/neo4j/gds/topologicalsort/TopologicalSort.java +++ b/algo/src/main/java/org/neo4j/gds/topologicalsort/TopologicalSort.java @@ -23,8 +23,8 @@ import org.neo4j.gds.Algorithm; import org.neo4j.gds.api.Graph; import org.neo4j.gds.collections.haa.HugeAtomicLongArray; +import org.neo4j.gds.core.concurrency.ExecutorServiceUtil; import org.neo4j.gds.core.concurrency.ParallelUtil; -import org.neo4j.gds.core.concurrency.Pools; import org.neo4j.gds.core.utils.TerminationFlag; import org.neo4j.gds.core.utils.paged.ParalleLongPageCreator; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; @@ -98,7 +98,7 @@ private void initializeInDegrees() { } private void traverse() { - ForkJoinPool forkJoinPool = Pools.createForkJoinPool(concurrency); + ForkJoinPool forkJoinPool = ExecutorServiceUtil.createForkJoinPool(concurrency); var tasks = ConcurrentHashMap.>newKeySet(); ParallelUtil.parallelForEachNode(nodeCount, concurrency, TerminationFlag.RUNNING_TRUE, nodeId -> { diff --git a/algo/src/main/java/org/neo4j/gds/traversal/RandomWalk.java b/algo/src/main/java/org/neo4j/gds/traversal/RandomWalk.java index fa93cf218e..2a6bd96103 100644 --- a/algo/src/main/java/org/neo4j/gds/traversal/RandomWalk.java +++ b/algo/src/main/java/org/neo4j/gds/traversal/RandomWalk.java @@ -22,7 +22,7 @@ import org.neo4j.gds.Algorithm; import org.neo4j.gds.api.Graph; import org.neo4j.gds.config.SourceNodesConfig; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServiceUtil; import org.neo4j.gds.core.concurrency.RunWithConcurrency; import org.neo4j.gds.core.utils.TerminationFlag; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; @@ -150,7 +150,7 @@ private void startWalkers( TOMB, terminationFlag ), - Pools.DEFAULT_SINGLE_THREAD_POOL + ExecutorServiceUtil.DEFAULT_SINGLE_THREAD_POOL ).whenComplete((__, ___) -> { progressTracker.endSubTask("RandomWalk"); }); diff --git a/algo/src/test/java/org/neo4j/gds/leiden/GraphAggregationPhaseTest.java b/algo/src/test/java/org/neo4j/gds/leiden/GraphAggregationPhaseTest.java index 10b2236fcf..93cd013212 100644 --- a/algo/src/test/java/org/neo4j/gds/leiden/GraphAggregationPhaseTest.java +++ b/algo/src/test/java/org/neo4j/gds/leiden/GraphAggregationPhaseTest.java @@ -22,7 +22,7 @@ import org.junit.jupiter.api.Test; import org.neo4j.gds.Orientation; import org.neo4j.gds.api.schema.Direction; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServiceUtil; import org.neo4j.gds.core.utils.TerminationFlag; import org.neo4j.gds.core.utils.paged.HugeLongArray; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; @@ -79,7 +79,7 @@ void testGraphAggregation() { Direction.UNDIRECTED, communities, 1L, - Pools.DEFAULT_SINGLE_THREAD_POOL, + ExecutorServiceUtil.DEFAULT_SINGLE_THREAD_POOL, 4, TerminationFlag.RUNNING_TRUE, ProgressTracker.NULL_TRACKER diff --git a/algo/src/test/java/org/neo4j/gds/leiden/GraphWithSelfLoopTest.java b/algo/src/test/java/org/neo4j/gds/leiden/GraphWithSelfLoopTest.java index a6e8baf301..7724f5827e 100644 --- a/algo/src/test/java/org/neo4j/gds/leiden/GraphWithSelfLoopTest.java +++ b/algo/src/test/java/org/neo4j/gds/leiden/GraphWithSelfLoopTest.java @@ -24,7 +24,7 @@ import org.neo4j.gds.Orientation; import org.neo4j.gds.api.schema.Direction; import org.neo4j.gds.core.concurrency.DefaultPool; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServiceUtil; import org.neo4j.gds.core.utils.TerminationFlag; import org.neo4j.gds.core.utils.paged.HugeDoubleArray; import org.neo4j.gds.core.utils.paged.HugeLongArray; @@ -87,7 +87,7 @@ void shouldAggregateCorrectly(){ Direction.UNDIRECTED, communities, 2L, - Pools.DEFAULT_SINGLE_THREAD_POOL, + ExecutorServiceUtil.DEFAULT_SINGLE_THREAD_POOL, 4, TerminationFlag.RUNNING_TRUE, ProgressTracker.NULL_TRACKER @@ -122,7 +122,7 @@ void shouldMaintainPartition() { Direction.UNDIRECTED, refinedCommunities, 2, - Pools.DEFAULT_SINGLE_THREAD_POOL, + ExecutorServiceUtil.DEFAULT_SINGLE_THREAD_POOL, 1, TerminationFlag.RUNNING_TRUE, ProgressTracker.NULL_TRACKER diff --git a/algo/src/test/java/org/neo4j/gds/leiden/LeidenTest.java b/algo/src/test/java/org/neo4j/gds/leiden/LeidenTest.java index 2685788beb..7f1d0c56d8 100644 --- a/algo/src/test/java/org/neo4j/gds/leiden/LeidenTest.java +++ b/algo/src/test/java/org/neo4j/gds/leiden/LeidenTest.java @@ -30,7 +30,7 @@ import org.neo4j.gds.compat.TestLog; import org.neo4j.gds.config.RandomGraphGeneratorConfig; import org.neo4j.gds.core.Aggregation; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServiceUtil; import org.neo4j.gds.core.utils.TerminationFlag; import org.neo4j.gds.core.utils.paged.HugeDoubleArray; import org.neo4j.gds.core.utils.paged.HugeLongArray; @@ -219,7 +219,7 @@ void shouldMaintainPartition() { Direction.UNDIRECTED, refinedCommunities, 4, - Pools.DEFAULT_SINGLE_THREAD_POOL, + ExecutorServiceUtil.DEFAULT_SINGLE_THREAD_POOL, 1, TerminationFlag.RUNNING_TRUE, ProgressTracker.NULL_TRACKER diff --git a/core-write/src/main/java/org/neo4j/gds/core/write/NativeRelationshipExporter.java b/core-write/src/main/java/org/neo4j/gds/core/write/NativeRelationshipExporter.java index cb56c73558..42f9d1b62f 100644 --- a/core-write/src/main/java/org/neo4j/gds/core/write/NativeRelationshipExporter.java +++ b/core-write/src/main/java/org/neo4j/gds/core/write/NativeRelationshipExporter.java @@ -26,7 +26,7 @@ import org.neo4j.gds.api.RelationshipIterator; import org.neo4j.gds.api.RelationshipWithPropertyConsumer; import org.neo4j.gds.core.concurrency.ParallelUtil; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServiceUtil; import org.neo4j.gds.core.utils.TerminationFlag; import org.neo4j.gds.core.utils.partition.Partition; import org.neo4j.gds.core.utils.partition.PartitionUtils; @@ -91,7 +91,7 @@ public static RelationshipExporterBuilder builder( this.propertyTranslator = propertyTranslator; this.terminationFlag = terminationFlag; this.progressTracker = progressTracker; - this.executorService = Pools.DEFAULT_SINGLE_THREAD_POOL; + this.executorService = ExecutorServiceUtil.DEFAULT_SINGLE_THREAD_POOL; } @Override diff --git a/core-write/src/main/java/org/neo4j/gds/core/write/NativeRelationshipPropertiesExporterBuilder.java b/core-write/src/main/java/org/neo4j/gds/core/write/NativeRelationshipPropertiesExporterBuilder.java index f1ebf2099b..0eb0fae947 100644 --- a/core-write/src/main/java/org/neo4j/gds/core/write/NativeRelationshipPropertiesExporterBuilder.java +++ b/core-write/src/main/java/org/neo4j/gds/core/write/NativeRelationshipPropertiesExporterBuilder.java @@ -19,7 +19,7 @@ */ package org.neo4j.gds.core.write; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServiceUtil; import org.neo4j.gds.transaction.TransactionContext; public class NativeRelationshipPropertiesExporterBuilder extends RelationshipPropertiesExporterBuilder { @@ -37,7 +37,7 @@ public RelationshipPropertiesExporter build() { graphStore, propertyTranslator, progressTracker, - Pools.DEFAULT_SINGLE_THREAD_POOL, + ExecutorServiceUtil.DEFAULT_SINGLE_THREAD_POOL, terminationFlag ); } diff --git a/core-write/src/test/java/org/neo4j/gds/core/write/NativeRelationshipPropertiesExporterTest.java b/core-write/src/test/java/org/neo4j/gds/core/write/NativeRelationshipPropertiesExporterTest.java index 2b98050c58..34899c4ab4 100644 --- a/core-write/src/test/java/org/neo4j/gds/core/write/NativeRelationshipPropertiesExporterTest.java +++ b/core-write/src/test/java/org/neo4j/gds/core/write/NativeRelationshipPropertiesExporterTest.java @@ -29,7 +29,7 @@ import org.neo4j.gds.api.DefaultValue; import org.neo4j.gds.api.GraphStore; import org.neo4j.gds.core.Aggregation; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServiceUtil; import org.neo4j.gds.core.utils.TerminationFlag; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.values.storable.Values; @@ -79,7 +79,7 @@ void shouldWriteRelationshipsWithMultipleProperties() { graphStore, Values::doubleValue, ProgressTracker.NULL_TRACKER, - Pools.DEFAULT_SINGLE_THREAD_POOL, + ExecutorServiceUtil.DEFAULT_SINGLE_THREAD_POOL, TerminationFlag.RUNNING_TRUE ); diff --git a/core/src/main/java/org/neo4j/gds/core/concurrency/DefaultPool.java b/core/src/main/java/org/neo4j/gds/core/concurrency/DefaultPool.java index e1ba5a6bd6..1ba55b09b2 100644 --- a/core/src/main/java/org/neo4j/gds/core/concurrency/DefaultPool.java +++ b/core/src/main/java/org/neo4j/gds/core/concurrency/DefaultPool.java @@ -38,8 +38,8 @@ private static ExecutorService createDefaultPool(PoolSizes poolSizes) { 30L, TimeUnit.SECONDS, new ArrayBlockingQueue<>(poolSizes.corePoolSize() * 50), - Pools.DEFAULT_THREAD_FACTORY, - new Pools.CallerBlocksPolicy() + ExecutorServiceUtil.DEFAULT_THREAD_FACTORY, + new ExecutorServiceUtil.CallerBlocksPolicy() ); } diff --git a/core/src/main/java/org/neo4j/gds/core/concurrency/Pools.java b/core/src/main/java/org/neo4j/gds/core/concurrency/ExecutorServiceUtil.java similarity index 95% rename from core/src/main/java/org/neo4j/gds/core/concurrency/Pools.java rename to core/src/main/java/org/neo4j/gds/core/concurrency/ExecutorServiceUtil.java index f25a3e2e84..111de7433e 100644 --- a/core/src/main/java/org/neo4j/gds/core/concurrency/Pools.java +++ b/core/src/main/java/org/neo4j/gds/core/concurrency/ExecutorServiceUtil.java @@ -33,14 +33,14 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; -public final class Pools { +public final class ExecutorServiceUtil { private static final String THREAD_NAME_PREFIX = "gds"; public static final ThreadFactory DEFAULT_THREAD_FACTORY = NamedThreadFactory.daemon(THREAD_NAME_PREFIX); public static final ExecutorService DEFAULT_SINGLE_THREAD_POOL = createSingleThreadPool("algo"); - private Pools() { + private ExecutorServiceUtil() { throw new UnsupportedOperationException(); } @@ -87,7 +87,7 @@ public void rejectedExecution(Runnable r, ThreadPoolExecutor executor) { private static final ForkJoinPool.ForkJoinWorkerThreadFactory FJ_WORKER_THREAD_FACTORY = pool -> { var worker = ForkJoinPool.defaultForkJoinWorkerThreadFactory.newThread(pool); - worker.setName(Pools.THREAD_NAME_PREFIX + "-forkjoin-" + worker.getPoolIndex()); + worker.setName(ExecutorServiceUtil.THREAD_NAME_PREFIX + "-forkjoin-" + worker.getPoolIndex()); return worker; }; } diff --git a/core/src/main/java/org/neo4j/gds/core/concurrency/ParallelUtil.java b/core/src/main/java/org/neo4j/gds/core/concurrency/ParallelUtil.java index 2f57d486a6..9e0bf799e8 100644 --- a/core/src/main/java/org/neo4j/gds/core/concurrency/ParallelUtil.java +++ b/core/src/main/java/org/neo4j/gds/core/concurrency/ParallelUtil.java @@ -70,7 +70,7 @@ private ParallelUtil() {} * The concurrency value is assumed to already be validated towards the edition limitation. */ public static , R> R parallelStream(T data, int concurrency, Function fn) { - ForkJoinPool pool = Pools.createForkJoinPool(concurrency); + ForkJoinPool pool = ExecutorServiceUtil.createForkJoinPool(concurrency); try { return pool.submit(() -> fn.apply(data.parallel())).get(); } catch (InterruptedException e) { diff --git a/core/src/main/java/org/neo4j/gds/core/utils/paged/HugeMergeSort.java b/core/src/main/java/org/neo4j/gds/core/utils/paged/HugeMergeSort.java index 20a3dc05ae..27760cc1de 100644 --- a/core/src/main/java/org/neo4j/gds/core/utils/paged/HugeMergeSort.java +++ b/core/src/main/java/org/neo4j/gds/core/utils/paged/HugeMergeSort.java @@ -20,7 +20,7 @@ package org.neo4j.gds.core.utils.paged; import org.jetbrains.annotations.Nullable; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServiceUtil; import java.util.concurrent.CountedCompleter; @@ -30,7 +30,7 @@ public final class HugeMergeSort { public static void sort(HugeLongArray array, int concurrency) { var temp = HugeLongArray.newArray(array.size()); - var forkJoinPool = Pools.createForkJoinPool(concurrency); + var forkJoinPool = ExecutorServiceUtil.createForkJoinPool(concurrency); try { forkJoinPool.invoke(new MergeSortTask(null, array, temp, 0, array.size() - 1)); } finally { diff --git a/core/src/test/java/org/neo4j/gds/core/concurrency/ParallelUtilTest.java b/core/src/test/java/org/neo4j/gds/core/concurrency/ParallelUtilTest.java index cc73aa2be2..a385bafdf3 100644 --- a/core/src/test/java/org/neo4j/gds/core/concurrency/ParallelUtilTest.java +++ b/core/src/test/java/org/neo4j/gds/core/concurrency/ParallelUtilTest.java @@ -358,7 +358,7 @@ void shouldBailOnTermination() { AtomicReference thrownException = new AtomicReference<>(); AtomicBoolean running = new AtomicBoolean(true); TerminationFlag isRunning = running::get; - var thread = Pools.newThread(() -> tasks.run(t -> + var thread = ExecutorServiceUtil.newThread(() -> tasks.run(t -> RunWithConcurrency.builder() .concurrency(2) .tasks(t) diff --git a/etc/forbidden-apis b/etc/forbidden-apis index 87ef4e5d7c..97d13d3492 100644 --- a/etc/forbidden-apis +++ b/etc/forbidden-apis @@ -1,6 +1,6 @@ org.neo4j.internal.helpers.collection.MapUtil @ Use the GDS MapUtil instead. -java.lang.Thread.Thread(java.lang.Runnable) @ Use org.neo4j.gds.core.concurrency.Pools.newThread to make sure that the new thread is properly named -java.lang.Thread.Thread(java.lang.ThreadGroup, java.lang.Runnable) @ Use org.neo4j.gds.core.concurrency.Pools.newThread to make sure that the new thread is properly named +java.lang.Thread.Thread(java.lang.Runnable) @ Use org.neo4j.gds.core.concurrency.ExecutorServiceUtil.newThread to make sure that the new thread is properly named +java.lang.Thread.Thread(java.lang.ThreadGroup, java.lang.Runnable) @ Use org.neo4j.gds.core.concurrency.ExecutorServiceUtil.newThread to make sure that the new thread is properly named org.neo4j.internal.batchimport.staging.ExecutionMonitor.Adapter @ Implement CompatExecutionMonitor and call Neo4jProxy.executionMonitor org.neo4j.logging.internal.LogService#getUserLogProvider() @ Pass the logService through and use Neo4jProxy at the end. diff --git a/pregel/src/main/java/org/neo4j/gds/beta/pregel/Pregel.java b/pregel/src/main/java/org/neo4j/gds/beta/pregel/Pregel.java index 56ce22a56c..8c8a788f7b 100644 --- a/pregel/src/main/java/org/neo4j/gds/beta/pregel/Pregel.java +++ b/pregel/src/main/java/org/neo4j/gds/beta/pregel/Pregel.java @@ -22,7 +22,7 @@ import org.immutables.value.Value; import org.neo4j.gds.api.Graph; import org.neo4j.gds.beta.pregel.context.MasterComputeContext; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServiceUtil; import org.neo4j.gds.core.utils.TerminationFlag; import org.neo4j.gds.core.utils.mem.MemoryEstimation; import org.neo4j.gds.core.utils.mem.MemoryEstimations; @@ -159,7 +159,7 @@ private Pregel( .messenger(messenger) .voteBits(HugeAtomicBitSet.create(graph.nodeCount())) .executorService(config.useForkJoin() - ? Pools.createForkJoinPool(config.concurrency()) + ? ExecutorServiceUtil.createForkJoinPool(config.concurrency()) : executor) .progressTracker(progressTracker) .build(); diff --git a/subgraph-filtering/src/test/java/org/neo4j/gds/beta/filter/NodesFilterTest.java b/subgraph-filtering/src/test/java/org/neo4j/gds/beta/filter/NodesFilterTest.java index ce8a9d31cb..e269fb13bb 100644 --- a/subgraph-filtering/src/test/java/org/neo4j/gds/beta/filter/NodesFilterTest.java +++ b/subgraph-filtering/src/test/java/org/neo4j/gds/beta/filter/NodesFilterTest.java @@ -23,7 +23,7 @@ import org.neo4j.gds.NodeLabel; import org.neo4j.gds.api.GraphStore; import org.neo4j.gds.beta.filter.expression.ExpressionParser; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServiceUtil; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.extension.GdlExtension; import org.neo4j.gds.extension.GdlGraph; @@ -60,7 +60,7 @@ void basicFiltering() throws ParseException { ExpressionParser.parse("n:A AND n.p > 1.0", Map.of()), 1, Map.of(), - Pools.DEFAULT_SINGLE_THREAD_POOL, + ExecutorServiceUtil.DEFAULT_SINGLE_THREAD_POOL, ProgressTracker.NULL_TRACKER ); From 1cd1b4e21ed1f3c6ace0f22ca3d7a1a17ee716f4 Mon Sep 17 00:00:00 2001 From: Mats Rydberg Date: Mon, 11 Sep 2023 11:45:39 +0200 Subject: [PATCH 231/273] Use OpenGDS pool sizes by default We kinda want the null initialisation, but there are many test dependencies that break it. Co-authored-by: Paul Horn --- .../java/org/neo4j/gds/concurrency/PoolSizesService.java | 2 +- .../neo4j/gds/core/concurrency/PoolSizesServiceTest.java | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/concurrency-validation-api/src/main/java/org/neo4j/gds/concurrency/PoolSizesService.java b/concurrency-validation-api/src/main/java/org/neo4j/gds/concurrency/PoolSizesService.java index 77714b6277..43744b9adf 100644 --- a/concurrency-validation-api/src/main/java/org/neo4j/gds/concurrency/PoolSizesService.java +++ b/concurrency-validation-api/src/main/java/org/neo4j/gds/concurrency/PoolSizesService.java @@ -23,7 +23,7 @@ public final class PoolSizesService { - private static PoolSizes instance = null; + private static PoolSizes instance = new OpenGdsPoolSizes(); private PoolSizesService() { } diff --git a/core/src/test/java/org/neo4j/gds/core/concurrency/PoolSizesServiceTest.java b/core/src/test/java/org/neo4j/gds/core/concurrency/PoolSizesServiceTest.java index 6189572cf5..c48e812200 100644 --- a/core/src/test/java/org/neo4j/gds/core/concurrency/PoolSizesServiceTest.java +++ b/core/src/test/java/org/neo4j/gds/core/concurrency/PoolSizesServiceTest.java @@ -22,13 +22,17 @@ import org.junit.jupiter.api.Test; import org.neo4j.gds.concurrency.PoolSizesService; +import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.AssertionsForClassTypes.assertThatThrownBy; class PoolSizesServiceTest { @Test - void throwOnAccessBeforeProperInitialisation() { - assertThatThrownBy(PoolSizesService::poolSizes).isInstanceOf(NullPointerException.class); + void openGdsPoolSizesByDefault() { + var poolSizes = PoolSizesService.poolSizes(); + + assertThat(poolSizes.corePoolSize()).isEqualTo(4); + assertThat(poolSizes.maxPoolSize()).isEqualTo(4); } } From 816b0d9880ea4434c6d720e80bdd84e7c4dea220 Mon Sep 17 00:00:00 2001 From: ioannispan Date: Mon, 11 Sep 2023 12:09:29 +0200 Subject: [PATCH 232/273] `didConverge ` is not integer --- doc/modules/ROOT/pages/algorithms/leiden.adoc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/modules/ROOT/pages/algorithms/leiden.adoc b/doc/modules/ROOT/pages/algorithms/leiden.adoc index 2bb39ceea9..c1976b37fe 100644 --- a/doc/modules/ROOT/pages/algorithms/leiden.adoc +++ b/doc/modules/ROOT/pages/algorithms/leiden.adoc @@ -151,7 +151,7 @@ YIELD modularity: Float, modularities: List of Float, nodeCount: Integer, - didConverge: Integer, + didConverge: Boolean, nodePropertiesWritten: Integer, communityDistribution: Map, configuration: Map @@ -207,7 +207,7 @@ YIELD modularity: Float, modularities: List of Float, nodeCount: Integer, - didConverge: Integer, + didConverge: Boolean, nodePropertiesWritten: Integer, communityDistribution: Map, configuration: Map From 8c82ff91143968242e62e82be8ddbba6068052df Mon Sep 17 00:00:00 2001 From: Mats Rydberg Date: Mon, 11 Sep 2023 12:36:17 +0200 Subject: [PATCH 233/273] Fix test assertion after refactor --- .../org/neo4j/gds/core/concurrency/PoolSizesServiceTest.java | 1 - .../org/neo4j/gds/pregel/generator/AlgorithmGeneratorTest.java | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/core/src/test/java/org/neo4j/gds/core/concurrency/PoolSizesServiceTest.java b/core/src/test/java/org/neo4j/gds/core/concurrency/PoolSizesServiceTest.java index c48e812200..635f2ed63b 100644 --- a/core/src/test/java/org/neo4j/gds/core/concurrency/PoolSizesServiceTest.java +++ b/core/src/test/java/org/neo4j/gds/core/concurrency/PoolSizesServiceTest.java @@ -23,7 +23,6 @@ import org.neo4j.gds.concurrency.PoolSizesService; import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.AssertionsForClassTypes.assertThatThrownBy; class PoolSizesServiceTest { diff --git a/pregel-proc-generator/src/test/java/org/neo4j/gds/pregel/generator/AlgorithmGeneratorTest.java b/pregel-proc-generator/src/test/java/org/neo4j/gds/pregel/generator/AlgorithmGeneratorTest.java index a69779c2e2..d04e631292 100644 --- a/pregel-proc-generator/src/test/java/org/neo4j/gds/pregel/generator/AlgorithmGeneratorTest.java +++ b/pregel-proc-generator/src/test/java/org/neo4j/gds/pregel/generator/AlgorithmGeneratorTest.java @@ -61,7 +61,7 @@ void shouldGenerateConstructor() { " org.neo4j.gds.core.utils.progress.tasks.ProgressTracker progressTracker) {" + NL + " super(progressTracker);" + NL + " var computation = new a.b.C();" + NL + - " this.pregelJob = org.neo4j.gds.beta.pregel.Pregel.create(graph, configuration, computation, org.neo4j.gds.core.concurrency.ExecutorServices.DEFAULT, progressTracker);" + NL + + " this.pregelJob = org.neo4j.gds.beta.pregel.Pregel.create(graph, configuration, computation, org.neo4j.gds.core.concurrency.DefaultPool.INSTANCE, progressTracker);" + NL + "}" + NL ); } From 837252ae3242c212cbc8529ff7e692030c15dffc Mon Sep 17 00:00:00 2001 From: Mats Rydberg Date: Tue, 12 Sep 2023 09:36:43 +0200 Subject: [PATCH 234/273] Revert ExecutorService change In PregelBootstrap we depend on the released version (2.4.5). This change will come when we release 2.4.6. --- .../test/java/gds/example/ExamplePregelComputationAlgoTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/pregel-bootstrap/src/test/java/gds/example/ExamplePregelComputationAlgoTest.java b/examples/pregel-bootstrap/src/test/java/gds/example/ExamplePregelComputationAlgoTest.java index 7ce2e5b753..26f51bb395 100644 --- a/examples/pregel-bootstrap/src/test/java/gds/example/ExamplePregelComputationAlgoTest.java +++ b/examples/pregel-bootstrap/src/test/java/gds/example/ExamplePregelComputationAlgoTest.java @@ -64,7 +64,7 @@ void runExamplePregelComputation() { graph, config, new ExamplePregelComputation(), - ExecutorServices.DEFAULT, + Pools.DEFAULT, ProgressTracker.NULL_TRACKER ); From 57be3c6fd2590912e9b84ccf87a719e0995cca74 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B6ren=20Reichardt?= Date: Thu, 31 Aug 2023 10:02:26 +0200 Subject: [PATCH 235/273] Replace removed DependencyResolver.Adapter --- .../gds/api/EmptyDependencyResolver.java | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/neo4j-api/src/main/java/org/neo4j/gds/api/EmptyDependencyResolver.java b/neo4j-api/src/main/java/org/neo4j/gds/api/EmptyDependencyResolver.java index c07f748570..8cf9c14c95 100644 --- a/neo4j-api/src/main/java/org/neo4j/gds/api/EmptyDependencyResolver.java +++ b/neo4j-api/src/main/java/org/neo4j/gds/api/EmptyDependencyResolver.java @@ -21,7 +21,9 @@ import org.neo4j.common.DependencyResolver; -public final class EmptyDependencyResolver extends DependencyResolver.Adapter { +import java.util.function.Supplier; + +public final class EmptyDependencyResolver implements DependencyResolver { public static final EmptyDependencyResolver INSTANCE = new EmptyDependencyResolver(); @@ -36,4 +38,19 @@ public T resolveDependency(Class type, SelectionStrategy selector) { public boolean containsDependency(Class type) { return false; } + + @Override + public T resolveDependency(Class type) { + return null; + } + + @Override + public Iterable resolveTypeDependencies(Class type) { + return null; + } + + @Override + public Supplier provideDependency(Class type) { + return null; + } } From 2d0488966289bbb0eca09e6cff647506ed04d109 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B6ren=20Reichardt?= Date: Thu, 31 Aug 2023 10:55:39 +0200 Subject: [PATCH 236/273] Move empty dependency resolver behind Neo4jProxy --- .../neo4j/gds/compat/_44/Neo4jProxyImpl.java | 15 +++++ .../neo4j/gds/compat/_510/Neo4jProxyImpl.java | 15 +++++ .../neo4j/gds/compat/_511/Neo4jProxyImpl.java | 15 +++++ .../neo4j/gds/compat/_512/Neo4jProxyImpl.java | 15 +++++ .../neo4j/gds/compat/_56/Neo4jProxyImpl.java | 15 +++++ .../neo4j/gds/compat/_57/Neo4jProxyImpl.java | 15 +++++ .../neo4j/gds/compat/_58/Neo4jProxyImpl.java | 15 +++++ .../neo4j/gds/compat/_59/Neo4jProxyImpl.java | 15 +++++ .../org/neo4j/gds/compat/Neo4jProxyApi.java | 3 + .../java/org/neo4j/gds/compat/Neo4jProxy.java | 5 ++ .../neo4j/gds/executor/ExecutionContext.java | 4 +- .../gds/executor/ProcedureExecutorTest.java | 3 +- .../gds/api/EmptyDependencyResolver.java | 56 ------------------- .../LinkPredictionTrainingPipelineTest.java | 8 +-- .../gds/catalog/GraphListOperatorTest.java | 3 +- .../gds/catalog/GraphMemoryUsageTest.java | 3 +- .../gds/catalog/NodeLabelMutatorTest.java | 3 +- .../gds/catalog/NodePropertiesWriterTest.java | 3 +- .../gds/catalog/SamplerOperatorTest.java | 3 +- ...PropertyComputationResultConsumerTest.java | 3 +- ...opertiesComputationResultConsumerTest.java | 3 +- 21 files changed, 142 insertions(+), 78 deletions(-) delete mode 100644 neo4j-api/src/main/java/org/neo4j/gds/api/EmptyDependencyResolver.java diff --git a/compatibility/4.4/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_44/Neo4jProxyImpl.java b/compatibility/4.4/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_44/Neo4jProxyImpl.java index b8a3076c66..ea972cfcd2 100644 --- a/compatibility/4.4/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_44/Neo4jProxyImpl.java +++ b/compatibility/4.4/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_44/Neo4jProxyImpl.java @@ -836,4 +836,19 @@ public Stream getAllAggregatingFunctions() { } }; } + + @Override + public DependencyResolver emptyDependencyResolver() { + return new DependencyResolver.Adapter() { + @Override + public T resolveDependency(Class type, SelectionStrategy selector) { + return null; + } + + @Override + public boolean containsDependency(Class type) { + return false; + } + }; + } } diff --git a/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/Neo4jProxyImpl.java b/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/Neo4jProxyImpl.java index 3c53d58a61..5d3ed7f272 100644 --- a/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/Neo4jProxyImpl.java +++ b/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/Neo4jProxyImpl.java @@ -948,4 +948,19 @@ public Stream getAllAggregatingFunctions() { } }; } + + @Override + public DependencyResolver emptyDependencyResolver() { + return new DependencyResolver.Adapter() { + @Override + public T resolveDependency(Class type, SelectionStrategy selector) { + return null; + } + + @Override + public boolean containsDependency(Class type) { + return false; + } + }; + } } diff --git a/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/Neo4jProxyImpl.java b/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/Neo4jProxyImpl.java index 28eb91d620..10b0e8d0e8 100644 --- a/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/Neo4jProxyImpl.java +++ b/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/Neo4jProxyImpl.java @@ -948,4 +948,19 @@ public Stream getAllAggregatingFunctions() { } }; } + + @Override + public DependencyResolver emptyDependencyResolver() { + return new DependencyResolver.Adapter() { + @Override + public T resolveDependency(Class type, SelectionStrategy selector) { + return null; + } + + @Override + public boolean containsDependency(Class type) { + return false; + } + }; + } } diff --git a/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/Neo4jProxyImpl.java b/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/Neo4jProxyImpl.java index 9a02e66c34..2ef6fdba2d 100644 --- a/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/Neo4jProxyImpl.java +++ b/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/Neo4jProxyImpl.java @@ -947,4 +947,19 @@ public Stream getAllAggregatingFunctions() { } }; } + + @Override + public DependencyResolver emptyDependencyResolver() { + return new DependencyResolver() { + @Override + public T resolveDependency(Class type, SelectionStrategy selector) { + return null; + } + + @Override + public boolean containsDependency(Class type) { + return false; + } + }; + } } diff --git a/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/Neo4jProxyImpl.java b/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/Neo4jProxyImpl.java index 9074bc65de..0156b205b2 100644 --- a/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/Neo4jProxyImpl.java +++ b/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/Neo4jProxyImpl.java @@ -952,4 +952,19 @@ public Stream getAllAggregatingFunctions() { } }; } + + @Override + public DependencyResolver emptyDependencyResolver() { + return new DependencyResolver.Adapter() { + @Override + public T resolveDependency(Class type, SelectionStrategy selector) { + return null; + } + + @Override + public boolean containsDependency(Class type) { + return false; + } + }; + } } diff --git a/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/Neo4jProxyImpl.java b/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/Neo4jProxyImpl.java index b53c1bdf1a..43b4b33eea 100644 --- a/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/Neo4jProxyImpl.java +++ b/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/Neo4jProxyImpl.java @@ -947,4 +947,19 @@ public Stream getAllAggregatingFunctions() { } }; } + + @Override + public DependencyResolver emptyDependencyResolver() { + return new DependencyResolver.Adapter() { + @Override + public T resolveDependency(Class type, SelectionStrategy selector) { + return null; + } + + @Override + public boolean containsDependency(Class type) { + return false; + } + }; + } } diff --git a/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/Neo4jProxyImpl.java b/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/Neo4jProxyImpl.java index e0762819ac..66abf167cb 100644 --- a/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/Neo4jProxyImpl.java +++ b/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/Neo4jProxyImpl.java @@ -947,4 +947,19 @@ public Stream getAllAggregatingFunctions() { } }; } + + @Override + public DependencyResolver emptyDependencyResolver() { + return new DependencyResolver.Adapter() { + @Override + public T resolveDependency(Class type, SelectionStrategy selector) { + return null; + } + + @Override + public boolean containsDependency(Class type) { + return false; + } + }; + } } diff --git a/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/Neo4jProxyImpl.java b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/Neo4jProxyImpl.java index cbd431f918..8661e69d1b 100644 --- a/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/Neo4jProxyImpl.java +++ b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/Neo4jProxyImpl.java @@ -947,4 +947,19 @@ public Stream getAllAggregatingFunctions() { } }; } + + @Override + public DependencyResolver emptyDependencyResolver() { + return new DependencyResolver.Adapter() { + @Override + public T resolveDependency(Class type, SelectionStrategy selector) { + return null; + } + + @Override + public boolean containsDependency(Class type) { + return false; + } + }; + } } diff --git a/compatibility/api/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/Neo4jProxyApi.java b/compatibility/api/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/Neo4jProxyApi.java index abe644512a..a15863019d 100644 --- a/compatibility/api/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/Neo4jProxyApi.java +++ b/compatibility/api/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/Neo4jProxyApi.java @@ -21,6 +21,7 @@ import org.jetbrains.annotations.Nullable; import org.jetbrains.annotations.TestOnly; +import org.neo4j.common.DependencyResolver; import org.neo4j.configuration.Config; import org.neo4j.configuration.connectors.ConnectorPortRegister; import org.neo4j.dbms.api.DatabaseManagementService; @@ -327,4 +328,6 @@ TransactionalContext newQueryContext( T lookupComponentProvider(Context ctx, Class component, boolean safe) throws ProcedureException; GlobalProcedureRegistry globalProcedureRegistry(GlobalProcedures globalProcedures); + + DependencyResolver emptyDependencyResolver(); } diff --git a/compatibility/common/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/Neo4jProxy.java b/compatibility/common/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/Neo4jProxy.java index 0cfb540669..6b42e16549 100644 --- a/compatibility/common/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/Neo4jProxy.java +++ b/compatibility/common/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/Neo4jProxy.java @@ -21,6 +21,7 @@ import org.jetbrains.annotations.Nullable; import org.jetbrains.annotations.TestOnly; +import org.neo4j.common.DependencyResolver; import org.neo4j.configuration.Config; import org.neo4j.configuration.connectors.ConnectorPortRegister; import org.neo4j.dbms.api.DatabaseManagementService; @@ -493,6 +494,10 @@ public static GlobalProcedureRegistry globalProcedureRegistry(GlobalProcedures g return IMPL.globalProcedureRegistry(globalProcedures); } + public static DependencyResolver emptyDependencyResolver() { + return IMPL.emptyDependencyResolver(); + } + private Neo4jProxy() { throw new UnsupportedOperationException("No instances"); } diff --git a/executor/src/main/java/org/neo4j/gds/executor/ExecutionContext.java b/executor/src/main/java/org/neo4j/gds/executor/ExecutionContext.java index 43c7476047..ee333204b6 100644 --- a/executor/src/main/java/org/neo4j/gds/executor/ExecutionContext.java +++ b/executor/src/main/java/org/neo4j/gds/executor/ExecutionContext.java @@ -25,10 +25,10 @@ import org.neo4j.gds.api.AlgorithmMetaDataSetter; import org.neo4j.gds.api.CloseableResourceRegistry; import org.neo4j.gds.api.DatabaseId; -import org.neo4j.gds.api.EmptyDependencyResolver; import org.neo4j.gds.api.NodeLookup; import org.neo4j.gds.api.ProcedureReturnColumns; import org.neo4j.gds.api.TerminationMonitor; +import org.neo4j.gds.compat.Neo4jProxy; import org.neo4j.gds.core.model.ModelCatalog; import org.neo4j.gds.core.utils.progress.EmptyTaskRegistryFactory; import org.neo4j.gds.core.utils.progress.TaskRegistryFactory; @@ -120,7 +120,7 @@ public DatabaseId databaseId() { @Override public DependencyResolver dependencyResolver() { - return EmptyDependencyResolver.INSTANCE; + return Neo4jProxy.emptyDependencyResolver(); } @Override diff --git a/executor/src/test/java/org/neo4j/gds/executor/ProcedureExecutorTest.java b/executor/src/test/java/org/neo4j/gds/executor/ProcedureExecutorTest.java index 3fb3a9af6e..4e7c4bb04e 100644 --- a/executor/src/test/java/org/neo4j/gds/executor/ProcedureExecutorTest.java +++ b/executor/src/test/java/org/neo4j/gds/executor/ProcedureExecutorTest.java @@ -25,7 +25,6 @@ import org.neo4j.gds.ProcedureCallContextReturnColumns; import org.neo4j.gds.api.AlgorithmMetaDataSetter; import org.neo4j.gds.api.CloseableResourceRegistry; -import org.neo4j.gds.api.EmptyDependencyResolver; import org.neo4j.gds.api.GraphStore; import org.neo4j.gds.api.NodeLookup; import org.neo4j.gds.api.TerminationMonitor; @@ -131,7 +130,7 @@ private ExecutionContext executionContext(TaskStore taskStore) { .username("") .terminationMonitor(TerminationMonitor.EMPTY) .isGdsAdmin(true) - .dependencyResolver(EmptyDependencyResolver.INSTANCE) + .dependencyResolver(Neo4jProxy.emptyDependencyResolver()) .modelCatalog(ModelCatalog.EMPTY) .closeableResourceRegistry(CloseableResourceRegistry.EMPTY) .algorithmMetaDataSetter(AlgorithmMetaDataSetter.EMPTY) diff --git a/neo4j-api/src/main/java/org/neo4j/gds/api/EmptyDependencyResolver.java b/neo4j-api/src/main/java/org/neo4j/gds/api/EmptyDependencyResolver.java deleted file mode 100644 index 8cf9c14c95..0000000000 --- a/neo4j-api/src/main/java/org/neo4j/gds/api/EmptyDependencyResolver.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Copyright (c) "Neo4j" - * Neo4j Sweden AB [http://neo4j.com] - * - * This file is part of Neo4j. - * - * Neo4j is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -package org.neo4j.gds.api; - -import org.neo4j.common.DependencyResolver; - -import java.util.function.Supplier; - -public final class EmptyDependencyResolver implements DependencyResolver { - - public static final EmptyDependencyResolver INSTANCE = new EmptyDependencyResolver(); - - private EmptyDependencyResolver() {} - - @Override - public T resolveDependency(Class type, SelectionStrategy selector) { - return null; - } - - @Override - public boolean containsDependency(Class type) { - return false; - } - - @Override - public T resolveDependency(Class type) { - return null; - } - - @Override - public Iterable resolveTypeDependencies(Class type) { - return null; - } - - @Override - public Supplier provideDependency(Class type) { - return null; - } -} diff --git a/pipeline/src/test/java/org/neo4j/gds/ml/pipeline/linkPipeline/LinkPredictionTrainingPipelineTest.java b/pipeline/src/test/java/org/neo4j/gds/ml/pipeline/linkPipeline/LinkPredictionTrainingPipelineTest.java index 9b3a7a99cd..e897aeb0b6 100644 --- a/pipeline/src/test/java/org/neo4j/gds/ml/pipeline/linkPipeline/LinkPredictionTrainingPipelineTest.java +++ b/pipeline/src/test/java/org/neo4j/gds/ml/pipeline/linkPipeline/LinkPredictionTrainingPipelineTest.java @@ -27,11 +27,11 @@ import org.neo4j.gds.api.AlgorithmMetaDataSetter; import org.neo4j.gds.api.CloseableResourceRegistry; import org.neo4j.gds.api.DatabaseId; -import org.neo4j.gds.api.EmptyDependencyResolver; import org.neo4j.gds.api.NodeLookup; import org.neo4j.gds.api.ProcedureReturnColumns; import org.neo4j.gds.api.TerminationMonitor; import org.neo4j.gds.api.schema.GraphSchema; +import org.neo4j.gds.compat.Neo4jProxy; import org.neo4j.gds.core.model.Model; import org.neo4j.gds.core.model.ModelCatalog; import org.neo4j.gds.core.model.OpenModelCatalog; @@ -182,7 +182,7 @@ void overridesTheSplitConfig() { void deriveRelationshipWeightProperty() { var executionContext = ImmutableExecutionContext.builder() .databaseId(DatabaseId.from("")) - .dependencyResolver(EmptyDependencyResolver.INSTANCE) + .dependencyResolver(Neo4jProxy.emptyDependencyResolver()) .username("") .terminationMonitor(TerminationMonitor.EMPTY) .closeableResourceRegistry(CloseableResourceRegistry.EMPTY) @@ -226,7 +226,7 @@ void deriveRelationshipWeightPropertyFromTrainedModel() { var executionContext = ImmutableExecutionContext.builder() .databaseId(DatabaseId.from("")) - .dependencyResolver(EmptyDependencyResolver.INSTANCE) + .dependencyResolver(Neo4jProxy.emptyDependencyResolver()) .username("") .modelCatalog(modelCatalog) .terminationMonitor(TerminationMonitor.EMPTY) @@ -270,7 +270,7 @@ void notDerivePropertyFromUnweightedTrainedModel() { var executionContext = ImmutableExecutionContext.builder() .databaseId(DatabaseId.from("")) - .dependencyResolver(EmptyDependencyResolver.INSTANCE) + .dependencyResolver(Neo4jProxy.emptyDependencyResolver()) .username("") .modelCatalog(modelCatalog) .terminationMonitor(TerminationMonitor.EMPTY) diff --git a/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphListOperatorTest.java b/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphListOperatorTest.java index f2cb58cee5..8f26877881 100644 --- a/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphListOperatorTest.java +++ b/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphListOperatorTest.java @@ -35,7 +35,6 @@ import org.neo4j.gds.RelationshipType; import org.neo4j.gds.api.AlgorithmMetaDataSetter; import org.neo4j.gds.api.CloseableResourceRegistry; -import org.neo4j.gds.api.EmptyDependencyResolver; import org.neo4j.gds.api.GraphStore; import org.neo4j.gds.api.NodeLookup; import org.neo4j.gds.api.TerminationMonitor; @@ -438,7 +437,7 @@ private ImmutableExecutionContext.Builder executionContextBuilder(String usernam return ImmutableExecutionContext .builder() .databaseId(graphStore.databaseId()) - .dependencyResolver(EmptyDependencyResolver.INSTANCE) + .dependencyResolver(Neo4jProxy.emptyDependencyResolver()) .returnColumns(fieldName -> returnFields.contains(fieldName)) .userLogRegistryFactory(EmptyUserLogRegistryFactory.INSTANCE) .taskRegistryFactory(EmptyTaskRegistryFactory.INSTANCE) diff --git a/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphMemoryUsageTest.java b/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphMemoryUsageTest.java index 4c5e8ec5fe..8181e9ec22 100644 --- a/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphMemoryUsageTest.java +++ b/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphMemoryUsageTest.java @@ -26,7 +26,6 @@ import org.junit.jupiter.params.provider.MethodSource; import org.neo4j.gds.api.AlgorithmMetaDataSetter; import org.neo4j.gds.api.CloseableResourceRegistry; -import org.neo4j.gds.api.EmptyDependencyResolver; import org.neo4j.gds.api.GraphStore; import org.neo4j.gds.api.NodeLookup; import org.neo4j.gds.api.ProcedureReturnColumns; @@ -127,7 +126,7 @@ private ImmutableExecutionContext.Builder executionContextBuilder() { return ImmutableExecutionContext .builder() .databaseId(graphStore.databaseId()) - .dependencyResolver(EmptyDependencyResolver.INSTANCE) + .dependencyResolver(Neo4jProxy.emptyDependencyResolver()) .returnColumns(ProcedureReturnColumns.EMPTY) .userLogRegistryFactory(EmptyUserLogRegistryFactory.INSTANCE) .taskRegistryFactory(EmptyTaskRegistryFactory.INSTANCE) diff --git a/proc/catalog/src/test/java/org/neo4j/gds/catalog/NodeLabelMutatorTest.java b/proc/catalog/src/test/java/org/neo4j/gds/catalog/NodeLabelMutatorTest.java index 1839ca8646..017e817b97 100644 --- a/proc/catalog/src/test/java/org/neo4j/gds/catalog/NodeLabelMutatorTest.java +++ b/proc/catalog/src/test/java/org/neo4j/gds/catalog/NodeLabelMutatorTest.java @@ -29,7 +29,6 @@ import org.neo4j.gds.NodeLabel; import org.neo4j.gds.api.AlgorithmMetaDataSetter; import org.neo4j.gds.api.CloseableResourceRegistry; -import org.neo4j.gds.api.EmptyDependencyResolver; import org.neo4j.gds.api.GraphStore; import org.neo4j.gds.api.NodeLookup; import org.neo4j.gds.api.ProcedureReturnColumns; @@ -298,7 +297,7 @@ private ImmutableExecutionContext.Builder executionContextBuilder() { return ImmutableExecutionContext .builder() .databaseId(graphStore.databaseId()) - .dependencyResolver(EmptyDependencyResolver.INSTANCE) + .dependencyResolver(Neo4jProxy.emptyDependencyResolver()) .returnColumns(ProcedureReturnColumns.EMPTY) .userLogRegistryFactory(EmptyUserLogRegistryFactory.INSTANCE) .taskRegistryFactory(EmptyTaskRegistryFactory.INSTANCE) diff --git a/proc/catalog/src/test/java/org/neo4j/gds/catalog/NodePropertiesWriterTest.java b/proc/catalog/src/test/java/org/neo4j/gds/catalog/NodePropertiesWriterTest.java index fe4e366675..5bffb6b069 100644 --- a/proc/catalog/src/test/java/org/neo4j/gds/catalog/NodePropertiesWriterTest.java +++ b/proc/catalog/src/test/java/org/neo4j/gds/catalog/NodePropertiesWriterTest.java @@ -32,7 +32,6 @@ import org.mockito.junit.jupiter.MockitoExtension; import org.neo4j.gds.api.AlgorithmMetaDataSetter; import org.neo4j.gds.api.CloseableResourceRegistry; -import org.neo4j.gds.api.EmptyDependencyResolver; import org.neo4j.gds.api.GraphStore; import org.neo4j.gds.api.NodeLookup; import org.neo4j.gds.api.ProcedureReturnColumns; @@ -310,7 +309,7 @@ private ImmutableExecutionContext.Builder executionContextBuilder() { return ImmutableExecutionContext .builder() .databaseId(graphStore.databaseId()) - .dependencyResolver(EmptyDependencyResolver.INSTANCE) + .dependencyResolver(Neo4jProxy.emptyDependencyResolver()) .returnColumns(ProcedureReturnColumns.EMPTY) .userLogRegistryFactory(EmptyUserLogRegistryFactory.INSTANCE) .taskRegistryFactory(EmptyTaskRegistryFactory.INSTANCE) diff --git a/proc/catalog/src/test/java/org/neo4j/gds/catalog/SamplerOperatorTest.java b/proc/catalog/src/test/java/org/neo4j/gds/catalog/SamplerOperatorTest.java index 1435d6b0ce..97151c0ba3 100644 --- a/proc/catalog/src/test/java/org/neo4j/gds/catalog/SamplerOperatorTest.java +++ b/proc/catalog/src/test/java/org/neo4j/gds/catalog/SamplerOperatorTest.java @@ -27,7 +27,6 @@ import org.junit.jupiter.params.provider.MethodSource; import org.neo4j.gds.api.AlgorithmMetaDataSetter; import org.neo4j.gds.api.CloseableResourceRegistry; -import org.neo4j.gds.api.EmptyDependencyResolver; import org.neo4j.gds.api.GraphStore; import org.neo4j.gds.api.NodeLookup; import org.neo4j.gds.api.ProcedureReturnColumns; @@ -233,7 +232,7 @@ private ImmutableExecutionContext.Builder executionContextBuilder() { return ImmutableExecutionContext .builder() .databaseId(graphStore.databaseId()) - .dependencyResolver(EmptyDependencyResolver.INSTANCE) + .dependencyResolver(Neo4jProxy.emptyDependencyResolver()) .returnColumns(ProcedureReturnColumns.EMPTY) .userLogRegistryFactory(EmptyUserLogRegistryFactory.INSTANCE) .taskRegistryFactory(EmptyTaskRegistryFactory.INSTANCE) diff --git a/proc/common/src/test/java/org/neo4j/gds/MutatePropertyComputationResultConsumerTest.java b/proc/common/src/test/java/org/neo4j/gds/MutatePropertyComputationResultConsumerTest.java index 4b99d26628..04b9a5db37 100644 --- a/proc/common/src/test/java/org/neo4j/gds/MutatePropertyComputationResultConsumerTest.java +++ b/proc/common/src/test/java/org/neo4j/gds/MutatePropertyComputationResultConsumerTest.java @@ -27,7 +27,6 @@ import org.neo4j.gds.api.CloseableResourceRegistry; import org.neo4j.gds.api.DatabaseId; import org.neo4j.gds.api.DefaultValue; -import org.neo4j.gds.api.EmptyDependencyResolver; import org.neo4j.gds.api.Graph; import org.neo4j.gds.api.GraphStore; import org.neo4j.gds.api.NodeLookup; @@ -77,7 +76,7 @@ class MutatePropertyComputationResultConsumerTest { private final ExecutionContext executionContext = ImmutableExecutionContext .builder() .databaseId(DatabaseId.from("")) - .dependencyResolver(EmptyDependencyResolver.INSTANCE) + .dependencyResolver(Neo4jProxy.emptyDependencyResolver()) .returnColumns(ProcedureReturnColumns.EMPTY) .log(Neo4jProxy.testLog()) .taskRegistryFactory(EmptyTaskRegistryFactory.INSTANCE) diff --git a/proc/common/src/test/java/org/neo4j/gds/WriteNodePropertiesComputationResultConsumerTest.java b/proc/common/src/test/java/org/neo4j/gds/WriteNodePropertiesComputationResultConsumerTest.java index ee6d39b3c2..2908cf294f 100644 --- a/proc/common/src/test/java/org/neo4j/gds/WriteNodePropertiesComputationResultConsumerTest.java +++ b/proc/common/src/test/java/org/neo4j/gds/WriteNodePropertiesComputationResultConsumerTest.java @@ -23,7 +23,6 @@ import org.neo4j.gds.api.AlgorithmMetaDataSetter; import org.neo4j.gds.api.CloseableResourceRegistry; import org.neo4j.gds.api.DatabaseId; -import org.neo4j.gds.api.EmptyDependencyResolver; import org.neo4j.gds.api.Graph; import org.neo4j.gds.api.GraphCharacteristics; import org.neo4j.gds.api.GraphStore; @@ -84,7 +83,7 @@ class WriteNodePropertiesComputationResultConsumerTest extends BaseTest { private final ExecutionContext executionContext = ImmutableExecutionContext .builder() .databaseId(DatabaseId.from("")) - .dependencyResolver(EmptyDependencyResolver.INSTANCE) + .dependencyResolver(Neo4jProxy.emptyDependencyResolver()) .returnColumns(ProcedureReturnColumns.EMPTY) .log(Neo4jProxy.testLog()) .taskRegistryFactory(EmptyTaskRegistryFactory.INSTANCE) From ef00f14fc4e9881f45951929540c052047869b0e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B6ren=20Reichardt?= Date: Fri, 1 Sep 2023 09:56:37 +0200 Subject: [PATCH 237/273] Remove redundant query param Co-authored-by: Veselin Nikolov --- .../neo4j/gds/embeddings/fastrp/FastRPStreamProcTest.java | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/proc/embeddings/src/test/java/org/neo4j/gds/embeddings/fastrp/FastRPStreamProcTest.java b/proc/embeddings/src/test/java/org/neo4j/gds/embeddings/fastrp/FastRPStreamProcTest.java index a90bb8d68c..b774387273 100644 --- a/proc/embeddings/src/test/java/org/neo4j/gds/embeddings/fastrp/FastRPStreamProcTest.java +++ b/proc/embeddings/src/test/java/org/neo4j/gds/embeddings/fastrp/FastRPStreamProcTest.java @@ -108,16 +108,14 @@ void shouldComputeNonZeroEmbeddingsWhenFirstWeightIsZero() { var propertyRatio = 0.5; int embeddingDimension = 128; var weights = List.of(0.0D, 1.0D, 2.0D, 4.0D); - GdsCypher.ParametersBuildStage queryBuilder = GdsCypher.call(FAST_RP_GRAPH) + var query = GdsCypher.call(FAST_RP_GRAPH) .algo("fastRP") .streamMode() .addParameter("embeddingDimension", embeddingDimension) .addParameter("propertyRatio", propertyRatio) .addParameter("featureProperties", featureProperties) - .addParameter("iterationWeights", weights); - - queryBuilder.addParameter("iterationWeights", weights); - String query = queryBuilder.yields(); + .addParameter("iterationWeights", weights) + .yields(); var rowCount = runQueryWithRowConsumer(query, row -> { assertThat(row.get("embedding")) From 24e90bb5a24ce62b24b2504292edc70cd43af86f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B6ren=20Reichardt?= Date: Fri, 1 Sep 2023 10:00:32 +0200 Subject: [PATCH 238/273] Use static empty dependency resolvers Co-authored-by: Mats Rydberg --- .../neo4j/gds/compat/_44/Neo4jProxyImpl.java | 24 +++++++------ .../neo4j/gds/compat/_510/Neo4jProxyImpl.java | 24 +++++++------ .../neo4j/gds/compat/_511/Neo4jProxyImpl.java | 35 ++++++++++++------- .../neo4j/gds/compat/_512/Neo4jProxyImpl.java | 24 +++++++------ .../neo4j/gds/compat/_56/Neo4jProxyImpl.java | 24 +++++++------ .../neo4j/gds/compat/_57/Neo4jProxyImpl.java | 24 +++++++------ .../neo4j/gds/compat/_58/Neo4jProxyImpl.java | 24 +++++++------ .../neo4j/gds/compat/_59/Neo4jProxyImpl.java | 24 +++++++------ 8 files changed, 113 insertions(+), 90 deletions(-) diff --git a/compatibility/4.4/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_44/Neo4jProxyImpl.java b/compatibility/4.4/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_44/Neo4jProxyImpl.java index ea972cfcd2..2d967bcf3e 100644 --- a/compatibility/4.4/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_44/Neo4jProxyImpl.java +++ b/compatibility/4.4/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_44/Neo4jProxyImpl.java @@ -837,18 +837,20 @@ public Stream getAllAggregatingFunctions() { }; } + private static final DependencyResolver EMPTY_DEPENDENCY_RESOLVER = new DependencyResolver.Adapter() { + @Override + public T resolveDependency(Class type, SelectionStrategy selector) { + return null; + } + + @Override + public boolean containsDependency(Class type) { + return false; + } + }; + @Override public DependencyResolver emptyDependencyResolver() { - return new DependencyResolver.Adapter() { - @Override - public T resolveDependency(Class type, SelectionStrategy selector) { - return null; - } - - @Override - public boolean containsDependency(Class type) { - return false; - } - }; + return EMPTY_DEPENDENCY_RESOLVER; } } diff --git a/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/Neo4jProxyImpl.java b/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/Neo4jProxyImpl.java index 5d3ed7f272..63b7019601 100644 --- a/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/Neo4jProxyImpl.java +++ b/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/Neo4jProxyImpl.java @@ -949,18 +949,20 @@ public Stream getAllAggregatingFunctions() { }; } + private static final DependencyResolver EMPTY_DEPENDENCY_RESOLVER = new DependencyResolver.Adapter() { + @Override + public T resolveDependency(Class type, SelectionStrategy selector) { + return null; + } + + @Override + public boolean containsDependency(Class type) { + return false; + } + }; + @Override public DependencyResolver emptyDependencyResolver() { - return new DependencyResolver.Adapter() { - @Override - public T resolveDependency(Class type, SelectionStrategy selector) { - return null; - } - - @Override - public boolean containsDependency(Class type) { - return false; - } - }; + return EMPTY_DEPENDENCY_RESOLVER; } } diff --git a/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/Neo4jProxyImpl.java b/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/Neo4jProxyImpl.java index 10b0e8d0e8..a360f37af7 100644 --- a/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/Neo4jProxyImpl.java +++ b/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/Neo4jProxyImpl.java @@ -649,7 +649,11 @@ public String versionLongToString(long storeVersion) { Bits bits = Bits.bitsFromLongs(new long[]{storeVersion}); int length = bits.getShort(8); if (length == 0 || length > 7) { - throw new IllegalArgumentException(format(Locale.ENGLISH, "The read version string length %d is not proper.", length)); + throw new IllegalArgumentException(format( + Locale.ENGLISH, + "The read version string length %d is not proper.", + length + )); } char[] result = new char[length]; for (int i = 0; i < length; i++) { @@ -925,7 +929,10 @@ public boolean isCompositeDatabase(GraphDatabaseService databaseService) { @Override public T lookupComponentProvider(Context ctx, Class component, boolean safe) throws ProcedureException { - var globalProcedures = GraphDatabaseApiProxy.resolveDependency(ctx.dependencyResolver(), GlobalProcedures.class); + var globalProcedures = GraphDatabaseApiProxy.resolveDependency( + ctx.dependencyResolver(), + GlobalProcedures.class + ); return globalProcedures.getCurrentView().lookupComponentProvider(component, safe).apply(ctx); } @@ -949,18 +956,20 @@ public Stream getAllAggregatingFunctions() { }; } + private static final DependencyResolver EMPTY_DEPENDENCY_RESOLVER = new DependencyResolver.Adapter() { + @Override + public T resolveDependency(Class type, SelectionStrategy selector) { + return null; + } + + @Override + public boolean containsDependency(Class type) { + return false; + } + }; + @Override public DependencyResolver emptyDependencyResolver() { - return new DependencyResolver.Adapter() { - @Override - public T resolveDependency(Class type, SelectionStrategy selector) { - return null; - } - - @Override - public boolean containsDependency(Class type) { - return false; - } - }; + return EMPTY_DEPENDENCY_RESOLVER; } } diff --git a/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/Neo4jProxyImpl.java b/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/Neo4jProxyImpl.java index 2ef6fdba2d..644e639d52 100644 --- a/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/Neo4jProxyImpl.java +++ b/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/Neo4jProxyImpl.java @@ -948,18 +948,20 @@ public Stream getAllAggregatingFunctions() { }; } + private static final DependencyResolver EMPTY_DEPENDENCY_RESOLVER = new DependencyResolver() { + @Override + public T resolveDependency(Class type, SelectionStrategy selector) { + return null; + } + + @Override + public boolean containsDependency(Class type) { + return false; + } + }; + @Override public DependencyResolver emptyDependencyResolver() { - return new DependencyResolver() { - @Override - public T resolveDependency(Class type, SelectionStrategy selector) { - return null; - } - - @Override - public boolean containsDependency(Class type) { - return false; - } - }; + return EMPTY_DEPENDENCY_RESOLVER; } } diff --git a/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/Neo4jProxyImpl.java b/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/Neo4jProxyImpl.java index 0156b205b2..3b63c5ed99 100644 --- a/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/Neo4jProxyImpl.java +++ b/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/Neo4jProxyImpl.java @@ -953,18 +953,20 @@ public Stream getAllAggregatingFunctions() { }; } + private static final DependencyResolver EMPTY_DEPENDENCY_RESOLVER = new DependencyResolver.Adapter() { + @Override + public T resolveDependency(Class type, SelectionStrategy selector) { + return null; + } + + @Override + public boolean containsDependency(Class type) { + return false; + } + }; + @Override public DependencyResolver emptyDependencyResolver() { - return new DependencyResolver.Adapter() { - @Override - public T resolveDependency(Class type, SelectionStrategy selector) { - return null; - } - - @Override - public boolean containsDependency(Class type) { - return false; - } - }; + return EMPTY_DEPENDENCY_RESOLVER; } } diff --git a/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/Neo4jProxyImpl.java b/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/Neo4jProxyImpl.java index 43b4b33eea..1a6813cce5 100644 --- a/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/Neo4jProxyImpl.java +++ b/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/Neo4jProxyImpl.java @@ -948,18 +948,20 @@ public Stream getAllAggregatingFunctions() { }; } + private static final DependencyResolver EMPTY_DEPENDENCY_RESOLVER = new DependencyResolver.Adapter() { + @Override + public T resolveDependency(Class type, SelectionStrategy selector) { + return null; + } + + @Override + public boolean containsDependency(Class type) { + return false; + } + }; + @Override public DependencyResolver emptyDependencyResolver() { - return new DependencyResolver.Adapter() { - @Override - public T resolveDependency(Class type, SelectionStrategy selector) { - return null; - } - - @Override - public boolean containsDependency(Class type) { - return false; - } - }; + return EMPTY_DEPENDENCY_RESOLVER; } } diff --git a/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/Neo4jProxyImpl.java b/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/Neo4jProxyImpl.java index 66abf167cb..ca9671f11d 100644 --- a/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/Neo4jProxyImpl.java +++ b/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/Neo4jProxyImpl.java @@ -948,18 +948,20 @@ public Stream getAllAggregatingFunctions() { }; } + private static final DependencyResolver EMPTY_DEPENDENCY_RESOLVER = new DependencyResolver.Adapter() { + @Override + public T resolveDependency(Class type, SelectionStrategy selector) { + return null; + } + + @Override + public boolean containsDependency(Class type) { + return false; + } + }; + @Override public DependencyResolver emptyDependencyResolver() { - return new DependencyResolver.Adapter() { - @Override - public T resolveDependency(Class type, SelectionStrategy selector) { - return null; - } - - @Override - public boolean containsDependency(Class type) { - return false; - } - }; + return EMPTY_DEPENDENCY_RESOLVER; } } diff --git a/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/Neo4jProxyImpl.java b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/Neo4jProxyImpl.java index 8661e69d1b..ce739eda4e 100644 --- a/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/Neo4jProxyImpl.java +++ b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/Neo4jProxyImpl.java @@ -948,18 +948,20 @@ public Stream getAllAggregatingFunctions() { }; } + private static final DependencyResolver EMPTY_DEPENDENCY_RESOLVER = new DependencyResolver.Adapter() { + @Override + public T resolveDependency(Class type, SelectionStrategy selector) { + return null; + } + + @Override + public boolean containsDependency(Class type) { + return false; + } + }; + @Override public DependencyResolver emptyDependencyResolver() { - return new DependencyResolver.Adapter() { - @Override - public T resolveDependency(Class type, SelectionStrategy selector) { - return null; - } - - @Override - public boolean containsDependency(Class type) { - return false; - } - }; + return EMPTY_DEPENDENCY_RESOLVER; } } From 915d2207201a28918d0c44601e1e37910033c81e Mon Sep 17 00:00:00 2001 From: Paul Horn Date: Tue, 12 Sep 2023 10:48:55 +0200 Subject: [PATCH 239/273] Add new compat method to old compat layers --- .../neo4j/gds/compat/_51/Neo4jProxyImpl.java | 17 +++++++++++++++++ .../neo4j/gds/compat/_52/Neo4jProxyImpl.java | 17 +++++++++++++++++ .../neo4j/gds/compat/_53/Neo4jProxyImpl.java | 17 +++++++++++++++++ .../neo4j/gds/compat/_54/Neo4jProxyImpl.java | 17 +++++++++++++++++ .../neo4j/gds/compat/_55/Neo4jProxyImpl.java | 17 +++++++++++++++++ 5 files changed, 85 insertions(+) diff --git a/compatibility/5.1/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_51/Neo4jProxyImpl.java b/compatibility/5.1/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_51/Neo4jProxyImpl.java index b323990b78..30a659e781 100644 --- a/compatibility/5.1/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_51/Neo4jProxyImpl.java +++ b/compatibility/5.1/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_51/Neo4jProxyImpl.java @@ -953,4 +953,21 @@ public Stream getAllAggregatingFunctions() { } }; } + + private static final DependencyResolver EMPTY_DEPENDENCY_RESOLVER = new DependencyResolver.Adapter() { + @Override + public T resolveDependency(Class type, DependencyResolver.SelectionStrategy selector) { + return null; + } + + @Override + public boolean containsDependency(Class type) { + return false; + } + }; + + @Override + public DependencyResolver emptyDependencyResolver() { + return EMPTY_DEPENDENCY_RESOLVER; + } } diff --git a/compatibility/5.2/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_52/Neo4jProxyImpl.java b/compatibility/5.2/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_52/Neo4jProxyImpl.java index 2b7c286b17..9cfa9eda30 100644 --- a/compatibility/5.2/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_52/Neo4jProxyImpl.java +++ b/compatibility/5.2/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_52/Neo4jProxyImpl.java @@ -951,4 +951,21 @@ public Stream getAllAggregatingFunctions() { } }; } + + private static final DependencyResolver EMPTY_DEPENDENCY_RESOLVER = new DependencyResolver.Adapter() { + @Override + public T resolveDependency(Class type, DependencyResolver.SelectionStrategy selector) { + return null; + } + + @Override + public boolean containsDependency(Class type) { + return false; + } + }; + + @Override + public DependencyResolver emptyDependencyResolver() { + return EMPTY_DEPENDENCY_RESOLVER; + } } diff --git a/compatibility/5.3/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_53/Neo4jProxyImpl.java b/compatibility/5.3/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_53/Neo4jProxyImpl.java index 13bdeedd6f..ee1a680d13 100644 --- a/compatibility/5.3/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_53/Neo4jProxyImpl.java +++ b/compatibility/5.3/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_53/Neo4jProxyImpl.java @@ -952,4 +952,21 @@ public Stream getAllAggregatingFunctions() { } }; } + + private static final DependencyResolver EMPTY_DEPENDENCY_RESOLVER = new DependencyResolver.Adapter() { + @Override + public T resolveDependency(Class type, DependencyResolver.SelectionStrategy selector) { + return null; + } + + @Override + public boolean containsDependency(Class type) { + return false; + } + }; + + @Override + public DependencyResolver emptyDependencyResolver() { + return EMPTY_DEPENDENCY_RESOLVER; + } } diff --git a/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/Neo4jProxyImpl.java b/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/Neo4jProxyImpl.java index 421cbabe2b..7161bde2b5 100644 --- a/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/Neo4jProxyImpl.java +++ b/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/Neo4jProxyImpl.java @@ -951,4 +951,21 @@ public Stream getAllAggregatingFunctions() { } }; } + + private static final DependencyResolver EMPTY_DEPENDENCY_RESOLVER = new DependencyResolver.Adapter() { + @Override + public T resolveDependency(Class type, DependencyResolver.SelectionStrategy selector) { + return null; + } + + @Override + public boolean containsDependency(Class type) { + return false; + } + }; + + @Override + public DependencyResolver emptyDependencyResolver() { + return EMPTY_DEPENDENCY_RESOLVER; + } } diff --git a/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/Neo4jProxyImpl.java b/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/Neo4jProxyImpl.java index b3e9f68823..2a1172c96c 100644 --- a/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/Neo4jProxyImpl.java +++ b/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/Neo4jProxyImpl.java @@ -952,4 +952,21 @@ public Stream getAllAggregatingFunctions() { } }; } + + private static final DependencyResolver EMPTY_DEPENDENCY_RESOLVER = new DependencyResolver.Adapter() { + @Override + public T resolveDependency(Class type, DependencyResolver.SelectionStrategy selector) { + return null; + } + + @Override + public boolean containsDependency(Class type) { + return false; + } + }; + + @Override + public DependencyResolver emptyDependencyResolver() { + return EMPTY_DEPENDENCY_RESOLVER; + } } From 59a2d2f080f3c75753432d094bd656389c30313d Mon Sep 17 00:00:00 2001 From: Veselin Nikolov Date: Tue, 12 Sep 2023 09:56:56 +0100 Subject: [PATCH 240/273] Increment Aura version --- gradle/version.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle/version.gradle b/gradle/version.gradle index df5d50fd06..9b6e5b1fee 100644 --- a/gradle/version.gradle +++ b/gradle/version.gradle @@ -1,6 +1,6 @@ ext { gdsBaseVersion = '2.4.6' - gdsAuraVersion = '33' + gdsAuraVersion = '34' gdsVersion = gdsBaseVersion + (rootProject.hasProperty('aurads') ? "+${gdsAuraVersion}" : "") } From 756ae79a6fef6c0ed8f9f1ec67990237ce4e207e Mon Sep 17 00:00:00 2001 From: Paul Horn Date: Tue, 12 Sep 2023 11:19:32 +0200 Subject: [PATCH 241/273] Remove duplicated 5.12 dependency entry --- gradle/dependencies.gradle | 1 - 1 file changed, 1 deletion(-) diff --git a/gradle/dependencies.gradle b/gradle/dependencies.gradle index c8ae007e83..37860badf3 100644 --- a/gradle/dependencies.gradle +++ b/gradle/dependencies.gradle @@ -12,7 +12,6 @@ ext { '5.9': properties.getOrDefault('neo4jVersion59', '5.9.0'), '5.10': properties.getOrDefault('neo4jVersion510', '5.10.0'), '5.11': properties.getOrDefault('neo4jVersion511', '5.11.0'), - '5.12': properties.getOrDefault('neo4jVersion511', '5.12.0'), '5.12': properties.getOrDefault('neo4jVersion512', '5.12.0'), ] From cc31933b6737086d89eaba9c047689e94602a871 Mon Sep 17 00:00:00 2001 From: yuval Date: Wed, 13 Sep 2023 11:35:12 +0300 Subject: [PATCH 242/273] bumped Aura version to 35 --- gradle/version.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle/version.gradle b/gradle/version.gradle index 9b6e5b1fee..8f5307c154 100644 --- a/gradle/version.gradle +++ b/gradle/version.gradle @@ -1,6 +1,6 @@ ext { gdsBaseVersion = '2.4.6' - gdsAuraVersion = '34' + gdsAuraVersion = '35' gdsVersion = gdsBaseVersion + (rootProject.hasProperty('aurads') ? "+${gdsAuraVersion}" : "") } From c2b2868d5e8ab6b9aa0d938f96086194ccafb454 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Wed, 13 Sep 2023 14:04:53 +0200 Subject: [PATCH 243/273] Handle empty relTypeCounts in graph_info.csv MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit before it resulted in a rel-type cannot be empty exception. Co-authored-by: Jonatan Jäderberg --- .../neo4j/gds/core/io/file/csv/CsvMapUtil.java | 4 ++++ .../gds/core/io/file/csv/GraphInfoLoaderTest.java | 15 +++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/CsvMapUtil.java b/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/CsvMapUtil.java index 101d1f0368..6e09c01c38 100644 --- a/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/CsvMapUtil.java +++ b/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/CsvMapUtil.java @@ -42,6 +42,10 @@ static Map fromString( Function keyParser, Function valueParser ) { + if (mapString.isEmpty()) { + return Map.of(); + } + var listElements = mapString.split(String.valueOf(LIST_DELIMITER)); var map = new HashMap(); for (int i = 0; i < listElements.length; i+=2) { diff --git a/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/GraphInfoLoaderTest.java b/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/GraphInfoLoaderTest.java index c889cae05d..9e2ae7abbe 100644 --- a/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/GraphInfoLoaderTest.java +++ b/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/GraphInfoLoaderTest.java @@ -69,6 +69,21 @@ void shouldLoadGraphInfo(@TempDir Path exportDir) throws IOException { assertThat(graphInfo.inverseIndexedRelationshipTypes()).containsExactly(RelationshipType.of("REL"), RelationshipType.of("REL1")); } + @Test + void shouldHandleEmptyRelCountsAndInverseIndexRelTypes(@TempDir Path exportDir) throws IOException { + var graphInfoFile = exportDir.resolve(GRAPH_INFO_FILE_NAME).toFile(); + var lines = List.of( + String.join(", ", "databaseName", "nodeCount", "maxOriginalId", "relTypeCounts", "inverseIndexedRelTypes"), + String.join(", ", "my-database", "19", "", "") + ); + FileUtils.writeLines(graphInfoFile, lines); + + var graphInfoLoader = new GraphInfoLoader(exportDir, CSV_MAPPER); + var graphInfo = graphInfoLoader.load(); + + assertThat(graphInfo.relationshipTypeCounts()).isEmpty(); + } + /** * Test for backwards compatibility by including `databaseId` */ From edd124267a15ddd2801a39c7a532a27bc4abe8eb Mon Sep 17 00:00:00 2001 From: ioannispan Date: Thu, 14 Sep 2023 11:54:06 +0200 Subject: [PATCH 244/273] Sort out progress tracking for eigenvector/articlerank Co-authored-by: Veselin Nikolov --- .../pagerank/PageRankAlgorithmFactory.java | 23 +++++++---- .../org/neo4j/gds/pagerank/PageRankTest.java | 40 ++++++++++++------- 2 files changed, 40 insertions(+), 23 deletions(-) diff --git a/algo/src/main/java/org/neo4j/gds/pagerank/PageRankAlgorithmFactory.java b/algo/src/main/java/org/neo4j/gds/pagerank/PageRankAlgorithmFactory.java index d62d9c6d84..6ead19c2b5 100644 --- a/algo/src/main/java/org/neo4j/gds/pagerank/PageRankAlgorithmFactory.java +++ b/algo/src/main/java/org/neo4j/gds/pagerank/PageRankAlgorithmFactory.java @@ -44,9 +44,6 @@ public class PageRankAlgorithmFactory extends GraphAlgorithmFactory { - static Task pagerankProgressTask(Graph graph, CONFIG config) { - return Pregel.progressTask(graph, config, "PageRank"); - } private static double averageDegree(Graph graph, int concurrency) { var degreeSum = new LongAdder(); @@ -60,9 +57,19 @@ private static double averageDegree(Graph graph, int concurrency) { } public enum Mode { - PAGE_RANK, - ARTICLE_RANK, - EIGENVECTOR, + PAGE_RANK("PageRank"), + ARTICLE_RANK("ArticleRank"), + EIGENVECTOR("EigenVector"); + + private final String taskName; + + Mode(String taskName) { + this.taskName = taskName; + } + + String taskName() { + return taskName; + } } private final Mode mode; @@ -77,7 +84,7 @@ public PageRankAlgorithmFactory(Mode mode) { @Override public String taskName() { - return mode.name(); + return mode.taskName(); } @Override @@ -128,7 +135,7 @@ public PageRankAlgorithm build( @Override public Task progressTask(Graph graph, CONFIG config) { - return pagerankProgressTask(graph, config); + return Pregel.progressTask(graph, config, taskName()); } @NotNull diff --git a/algo/src/test/java/org/neo4j/gds/pagerank/PageRankTest.java b/algo/src/test/java/org/neo4j/gds/pagerank/PageRankTest.java index 5db2c5025b..e7b63ab09d 100644 --- a/algo/src/test/java/org/neo4j/gds/pagerank/PageRankTest.java +++ b/algo/src/test/java/org/neo4j/gds/pagerank/PageRankTest.java @@ -169,15 +169,19 @@ void withSourceNodes(String sourceNodesString, String expectedPropertyKey) { } } - @Test - void shouldLogProgress() { + @ParameterizedTest + @EnumSource(Mode.class) + void shouldLogProgress(Mode mode) { var maxIterations = 10; var config = ImmutablePageRankConfig.builder() .maxIterations(maxIterations) .build(); - var progressTask = PageRankAlgorithmFactory.pagerankProgressTask(graph, config); + var factory = new PageRankAlgorithmFactory<>(mode); + + var progressTask = factory.progressTask(graph, config); var log = Neo4jProxy.testLog(); + var progressTracker = new TestProgressTracker( progressTask, log, @@ -185,7 +189,12 @@ void shouldLogProgress() { EmptyTaskRegistryFactory.INSTANCE ); - runOnPregel(graph, config, Mode.PAGE_RANK, progressTracker); + factory.build( + graph, + config, + progressTracker + ) + .compute(); var progresses = progressTracker.getProgresses().stream() .filter(it -> it.get() > 0) @@ -207,22 +216,26 @@ void shouldLogProgress() { .extracting(removingThreadId()) .contains( formatWithLocale( - "PageRank :: Compute iteration %d of %d :: Start", + "%s :: Compute iteration %d of %d :: Start", + mode.taskName(), iteration, config.maxIterations() ), formatWithLocale( - "PageRank :: Compute iteration %d of %d :: Finished", + "%s :: Compute iteration %d of %d :: Finished", + mode.taskName(), iteration, config.maxIterations() ), formatWithLocale( - "PageRank :: Master compute iteration %d of %d :: Start", + "%s :: Master compute iteration %d of %d :: Start", + mode.taskName(), iteration, config.maxIterations() ), formatWithLocale( - "PageRank :: Master compute iteration %d of %d :: Finished", + "%s :: Master compute iteration %d of %d :: Finished", + mode.taskName(), iteration, config.maxIterations() ) @@ -231,8 +244,8 @@ void shouldLogProgress() { assertThat(messages) .extracting(removingThreadId()) .contains( - "PageRank :: Start", - "PageRank :: Finished" + formatWithLocale("%s :: Start", mode.taskName()), + formatWithLocale("%s :: Finished", mode.taskName()) ); } @@ -681,16 +694,13 @@ PageRankResult runOnPregel(Graph graph, PageRankConfig config) { } PageRankResult runOnPregel(Graph graph, PageRankConfig config, Mode mode) { - return runOnPregel(graph, config, mode, ProgressTracker.NULL_TRACKER); - } - - PageRankResult runOnPregel(Graph graph, PageRankConfig config, Mode mode, ProgressTracker progressTracker) { return new PageRankAlgorithmFactory<>(mode) .build( graph, config, - progressTracker + ProgressTracker.NULL_TRACKER ) .compute(); } + } From 025bd8a55537d43f39d0a37d34f3b4e188248c79 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Thu, 14 Sep 2023 14:58:07 +0200 Subject: [PATCH 245/273] Post 2.4.6 --- README.adoc | 16 ++++++++-------- .../pages/management-ops/utility-functions.adoc | 2 +- examples/pregel-bootstrap/build.gradle | 2 +- gradle/version.gradle | 2 +- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/README.adoc b/README.adoc index ed2399fc82..41e0803e94 100644 --- a/README.adoc +++ b/README.adoc @@ -96,7 +96,7 @@ For the most basic set of features, like graph loading and the graph representat org.neo4j.gds core - 2.4.5 + 2.4.6 ---- @@ -108,21 +108,21 @@ The algorithms are located in the `algo-common`, `algo` and `alpha-algo` modules org.neo4j.gds algo-common - 2.4.5 + 2.4.6 org.neo4j.gds algo - 2.4.5 + 2.4.6 org.neo4j.gds alpha-algo - 2.4.5 + 2.4.6 ---- @@ -134,28 +134,28 @@ The procedures are located in the `proc-common`, `proc` and `alpha-proc` modules org.neo4j.gds proc-common - 2.4.5 + 2.4.6 org.neo4j.gds proc - 2.4.5 + 2.4.6 org.neo4j.gds alpha-proc - 2.4.5 + 2.4.6 org.neo4j.gds open-write-services - 2.4.5 + 2.4.6 ---- diff --git a/doc/modules/ROOT/pages/management-ops/utility-functions.adoc b/doc/modules/ROOT/pages/management-ops/utility-functions.adoc index ac95df788a..157de93cb2 100644 --- a/doc/modules/ROOT/pages/management-ops/utility-functions.adoc +++ b/doc/modules/ROOT/pages/management-ops/utility-functions.adoc @@ -27,7 +27,7 @@ RETURN gds.version() AS version [opts="header"] |=== | version -| "2.4.6" +| "2.4.7" |=== -- diff --git a/examples/pregel-bootstrap/build.gradle b/examples/pregel-bootstrap/build.gradle index 867250985f..d0bf8c797f 100644 --- a/examples/pregel-bootstrap/build.gradle +++ b/examples/pregel-bootstrap/build.gradle @@ -7,7 +7,7 @@ plugins { ext { // Make sure these are the same as your installation of GDS and Neo4j - gdsVersion = '2.4.5' + gdsVersion = '2.4.6' neo4jVersion = '5.10.0' // Necessary to generate value classes for Pregel configs diff --git a/gradle/version.gradle b/gradle/version.gradle index 8f5307c154..ac5467d106 100644 --- a/gradle/version.gradle +++ b/gradle/version.gradle @@ -1,5 +1,5 @@ ext { - gdsBaseVersion = '2.4.6' + gdsBaseVersion = '2.4.7' gdsAuraVersion = '35' gdsVersion = gdsBaseVersion + (rootProject.hasProperty('aurads') ? "+${gdsAuraVersion}" : "") From fcb4b3497f6ced1c4802023d8e86f73a8a2d214d Mon Sep 17 00:00:00 2001 From: Veselin Nikolov Date: Fri, 15 Sep 2023 08:03:27 +0100 Subject: [PATCH 246/273] Use the correct ThreadPool --- .../java/gds/example/ExamplePregelComputationAlgoTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/pregel-bootstrap/src/test/java/gds/example/ExamplePregelComputationAlgoTest.java b/examples/pregel-bootstrap/src/test/java/gds/example/ExamplePregelComputationAlgoTest.java index 26f51bb395..7143e16fcb 100644 --- a/examples/pregel-bootstrap/src/test/java/gds/example/ExamplePregelComputationAlgoTest.java +++ b/examples/pregel-bootstrap/src/test/java/gds/example/ExamplePregelComputationAlgoTest.java @@ -22,7 +22,7 @@ import org.junit.jupiter.api.Test; import org.neo4j.gds.TestSupport; import org.neo4j.gds.beta.pregel.Pregel; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.extension.GdlExtension; import org.neo4j.gds.extension.GdlGraph; @@ -64,7 +64,7 @@ void runExamplePregelComputation() { graph, config, new ExamplePregelComputation(), - Pools.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); From 3b404503a9a83bbb9fddcc0ab9f4baccd12aefe1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Fri, 15 Sep 2023 10:22:20 +0200 Subject: [PATCH 247/273] Post 5.12 neo4j release --- README.adoc | 5 +++-- build.gradle | 2 ++ .../ROOT/pages/installation/supported-neo4j-versions.adoc | 1 + settings.gradle | 6 ++++++ 4 files changed, 12 insertions(+), 2 deletions(-) diff --git a/README.adoc b/README.adoc index 41e0803e94..ef66c03632 100644 --- a/README.adoc +++ b/README.adoc @@ -26,9 +26,10 @@ When installing GDS manually, please refer to the below compatibility matrix: .Compatibility matrix (italicized version is in development) |=== |GDS version | Neo4j version | Java Version -.11+<.^|_GDS 2.4.x_ +.12+<.^|_GDS 2.4.x_ +|Neo4j 5.12.0 +.12+.^|Java 17 |Neo4j 5.11.0 -.11+.^|Java 17 |Neo4j 5.10.0 |Neo4j 5.9.0 |Neo4j 5.8.0 diff --git a/build.gradle b/build.gradle index 2638b2315b..bc8e4fd1d0 100644 --- a/build.gradle +++ b/build.gradle @@ -38,6 +38,7 @@ ext { project(':neo4j-kernel-adapter-5.9'), project(':neo4j-kernel-adapter-5.10'), project(':neo4j-kernel-adapter-5.11'), + project(':neo4j-kernel-adapter-5.12'), ], 'storage-engine-adapter': [ project(':storage-engine-adapter-4.4'), @@ -52,6 +53,7 @@ ext { project(':storage-engine-adapter-5.9'), project(':storage-engine-adapter-5.10'), project(':storage-engine-adapter-5.11'), + project(':storage-engine-adapter-5.12'), ] ] } diff --git a/doc/modules/ROOT/pages/installation/supported-neo4j-versions.adoc b/doc/modules/ROOT/pages/installation/supported-neo4j-versions.adoc index 37de15a2ca..ffe2e38db4 100644 --- a/doc/modules/ROOT/pages/installation/supported-neo4j-versions.adoc +++ b/doc/modules/ROOT/pages/installation/supported-neo4j-versions.adoc @@ -10,6 +10,7 @@ If your version of GDS or Neo4j is not listed in the matrix, you should upgrade. [opts=header] |=== | Neo4j version | Neo4j Graph Data Science +| `5.12` | `2.4.6` or later | `5.11` | `2.4.4` or later | `5.10` | `2.4.2` or later | `5.9` | `2.4`, `2.3.9` or later footnote:eol[This version series is end-of-life and will not receive further patches. Please use a later version.] diff --git a/settings.gradle b/settings.gradle index 1ea5e666db..cf9d1efecf 100644 --- a/settings.gradle +++ b/settings.gradle @@ -163,6 +163,9 @@ project(':neo4j-kernel-adapter-5.10').projectDir = file('compatibility/5.10/neo4 include('neo4j-kernel-adapter-5.11') project(':neo4j-kernel-adapter-5.11').projectDir = file('compatibility/5.11/neo4j-kernel-adapter') +include('neo4j-kernel-adapter-5.12') +project(':neo4j-kernel-adapter-5.12').projectDir = file('compatibility/5.12/neo4j-kernel-adapter') + include('neo4j-kernel-adapter-api') project(':neo4j-kernel-adapter-api').projectDir = file('compatibility/api/neo4j-kernel-adapter') @@ -274,6 +277,9 @@ project(':storage-engine-adapter-5.10').projectDir = file('compatibility/5.10/st include('storage-engine-adapter-5.11') project(':storage-engine-adapter-5.11').projectDir = file('compatibility/5.11/storage-engine-adapter') +include('storage-engine-adapter-5.12') +project(':storage-engine-adapter-5.12').projectDir = file('compatibility/5.12/storage-engine-adapter') + include('storage-engine-adapter-api') project(':storage-engine-adapter-api').projectDir = file('compatibility/api/storage-engine-adapter') From 41ebd4f5cc376d41623e21798b596366bc106303 Mon Sep 17 00:00:00 2001 From: Veselin Nikolov Date: Thu, 21 Sep 2023 14:27:42 +0100 Subject: [PATCH 248/273] Update Neo4j version to 4.4.26 --- gradle/dependencies.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle/dependencies.gradle b/gradle/dependencies.gradle index 37860badf3..d90da51661 100644 --- a/gradle/dependencies.gradle +++ b/gradle/dependencies.gradle @@ -1,6 +1,6 @@ ext { neos = [ - '4.4': properties.getOrDefault('neo4jVersion44', '4.4.25'), + '4.4': properties.getOrDefault('neo4jVersion44', '4.4.26'), '5.1': properties.getOrDefault('neo4jVersion51', '5.1.0'), '5.2': properties.getOrDefault('neo4jVersion52', '5.2.0'), '5.3': properties.getOrDefault('neo4jVersion53', '5.3.0'), From 690384dc1a9ac1ed2fd455943558fcfc700595f0 Mon Sep 17 00:00:00 2001 From: Veselin Nikolov Date: Thu, 21 Sep 2023 14:37:33 +0100 Subject: [PATCH 249/273] Reflect Neo4j 4.4.26 support --- README.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.adoc b/README.adoc index ef66c03632..13dc641038 100644 --- a/README.adoc +++ b/README.adoc @@ -40,7 +40,7 @@ When installing GDS manually, please refer to the below compatibility matrix: |Neo4j 5.4.0 |Neo4j 5.2.0 |Neo4j 5.1.0 -|Neo4j 4.4.9 - 4.4.25 +|Neo4j 4.4.9 - 4.4.26 .1+.^|Java 11 .9+<.^|GDS 2.3.x |Neo4j 5.8.0 From 088a3857c6d48b396aa28f7d2d495f632689e3bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Tue, 26 Sep 2023 12:09:50 +0200 Subject: [PATCH 250/273] Bump Aura version --- gradle/version.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle/version.gradle b/gradle/version.gradle index ac5467d106..d9dfb7419a 100644 --- a/gradle/version.gradle +++ b/gradle/version.gradle @@ -1,6 +1,6 @@ ext { gdsBaseVersion = '2.4.7' - gdsAuraVersion = '35' + gdsAuraVersion = '36' gdsVersion = gdsBaseVersion + (rootProject.hasProperty('aurads') ? "+${gdsAuraVersion}" : "") } From c4f423b069215c863d4ca77aa59d0ec391410c65 Mon Sep 17 00:00:00 2001 From: Martin Junghanns Date: Tue, 26 Sep 2023 12:03:59 +0200 Subject: [PATCH 251/273] Do not log once global percentage exceeded 100 percent --- .../core/utils/progress/BatchingProgressLogger.java | 2 +- .../gds/core/utils/BatchingProgressLoggerTest.java | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/progress-tracking/src/main/java/org/neo4j/gds/core/utils/progress/BatchingProgressLogger.java b/progress-tracking/src/main/java/org/neo4j/gds/core/utils/progress/BatchingProgressLogger.java index ef70ed9f6f..005e0c70b3 100644 --- a/progress-tracking/src/main/java/org/neo4j/gds/core/utils/progress/BatchingProgressLogger.java +++ b/progress-tracking/src/main/java/org/neo4j/gds/core/utils/progress/BatchingProgressLogger.java @@ -124,7 +124,7 @@ private synchronized void doLogPercentage(Supplier msgFactory, long prog String message = msgFactory != NO_MESSAGE ? msgFactory.get() : null; progressCounter.add(progress); int nextPercentage = (int) ((progressCounter.sum() / (double) taskVolume) * 100); - if (globalPercentage < nextPercentage) { + if (globalPercentage < nextPercentage && globalPercentage < 100) { globalPercentage = nextPercentage; if (message == null || message.isEmpty()) { logProgress(nextPercentage); diff --git a/progress-tracking/src/test/java/org/neo4j/gds/core/utils/BatchingProgressLoggerTest.java b/progress-tracking/src/test/java/org/neo4j/gds/core/utils/BatchingProgressLoggerTest.java index 4fc44bf7ab..027ed7df69 100644 --- a/progress-tracking/src/test/java/org/neo4j/gds/core/utils/BatchingProgressLoggerTest.java +++ b/progress-tracking/src/test/java/org/neo4j/gds/core/utils/BatchingProgressLoggerTest.java @@ -182,6 +182,18 @@ void shouldLog100OnlyOnce() { .containsExactly("Test 100%"); } + @Test + void shouldNotExceed100Percent() { + TestLog log = Neo4jProxy.testLog(); + var testProgressLogger = new BatchingProgressLogger(log, Tasks.leaf("Test"), 1); + testProgressLogger.reset(1); + testProgressLogger.logProgress(1); // reaches 100 % + testProgressLogger.logProgress(1); // exceeds 100 % + assertThat(log.getMessages(TestLog.INFO)) + .extracting(Extractors.removingThreadId()) + .containsExactly("Test 100%"); + } + @Test void closesThreadLocal() { var logger = new BatchingProgressLogger( From 7004ed3fdf886283381a84c362ca265baa993573 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Wed, 27 Sep 2023 14:13:13 +0200 Subject: [PATCH 252/273] Add 5.13.0 as potential Neo4j version Co-authored-by: Paul Horn --- gradle/dependencies.gradle | 3 +++ 1 file changed, 3 insertions(+) diff --git a/gradle/dependencies.gradle b/gradle/dependencies.gradle index d90da51661..fbf7d90509 100644 --- a/gradle/dependencies.gradle +++ b/gradle/dependencies.gradle @@ -13,6 +13,7 @@ ext { '5.10': properties.getOrDefault('neo4jVersion510', '5.10.0'), '5.11': properties.getOrDefault('neo4jVersion511', '5.11.0'), '5.12': properties.getOrDefault('neo4jVersion512', '5.12.0'), + '5.13': properties.getOrDefault('neo4jVersion513', '5.13.0'), ] neo4jDefault = neos.'4.4' @@ -34,6 +35,7 @@ ext { '5.10': '2.13.10', '5.11': '2.13.10', '5.12': '2.13.10', + '5.13': '2.13.11', ] log4js = [ @@ -50,6 +52,7 @@ ext { '5.10': '2.20.0', '5.11': '2.20.0', '5.12': '2.20.0', + '5.13': '2.20.0', ] ver = [ From 5846363a9017c296a199c1d07ed3feb086a2e41f Mon Sep 17 00:00:00 2001 From: yuval Date: Tue, 3 Oct 2023 14:41:03 +0200 Subject: [PATCH 253/273] compiling locally with 5.13 --- .../src/main/java/org/neo4j/gds/compat/Neo4jVersion.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/neo4j-adapter/src/main/java/org/neo4j/gds/compat/Neo4jVersion.java b/neo4j-adapter/src/main/java/org/neo4j/gds/compat/Neo4jVersion.java index 298009dce6..0eee2223b3 100644 --- a/neo4j-adapter/src/main/java/org/neo4j/gds/compat/Neo4jVersion.java +++ b/neo4j-adapter/src/main/java/org/neo4j/gds/compat/Neo4jVersion.java @@ -80,7 +80,7 @@ public String toString() { public MajorMinorVersion semanticVersion() { if (this == V_RC) { - return ImmutableMajorMinorVersion.of(5, 12); + return ImmutableMajorMinorVersion.of(5, 13); } String version = toString(); From 64bcdcb1a6a3b7947a0a1cb561d883b05dd6a8a2 Mon Sep 17 00:00:00 2001 From: yuval Date: Tue, 3 Oct 2023 15:28:40 +0200 Subject: [PATCH 254/273] 5.13 compat layer --- .../5.13/neo4j-kernel-adapter/build.gradle | 65 ++ .../compat/_513/Neo4jProxyFactoryImpl.java | 44 + .../compat/_513/SettingProxyFactoryImpl.java | 44 + .../_513/BoltTransactionRunnerImpl.java | 97 ++ .../compat/_513/CallableProcedureImpl.java | 53 + .../CallableUserAggregationFunctionImpl.java | 77 ++ .../gds/compat/_513/CompatAccessModeImpl.java | 44 + .../_513/CompatGraphDatabaseAPIImpl.java | 74 ++ .../gds/compat/_513/CompatIndexQueryImpl.java | 31 + .../_513/CompatUsernameAuthSubjectImpl.java | 35 + .../compat/_513/CompositeNodeCursorImpl.java | 32 + .../compat/_513/GdsDatabaseLayoutImpl.java | 50 + ...sDatabaseManagementServiceBuilderImpl.java | 54 + .../compat/_513/Neo4jProxyFactoryImpl.java | 44 + .../neo4j/gds/compat/_513/Neo4jProxyImpl.java | 967 ++++++++++++++++++ .../compat/_513/NodeLabelIndexLookupImpl.java | 68 ++ .../gds/compat/_513/PartitionedStoreScan.java | 58 ++ .../_513/ReferencePropertyReference.java | 49 + .../compat/_513/ScanBasedStoreScanImpl.java | 40 + .../compat/_513/SettingProxyFactoryImpl.java | 44 + .../gds/compat/_513/SettingProxyImpl.java | 87 ++ .../neo4j/gds/compat/_513/TestLogImpl.java | 146 +++ .../compat/_513/VirtualRelationshipImpl.java | 41 + .../5.13/storage-engine-adapter/build.gradle | 68 ++ .../_513/InMemoryStorageEngineFactory.java | 268 +++++ .../_513/StorageEngineProxyFactoryImpl.java | 44 + .../InMemoryCommandCreationContextImpl.java | 107 ++ .../compat/_513/InMemoryCountsStoreImpl.java | 111 ++ .../_513/InMemoryMetaDataProviderImpl.java | 201 ++++ .../gds/compat/_513/InMemoryNodeCursor.java | 82 ++ .../_513/InMemoryNodePropertyCursor.java | 45 + .../compat/_513/InMemoryPropertyCursor.java | 71 ++ .../_513/InMemoryPropertySelectionImpl.java | 55 + .../InMemoryRelationshipPropertyCursor.java | 60 ++ .../_513/InMemoryRelationshipScanCursor.java | 61 ++ .../InMemoryRelationshipTraversalCursor.java | 47 + .../_513/InMemoryStorageEngineFactory.java | 572 +++++++++++ .../_513/InMemoryStorageEngineImpl.java | 366 +++++++ .../compat/_513/InMemoryStorageLocksImpl.java | 86 ++ .../gds/compat/_513/InMemoryStoreVersion.java | 70 ++ .../_513/InMemoryTransactionIdStoreImpl.java | 117 +++ .../gds/compat/_513/InMemoryVersionCheck.java | 72 ++ .../_513/StorageEngineProxyFactoryImpl.java | 44 + .../compat/_513/StorageEngineProxyImpl.java | 140 +++ .../InMemoryLogVersionRepository513.java | 71 ++ ...nMemoryStorageCommandReaderFactory513.java | 43 + .../InMemoryStorageReader513.java | 333 ++++++ gradle/dependencies.gradle | 1 + .../org/neo4j/gds/compat/Neo4jVersion.java | 5 + .../neo4j/gds/compat/Neo4jVersionTest.java | 8 +- .../java/org/neo4j/gds/SysInfoProcTest.java | 13 + 51 files changed, 5404 insertions(+), 1 deletion(-) create mode 100644 compatibility/5.13/neo4j-kernel-adapter/build.gradle create mode 100644 compatibility/5.13/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_513/Neo4jProxyFactoryImpl.java create mode 100644 compatibility/5.13/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_513/SettingProxyFactoryImpl.java create mode 100644 compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/BoltTransactionRunnerImpl.java create mode 100644 compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/CallableProcedureImpl.java create mode 100644 compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/CallableUserAggregationFunctionImpl.java create mode 100644 compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/CompatAccessModeImpl.java create mode 100644 compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/CompatGraphDatabaseAPIImpl.java create mode 100644 compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/CompatIndexQueryImpl.java create mode 100644 compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/CompatUsernameAuthSubjectImpl.java create mode 100644 compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/CompositeNodeCursorImpl.java create mode 100644 compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/GdsDatabaseLayoutImpl.java create mode 100644 compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/GdsDatabaseManagementServiceBuilderImpl.java create mode 100644 compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/Neo4jProxyFactoryImpl.java create mode 100644 compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/Neo4jProxyImpl.java create mode 100644 compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/NodeLabelIndexLookupImpl.java create mode 100644 compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/PartitionedStoreScan.java create mode 100644 compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/ReferencePropertyReference.java create mode 100644 compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/ScanBasedStoreScanImpl.java create mode 100644 compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/SettingProxyFactoryImpl.java create mode 100644 compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/SettingProxyImpl.java create mode 100644 compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/TestLogImpl.java create mode 100644 compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/VirtualRelationshipImpl.java create mode 100644 compatibility/5.13/storage-engine-adapter/build.gradle create mode 100644 compatibility/5.13/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_513/InMemoryStorageEngineFactory.java create mode 100644 compatibility/5.13/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_513/StorageEngineProxyFactoryImpl.java create mode 100644 compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/InMemoryCommandCreationContextImpl.java create mode 100644 compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/InMemoryCountsStoreImpl.java create mode 100644 compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/InMemoryMetaDataProviderImpl.java create mode 100644 compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/InMemoryNodeCursor.java create mode 100644 compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/InMemoryNodePropertyCursor.java create mode 100644 compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/InMemoryPropertyCursor.java create mode 100644 compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/InMemoryPropertySelectionImpl.java create mode 100644 compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/InMemoryRelationshipPropertyCursor.java create mode 100644 compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/InMemoryRelationshipScanCursor.java create mode 100644 compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/InMemoryRelationshipTraversalCursor.java create mode 100644 compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/InMemoryStorageEngineFactory.java create mode 100644 compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/InMemoryStorageEngineImpl.java create mode 100644 compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/InMemoryStorageLocksImpl.java create mode 100644 compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/InMemoryStoreVersion.java create mode 100644 compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/InMemoryTransactionIdStoreImpl.java create mode 100644 compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/InMemoryVersionCheck.java create mode 100644 compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/StorageEngineProxyFactoryImpl.java create mode 100644 compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/StorageEngineProxyImpl.java create mode 100644 compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository513.java create mode 100644 compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory513.java create mode 100644 compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageReader513.java diff --git a/compatibility/5.13/neo4j-kernel-adapter/build.gradle b/compatibility/5.13/neo4j-kernel-adapter/build.gradle new file mode 100644 index 0000000000..a7828abf67 --- /dev/null +++ b/compatibility/5.13/neo4j-kernel-adapter/build.gradle @@ -0,0 +1,65 @@ +apply plugin: 'java-library' +apply plugin: 'me.champeau.mrjar' + +description = 'Neo4j Graph Data Science :: Neo4j Kernel Adapter 5.13' + +group = 'org.neo4j.gds' + +// for all 5.x versions +if (ver.'neo4j'.startsWith('5.')) { + sourceSets { + main { + java { + srcDirs = ['src/main/java17'] + } + } + } + + dependencies { + annotationProcessor project(':annotations') + annotationProcessor group: 'org.immutables', name: 'value', version: ver.'immutables' + annotationProcessor group: 'org.neo4j', name: 'annotations', version: neos.'5.13' + + compileOnly project(':annotations') + compileOnly group: 'com.github.spotbugs', name: 'spotbugs-annotations', version: ver.'spotbugsToolVersion' + compileOnly group: 'org.immutables', name: 'value-annotations', version: ver.'immutables' + compileOnly group: 'org.neo4j', name: 'annotations', version: neos.'5.13' + compileOnly group: 'org.neo4j', name: 'neo4j', version: neos.'5.13' + compileOnly group: 'org.neo4j', name: 'neo4j-record-storage-engine', version: neos.'5.13' + compileOnly group: 'org.neo4j.community', name: 'it-test-support', version: neos.'5.13' + + implementation project(':neo4j-kernel-adapter-api') + } +} else { + multiRelease { + targetVersions 11, 17 + } + + if (!project.hasProperty('no-forbidden-apis')) { + forbiddenApisJava17 { + exclude('**') + } + } + + dependencies { + annotationProcessor group: 'org.neo4j', name: 'annotations', version: ver.'neo4j' + + compileOnly project(':annotations') + compileOnly group: 'org.neo4j', name: 'annotations', version: ver.'neo4j' + + implementation project(':neo4j-kernel-adapter-api') + + java17AnnotationProcessor project(':annotations') + java17AnnotationProcessor group: 'org.immutables', name: 'value', version: ver.'immutables' + java17AnnotationProcessor group: 'org.neo4j', name: 'annotations', version: neos.'5.13' + + java17CompileOnly project(':annotations') + java17CompileOnly group: 'org.immutables', name: 'value-annotations', version: ver.'immutables' + java17CompileOnly group: 'org.neo4j', name: 'neo4j', version: neos.'5.13' + java17CompileOnly group: 'org.neo4j', name: 'neo4j-record-storage-engine', version: neos.'5.13' + java17CompileOnly group: 'org.neo4j.community', name: 'it-test-support', version: neos.'5.13' + java17CompileOnly group: 'com.github.spotbugs', name: 'spotbugs-annotations', version: ver.'spotbugsToolVersion' + + java17Implementation project(':neo4j-kernel-adapter-api') + } +} diff --git a/compatibility/5.13/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_513/Neo4jProxyFactoryImpl.java b/compatibility/5.13/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_513/Neo4jProxyFactoryImpl.java new file mode 100644 index 0000000000..668351e0ed --- /dev/null +++ b/compatibility/5.13/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_513/Neo4jProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._513; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jProxyApi; +import org.neo4j.gds.compat.Neo4jProxyFactory; +import org.neo4j.gds.compat.Neo4jVersion; + +@ServiceProvider +public final class Neo4jProxyFactoryImpl implements Neo4jProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return false; + } + + @Override + public Neo4jProxyApi load() { + throw new UnsupportedOperationException("5.13 compatibility requires JDK17"); + } + + @Override + public String description() { + return "Neo4j 5.13 (placeholder)"; + } +} diff --git a/compatibility/5.13/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_513/SettingProxyFactoryImpl.java b/compatibility/5.13/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_513/SettingProxyFactoryImpl.java new file mode 100644 index 0000000000..5abd5f6fcc --- /dev/null +++ b/compatibility/5.13/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_513/SettingProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._513; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jVersion; +import org.neo4j.gds.compat.SettingProxyApi; +import org.neo4j.gds.compat.SettingProxyFactory; + +@ServiceProvider +public final class SettingProxyFactoryImpl implements SettingProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return false; + } + + @Override + public SettingProxyApi load() { + throw new UnsupportedOperationException("5.13 compatibility requires JDK17"); + } + + @Override + public String description() { + return "Neo4j Settings 5.13 (placeholder)"; + } +} diff --git a/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/BoltTransactionRunnerImpl.java b/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/BoltTransactionRunnerImpl.java new file mode 100644 index 0000000000..1c9014edbf --- /dev/null +++ b/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/BoltTransactionRunnerImpl.java @@ -0,0 +1,97 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._513; + +import org.neo4j.bolt.dbapi.BoltGraphDatabaseServiceSPI; +import org.neo4j.bolt.dbapi.BoltTransaction; +import org.neo4j.bolt.protocol.common.message.AccessMode; +import org.neo4j.bolt.protocol.common.message.request.connection.RoutingContext; +import org.neo4j.bolt.tx.statement.StatementQuerySubscriber; +import org.neo4j.exceptions.KernelException; +import org.neo4j.gds.compat.BoltQuerySubscriber; +import org.neo4j.gds.compat.BoltTransactionRunner; +import org.neo4j.graphdb.QueryStatistics; +import org.neo4j.internal.kernel.api.connectioninfo.ClientConnectionInfo; +import org.neo4j.internal.kernel.api.security.LoginContext; +import org.neo4j.kernel.api.KernelTransaction; +import org.neo4j.kernel.impl.query.QueryExecutionConfiguration; +import org.neo4j.kernel.impl.query.QueryExecutionKernelException; +import org.neo4j.values.virtual.MapValue; + +import java.time.Duration; +import java.util.List; +import java.util.Map; + +public class BoltTransactionRunnerImpl extends BoltTransactionRunner { + + @Override + protected BoltQuerySubscriber boltQuerySubscriber() { + var subscriber = new StatementQuerySubscriber(); + return new BoltQuerySubscriber<>() { + @Override + public void assertSucceeded() throws KernelException { + subscriber.assertSuccess(); + } + + @Override + public QueryStatistics queryStatistics() { + return subscriber.getStatistics(); + } + + @Override + public StatementQuerySubscriber innerSubscriber() { + return subscriber; + } + }; + } + + @Override + protected void executeQuery( + BoltTransaction boltTransaction, + String query, + MapValue parameters, + StatementQuerySubscriber querySubscriber + ) throws QueryExecutionKernelException { + boltTransaction.executeQuery(query, parameters, true, querySubscriber); + } + + @Override + protected BoltTransaction beginBoltWriteTransaction( + BoltGraphDatabaseServiceSPI fabricDb, + LoginContext loginContext, + KernelTransaction.Type kernelTransactionType, + ClientConnectionInfo clientConnectionInfo, + List bookmarks, + Duration txTimeout, + Map txMetadata + ) { + return fabricDb.beginTransaction( + kernelTransactionType, + loginContext, + clientConnectionInfo, + bookmarks, + txTimeout, + AccessMode.WRITE, + txMetadata, + new RoutingContext(true, Map.of()), + QueryExecutionConfiguration.DEFAULT_CONFIG + ); + } +} diff --git a/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/CallableProcedureImpl.java b/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/CallableProcedureImpl.java new file mode 100644 index 0000000000..ed8ee50b24 --- /dev/null +++ b/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/CallableProcedureImpl.java @@ -0,0 +1,53 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._513; + +import org.neo4j.collection.RawIterator; +import org.neo4j.gds.annotation.SuppressForbidden; +import org.neo4j.gds.compat.CompatCallableProcedure; +import org.neo4j.internal.kernel.api.exceptions.ProcedureException; +import org.neo4j.internal.kernel.api.procs.ProcedureSignature; +import org.neo4j.kernel.api.ResourceMonitor; +import org.neo4j.kernel.api.procedure.CallableProcedure; +import org.neo4j.kernel.api.procedure.Context; +import org.neo4j.values.AnyValue; + +@SuppressForbidden(reason = "This is the compat API") +public final class CallableProcedureImpl implements CallableProcedure { + private final CompatCallableProcedure procedure; + + CallableProcedureImpl(CompatCallableProcedure procedure) { + this.procedure = procedure; + } + + @Override + public ProcedureSignature signature() { + return this.procedure.signature(); + } + + @Override + public RawIterator apply( + Context ctx, + AnyValue[] input, + ResourceMonitor resourceMonitor + ) throws ProcedureException { + return this.procedure.apply(ctx, input); + } +} diff --git a/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/CallableUserAggregationFunctionImpl.java b/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/CallableUserAggregationFunctionImpl.java new file mode 100644 index 0000000000..3c51894b2e --- /dev/null +++ b/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/CallableUserAggregationFunctionImpl.java @@ -0,0 +1,77 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._513; + +import org.neo4j.gds.annotation.SuppressForbidden; +import org.neo4j.gds.compat.CompatUserAggregationFunction; +import org.neo4j.gds.compat.CompatUserAggregator; +import org.neo4j.internal.kernel.api.exceptions.ProcedureException; +import org.neo4j.internal.kernel.api.procs.UserAggregationReducer; +import org.neo4j.internal.kernel.api.procs.UserAggregationUpdater; +import org.neo4j.internal.kernel.api.procs.UserFunctionSignature; +import org.neo4j.kernel.api.procedure.CallableUserAggregationFunction; +import org.neo4j.kernel.api.procedure.Context; +import org.neo4j.values.AnyValue; + +@SuppressForbidden(reason = "This is the compat API") +public final class CallableUserAggregationFunctionImpl implements CallableUserAggregationFunction { + private final CompatUserAggregationFunction function; + + CallableUserAggregationFunctionImpl(CompatUserAggregationFunction function) { + this.function = function; + } + + @Override + public UserFunctionSignature signature() { + return this.function.signature(); + } + + @Override + public UserAggregationReducer createReducer(Context ctx) throws ProcedureException { + return new UserAggregatorImpl(this.function.create(ctx)); + } + + private static final class UserAggregatorImpl implements UserAggregationReducer, UserAggregationUpdater { + private final CompatUserAggregator aggregator; + + private UserAggregatorImpl(CompatUserAggregator aggregator) { + this.aggregator = aggregator; + } + + @Override + public UserAggregationUpdater newUpdater() { + return this; + } + + @Override + public void update(AnyValue[] input) throws ProcedureException { + this.aggregator.update(input); + } + + @Override + public void applyUpdates() { + } + + @Override + public AnyValue result() throws ProcedureException { + return this.aggregator.result(); + } + } +} diff --git a/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/CompatAccessModeImpl.java b/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/CompatAccessModeImpl.java new file mode 100644 index 0000000000..d5b41cf544 --- /dev/null +++ b/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/CompatAccessModeImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._513; + +import org.neo4j.gds.compat.CompatAccessMode; +import org.neo4j.gds.compat.CustomAccessMode; +import org.neo4j.internal.kernel.api.RelTypeSupplier; +import org.neo4j.internal.kernel.api.TokenSet; + +import java.util.function.Supplier; + +public final class CompatAccessModeImpl extends CompatAccessMode { + + CompatAccessModeImpl(CustomAccessMode custom) { + super(custom); + } + + @Override + public boolean allowsReadNodeProperty(Supplier labels, int propertyKey) { + return custom.allowsReadNodeProperty(propertyKey); + } + + @Override + public boolean allowsReadRelationshipProperty(RelTypeSupplier relType, int propertyKey) { + return custom.allowsReadRelationshipProperty(propertyKey); + } +} diff --git a/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/CompatGraphDatabaseAPIImpl.java b/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/CompatGraphDatabaseAPIImpl.java new file mode 100644 index 0000000000..0cbc818528 --- /dev/null +++ b/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/CompatGraphDatabaseAPIImpl.java @@ -0,0 +1,74 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._513; + +import org.neo4j.dbms.api.DatabaseManagementService; +import org.neo4j.dbms.systemgraph.TopologyGraphDbmsModel; +import org.neo4j.gds.compat.GdsGraphDatabaseAPI; +import org.neo4j.internal.kernel.api.connectioninfo.ClientConnectionInfo; +import org.neo4j.internal.kernel.api.security.LoginContext; +import org.neo4j.kernel.api.KernelTransaction; +import org.neo4j.kernel.api.exceptions.Status; +import org.neo4j.kernel.impl.coreapi.InternalTransaction; +import org.neo4j.kernel.impl.coreapi.TransactionExceptionMapper; + +import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; + +final class CompatGraphDatabaseAPIImpl extends GdsGraphDatabaseAPI { + + CompatGraphDatabaseAPIImpl(DatabaseManagementService dbms) { + super(dbms); + } + + @Override + public boolean isAvailable() { + return api.isAvailable(); + } + + @Override + public TopologyGraphDbmsModel.HostedOnMode mode() { + // NOTE: This means we can never start clusters locally, which is probably fine since: + // 1) We never did this before + // 2) We only use this for tests and benchmarks + return TopologyGraphDbmsModel.HostedOnMode.SINGLE; + } + + @Override + public InternalTransaction beginTransaction( + KernelTransaction.Type type, + LoginContext loginContext, + ClientConnectionInfo clientInfo, + long timeout, + TimeUnit unit, + Consumer terminationCallback, + TransactionExceptionMapper transactionExceptionMapper + ) { + return api.beginTransaction( + type, + loginContext, + clientInfo, + timeout, + unit, + terminationCallback, + transactionExceptionMapper + ); + } +} diff --git a/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/CompatIndexQueryImpl.java b/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/CompatIndexQueryImpl.java new file mode 100644 index 0000000000..9265647abb --- /dev/null +++ b/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/CompatIndexQueryImpl.java @@ -0,0 +1,31 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._513; + +import org.neo4j.gds.compat.CompatIndexQuery; +import org.neo4j.internal.kernel.api.PropertyIndexQuery; + +final class CompatIndexQueryImpl implements CompatIndexQuery { + final PropertyIndexQuery indexQuery; + + CompatIndexQueryImpl(PropertyIndexQuery indexQuery) { + this.indexQuery = indexQuery; + } +} diff --git a/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/CompatUsernameAuthSubjectImpl.java b/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/CompatUsernameAuthSubjectImpl.java new file mode 100644 index 0000000000..090434df0d --- /dev/null +++ b/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/CompatUsernameAuthSubjectImpl.java @@ -0,0 +1,35 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._513; + +import org.neo4j.gds.compat.CompatUsernameAuthSubject; +import org.neo4j.internal.kernel.api.security.AuthSubject; + +final class CompatUsernameAuthSubjectImpl extends CompatUsernameAuthSubject { + + CompatUsernameAuthSubjectImpl(String username, AuthSubject authSubject) { + super(username, authSubject); + } + + @Override + public String executingUser() { + return username; + } +} diff --git a/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/CompositeNodeCursorImpl.java b/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/CompositeNodeCursorImpl.java new file mode 100644 index 0000000000..7a12da5dfc --- /dev/null +++ b/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/CompositeNodeCursorImpl.java @@ -0,0 +1,32 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._513; + +import org.neo4j.gds.compat.CompositeNodeCursor; +import org.neo4j.internal.kernel.api.NodeLabelIndexCursor; + +import java.util.List; + +public final class CompositeNodeCursorImpl extends CompositeNodeCursor { + + CompositeNodeCursorImpl(List cursors, int[] labelIds) { + super(cursors, labelIds); + } +} diff --git a/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/GdsDatabaseLayoutImpl.java b/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/GdsDatabaseLayoutImpl.java new file mode 100644 index 0000000000..0f2d80d043 --- /dev/null +++ b/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/GdsDatabaseLayoutImpl.java @@ -0,0 +1,50 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._513; + +import org.neo4j.gds.compat.GdsDatabaseLayout; +import org.neo4j.io.layout.DatabaseLayout; + +import java.nio.file.Path; + +public class GdsDatabaseLayoutImpl implements GdsDatabaseLayout { + private final DatabaseLayout databaseLayout; + + public GdsDatabaseLayoutImpl(DatabaseLayout databaseLayout) {this.databaseLayout = databaseLayout;} + + @Override + public Path databaseDirectory() { + return databaseLayout.databaseDirectory(); + } + + @Override + public Path getTransactionLogsDirectory() { + return databaseLayout.getTransactionLogsDirectory(); + } + + @Override + public Path metadataStore() { + return databaseLayout.metadataStore(); + } + + public DatabaseLayout databaseLayout() { + return databaseLayout; + } +} diff --git a/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/GdsDatabaseManagementServiceBuilderImpl.java b/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/GdsDatabaseManagementServiceBuilderImpl.java new file mode 100644 index 0000000000..c35f5b69d2 --- /dev/null +++ b/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/GdsDatabaseManagementServiceBuilderImpl.java @@ -0,0 +1,54 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._513; + +import org.neo4j.dbms.api.DatabaseManagementService; +import org.neo4j.dbms.api.DatabaseManagementServiceBuilderImplementation; +import org.neo4j.gds.compat.GdsDatabaseManagementServiceBuilder; +import org.neo4j.graphdb.config.Setting; + +import java.nio.file.Path; +import java.util.Map; + +public class GdsDatabaseManagementServiceBuilderImpl implements GdsDatabaseManagementServiceBuilder { + + private final DatabaseManagementServiceBuilderImplementation dbmsBuilder; + + GdsDatabaseManagementServiceBuilderImpl(Path storeDir) { + this.dbmsBuilder = new DatabaseManagementServiceBuilderImplementation(storeDir); + } + + @Override + public GdsDatabaseManagementServiceBuilder setConfigRaw(Map configMap) { + dbmsBuilder.setConfigRaw(configMap); + return this; + } + + @Override + public GdsDatabaseManagementServiceBuilder setConfig(Setting setting, S value) { + dbmsBuilder.setConfig(setting, value); + return this; + } + + @Override + public DatabaseManagementService build() { + return dbmsBuilder.build(); + } +} diff --git a/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/Neo4jProxyFactoryImpl.java b/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/Neo4jProxyFactoryImpl.java new file mode 100644 index 0000000000..a66b1469ed --- /dev/null +++ b/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/Neo4jProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._513; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jProxyApi; +import org.neo4j.gds.compat.Neo4jProxyFactory; +import org.neo4j.gds.compat.Neo4jVersion; + +@ServiceProvider +public final class Neo4jProxyFactoryImpl implements Neo4jProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return version == Neo4jVersion.V_5_13; + } + + @Override + public Neo4jProxyApi load() { + return new Neo4jProxyImpl(); + } + + @Override + public String description() { + return "Neo4j 5.13"; + } +} diff --git a/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/Neo4jProxyImpl.java b/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/Neo4jProxyImpl.java new file mode 100644 index 0000000000..d3b2d662ea --- /dev/null +++ b/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/Neo4jProxyImpl.java @@ -0,0 +1,967 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._513; + +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import org.neo4j.common.DependencyResolver; +import org.neo4j.common.EntityType; +import org.neo4j.configuration.BootloaderSettings; +import org.neo4j.configuration.Config; +import org.neo4j.configuration.GraphDatabaseSettings; +import org.neo4j.configuration.SettingValueParsers; +import org.neo4j.configuration.connectors.ConnectorPortRegister; +import org.neo4j.configuration.connectors.ConnectorType; +import org.neo4j.configuration.helpers.DatabaseNameValidator; +import org.neo4j.dbms.api.DatabaseManagementService; +import org.neo4j.exceptions.KernelException; +import org.neo4j.fabric.FabricDatabaseManager; +import org.neo4j.gds.annotation.SuppressForbidden; +import org.neo4j.gds.compat.BoltTransactionRunner; +import org.neo4j.gds.compat.CompatCallableProcedure; +import org.neo4j.gds.compat.CompatExecutionMonitor; +import org.neo4j.gds.compat.CompatIndexQuery; +import org.neo4j.gds.compat.CompatInput; +import org.neo4j.gds.compat.CompatUserAggregationFunction; +import org.neo4j.gds.compat.CompositeNodeCursor; +import org.neo4j.gds.compat.CustomAccessMode; +import org.neo4j.gds.compat.GdsDatabaseLayout; +import org.neo4j.gds.compat.GdsDatabaseManagementServiceBuilder; +import org.neo4j.gds.compat.GdsGraphDatabaseAPI; +import org.neo4j.gds.compat.GlobalProcedureRegistry; +import org.neo4j.gds.compat.GraphDatabaseApiProxy; +import org.neo4j.gds.compat.InputEntityIdVisitor; +import org.neo4j.gds.compat.Neo4jProxyApi; +import org.neo4j.gds.compat.PropertyReference; +import org.neo4j.gds.compat.StoreScan; +import org.neo4j.gds.compat.TestLog; +import org.neo4j.graphdb.GraphDatabaseService; +import org.neo4j.graphdb.Node; +import org.neo4j.graphdb.Relationship; +import org.neo4j.graphdb.RelationshipType; +import org.neo4j.graphdb.config.Setting; +import org.neo4j.internal.batchimport.AdditionalInitialIds; +import org.neo4j.internal.batchimport.BatchImporter; +import org.neo4j.internal.batchimport.BatchImporterFactory; +import org.neo4j.internal.batchimport.Configuration; +import org.neo4j.internal.batchimport.IndexConfig; +import org.neo4j.internal.batchimport.InputIterable; +import org.neo4j.internal.batchimport.Monitor; +import org.neo4j.internal.batchimport.input.Collector; +import org.neo4j.internal.batchimport.input.IdType; +import org.neo4j.internal.batchimport.input.Input; +import org.neo4j.internal.batchimport.input.InputEntityVisitor; +import org.neo4j.internal.batchimport.input.PropertySizeCalculator; +import org.neo4j.internal.batchimport.input.ReadableGroups; +import org.neo4j.internal.batchimport.staging.ExecutionMonitor; +import org.neo4j.internal.batchimport.staging.StageExecution; +import org.neo4j.internal.helpers.HostnamePort; +import org.neo4j.internal.id.IdGenerator; +import org.neo4j.internal.id.IdGeneratorFactory; +import org.neo4j.internal.kernel.api.Cursor; +import org.neo4j.internal.kernel.api.IndexQueryConstraints; +import org.neo4j.internal.kernel.api.IndexReadSession; +import org.neo4j.internal.kernel.api.NodeCursor; +import org.neo4j.internal.kernel.api.NodeLabelIndexCursor; +import org.neo4j.internal.kernel.api.NodeValueIndexCursor; +import org.neo4j.internal.kernel.api.PropertyCursor; +import org.neo4j.internal.kernel.api.PropertyIndexQuery; +import org.neo4j.internal.kernel.api.QueryContext; +import org.neo4j.internal.kernel.api.Read; +import org.neo4j.internal.kernel.api.RelationshipScanCursor; +import org.neo4j.internal.kernel.api.Scan; +import org.neo4j.internal.kernel.api.TokenPredicate; +import org.neo4j.internal.kernel.api.connectioninfo.ClientConnectionInfo; +import org.neo4j.internal.kernel.api.exceptions.ProcedureException; +import org.neo4j.internal.kernel.api.procs.FieldSignature; +import org.neo4j.internal.kernel.api.procs.Neo4jTypes; +import org.neo4j.internal.kernel.api.procs.ProcedureSignature; +import org.neo4j.internal.kernel.api.procs.QualifiedName; +import org.neo4j.internal.kernel.api.procs.UserFunctionSignature; +import org.neo4j.internal.kernel.api.security.AccessMode; +import org.neo4j.internal.kernel.api.security.AuthSubject; +import org.neo4j.internal.kernel.api.security.SecurityContext; +import org.neo4j.internal.recordstorage.RecordIdType; +import org.neo4j.internal.schema.IndexCapability; +import org.neo4j.internal.schema.IndexDescriptor; +import org.neo4j.internal.schema.IndexOrder; +import org.neo4j.internal.schema.SchemaDescriptors; +import org.neo4j.io.fs.FileSystemAbstraction; +import org.neo4j.io.layout.DatabaseLayout; +import org.neo4j.io.layout.Neo4jLayout; +import org.neo4j.io.layout.recordstorage.RecordDatabaseLayout; +import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.context.CursorContextFactory; +import org.neo4j.io.pagecache.context.FixedVersionContextSupplier; +import org.neo4j.io.pagecache.tracing.PageCacheTracer; +import org.neo4j.kernel.api.KernelTransaction; +import org.neo4j.kernel.api.KernelTransactionHandle; +import org.neo4j.kernel.api.procedure.CallableProcedure; +import org.neo4j.kernel.api.procedure.CallableUserAggregationFunction; +import org.neo4j.kernel.api.procedure.Context; +import org.neo4j.kernel.api.procedure.GlobalProcedures; +import org.neo4j.kernel.database.NormalizedDatabaseName; +import org.neo4j.kernel.impl.coreapi.InternalTransaction; +import org.neo4j.kernel.impl.index.schema.IndexImporterFactoryImpl; +import org.neo4j.kernel.impl.query.QueryExecutionConfiguration; +import org.neo4j.kernel.impl.query.TransactionalContext; +import org.neo4j.kernel.impl.query.TransactionalContextFactory; +import org.neo4j.kernel.impl.store.RecordStore; +import org.neo4j.kernel.impl.store.format.RecordFormatSelector; +import org.neo4j.kernel.impl.store.format.RecordFormats; +import org.neo4j.kernel.impl.store.record.AbstractBaseRecord; +import org.neo4j.kernel.impl.transaction.log.EmptyLogTailMetadata; +import org.neo4j.kernel.impl.transaction.log.files.TransactionLogInitializer; +import org.neo4j.logging.Log; +import org.neo4j.logging.internal.LogService; +import org.neo4j.memory.EmptyMemoryTracker; +import org.neo4j.procedure.Mode; +import org.neo4j.scheduler.JobScheduler; +import org.neo4j.ssl.config.SslPolicyLoader; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.StorageEngineFactory; +import org.neo4j.util.BitBuffer; +import org.neo4j.values.storable.TextArray; +import org.neo4j.values.storable.ValueCategory; +import org.neo4j.values.storable.Values; +import org.neo4j.values.virtual.MapValue; +import org.neo4j.values.virtual.NodeValue; +import org.neo4j.values.virtual.VirtualValues; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Locale; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static java.lang.String.format; +import static org.neo4j.gds.compat.InternalReadOps.countByIdGenerator; + +public final class Neo4jProxyImpl implements Neo4jProxyApi { + + @Override + public GdsGraphDatabaseAPI newDb(DatabaseManagementService dbms) { + return new CompatGraphDatabaseAPIImpl(dbms); + } + + @Override + public String validateExternalDatabaseName(String databaseName) { + var normalizedName = new NormalizedDatabaseName(databaseName); + DatabaseNameValidator.validateExternalDatabaseName(normalizedName); + return normalizedName.name(); + } + + @Override + public AccessMode accessMode(CustomAccessMode customAccessMode) { + return new CompatAccessModeImpl(customAccessMode); + } + + @Override + public String username(AuthSubject subject) { + return subject.executingUser(); + } + + @Override + public SecurityContext securityContext( + String username, + AuthSubject authSubject, + AccessMode mode, + String databaseName + ) { + return new SecurityContext( + new CompatUsernameAuthSubjectImpl(username, authSubject), + mode, + // GDS is always operating from an embedded context + ClientConnectionInfo.EMBEDDED_CONNECTION, + databaseName + ); + } + + @Override + public long getHighId(RecordStore recordStore) { + return recordStore.getIdGenerator().getHighId(); + } + + @Override + public List> entityCursorScan( + KernelTransaction transaction, + int[] labelIds, + int batchSize, + boolean allowPartitionedScan + ) { + if (allowPartitionedScan) { + return partitionedNodeLabelIndexScan(transaction, batchSize, labelIds); + } else { + var read = transaction.dataRead(); + return Arrays + .stream(labelIds) + .mapToObj(read::nodeLabelScan) + .map(scan -> scanToStoreScan(scan, batchSize)) + .collect(Collectors.toList()); + } + } + + @Override + public PropertyCursor allocatePropertyCursor(KernelTransaction kernelTransaction) { + return kernelTransaction + .cursors() + .allocatePropertyCursor(kernelTransaction.cursorContext(), kernelTransaction.memoryTracker()); + } + + @Override + public PropertyReference propertyReference(NodeCursor nodeCursor) { + return ReferencePropertyReference.of(nodeCursor.propertiesReference()); + } + + @Override + public PropertyReference propertyReference(RelationshipScanCursor relationshipScanCursor) { + return ReferencePropertyReference.of(relationshipScanCursor.propertiesReference()); + } + + @Override + public PropertyReference noPropertyReference() { + return ReferencePropertyReference.empty(); + } + + @Override + public void nodeProperties( + KernelTransaction kernelTransaction, + long nodeReference, + PropertyReference reference, + PropertyCursor cursor + ) { + var neoReference = ((ReferencePropertyReference) reference).reference; + kernelTransaction + .dataRead() + .nodeProperties(nodeReference, neoReference, PropertySelection.ALL_PROPERTIES, cursor); + } + + @Override + public void relationshipProperties( + KernelTransaction kernelTransaction, + long relationshipReference, + PropertyReference reference, + PropertyCursor cursor + ) { + var neoReference = ((ReferencePropertyReference) reference).reference; + kernelTransaction + .dataRead() + .relationshipProperties(relationshipReference, neoReference, PropertySelection.ALL_PROPERTIES, cursor); + } + + @Override + public NodeCursor allocateNodeCursor(KernelTransaction kernelTransaction) { + return kernelTransaction.cursors().allocateNodeCursor(kernelTransaction.cursorContext()); + } + + @Override + public RelationshipScanCursor allocateRelationshipScanCursor(KernelTransaction kernelTransaction) { + return kernelTransaction.cursors().allocateRelationshipScanCursor(kernelTransaction.cursorContext()); + } + + @Override + public NodeLabelIndexCursor allocateNodeLabelIndexCursor(KernelTransaction kernelTransaction) { + return kernelTransaction.cursors().allocateNodeLabelIndexCursor(kernelTransaction.cursorContext()); + } + + @Override + public NodeValueIndexCursor allocateNodeValueIndexCursor(KernelTransaction kernelTransaction) { + return kernelTransaction + .cursors() + .allocateNodeValueIndexCursor(kernelTransaction.cursorContext(), kernelTransaction.memoryTracker()); + } + + @Override + public boolean hasNodeLabelIndex(KernelTransaction kernelTransaction) { + return NodeLabelIndexLookupImpl.hasNodeLabelIndex(kernelTransaction); + } + + @Override + public StoreScan nodeLabelIndexScan( + KernelTransaction transaction, + int labelId, + int batchSize, + boolean allowPartitionedScan + ) { + if (allowPartitionedScan) { + return partitionedNodeLabelIndexScan(transaction, batchSize, labelId).get(0); + } else { + var read = transaction.dataRead(); + return scanToStoreScan(read.nodeLabelScan(labelId), batchSize); + } + } + + @Override + public StoreScan scanToStoreScan(Scan scan, int batchSize) { + return new ScanBasedStoreScanImpl<>(scan, batchSize); + } + + private List> partitionedNodeLabelIndexScan( + KernelTransaction transaction, + int batchSize, + int... labelIds + ) { + var indexDescriptor = NodeLabelIndexLookupImpl.findUsableMatchingIndex( + transaction, + SchemaDescriptors.forAnyEntityTokens(EntityType.NODE) + ); + + if (indexDescriptor == IndexDescriptor.NO_INDEX) { + throw new IllegalStateException("There is no index that can back a node label scan."); + } + + var read = transaction.dataRead(); + + // Our current strategy is to select the token with the highest count + // and use that one as the driving partitioned index scan. The partitions + // of all other partitioned index scans will be aligned to that one. + int maxToken = labelIds[0]; + long maxCount = read.countsForNodeWithoutTxState(labelIds[0]); + + for (int i = 1; i < labelIds.length; i++) { + long count = read.countsForNodeWithoutTxState(labelIds[i]); + if (count > maxCount) { + maxCount = count; + maxToken = labelIds[i]; + } + } + + int numberOfPartitions = PartitionedStoreScan.getNumberOfPartitions(maxCount, batchSize); + + try { + var session = read.tokenReadSession(indexDescriptor); + + var partitionedScan = read.nodeLabelScan( + session, + numberOfPartitions, + transaction.cursorContext(), + new TokenPredicate(maxToken) + ); + + var scans = new ArrayList>(labelIds.length); + scans.add(new PartitionedStoreScan(partitionedScan)); + + // Initialize the remaining index scans with the partitioning of the first scan. + for (int labelToken : labelIds) { + if (labelToken != maxToken) { + var scan = read.nodeLabelScan(session, partitionedScan, new TokenPredicate(labelToken)); + scans.add(new PartitionedStoreScan(scan)); + } + } + + return scans; + } catch (KernelException e) { + // should not happen, we check for the index existence and applicability + // before reading it + throw new RuntimeException("Unexpected error while initialising reading from node label index", e); + } + } + + @Override + public CompatIndexQuery rangeIndexQuery( + int propertyKeyId, + double from, + boolean fromInclusive, + double to, + boolean toInclusive + ) { + return new CompatIndexQueryImpl(PropertyIndexQuery.range(propertyKeyId, from, fromInclusive, to, toInclusive)); + } + + @Override + public CompatIndexQuery rangeAllIndexQuery(int propertyKeyId) { + var rangePredicate = PropertyIndexQuery.range( + propertyKeyId, + Values.doubleValue(Double.NEGATIVE_INFINITY), + true, + Values.doubleValue(Double.POSITIVE_INFINITY), + true + ); + return new CompatIndexQueryImpl(rangePredicate); + } + + @Override + public void nodeIndexSeek( + Read dataRead, + IndexReadSession index, + NodeValueIndexCursor cursor, + IndexOrder indexOrder, + boolean needsValues, + CompatIndexQuery query + ) throws KernelException { + var indexQueryConstraints = indexOrder == IndexOrder.NONE + ? IndexQueryConstraints.unordered(needsValues) + : IndexQueryConstraints.constrained(indexOrder, needsValues); + + dataRead.nodeIndexSeek( + (QueryContext) dataRead, + index, + cursor, + indexQueryConstraints, + ((CompatIndexQueryImpl) query).indexQuery + ); + } + + @Override + public CompositeNodeCursor compositeNodeCursor(List cursors, int[] labelIds) { + return new CompositeNodeCursorImpl(cursors, labelIds); + } + + @Override + public Configuration batchImporterConfig( + int batchSize, + int writeConcurrency, + Optional pageCacheMemory, + boolean highIO, + IndexConfig indexConfig + ) { + return new org.neo4j.internal.batchimport.Configuration() { + @Override + public int batchSize() { + return batchSize; + } + + @Override + public int maxNumberOfWorkerThreads() { + return writeConcurrency; + } + + @Override + public boolean highIO() { + return highIO; + } + + @Override + public IndexConfig indexConfig() { + return indexConfig; + } + }; + } + + @Override + public int writeConcurrency(Configuration batchImportConfiguration) { + return batchImportConfiguration.maxNumberOfWorkerThreads(); + } + + @Override + public BatchImporter instantiateBatchImporter( + BatchImporterFactory factory, + GdsDatabaseLayout directoryStructure, + FileSystemAbstraction fileSystem, + PageCacheTracer pageCacheTracer, + Configuration configuration, + LogService logService, + ExecutionMonitor executionMonitor, + AdditionalInitialIds additionalInitialIds, + Config dbConfig, + RecordFormats recordFormats, + JobScheduler jobScheduler, + Collector badCollector + ) { + dbConfig.set(GraphDatabaseSettings.db_format, recordFormats.name()); + var databaseLayout = ((GdsDatabaseLayoutImpl) directoryStructure).databaseLayout(); + return factory.instantiate( + databaseLayout, + fileSystem, + pageCacheTracer, + configuration, + logService, + executionMonitor, + additionalInitialIds, + new EmptyLogTailMetadata(dbConfig), + dbConfig, + Monitor.NO_MONITOR, + jobScheduler, + badCollector, + TransactionLogInitializer.getLogFilesInitializer(), + new IndexImporterFactoryImpl(), + EmptyMemoryTracker.INSTANCE, + new CursorContextFactory(PageCacheTracer.NULL, FixedVersionContextSupplier.EMPTY_CONTEXT_SUPPLIER) + ); + } + + @Override + public Input batchInputFrom(CompatInput compatInput) { + return new InputFromCompatInput(compatInput); + } + + @Override + public InputEntityIdVisitor.Long inputEntityLongIdVisitor(IdType idType, ReadableGroups groups) { + switch (idType) { + case ACTUAL -> { + return new InputEntityIdVisitor.Long() { + @Override + public void visitNodeId(InputEntityVisitor visitor, long id) { + visitor.id(id); + } + + @Override + public void visitSourceId(InputEntityVisitor visitor, long id) { + visitor.startId(id); + } + + @Override + public void visitTargetId(InputEntityVisitor visitor, long id) { + visitor.endId(id); + } + }; + } + case INTEGER -> { + var globalGroup = groups.get(null); + + return new InputEntityIdVisitor.Long() { + @Override + public void visitNodeId(InputEntityVisitor visitor, long id) { + visitor.id(id, globalGroup); + } + + @Override + public void visitSourceId(InputEntityVisitor visitor, long id) { + visitor.startId(id, globalGroup); + } + + @Override + public void visitTargetId(InputEntityVisitor visitor, long id) { + visitor.endId(id, globalGroup); + } + }; + } + default -> throw new IllegalStateException("Unexpected value: " + idType); + } + } + + @Override + public InputEntityIdVisitor.String inputEntityStringIdVisitor(ReadableGroups groups) { + var globalGroup = groups.get(null); + + return new InputEntityIdVisitor.String() { + @Override + public void visitNodeId(InputEntityVisitor visitor, String id) { + visitor.id(id, globalGroup); + } + + @Override + public void visitSourceId(InputEntityVisitor visitor, String id) { + visitor.startId(id, globalGroup); + } + + @Override + public void visitTargetId(InputEntityVisitor visitor, String id) { + visitor.endId(id, globalGroup); + } + }; + } + + @Override + public Setting additionalJvm() { + return BootloaderSettings.additional_jvm; + } + + @Override + public Setting pageCacheMemory() { + return GraphDatabaseSettings.pagecache_memory; + } + + @Override + public Long pageCacheMemoryValue(String value) { + return SettingValueParsers.BYTES.parse(value); + } + + @Override + public ProcedureSignature procedureSignature( + QualifiedName name, + List inputSignature, + List outputSignature, + Mode mode, + boolean admin, + String deprecated, + String description, + String warning, + boolean eager, + boolean caseInsensitive, + boolean systemProcedure, + boolean internal, + boolean allowExpiredCredentials, + boolean threadSafe + ) { + return new ProcedureSignature( + name, + inputSignature, + outputSignature, + mode, + admin, + deprecated, + description, + warning, + eager, + caseInsensitive, + systemProcedure, + internal, + allowExpiredCredentials, + threadSafe + ); + } + + @Override + public long getHighestPossibleNodeCount( + Read read, IdGeneratorFactory idGeneratorFactory + ) { + return countByIdGenerator(idGeneratorFactory, RecordIdType.NODE).orElseGet(read::nodesGetCount); + } + + @Override + public long getHighestPossibleRelationshipCount( + Read read, IdGeneratorFactory idGeneratorFactory + ) { + return countByIdGenerator(idGeneratorFactory, RecordIdType.RELATIONSHIP).orElseGet(read::relationshipsGetCount); + } + + @Override + public String versionLongToString(long storeVersion) { + // copied from org.neo4j.kernel.impl.store.LegacyMetadataHandler.versionLongToString which is private + if (storeVersion == -1) { + return "Unknown"; + } + BitBuffer bits = BitBuffer.bitsFromLongs(new long[]{storeVersion}); + int length = bits.getShort(8); + if (length == 0 || length > 7) { + throw new IllegalArgumentException(format(Locale.ENGLISH, "The read version string length %d is not proper.", length)); + } + char[] result = new char[length]; + for (int i = 0; i < length; i++) { + result[i] = (char) bits.getShort(8); + } + return new String(result); + } + + private static final class InputFromCompatInput implements Input { + private final CompatInput delegate; + + private InputFromCompatInput(CompatInput delegate) { + this.delegate = delegate; + } + + @Override + public InputIterable nodes(Collector badCollector) { + return delegate.nodes(badCollector); + } + + @Override + public InputIterable relationships(Collector badCollector) { + return delegate.relationships(badCollector); + } + + @Override + public IdType idType() { + return delegate.idType(); + } + + @Override + public ReadableGroups groups() { + return delegate.groups(); + } + + @Override + public Estimates calculateEstimates(PropertySizeCalculator propertySizeCalculator) throws IOException { + return delegate.calculateEstimates((values, kernelTransaction) -> propertySizeCalculator.calculateSize( + values, + kernelTransaction.cursorContext(), + kernelTransaction.memoryTracker() + )); + } + } + + @Override + public TestLog testLog() { + return new TestLogImpl(); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public Log getUserLog(LogService logService, Class loggingClass) { + return logService.getUserLog(loggingClass); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public Log getInternalLog(LogService logService, Class loggingClass) { + return logService.getInternalLog(loggingClass); + } + + @Override + public NodeValue nodeValue(long id, TextArray labels, MapValue properties) { + return VirtualValues.nodeValue(id, String.valueOf(id), labels, properties); + } + + @Override + public Relationship virtualRelationship(long id, Node startNode, Node endNode, RelationshipType type) { + return new VirtualRelationshipImpl(id, startNode, endNode, type); + } + + @Override + public GdsDatabaseManagementServiceBuilder databaseManagementServiceBuilder(Path storeDir) { + return new GdsDatabaseManagementServiceBuilderImpl(storeDir); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public RecordFormats selectRecordFormatForStore( + DatabaseLayout databaseLayout, + FileSystemAbstraction fs, + PageCache pageCache, + LogService logService, + PageCacheTracer pageCacheTracer + ) { + return RecordFormatSelector.selectForStore( + (RecordDatabaseLayout) databaseLayout, + fs, + pageCache, + logService.getInternalLogProvider(), + new CursorContextFactory(pageCacheTracer, FixedVersionContextSupplier.EMPTY_CONTEXT_SUPPLIER) + ); + } + + @Override + public boolean isNotNumericIndex(IndexCapability indexCapability) { + return !indexCapability.areValueCategoriesAccepted(ValueCategory.NUMBER); + } + + @Override + public void setAllowUpgrades(Config.Builder configBuilder, boolean value) { + } + + @Override + public String defaultRecordFormatSetting() { + return GraphDatabaseSettings.db_format.defaultValue(); + } + + @Override + public void configureRecordFormat(Config.Builder configBuilder, String recordFormat) { + var databaseRecordFormat = recordFormat.toLowerCase(Locale.ENGLISH); + configBuilder.set(GraphDatabaseSettings.db_format, databaseRecordFormat); + } + + @Override + public GdsDatabaseLayout databaseLayout(Config config, String databaseName) { + var storageEngineFactory = StorageEngineFactory.selectStorageEngine(config); + var dbLayout = neo4jLayout(config).databaseLayout(databaseName); + var databaseLayout = storageEngineFactory.formatSpecificDatabaseLayout(dbLayout); + return new GdsDatabaseLayoutImpl(databaseLayout); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public Neo4jLayout neo4jLayout(Config config) { + return Neo4jLayout.of(config); + } + + @Override + public BoltTransactionRunner boltTransactionRunner() { + return new BoltTransactionRunnerImpl(); + } + + @Override + public HostnamePort getLocalBoltAddress(ConnectorPortRegister connectorPortRegister) { + return connectorPortRegister.getLocalAddress(ConnectorType.BOLT); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public SslPolicyLoader createSllPolicyLoader( + FileSystemAbstraction fileSystem, + Config config, + LogService logService + ) { + return SslPolicyLoader.create(fileSystem, config, logService.getInternalLogProvider()); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public RecordFormats recordFormatSelector( + String databaseName, + Config databaseConfig, + FileSystemAbstraction fs, + LogService logService, + GraphDatabaseService databaseService + ) { + var neo4jLayout = Neo4jLayout.of(databaseConfig); + var recordDatabaseLayout = RecordDatabaseLayout.of(neo4jLayout, databaseName); + return RecordFormatSelector.selectForStoreOrConfigForNewDbs( + databaseConfig, + recordDatabaseLayout, + fs, + GraphDatabaseApiProxy.resolveDependency(databaseService, PageCache.class), + logService.getInternalLogProvider(), + GraphDatabaseApiProxy.resolveDependency(databaseService, CursorContextFactory.class) + ); + } + + @Override + public ExecutionMonitor executionMonitor(CompatExecutionMonitor compatExecutionMonitor) { + return new ExecutionMonitor.Adapter( + compatExecutionMonitor.checkIntervalMillis(), + TimeUnit.MILLISECONDS + ) { + + @Override + public void initialize(DependencyResolver dependencyResolver) { + compatExecutionMonitor.initialize(dependencyResolver); + } + + @Override + public void start(StageExecution execution) { + compatExecutionMonitor.start(execution); + } + + @Override + public void end(StageExecution execution, long totalTimeMillis) { + compatExecutionMonitor.end(execution, totalTimeMillis); + } + + @Override + public void done(boolean successful, long totalTimeMillis, String additionalInformation) { + compatExecutionMonitor.done(successful, totalTimeMillis, additionalInformation); + } + + @Override + public void check(StageExecution execution) { + compatExecutionMonitor.check(execution); + } + }; + } + + @Override + @SuppressFBWarnings("NP_LOAD_OF_KNOWN_NULL_VALUE") // We assign nulls because it makes the code more readable + public UserFunctionSignature userFunctionSignature( + QualifiedName name, + List inputSignature, + Neo4jTypes.AnyType type, + String description, + boolean internal, + boolean threadSafe, + Optional deprecatedBy + ) { + String category = null; // No predefined categpry (like temporal or math) + var caseInsensitive = false; // case sensitive name match + var isBuiltIn = false; // is built in; never true for GDS + + return new UserFunctionSignature( + name, + inputSignature, + type, + deprecatedBy.orElse(null), + description, + category, + caseInsensitive, + isBuiltIn, + internal, + threadSafe + ); + } + + @Override + @SuppressForbidden(reason = "This is the compat API") + public CallableProcedure callableProcedure(CompatCallableProcedure procedure) { + return new CallableProcedureImpl(procedure); + } + + @Override + @SuppressForbidden(reason = "This is the compat API") + public CallableUserAggregationFunction callableUserAggregationFunction(CompatUserAggregationFunction function) { + return new CallableUserAggregationFunctionImpl(function); + } + + @Override + public long transactionId(KernelTransactionHandle kernelTransactionHandle) { + return kernelTransactionHandle.getTransactionSequenceNumber(); + } + + @Override + public void reserveNeo4jIds(IdGeneratorFactory generatorFactory, int size, CursorContext cursorContext) { + IdGenerator idGenerator = generatorFactory.get(RecordIdType.NODE); + + idGenerator.nextConsecutiveIdRange(size, false, cursorContext); + } + + @Override + public TransactionalContext newQueryContext( + TransactionalContextFactory contextFactory, + InternalTransaction tx, + String queryText, + MapValue queryParameters + ) { + return contextFactory.newContext(tx, queryText, queryParameters, QueryExecutionConfiguration.DEFAULT_CONFIG); + } + + @Override + public boolean isCompositeDatabase(GraphDatabaseService databaseService) { + var databaseManager = GraphDatabaseApiProxy.resolveDependency(databaseService, FabricDatabaseManager.class); + return databaseManager.isFabricDatabase(GraphDatabaseApiProxy.databaseId(databaseService)); + } + + @Override + public T lookupComponentProvider(Context ctx, Class component, boolean safe) throws ProcedureException { + var globalProcedures = GraphDatabaseApiProxy.resolveDependency(ctx.dependencyResolver(), GlobalProcedures.class); + return globalProcedures.getCurrentView().lookupComponentProvider(component, safe).apply(ctx); + } + + @Override + public GlobalProcedureRegistry globalProcedureRegistry(GlobalProcedures globalProcedures) { + return new GlobalProcedureRegistry() { + @Override + public Set getAllProcedures() { + return globalProcedures.getCurrentView().getAllProcedures(); + } + + @Override + public Stream getAllNonAggregatingFunctions() { + return globalProcedures.getCurrentView().getAllNonAggregatingFunctions(); + } + + @Override + public Stream getAllAggregatingFunctions() { + return globalProcedures.getCurrentView().getAllAggregatingFunctions(); + } + }; + } + + private static final DependencyResolver EMPTY_DEPENDENCY_RESOLVER = new DependencyResolver() { + @Override + public T resolveDependency(Class type, DependencyResolver.SelectionStrategy selector) { + return null; + } + + @Override + public boolean containsDependency(Class type) { + return false; + } + }; + + @Override + public DependencyResolver emptyDependencyResolver() { + return EMPTY_DEPENDENCY_RESOLVER; + } +} diff --git a/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/NodeLabelIndexLookupImpl.java b/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/NodeLabelIndexLookupImpl.java new file mode 100644 index 0000000000..23bd3ab65b --- /dev/null +++ b/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/NodeLabelIndexLookupImpl.java @@ -0,0 +1,68 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._513; + +import org.neo4j.common.EntityType; +import org.neo4j.internal.kernel.api.InternalIndexState; +import org.neo4j.internal.kernel.api.SchemaRead; +import org.neo4j.internal.kernel.api.exceptions.schema.IndexNotFoundKernelException; +import org.neo4j.internal.schema.IndexDescriptor; +import org.neo4j.internal.schema.IndexType; +import org.neo4j.internal.schema.SchemaDescriptor; +import org.neo4j.internal.schema.SchemaDescriptors; +import org.neo4j.kernel.api.KernelTransaction; + +final class NodeLabelIndexLookupImpl { + + static boolean hasNodeLabelIndex(KernelTransaction transaction) { + return NodeLabelIndexLookupImpl.findUsableMatchingIndex( + transaction, + SchemaDescriptors.forAnyEntityTokens(EntityType.NODE) + ) != IndexDescriptor.NO_INDEX; + } + + static IndexDescriptor findUsableMatchingIndex( + KernelTransaction transaction, + SchemaDescriptor schemaDescriptor + ) { + var schemaRead = transaction.schemaRead(); + var iterator = schemaRead.index(schemaDescriptor); + while (iterator.hasNext()) { + var index = iterator.next(); + if (index.getIndexType() == IndexType.LOOKUP && indexIsOnline(schemaRead, index)) { + return index; + } + } + return IndexDescriptor.NO_INDEX; + } + + private static boolean indexIsOnline(SchemaRead schemaRead, IndexDescriptor index) { + var state = InternalIndexState.FAILED; + try { + state = schemaRead.indexGetState(index); + } catch (IndexNotFoundKernelException e) { + // Well the index should always exist here, but if we didn't find it while checking the state, + // then we obviously don't want to use it. + } + return state == InternalIndexState.ONLINE; + } + + private NodeLabelIndexLookupImpl() {} +} diff --git a/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/PartitionedStoreScan.java b/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/PartitionedStoreScan.java new file mode 100644 index 0000000000..12e483dea5 --- /dev/null +++ b/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/PartitionedStoreScan.java @@ -0,0 +1,58 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._513; + +import org.neo4j.gds.compat.StoreScan; +import org.neo4j.internal.kernel.api.NodeLabelIndexCursor; +import org.neo4j.internal.kernel.api.PartitionedScan; +import org.neo4j.kernel.api.KernelTransaction; + +final class PartitionedStoreScan implements StoreScan { + private final PartitionedScan scan; + + PartitionedStoreScan(PartitionedScan scan) { + this.scan = scan; + } + + static int getNumberOfPartitions(long nodeCount, int batchSize) { + int numberOfPartitions; + if (nodeCount > 0) { + // ceil div to try to get enough partitions so a single one does + // not include more nodes than batchSize + long partitions = ((nodeCount - 1) / batchSize) + 1; + + // value must be positive + if (partitions < 1) { + partitions = 1; + } + + numberOfPartitions = (int) Long.min(Integer.MAX_VALUE, partitions); + } else { + // we have no partitions to scan, but the value must still be positive + numberOfPartitions = 1; + } + return numberOfPartitions; + } + + @Override + public boolean reserveBatch(NodeLabelIndexCursor cursor, KernelTransaction ktx) { + return scan.reservePartition(cursor, ktx.cursorContext(), ktx.securityContext().mode()); + } +} diff --git a/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/ReferencePropertyReference.java b/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/ReferencePropertyReference.java new file mode 100644 index 0000000000..93175c637a --- /dev/null +++ b/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/ReferencePropertyReference.java @@ -0,0 +1,49 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._513; + +import org.neo4j.gds.compat.PropertyReference; +import org.neo4j.storageengine.api.Reference; + +import java.util.Objects; + +public final class ReferencePropertyReference implements PropertyReference { + + private static final PropertyReference EMPTY = new ReferencePropertyReference(null); + + public final Reference reference; + + private ReferencePropertyReference(Reference reference) { + this.reference = reference; + } + + public static PropertyReference of(Reference reference) { + return new ReferencePropertyReference(Objects.requireNonNull(reference)); + } + + public static PropertyReference empty() { + return EMPTY; + } + + @Override + public boolean isEmpty() { + return reference == null; + } +} diff --git a/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/ScanBasedStoreScanImpl.java b/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/ScanBasedStoreScanImpl.java new file mode 100644 index 0000000000..61254cbe91 --- /dev/null +++ b/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/ScanBasedStoreScanImpl.java @@ -0,0 +1,40 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._513; + +import org.neo4j.gds.compat.StoreScan; +import org.neo4j.internal.kernel.api.Cursor; +import org.neo4j.internal.kernel.api.Scan; +import org.neo4j.kernel.api.KernelTransaction; + +public final class ScanBasedStoreScanImpl implements StoreScan { + private final Scan scan; + private final int batchSize; + + public ScanBasedStoreScanImpl(Scan scan, int batchSize) { + this.scan = scan; + this.batchSize = batchSize; + } + + @Override + public boolean reserveBatch(C cursor, KernelTransaction ktx) { + return scan.reserveBatch(cursor, batchSize, ktx.cursorContext(), ktx.securityContext().mode()); + } +} diff --git a/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/SettingProxyFactoryImpl.java b/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/SettingProxyFactoryImpl.java new file mode 100644 index 0000000000..ed12846cab --- /dev/null +++ b/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/SettingProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._513; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jVersion; +import org.neo4j.gds.compat.SettingProxyApi; +import org.neo4j.gds.compat.SettingProxyFactory; + +@ServiceProvider +public final class SettingProxyFactoryImpl implements SettingProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return version == Neo4jVersion.V_5_13; + } + + @Override + public SettingProxyApi load() { + return new SettingProxyImpl(); + } + + @Override + public String description() { + return "Neo4j Settings 5.13"; + } +} diff --git a/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/SettingProxyImpl.java b/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/SettingProxyImpl.java new file mode 100644 index 0000000000..7dd33b8b54 --- /dev/null +++ b/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/SettingProxyImpl.java @@ -0,0 +1,87 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._513; + +import org.neo4j.configuration.Config; +import org.neo4j.configuration.SettingBuilder; +import org.neo4j.dbms.systemgraph.TopologyGraphDbmsModel; +import org.neo4j.gds.compat.DatabaseMode; +import org.neo4j.gds.compat.SettingProxyApi; +import org.neo4j.graphdb.GraphDatabaseService; +import org.neo4j.graphdb.config.Setting; +import org.neo4j.kernel.impl.factory.GraphDatabaseFacade; +import org.neo4j.kernel.internal.GraphDatabaseAPI; + +public class SettingProxyImpl implements SettingProxyApi { + + @Override + public Setting setting(org.neo4j.gds.compat.Setting setting) { + var builder = SettingBuilder.newBuilder(setting.name(), setting.parser(), setting.defaultValue()); + if (setting.dynamic()) { + builder = builder.dynamic(); + } + if (setting.immutable()) { + builder = builder.immutable(); + } + setting.dependency().ifPresent(builder::setDependency); + setting.constraints().forEach(builder::addConstraint); + return builder.build(); + } + + @Override + public DatabaseMode databaseMode(Config config, GraphDatabaseService databaseService) { + return switch (((GraphDatabaseAPI) databaseService).mode()) { + case RAFT -> DatabaseMode.CORE; + case REPLICA -> DatabaseMode.READ_REPLICA; + case SINGLE -> DatabaseMode.SINGLE; + case VIRTUAL -> throw new UnsupportedOperationException("What's a virtual database anyway?"); + }; + } + + @Override + public void setDatabaseMode(Config config, DatabaseMode databaseMode, GraphDatabaseService databaseService) { + // super hacky, there is no way to set the mode of a database without restarting it + if (!(databaseService instanceof GraphDatabaseFacade db)) { + throw new IllegalArgumentException( + "Cannot set database mode on a database that is not a GraphDatabaseFacade"); + } + try { + var modeField = GraphDatabaseFacade.class.getDeclaredField("mode"); + modeField.setAccessible(true); + modeField.set(db, switch (databaseMode) { + case CORE -> TopologyGraphDbmsModel.HostedOnMode.RAFT; + case READ_REPLICA -> TopologyGraphDbmsModel.HostedOnMode.REPLICA; + case SINGLE -> TopologyGraphDbmsModel.HostedOnMode.SINGLE; + }); + } catch (NoSuchFieldException e) { + throw new RuntimeException( + "Could not set the mode field because it no longer exists. This compat layer needs to be updated.", + e + ); + } catch (IllegalAccessException e) { + throw new RuntimeException("Could not get the permissions to set the mode field.", e); + } + } + + @Override + public String secondaryModeName() { + return "Secondary"; + } +} diff --git a/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/TestLogImpl.java b/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/TestLogImpl.java new file mode 100644 index 0000000000..b6f3c9c50c --- /dev/null +++ b/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/TestLogImpl.java @@ -0,0 +1,146 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._513; + +import org.neo4j.gds.annotation.SuppressForbidden; +import org.neo4j.gds.compat.TestLog; + +import java.util.ArrayList; +import java.util.Locale; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.ConcurrentMap; + +public class TestLogImpl implements TestLog { + + private final ConcurrentMap> messages; + + TestLogImpl() { + messages = new ConcurrentHashMap<>(3); + } + + @Override + public void assertContainsMessage(String level, String fragment) { + if (!containsMessage(level, fragment)) { + throw new RuntimeException( + String.format( + Locale.US, + "Expected log output to contain `%s` for log level `%s`%nLog messages:%n%s", + fragment, + level, + String.join("\n", messages.get(level)) + ) + ); + } + } + + @Override + public boolean containsMessage(String level, String fragment) { + ConcurrentLinkedQueue messageList = messages.getOrDefault(level, new ConcurrentLinkedQueue<>()); + return messageList.stream().anyMatch((message) -> message.contains(fragment)); + } + + @Override + public boolean hasMessages(String level) { + return !messages.getOrDefault(level, new ConcurrentLinkedQueue<>()).isEmpty(); + } + + @Override + public ArrayList getMessages(String level) { + return new ArrayList<>(messages.getOrDefault(level, new ConcurrentLinkedQueue<>())); + } + + @SuppressForbidden(reason = "test log can print") + public void printMessages() { + System.out.println("TestLog Messages: " + messages); + } + + @Override + public boolean isDebugEnabled() { + return true; + } + + @Override + public void debug(String message) { + logMessage(DEBUG, message); + } + + @Override + public void debug(String message, Throwable throwable) { + debug(String.format(Locale.US, "%s - %s", message, throwable.getMessage())); + } + + @Override + public void debug(String format, Object... arguments) { + debug(String.format(Locale.US, format, arguments)); + } + + @Override + public void info(String message) { + logMessage(INFO, message); + } + + @Override + public void info(String message, Throwable throwable) { + info(String.format(Locale.US, "%s - %s", message, throwable.getMessage())); + } + + @Override + public void info(String format, Object... arguments) { + info(String.format(Locale.US, format, arguments)); + } + + @Override + public void warn(String message) { + logMessage(WARN, message); + } + + @Override + public void warn(String message, Throwable throwable) { + warn(String.format(Locale.US, "%s - %s", message, throwable.getMessage())); + } + + @Override + public void warn(String format, Object... arguments) { + warn(String.format(Locale.US, format, arguments)); + } + + @Override + public void error(String message) { + logMessage(ERROR, message); + } + + @Override + public void error(String message, Throwable throwable) { + error(String.format(Locale.US, "%s - %s", message, throwable.getMessage())); + } + + @Override + public void error(String format, Object... arguments) { + error(String.format(Locale.US, format, arguments)); + } + + private void logMessage(String level, String message) { + messages.computeIfAbsent( + level, + (ignore) -> new ConcurrentLinkedQueue<>() + ).add(message); + } +} diff --git a/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/VirtualRelationshipImpl.java b/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/VirtualRelationshipImpl.java new file mode 100644 index 0000000000..25372741c2 --- /dev/null +++ b/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/VirtualRelationshipImpl.java @@ -0,0 +1,41 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._513; + +import org.neo4j.gds.compat.VirtualRelationship; +import org.neo4j.graphdb.Node; +import org.neo4j.graphdb.RelationshipType; + +public class VirtualRelationshipImpl extends VirtualRelationship { + + VirtualRelationshipImpl( + long id, + Node startNode, + Node endNode, + RelationshipType type + ) { + super(id, startNode, endNode, type); + } + + @Override + public String getElementId() { + return Long.toString(getId()); + } +} diff --git a/compatibility/5.13/storage-engine-adapter/build.gradle b/compatibility/5.13/storage-engine-adapter/build.gradle new file mode 100644 index 0000000000..7326e65721 --- /dev/null +++ b/compatibility/5.13/storage-engine-adapter/build.gradle @@ -0,0 +1,68 @@ +apply plugin: 'java-library' +apply plugin: 'me.champeau.mrjar' + +description = 'Neo4j Graph Data Science :: Storage Engine Adapter 5.13' + +group = 'org.neo4j.gds' + +// for all 5.x versions +if (ver.'neo4j'.startsWith('5.')) { + sourceSets { + main { + java { + srcDirs = ['src/main/java17'] + } + } + } + + dependencies { + annotationProcessor project(':annotations') + annotationProcessor group: 'org.immutables', name: 'value', version: ver.'immutables' + annotationProcessor group: 'org.neo4j', name: 'annotations', version: neos.'5.13' + + compileOnly project(':annotations') + compileOnly project(':progress-tracking') + compileOnly group: 'org.immutables', name: 'value-annotations', version: ver.'immutables' + compileOnly group: 'org.neo4j', name: 'neo4j', version: neos.'5.13' + compileOnly group: 'org.neo4j', name: 'neo4j-record-storage-engine', version: neos.'5.13' + + implementation project(':core') + implementation project(':storage-engine-adapter-api') + implementation project(':config-api') + implementation project(':string-formatting') + } +} else { + multiRelease { + targetVersions 11, 17 + } + + if (!project.hasProperty('no-forbidden-apis')) { + forbiddenApisJava17 { + exclude('**') + } + } + + dependencies { + annotationProcessor group: 'org.neo4j', name: 'annotations', version: ver.'neo4j' + compileOnly group: 'org.neo4j', name: 'annotations', version: ver.'neo4j' + compileOnly group: 'org.neo4j', name: 'neo4j-kernel-api', version: ver.'neo4j' + + implementation project(':neo4j-adapter') + implementation project(':storage-engine-adapter-api') + + java17AnnotationProcessor project(':annotations') + java17AnnotationProcessor group: 'org.immutables', name: 'value', version: ver.'immutables' + java17AnnotationProcessor group: 'org.neo4j', name: 'annotations', version: neos.'5.13' + + java17CompileOnly project(':annotations') + java17CompileOnly project(':progress-tracking') + java17CompileOnly group: 'org.immutables', name: 'value-annotations', version: ver.'immutables' + java17CompileOnly group: 'org.neo4j', name: 'neo4j', version: neos.'5.13' + java17CompileOnly group: 'org.neo4j', name: 'neo4j-record-storage-engine', version: neos.'5.13' + + java17Implementation project(':core') + java17Implementation project(':storage-engine-adapter-api') + java17Implementation project(':config-api') + java17Implementation project(':string-formatting') + } +} diff --git a/compatibility/5.13/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_513/InMemoryStorageEngineFactory.java b/compatibility/5.13/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_513/InMemoryStorageEngineFactory.java new file mode 100644 index 0000000000..b2d569c80c --- /dev/null +++ b/compatibility/5.13/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_513/InMemoryStorageEngineFactory.java @@ -0,0 +1,268 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._513; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.configuration.Config; +import org.neo4j.dbms.database.readonly.DatabaseReadOnlyChecker; +import org.neo4j.index.internal.gbptree.RecoveryCleanupWorkCollector; +import org.neo4j.internal.id.IdController; +import org.neo4j.internal.id.IdGeneratorFactory; +import org.neo4j.internal.schema.IndexConfigCompleter; +import org.neo4j.internal.schema.SchemaRule; +import org.neo4j.internal.schema.SchemaState; +import org.neo4j.io.fs.FileSystemAbstraction; +import org.neo4j.io.layout.DatabaseLayout; +import org.neo4j.io.layout.Neo4jLayout; +import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.tracing.PageCacheTracer; +import org.neo4j.lock.LockService; +import org.neo4j.logging.LogProvider; +import org.neo4j.logging.internal.LogService; +import org.neo4j.memory.MemoryTracker; +import org.neo4j.monitoring.DatabaseHealth; +import org.neo4j.scheduler.JobScheduler; +import org.neo4j.storageengine.api.CommandReaderFactory; +import org.neo4j.storageengine.api.ConstraintRuleAccessor; +import org.neo4j.storageengine.api.LogVersionRepository; +import org.neo4j.storageengine.api.MetadataProvider; +import org.neo4j.storageengine.api.StorageEngine; +import org.neo4j.storageengine.api.StorageEngineFactory; +import org.neo4j.storageengine.api.StorageFilesState; +import org.neo4j.storageengine.api.StoreId; +import org.neo4j.storageengine.api.StoreVersion; +import org.neo4j.storageengine.api.StoreVersionCheck; +import org.neo4j.storageengine.api.TransactionIdStore; +import org.neo4j.storageengine.migration.RollingUpgradeCompatibility; +import org.neo4j.storageengine.migration.SchemaRuleMigrationAccess; +import org.neo4j.storageengine.migration.StoreMigrationParticipant; +import org.neo4j.token.TokenHolders; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.List; +import java.util.Optional; +import java.util.UUID; + +@ServiceProvider +public class InMemoryStorageEngineFactory implements StorageEngineFactory { + + @Override + public String name() { + return "unsupported513"; + } + + @Override + public StoreVersionCheck versionCheck( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + LogService logService, + PageCacheTracer pageCacheTracer + ) { + throw new UnsupportedOperationException("5.13 storage engine requires JDK17"); + } + + @Override + public StoreVersion versionInformation(String storeVersion) { + throw new UnsupportedOperationException("5.13 storage engine requires JDK17"); + } + + @Override + public StoreVersion versionInformation(StoreId storeId) { + throw new UnsupportedOperationException("5.13 storage engine requires JDK17"); + } + + @Override + public RollingUpgradeCompatibility rollingUpgradeCompatibility() { + throw new UnsupportedOperationException("5.13 storage engine requires JDK17"); + } + + @Override + public List migrationParticipants( + FileSystemAbstraction fs, + Config config, + PageCache pageCache, + JobScheduler jobScheduler, + LogService logService, + PageCacheTracer cacheTracer, + MemoryTracker memoryTracker + ) { + throw new UnsupportedOperationException("5.13 storage engine requires JDK17"); + } + + @Override + public StorageEngine instantiate( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + TokenHolders tokenHolders, + SchemaState schemaState, + ConstraintRuleAccessor constraintSemantics, + IndexConfigCompleter indexConfigCompleter, + LockService lockService, + IdGeneratorFactory idGeneratorFactory, + IdController idController, + DatabaseHealth databaseHealth, + LogProvider internalLogProvider, + LogProvider userLogProvider, + RecoveryCleanupWorkCollector recoveryCleanupWorkCollector, + PageCacheTracer cacheTracer, + boolean createStoreIfNotExists, + DatabaseReadOnlyChecker readOnlyChecker, + MemoryTracker memoryTracker + ) { + throw new UnsupportedOperationException("5.13 storage engine requires JDK17"); + } + + @Override + public List listStorageFiles(FileSystemAbstraction fileSystem, DatabaseLayout databaseLayout) throws + IOException { + throw new UnsupportedOperationException("5.13 storage engine requires JDK17"); + } + + @Override + public boolean storageExists(FileSystemAbstraction fileSystem, DatabaseLayout databaseLayout, PageCache pageCache) { + return false; + } + + @Override + public TransactionIdStore readOnlyTransactionIdStore( + FileSystemAbstraction filySystem, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) throws IOException { + throw new UnsupportedOperationException("5.13 storage engine requires JDK17"); + } + + @Override + public LogVersionRepository readOnlyLogVersionRepository( + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) throws IOException { + throw new UnsupportedOperationException("5.13 storage engine requires JDK17"); + } + + @Override + public MetadataProvider transactionMetaDataStore( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + PageCacheTracer cacheTracer, + DatabaseReadOnlyChecker readOnlyChecker + ) throws IOException { + throw new UnsupportedOperationException("5.13 storage engine requires JDK17"); + } + + @Override + public StoreId storeId( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) throws IOException { + throw new UnsupportedOperationException("5.13 storage engine requires JDK17"); + } + + @Override + public void setStoreId( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext, + StoreId storeId, + long upgradeTxChecksum, + long upgradeTxCommitTimestamp + ) throws IOException { + throw new UnsupportedOperationException("5.13 storage engine requires JDK17"); + } + + @Override + public void setExternalStoreUUID( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext, + UUID externalStoreId + ) throws IOException { + throw new UnsupportedOperationException("5.13 storage engine requires JDK17"); + } + + @Override + public Optional databaseIdUuid( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) { + throw new UnsupportedOperationException("5.13 storage engine requires JDK17"); + } + + @Override + public SchemaRuleMigrationAccess schemaRuleMigrationAccess( + FileSystemAbstraction fs, + PageCache pageCache, + Config config, + DatabaseLayout databaseLayout, + LogService logService, + String recordFormats, + PageCacheTracer cacheTracer, + CursorContext cursorContext, + MemoryTracker memoryTracker + ) { + throw new UnsupportedOperationException("5.13 storage engine requires JDK17"); + } + + @Override + public List loadSchemaRules( + FileSystemAbstraction fs, + PageCache pageCache, + Config config, + DatabaseLayout databaseLayout, + CursorContext cursorContext + ) { + throw new UnsupportedOperationException("5.13 storage engine requires JDK17"); + } + + @Override + public StorageFilesState checkStoreFileState( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache + ) { + throw new UnsupportedOperationException("5.13 storage engine requires JDK17"); + } + + @Override + public CommandReaderFactory commandReaderFactory() { + throw new UnsupportedOperationException("5.13 storage engine requires JDK17"); + } + + @Override + public DatabaseLayout databaseLayout(Neo4jLayout neo4jLayout, String databaseName) { + throw new UnsupportedOperationException("5.13 storage engine requires JDK17"); + } +} diff --git a/compatibility/5.13/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_513/StorageEngineProxyFactoryImpl.java b/compatibility/5.13/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_513/StorageEngineProxyFactoryImpl.java new file mode 100644 index 0000000000..85fbf8d91c --- /dev/null +++ b/compatibility/5.13/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_513/StorageEngineProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._513; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jVersion; +import org.neo4j.gds.compat.StorageEngineProxyApi; +import org.neo4j.gds.compat.StorageEngineProxyFactory; + +@ServiceProvider +public class StorageEngineProxyFactoryImpl implements StorageEngineProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return false; + } + + @Override + public StorageEngineProxyApi load() { + throw new UnsupportedOperationException("5.13 storage engine requires JDK17"); + } + + @Override + public String description() { + return "Storage Engine 5.13"; + } +} diff --git a/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/InMemoryCommandCreationContextImpl.java b/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/InMemoryCommandCreationContextImpl.java new file mode 100644 index 0000000000..1662f5761e --- /dev/null +++ b/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/InMemoryCommandCreationContextImpl.java @@ -0,0 +1,107 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._513; + +import org.neo4j.configuration.Config; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.kernel.KernelVersion; +import org.neo4j.kernel.KernelVersionProvider; +import org.neo4j.lock.LockTracer; +import org.neo4j.lock.ResourceLocker; +import org.neo4j.storageengine.api.CommandCreationContext; +import org.neo4j.storageengine.api.cursor.StoreCursors; + +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Supplier; + +public class InMemoryCommandCreationContextImpl implements CommandCreationContext { + + private final AtomicLong schemaTokens; + private final AtomicInteger propertyTokens; + private final AtomicInteger labelTokens; + private final AtomicInteger typeTokens; + + InMemoryCommandCreationContextImpl() { + this.schemaTokens = new AtomicLong(0); + this.propertyTokens = new AtomicInteger(0); + this.labelTokens = new AtomicInteger(0); + this.typeTokens = new AtomicInteger(0); + } + + @Override + public long reserveNode() { + throw new UnsupportedOperationException("Creating nodes is not supported"); + } + + @Override + public long reserveRelationship( + long sourceNode, + long targetNode, + int relationshipType, + boolean sourceNodeAddedInTx, + boolean targetNodeAddedInTx + ) { + throw new UnsupportedOperationException("Creating relationships is not supported"); + } + + @Override + public long reserveSchema() { + return schemaTokens.getAndIncrement(); + } + + @Override + public int reserveLabelTokenId() { + return labelTokens.getAndIncrement(); + } + + @Override + public int reservePropertyKeyTokenId() { + return propertyTokens.getAndIncrement(); + } + + @Override + public int reserveRelationshipTypeTokenId() { + return typeTokens.getAndIncrement(); + } + + @Override + public void close() { + + } + + @Override + public void initialize( + KernelVersionProvider kernelVersionProvider, + CursorContext cursorContext, + StoreCursors storeCursors, + Supplier oldestActiveTransactionSequenceNumber, + ResourceLocker locks, + Supplier lockTracer + ) { + + } + + @Override + public KernelVersion kernelVersion() { + // NOTE: Double-check if this is still correct when you copy this into a new compat layer + return KernelVersion.getLatestVersion(Config.newBuilder().build()); + } +} diff --git a/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/InMemoryCountsStoreImpl.java b/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/InMemoryCountsStoreImpl.java new file mode 100644 index 0000000000..149e6c2458 --- /dev/null +++ b/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/InMemoryCountsStoreImpl.java @@ -0,0 +1,111 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._513; + +import org.neo4j.annotations.documented.ReporterFactory; +import org.neo4j.counts.CountsStore; +import org.neo4j.counts.CountsUpdater; +import org.neo4j.counts.CountsVisitor; +import org.neo4j.gds.NodeLabel; +import org.neo4j.gds.api.GraphStore; +import org.neo4j.internal.helpers.progress.ProgressMonitorFactory; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.context.CursorContextFactory; +import org.neo4j.io.pagecache.tracing.FileFlushEvent; +import org.neo4j.memory.MemoryTracker; +import org.neo4j.token.TokenHolders; +import org.neo4j.token.api.TokenNotFoundException; + +import java.io.IOException; + +public class InMemoryCountsStoreImpl implements CountsStore { + + private final GraphStore graphStore; + private final TokenHolders tokenHolders; + + public InMemoryCountsStoreImpl( + GraphStore graphStore, + TokenHolders tokenHolders + ) { + + this.graphStore = graphStore; + this.tokenHolders = tokenHolders; + } + + @Override + public void start(CursorContext cursorContext, MemoryTracker memoryTracker) throws IOException { + + } + + @Override + public CountsUpdater updater(long txId, boolean isLast, CursorContext cursorContext) { + throw new UnsupportedOperationException("Updates are not supported"); + } + + @Override + public void checkpoint(FileFlushEvent fileFlushEvent, CursorContext cursorContext) { + + } + + @Override + public long nodeCount(int labelId, CursorContext cursorContext) { + if (labelId == -1) { + return graphStore.nodeCount(); + } + + String nodeLabel; + try { + nodeLabel = tokenHolders.labelTokens().getTokenById(labelId).name(); + } catch (TokenNotFoundException e) { + throw new RuntimeException(e); + } + return graphStore.nodes().nodeCount(NodeLabel.of(nodeLabel)); + } + + @Override + public long relationshipCount(int startLabelId, int typeId, int endLabelId, CursorContext cursorContext) { + // TODO: this is quite wrong + return graphStore.relationshipCount(); + } + + @Override + public boolean consistencyCheck( + ReporterFactory reporterFactory, + CursorContextFactory cursorContextFactory, + int i, + ProgressMonitorFactory progressMonitorFactory + ) { + return true; + } + + @Override + public void close() { + + } + + @Override + public void accept(CountsVisitor visitor, CursorContext cursorContext) { + tokenHolders.labelTokens().getAllTokens().forEach(labelToken -> { + visitor.visitNodeCount(labelToken.id(), nodeCount(labelToken.id(), cursorContext)); + }); + + visitor.visitRelationshipCount(-1, -1, -1, graphStore.relationshipCount()); + } +} diff --git a/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/InMemoryMetaDataProviderImpl.java b/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/InMemoryMetaDataProviderImpl.java new file mode 100644 index 0000000000..48585f772e --- /dev/null +++ b/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/InMemoryMetaDataProviderImpl.java @@ -0,0 +1,201 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._513; + +import org.neo4j.internal.recordstorage.InMemoryLogVersionRepository513; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.context.TransactionIdSnapshot; +import org.neo4j.storageengine.api.ClosedTransactionMetadata; +import org.neo4j.storageengine.api.ExternalStoreId; +import org.neo4j.storageengine.api.MetadataProvider; +import org.neo4j.storageengine.api.StoreId; +import org.neo4j.storageengine.api.TransactionId; + +import java.io.IOException; +import java.util.Optional; +import java.util.UUID; + +public class InMemoryMetaDataProviderImpl implements MetadataProvider { + + private final ExternalStoreId externalStoreId; + private final InMemoryLogVersionRepository513 logVersionRepository; + private final InMemoryTransactionIdStoreImpl transactionIdStore; + + InMemoryMetaDataProviderImpl() { + this.logVersionRepository = new InMemoryLogVersionRepository513(); + this.externalStoreId = new ExternalStoreId(UUID.randomUUID()); + this.transactionIdStore = new InMemoryTransactionIdStoreImpl(); + } + + @Override + public ExternalStoreId getExternalStoreId() { + return this.externalStoreId; + } + + @Override + public ClosedTransactionMetadata getLastClosedTransaction() { + return this.transactionIdStore.getLastClosedTransaction(); + } + + @Override + public void setCurrentLogVersion(long version) { + logVersionRepository.setCurrentLogVersion(version); + } + + @Override + public long incrementAndGetVersion() { + return logVersionRepository.incrementAndGetVersion(); + } + + @Override + public void setCheckpointLogVersion(long version) { + logVersionRepository.setCheckpointLogVersion(version); + } + + @Override + public long incrementAndGetCheckpointLogVersion() { + return logVersionRepository.incrementAndGetCheckpointLogVersion(); + } + + @Override + public void transactionCommitted(long transactionId, int checksum, long commitTimestamp, long consensusIndex) { + transactionIdStore.transactionCommitted(transactionId, checksum, commitTimestamp, consensusIndex); + } + + @Override + public void setLastCommittedAndClosedTransactionId( + long transactionId, + int checksum, + long commitTimestamp, + long consensusIndex, + long byteOffset, + long logVersion + ) { + transactionIdStore.setLastCommittedAndClosedTransactionId( + transactionId, + checksum, + commitTimestamp, + consensusIndex, + byteOffset, + logVersion + ); + } + + @Override + public void transactionClosed( + long transactionId, + long logVersion, + long byteOffset, + int checksum, + long commitTimestamp, + long consensusIndex + ) { + this.transactionIdStore.transactionClosed( + transactionId, + logVersion, + byteOffset, + checksum, + commitTimestamp, + consensusIndex + ); + } + + @Override + public void resetLastClosedTransaction( + long transactionId, + long logVersion, + long byteOffset, + int checksum, + long commitTimestamp, + long consensusIndex + ) { + this.transactionIdStore.resetLastClosedTransaction( + transactionId, + logVersion, + byteOffset, + checksum, + commitTimestamp, + consensusIndex + ); + } + + @Override + public TransactionIdSnapshot getClosedTransactionSnapshot() { + return new TransactionIdSnapshot(this.getLastClosedTransactionId()); + } + + @Override + public void regenerateMetadata(StoreId storeId, UUID externalStoreUUID, CursorContext cursorContext) { + } + + @Override + public StoreId getStoreId() { + return StoreId.UNKNOWN; + } + + @Override + public void close() throws IOException { + } + + @Override + public long getCurrentLogVersion() { + return this.logVersionRepository.getCurrentLogVersion(); + } + + @Override + public long getCheckpointLogVersion() { + return this.logVersionRepository.getCheckpointLogVersion(); + } + + @Override + public long nextCommittingTransactionId() { + return this.transactionIdStore.nextCommittingTransactionId(); + } + + @Override + public long committingTransactionId() { + return this.transactionIdStore.committingTransactionId(); + } + + @Override + public long getLastCommittedTransactionId() { + return this.transactionIdStore.getLastCommittedTransactionId(); + } + + @Override + public TransactionId getLastCommittedTransaction() { + return this.transactionIdStore.getLastCommittedTransaction(); + } + + @Override + public long getLastClosedTransactionId() { + return this.transactionIdStore.getLastClosedTransactionId(); + } + + @Override + public Optional getDatabaseIdUuid(CursorContext cursorTracer) { + throw new IllegalStateException("Not supported"); + } + + @Override + public void setDatabaseIdUuid(UUID uuid, CursorContext cursorContext) { + throw new IllegalStateException("Not supported"); + } +} diff --git a/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/InMemoryNodeCursor.java b/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/InMemoryNodeCursor.java new file mode 100644 index 0000000000..02c848b6e4 --- /dev/null +++ b/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/InMemoryNodeCursor.java @@ -0,0 +1,82 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._513; + +import org.neo4j.gds.api.GraphStore; +import org.neo4j.gds.compat.AbstractInMemoryNodeCursor; +import org.neo4j.storageengine.api.AllNodeScan; +import org.neo4j.storageengine.api.Degrees; +import org.neo4j.storageengine.api.LongReference; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.storageengine.api.RelationshipSelection; +import org.neo4j.storageengine.api.StoragePropertyCursor; +import org.neo4j.storageengine.api.StorageRelationshipTraversalCursor; +import org.neo4j.token.TokenHolders; + +public class InMemoryNodeCursor extends AbstractInMemoryNodeCursor { + + public InMemoryNodeCursor(GraphStore graphStore, TokenHolders tokenHolders) { + super(graphStore, tokenHolders); + } + + @Override + public boolean hasLabel() { + return hasAtLeastOneLabelForCurrentNode(); + } + + @Override + public Reference propertiesReference() { + return LongReference.longReference(getId()); + } + + @Override + public void properties(StoragePropertyCursor propertyCursor, PropertySelection selection) { + propertyCursor.initNodeProperties(propertiesReference(), selection); + } + + @Override + public void properties(StoragePropertyCursor propertyCursor) { + properties(propertyCursor, PropertySelection.ALL_PROPERTIES); + } + + @Override + public boolean supportsFastRelationshipsTo() { + return false; + } + + @Override + public void relationshipsTo( + StorageRelationshipTraversalCursor storageRelationshipTraversalCursor, + RelationshipSelection relationshipSelection, + long neighbourNodeReference + ) { + throw new UnsupportedOperationException(); + } + + @Override + public void degrees(RelationshipSelection selection, Degrees.Mutator mutator) { + } + + @Override + public boolean scanBatch(AllNodeScan allNodeScan, long sizeHint) { + return super.scanBatch(allNodeScan, (int) sizeHint); + } +} diff --git a/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/InMemoryNodePropertyCursor.java b/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/InMemoryNodePropertyCursor.java new file mode 100644 index 0000000000..63ef42897f --- /dev/null +++ b/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/InMemoryNodePropertyCursor.java @@ -0,0 +1,45 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._513; + +import org.neo4j.gds.compat.AbstractInMemoryNodePropertyCursor; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.storageengine.api.LongReference; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.token.TokenHolders; + +public class InMemoryNodePropertyCursor extends AbstractInMemoryNodePropertyCursor { + + public InMemoryNodePropertyCursor(CypherGraphStore graphStore, TokenHolders tokenHolders) { + super(graphStore, tokenHolders); + } + + @Override + public void initNodeProperties(Reference reference, PropertySelection selection, long ownerReference) { + reset(); + setId(((LongReference) reference).id); + setPropertySelection(new InMemoryPropertySelectionImpl(selection)); + } + + @Override + public void initRelationshipProperties(Reference reference, PropertySelection selection, long ownerReference) { + } +} diff --git a/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/InMemoryPropertyCursor.java b/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/InMemoryPropertyCursor.java new file mode 100644 index 0000000000..a44b5cbaab --- /dev/null +++ b/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/InMemoryPropertyCursor.java @@ -0,0 +1,71 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._513; + +import org.neo4j.gds.compat.AbstractInMemoryPropertyCursor; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.storageengine.api.StorageNodeCursor; +import org.neo4j.storageengine.api.StorageRelationshipCursor; +import org.neo4j.token.TokenHolders; + +public class InMemoryPropertyCursor extends AbstractInMemoryPropertyCursor { + + public InMemoryPropertyCursor(CypherGraphStore graphStore, TokenHolders tokenHolders) { + super(graphStore, tokenHolders); + } + + @Override + public void initNodeProperties(Reference reference, PropertySelection selection, long ownerReference) { + if (this.delegate == null || !(this.delegate instanceof InMemoryNodePropertyCursor)) { + this.delegate = new InMemoryNodePropertyCursor(graphStore, tokenHolders); + } + + ((InMemoryNodePropertyCursor) delegate).initNodeProperties(reference, selection); + } + + @Override + public void initNodeProperties(StorageNodeCursor nodeCursor, PropertySelection selection) { + if (this.delegate == null || !(this.delegate instanceof InMemoryNodePropertyCursor)) { + this.delegate = new InMemoryNodePropertyCursor(graphStore, tokenHolders); + } + + ((InMemoryNodePropertyCursor) delegate).initNodeProperties(nodeCursor, selection); + } + + @Override + public void initRelationshipProperties(StorageRelationshipCursor relationshipCursor, PropertySelection selection) { + if (this.delegate == null || !(this.delegate instanceof InMemoryRelationshipPropertyCursor)) { + this.delegate = new InMemoryRelationshipPropertyCursor(graphStore, tokenHolders); + } + + ((InMemoryRelationshipPropertyCursor) delegate).initRelationshipProperties(relationshipCursor, selection); + } + + @Override + public void initRelationshipProperties(Reference reference, PropertySelection selection, long ownerReference) { + if (this.delegate == null || !(this.delegate instanceof InMemoryRelationshipPropertyCursor)) { + this.delegate = new InMemoryRelationshipPropertyCursor(graphStore, tokenHolders); + } + + ((InMemoryRelationshipPropertyCursor) delegate).initRelationshipProperties(reference, selection); + } +} diff --git a/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/InMemoryPropertySelectionImpl.java b/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/InMemoryPropertySelectionImpl.java new file mode 100644 index 0000000000..60e592442c --- /dev/null +++ b/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/InMemoryPropertySelectionImpl.java @@ -0,0 +1,55 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._513; + +import org.neo4j.gds.compat.InMemoryPropertySelection; +import org.neo4j.storageengine.api.PropertySelection; + +public class InMemoryPropertySelectionImpl implements InMemoryPropertySelection { + + private final PropertySelection propertySelection; + + public InMemoryPropertySelectionImpl(PropertySelection propertySelection) {this.propertySelection = propertySelection;} + + @Override + public boolean isLimited() { + return propertySelection.isLimited(); + } + + @Override + public int numberOfKeys() { + return propertySelection.numberOfKeys(); + } + + @Override + public int key(int index) { + return propertySelection.key(index); + } + + @Override + public boolean test(int key) { + return propertySelection.test(key); + } + + @Override + public boolean isKeysOnly() { + return propertySelection.isKeysOnly(); + } +} diff --git a/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/InMemoryRelationshipPropertyCursor.java b/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/InMemoryRelationshipPropertyCursor.java new file mode 100644 index 0000000000..6571e0187a --- /dev/null +++ b/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/InMemoryRelationshipPropertyCursor.java @@ -0,0 +1,60 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._513; + +import org.neo4j.gds.compat.AbstractInMemoryRelationshipPropertyCursor; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.gds.storageengine.InMemoryRelationshipCursor; +import org.neo4j.storageengine.api.LongReference; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.storageengine.api.StorageRelationshipCursor; +import org.neo4j.token.TokenHolders; + +public class InMemoryRelationshipPropertyCursor extends AbstractInMemoryRelationshipPropertyCursor { + + InMemoryRelationshipPropertyCursor(CypherGraphStore graphStore, TokenHolders tokenHolders) { + super(graphStore, tokenHolders); + } + + @Override + public void initNodeProperties( + Reference reference, PropertySelection propertySelection, long ownerReference + ) { + + } + + @Override + public void initRelationshipProperties( + Reference reference, PropertySelection propertySelection, long ownerReference + ) { + var relationshipId = ((LongReference) reference).id; + var relationshipCursor = new InMemoryRelationshipScanCursor(graphStore, tokenHolders); + relationshipCursor.single(relationshipId); + relationshipCursor.next(); + relationshipCursor.properties(this, new InMemoryPropertySelectionImpl(propertySelection)); + } + + @Override + public void initRelationshipProperties(StorageRelationshipCursor relationshipCursor, PropertySelection selection) { + var inMemoryRelationshipCursor = (InMemoryRelationshipCursor) relationshipCursor; + inMemoryRelationshipCursor.properties(this, selection); + } +} diff --git a/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/InMemoryRelationshipScanCursor.java b/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/InMemoryRelationshipScanCursor.java new file mode 100644 index 0000000000..2e6afb679b --- /dev/null +++ b/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/InMemoryRelationshipScanCursor.java @@ -0,0 +1,61 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._513; + +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.internal.recordstorage.AbstractInMemoryRelationshipScanCursor; +import org.neo4j.storageengine.api.AllRelationshipsScan; +import org.neo4j.storageengine.api.LongReference; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.storageengine.api.StoragePropertyCursor; +import org.neo4j.token.TokenHolders; + +public class InMemoryRelationshipScanCursor extends AbstractInMemoryRelationshipScanCursor { + + public InMemoryRelationshipScanCursor( + CypherGraphStore graphStore, + TokenHolders tokenHolders + ) { + super(graphStore, tokenHolders); + } + + @Override + public void single(long reference, long sourceNodeReference, int type, long targetNodeReference) { + single(reference); + } + + @Override + public Reference propertiesReference() { + return LongReference.longReference(getId()); + } + + @Override + public void properties( + StoragePropertyCursor storagePropertyCursor, PropertySelection propertySelection + ) { + properties(storagePropertyCursor, new InMemoryPropertySelectionImpl(propertySelection)); + } + + @Override + public boolean scanBatch(AllRelationshipsScan allRelationshipsScan, long sizeHint) { + return super.scanBatch(allRelationshipsScan, (int) sizeHint); + } +} diff --git a/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/InMemoryRelationshipTraversalCursor.java b/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/InMemoryRelationshipTraversalCursor.java new file mode 100644 index 0000000000..41a33a5f44 --- /dev/null +++ b/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/InMemoryRelationshipTraversalCursor.java @@ -0,0 +1,47 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._513; + +import org.neo4j.gds.compat.AbstractInMemoryRelationshipTraversalCursor; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.storageengine.api.LongReference; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.storageengine.api.StoragePropertyCursor; +import org.neo4j.token.TokenHolders; + +public class InMemoryRelationshipTraversalCursor extends AbstractInMemoryRelationshipTraversalCursor { + + public InMemoryRelationshipTraversalCursor(CypherGraphStore graphStore, TokenHolders tokenHolders) { + super(graphStore, tokenHolders); + } + + @Override + public Reference propertiesReference() { + return LongReference.longReference(getId()); + } + + @Override + public void properties( + StoragePropertyCursor propertyCursor, PropertySelection selection + ) { + properties(propertyCursor, new InMemoryPropertySelectionImpl(selection)); + } +} diff --git a/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/InMemoryStorageEngineFactory.java b/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/InMemoryStorageEngineFactory.java new file mode 100644 index 0000000000..7c098e6f19 --- /dev/null +++ b/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/InMemoryStorageEngineFactory.java @@ -0,0 +1,572 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._513; + +import org.eclipse.collections.api.factory.Sets; +import org.eclipse.collections.api.set.ImmutableSet; +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.configuration.Config; +import org.neo4j.consistency.checking.ConsistencyFlags; +import org.neo4j.consistency.report.ConsistencySummaryStatistics; +import org.neo4j.dbms.database.readonly.DatabaseReadOnlyChecker; +import org.neo4j.function.ThrowingSupplier; +import org.neo4j.gds.annotation.SuppressForbidden; +import org.neo4j.gds.compat.Neo4jVersion; +import org.neo4j.gds.compat.StorageEngineFactoryIdProvider; +import org.neo4j.gds.compat.StorageEngineProxyApi; +import org.neo4j.index.internal.gbptree.RecoveryCleanupWorkCollector; +import org.neo4j.internal.batchimport.AdditionalInitialIds; +import org.neo4j.internal.batchimport.BatchImporter; +import org.neo4j.internal.batchimport.Configuration; +import org.neo4j.internal.batchimport.IncrementalBatchImporter; +import org.neo4j.internal.batchimport.IndexImporterFactory; +import org.neo4j.internal.batchimport.Monitor; +import org.neo4j.internal.batchimport.ReadBehaviour; +import org.neo4j.internal.batchimport.input.Collector; +import org.neo4j.internal.batchimport.input.Input; +import org.neo4j.internal.batchimport.input.LenientStoreInput; +import org.neo4j.internal.id.IdGeneratorFactory; +import org.neo4j.internal.id.ScanOnOpenReadOnlyIdGeneratorFactory; +import org.neo4j.internal.recordstorage.InMemoryStorageCommandReaderFactory513; +import org.neo4j.internal.recordstorage.StoreTokens; +import org.neo4j.internal.schema.IndexConfigCompleter; +import org.neo4j.internal.schema.SchemaRule; +import org.neo4j.internal.schema.SchemaState; +import org.neo4j.io.fs.FileSystemAbstraction; +import org.neo4j.io.layout.DatabaseLayout; +import org.neo4j.io.layout.Neo4jLayout; +import org.neo4j.io.layout.recordstorage.RecordDatabaseLayout; +import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.context.CursorContextFactory; +import org.neo4j.io.pagecache.tracing.PageCacheTracer; +import org.neo4j.kernel.KernelVersionRepository; +import org.neo4j.kernel.api.index.IndexProvidersAccess; +import org.neo4j.kernel.impl.api.index.IndexProviderMap; +import org.neo4j.kernel.impl.locking.LockManager; +import org.neo4j.kernel.impl.store.MetaDataStore; +import org.neo4j.kernel.impl.store.NeoStores; +import org.neo4j.kernel.impl.store.StoreFactory; +import org.neo4j.kernel.impl.store.StoreType; +import org.neo4j.kernel.impl.store.cursor.CachedStoreCursors; +import org.neo4j.kernel.impl.transaction.log.LogTailLogVersionsMetadata; +import org.neo4j.kernel.impl.transaction.log.LogTailMetadata; +import org.neo4j.lock.LockService; +import org.neo4j.logging.InternalLog; +import org.neo4j.logging.InternalLogProvider; +import org.neo4j.logging.NullLogProvider; +import org.neo4j.logging.internal.LogService; +import org.neo4j.memory.MemoryTracker; +import org.neo4j.monitoring.DatabaseHealth; +import org.neo4j.scheduler.JobScheduler; +import org.neo4j.storageengine.api.CommandReaderFactory; +import org.neo4j.storageengine.api.ConstraintRuleAccessor; +import org.neo4j.storageengine.api.LogFilesInitializer; +import org.neo4j.storageengine.api.MetadataProvider; +import org.neo4j.storageengine.api.SchemaRule44; +import org.neo4j.storageengine.api.StorageEngine; +import org.neo4j.storageengine.api.StorageEngineFactory; +import org.neo4j.storageengine.api.StorageFilesState; +import org.neo4j.storageengine.api.StoreId; +import org.neo4j.storageengine.api.StoreVersion; +import org.neo4j.storageengine.api.StoreVersionCheck; +import org.neo4j.storageengine.api.StoreVersionIdentifier; +import org.neo4j.storageengine.migration.SchemaRuleMigrationAccessExtended; +import org.neo4j.storageengine.migration.StoreMigrationParticipant; +import org.neo4j.time.SystemNanoClock; +import org.neo4j.token.DelegatingTokenHolder; +import org.neo4j.token.ReadOnlyTokenCreator; +import org.neo4j.token.TokenHolders; +import org.neo4j.token.api.NamedToken; +import org.neo4j.token.api.TokenHolder; +import org.neo4j.token.api.TokensLoader; + +import java.io.IOException; +import java.io.OutputStream; +import java.io.PrintStream; +import java.io.UncheckedIOException; +import java.nio.file.OpenOption; +import java.nio.file.Path; +import java.time.Clock; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.UUID; +import java.util.function.Function; + +@ServiceProvider +public class InMemoryStorageEngineFactory implements StorageEngineFactory { + + static final String IN_MEMORY_STORAGE_ENGINE_NAME = "in-memory-513"; + + public InMemoryStorageEngineFactory() { + StorageEngineProxyApi.requireNeo4jVersion(Neo4jVersion.V_5_13, StorageEngineFactory.class); + } + + @Override + public byte id() { + return StorageEngineFactoryIdProvider.ID; + } + + @Override + public boolean storageExists(FileSystemAbstraction fileSystem, DatabaseLayout databaseLayout) { + return false; + } + + @Override + public StorageEngine instantiate( + FileSystemAbstraction fs, + Clock clock, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + TokenHolders tokenHolders, + SchemaState schemaState, + ConstraintRuleAccessor constraintSemantics, + IndexConfigCompleter indexConfigCompleter, + LockService lockService, + IdGeneratorFactory idGeneratorFactory, + DatabaseHealth databaseHealth, + InternalLogProvider internalLogProvider, + InternalLogProvider userLogProvider, + RecoveryCleanupWorkCollector recoveryCleanupWorkCollector, + LogTailMetadata logTailMetadata, + KernelVersionRepository kernelVersionRepository, + MemoryTracker memoryTracker, + CursorContextFactory contextFactory, + PageCacheTracer pageCacheTracer + ) { + StoreFactory factory = new StoreFactory( + databaseLayout, + config, + idGeneratorFactory, + pageCache, + pageCacheTracer, + fs, + internalLogProvider, + contextFactory, + false, + logTailMetadata + ); + + factory.openNeoStores(StoreType.LABEL_TOKEN).close(); + + return new InMemoryStorageEngineImpl( + databaseLayout, + tokenHolders + ); + } + + @Override + public Optional databaseIdUuid( + FileSystemAbstraction fs, DatabaseLayout databaseLayout, PageCache pageCache, CursorContext cursorContext + ) { + var fieldAccess = MetaDataStore.getFieldAccess( + pageCache, + RecordDatabaseLayout.convert(databaseLayout).metadataStore(), + databaseLayout.getDatabaseName(), + cursorContext + ); + + try { + return fieldAccess.readDatabaseUUID(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + @Override + public List migrationParticipants( + FileSystemAbstraction fileSystemAbstraction, + Config config, + PageCache pageCache, + JobScheduler jobScheduler, + LogService logService, + MemoryTracker memoryTracker, + PageCacheTracer pageCacheTracer, + CursorContextFactory cursorContextFactory, + boolean b + ) { + return List.of(); + } + + @Override + public DatabaseLayout databaseLayout( + Neo4jLayout neo4jLayout, String databaseName + ) { + return RecordDatabaseLayout.of(neo4jLayout, databaseName); + } + + @Override + public DatabaseLayout formatSpecificDatabaseLayout(DatabaseLayout plainLayout) { + return databaseLayout(plainLayout.getNeo4jLayout(), plainLayout.getDatabaseName()); + } + + @SuppressForbidden(reason = "This is the compat layer and we don't really need to go through the proxy") + @Override + public BatchImporter batchImporter( + DatabaseLayout databaseLayout, + FileSystemAbstraction fileSystemAbstraction, + PageCacheTracer pageCacheTracer, + Configuration configuration, + LogService logService, + PrintStream printStream, + boolean b, + AdditionalInitialIds additionalInitialIds, + Config config, + Monitor monitor, + JobScheduler jobScheduler, + Collector collector, + LogFilesInitializer logFilesInitializer, + IndexImporterFactory indexImporterFactory, + MemoryTracker memoryTracker, + CursorContextFactory cursorContextFactory + ) { + throw new UnsupportedOperationException("Batch Import into GDS is not supported"); + } + + @Override + public Input asBatchImporterInput( + DatabaseLayout databaseLayout, + FileSystemAbstraction fileSystemAbstraction, + PageCache pageCache, + PageCacheTracer pageCacheTracer, + Config config, + MemoryTracker memoryTracker, + ReadBehaviour readBehaviour, + boolean b, + CursorContextFactory cursorContextFactory, + LogTailMetadata logTailMetadata + ) { + NeoStores neoStores = (new StoreFactory( + databaseLayout, + config, + new ScanOnOpenReadOnlyIdGeneratorFactory(), + pageCache, + pageCacheTracer, + fileSystemAbstraction, + NullLogProvider.getInstance(), + cursorContextFactory, + false, + logTailMetadata + )).openAllNeoStores(); + return new LenientStoreInput( + neoStores, + readBehaviour.decorateTokenHolders(this.loadReadOnlyTokens(neoStores, true, cursorContextFactory)), + true, + cursorContextFactory, + readBehaviour + ); + } + + @Override + public long optimalAvailableConsistencyCheckerMemory( + FileSystemAbstraction fileSystemAbstraction, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache + ) { + return 0; + } + + @Override + public String name() { + return IN_MEMORY_STORAGE_ENGINE_NAME; + } + + @Override + public Set supportedFormats(boolean includeFormatsUnderDevelopment) { + return Set.of(IN_MEMORY_STORAGE_ENGINE_NAME); + } + + @Override + public boolean supportedFormat(String format, boolean includeFormatsUnderDevelopment) { + return format.equals(IN_MEMORY_STORAGE_ENGINE_NAME); + } + + @Override + public MetadataProvider transactionMetaDataStore( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + DatabaseReadOnlyChecker readOnlyChecker, + CursorContextFactory contextFactory, + LogTailLogVersionsMetadata logTailMetadata, + PageCacheTracer pageCacheTracer + ) throws IOException { + return new InMemoryMetaDataProviderImpl(); + } + + @Override + public StoreVersionCheck versionCheck( + FileSystemAbstraction fileSystemAbstraction, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + LogService logService, + CursorContextFactory cursorContextFactory + ) { + return new InMemoryVersionCheck(); + } + + @Override + public List loadSchemaRules( + FileSystemAbstraction fileSystemAbstraction, + PageCache pageCache, + PageCacheTracer pageCacheTracer, + Config config, + DatabaseLayout databaseLayout, + boolean b, + Function function, + CursorContextFactory cursorContextFactory + ) { + return List.of(); + } + + @Override + public List load44SchemaRules( + FileSystemAbstraction fs, + PageCache pageCache, + PageCacheTracer pageCacheTracer, + Config config, + DatabaseLayout databaseLayout, + CursorContextFactory contextFactory, + LogTailLogVersionsMetadata logTailMetadata + ) { + return List.of(); + } + + @Override + public TokenHolders loadReadOnlyTokens( + FileSystemAbstraction fileSystemAbstraction, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + PageCacheTracer pageCacheTracer, + boolean lenient, + CursorContextFactory cursorContextFactory + ) { + StoreFactory factory = new StoreFactory( + databaseLayout, + config, + new ScanOnOpenReadOnlyIdGeneratorFactory(), + pageCache, + pageCacheTracer, + fileSystemAbstraction, + NullLogProvider.getInstance(), + cursorContextFactory, + false, + LogTailMetadata.EMPTY_LOG_TAIL + ); + try ( + NeoStores stores = factory.openNeoStores( + StoreType.PROPERTY_KEY_TOKEN, StoreType.PROPERTY_KEY_TOKEN_NAME, + StoreType.LABEL_TOKEN, StoreType.LABEL_TOKEN_NAME, + StoreType.RELATIONSHIP_TYPE_TOKEN, StoreType.RELATIONSHIP_TYPE_TOKEN_NAME + ) + ) { + return loadReadOnlyTokens(stores, lenient, cursorContextFactory); + } + } + + @Override + public SchemaRuleMigrationAccessExtended schemaRuleMigrationAccess( + FileSystemAbstraction fs, + PageCache pageCache, + PageCacheTracer pageCacheTracer, + Config config, + DatabaseLayout databaseLayout, + CursorContextFactory contextFactory, + MemoryTracker memoryTracker, + LogTailMetadata logTail + ) { + // this is used by store copy, which is not supported for GDS storage engine + return null; + } + + private TokenHolders loadReadOnlyTokens( + NeoStores stores, + boolean lenient, + CursorContextFactory cursorContextFactory + ) { + try ( + var cursorContext = cursorContextFactory.create("loadReadOnlyTokens"); + var storeCursors = new CachedStoreCursors(stores, cursorContext) + ) { + stores.start( cursorContext ); + TokensLoader loader = lenient ? StoreTokens.allReadableTokens( stores ) : StoreTokens.allTokens( stores ); + TokenHolder propertyKeys = new DelegatingTokenHolder( ReadOnlyTokenCreator.READ_ONLY, TokenHolder.TYPE_PROPERTY_KEY ); + TokenHolder labels = new DelegatingTokenHolder( ReadOnlyTokenCreator.READ_ONLY, TokenHolder.TYPE_LABEL ); + TokenHolder relationshipTypes = new DelegatingTokenHolder( ReadOnlyTokenCreator.READ_ONLY, TokenHolder.TYPE_RELATIONSHIP_TYPE ); + + propertyKeys.setInitialTokens( lenient ? unique( loader.getPropertyKeyTokens( storeCursors ) ) : loader.getPropertyKeyTokens( storeCursors ) ); + labels.setInitialTokens( lenient ? unique( loader.getLabelTokens( storeCursors ) ) : loader.getLabelTokens( storeCursors ) ); + relationshipTypes.setInitialTokens( + lenient ? unique( loader.getRelationshipTypeTokens( storeCursors ) ) : loader.getRelationshipTypeTokens( storeCursors ) ); + return new TokenHolders( propertyKeys, labels, relationshipTypes ); + } + catch ( IOException e ) + { + throw new UncheckedIOException( e ); + } + } + + private static List unique( List tokens ) + { + if ( !tokens.isEmpty() ) + { + Set names = new HashSet<>( tokens.size() ); + int i = 0; + while ( i < tokens.size() ) + { + if ( names.add( tokens.get( i ).name() ) ) + { + i++; + } + else + { + // Remove the token at the given index, by replacing it with the last token in the list. + // This changes the order of elements, but can be done in constant time instead of linear time. + int lastIndex = tokens.size() - 1; + NamedToken endToken = tokens.remove( lastIndex ); + if ( i < lastIndex ) + { + tokens.set( i, endToken ); + } + } + } + } + return tokens; + } + + @Override + public CommandReaderFactory commandReaderFactory() { + return InMemoryStorageCommandReaderFactory513.INSTANCE; + } + @Override + public void consistencyCheck( + FileSystemAbstraction fileSystem, + DatabaseLayout layout, + Config config, + PageCache pageCache, + IndexProviderMap indexProviders, + InternalLog reportLog, + InternalLog verboseLog, + ConsistencySummaryStatistics summary, + int numberOfThreads, + long maxOffHeapCachingMemory, + OutputStream progressOutput, + boolean verbose, + ConsistencyFlags flags, + CursorContextFactory contextFactory, + PageCacheTracer pageCacheTracer, + LogTailMetadata logTailMetadata + ) { + // we can do no-op, since our "database" is _always_ consistent + } + + @Override + public ImmutableSet getStoreOpenOptions( + FileSystemAbstraction fs, + PageCache pageCache, + DatabaseLayout layout, + CursorContextFactory contextFactory + ) { + // Not sure about this, empty set is returned when the store files are in `little-endian` format + // See: `org.neo4j.kernel.impl.store.format.PageCacheOptionsSelector.select` + return Sets.immutable.empty(); + } + + @Override + public StoreId retrieveStoreId( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) throws IOException { + return StoreId.retrieveFromStore(fs, databaseLayout, pageCache, cursorContext); + } + + + @Override + public Optional versionInformation(StoreVersionIdentifier storeVersionIdentifier) { + return Optional.of(new InMemoryStoreVersion()); + } + + @Override + public void resetMetadata( + FileSystemAbstraction fileSystemAbstraction, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + CursorContextFactory cursorContextFactory, + PageCacheTracer pageCacheTracer, + StoreId storeId, + UUID externalStoreId + ) { + throw new UnsupportedOperationException(); + } + + @Override + public IncrementalBatchImporter incrementalBatchImporter( + DatabaseLayout databaseLayout, + FileSystemAbstraction fileSystemAbstraction, + PageCacheTracer pageCacheTracer, + Configuration configuration, + LogService logService, + PrintStream printStream, + boolean b, + AdditionalInitialIds additionalInitialIds, + ThrowingSupplier throwingSupplier, + Config config, + Monitor monitor, + JobScheduler jobScheduler, + Collector collector, + LogFilesInitializer logFilesInitializer, + IndexImporterFactory indexImporterFactory, + MemoryTracker memoryTracker, + CursorContextFactory cursorContextFactory, + IndexProvidersAccess indexProvidersAccess + ) { + throw new UnsupportedOperationException(); + } + + @Override + public LockManager createLockManager(Config config, SystemNanoClock systemNanoClock) { + return LockManager.NO_LOCKS_LOCK_MANAGER; + } + + @Override + public List listStorageFiles( + FileSystemAbstraction fileSystem, DatabaseLayout databaseLayout + ) { + return Collections.emptyList(); + } + + @Override + public StorageFilesState checkStoreFileState( + FileSystemAbstraction fs, DatabaseLayout databaseLayout, PageCache pageCache + ) { + return StorageFilesState.recoveredState(); + } +} diff --git a/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/InMemoryStorageEngineImpl.java b/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/InMemoryStorageEngineImpl.java new file mode 100644 index 0000000000..2f3769728e --- /dev/null +++ b/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/InMemoryStorageEngineImpl.java @@ -0,0 +1,366 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._513; + +import org.neo4j.configuration.Config; +import org.neo4j.counts.CountsStore; +import org.neo4j.exceptions.KernelException; +import org.neo4j.gds.compat.TokenManager; +import org.neo4j.gds.config.GraphProjectConfig; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.gds.core.loading.GraphStoreCatalog; +import org.neo4j.gds.storageengine.InMemoryDatabaseCreationCatalog; +import org.neo4j.gds.storageengine.InMemoryTransactionStateVisitor; +import org.neo4j.internal.diagnostics.DiagnosticsLogger; +import org.neo4j.internal.recordstorage.InMemoryStorageReader513; +import org.neo4j.internal.schema.StorageEngineIndexingBehaviour; +import org.neo4j.io.fs.WritableChannel; +import org.neo4j.io.layout.DatabaseLayout; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.tracing.DatabaseFlushEvent; +import org.neo4j.kernel.KernelVersion; +import org.neo4j.kernel.impl.store.stats.StoreEntityCounters; +import org.neo4j.kernel.lifecycle.Lifecycle; +import org.neo4j.kernel.lifecycle.LifecycleAdapter; +import org.neo4j.lock.LockGroup; +import org.neo4j.lock.LockService; +import org.neo4j.lock.LockTracer; +import org.neo4j.lock.ResourceLocker; +import org.neo4j.logging.InternalLog; +import org.neo4j.memory.MemoryTracker; +import org.neo4j.storageengine.api.CommandBatchToApply; +import org.neo4j.storageengine.api.CommandCreationContext; +import org.neo4j.storageengine.api.CommandStream; +import org.neo4j.storageengine.api.IndexUpdateListener; +import org.neo4j.storageengine.api.InternalErrorTracer; +import org.neo4j.storageengine.api.MetadataProvider; +import org.neo4j.storageengine.api.StorageCommand; +import org.neo4j.storageengine.api.StorageEngine; +import org.neo4j.storageengine.api.StorageEngineFactory; +import org.neo4j.storageengine.api.StorageLocks; +import org.neo4j.storageengine.api.StorageReader; +import org.neo4j.storageengine.api.StoreFileMetadata; +import org.neo4j.storageengine.api.StoreId; +import org.neo4j.storageengine.api.TransactionApplicationMode; +import org.neo4j.storageengine.api.cursor.StoreCursors; +import org.neo4j.storageengine.api.enrichment.Enrichment; +import org.neo4j.storageengine.api.enrichment.EnrichmentCommand; +import org.neo4j.storageengine.api.txstate.ReadableTransactionState; +import org.neo4j.storageengine.api.txstate.TxStateVisitor; +import org.neo4j.storageengine.api.txstate.validation.TransactionValidatorFactory; +import org.neo4j.time.SystemNanoClock; +import org.neo4j.token.TokenHolders; + +import java.util.Collection; +import java.util.List; +import java.util.stream.Collectors; + +import static org.neo4j.gds.utils.StringFormatting.formatWithLocale; + +public final class InMemoryStorageEngineImpl implements StorageEngine { + + public static final byte ID = 42; + private final MetadataProvider metadataProvider; + private final CypherGraphStore graphStore; + private final DatabaseLayout databaseLayout; + private final InMemoryTransactionStateVisitor txStateVisitor; + + private final CommandCreationContext commandCreationContext; + + private final TokenManager tokenManager; + private final InMemoryCountsStoreImpl countsStore; + + private static final StorageEngineIndexingBehaviour INDEXING_BEHAVIOUR = new StorageEngineIndexingBehaviour() { + @Override + public boolean useNodeIdsInRelationshipTokenIndex() { + return false; + } + + @Override + public boolean requireCoordinationLocks() { + return false; + } + + @Override + public int nodesPerPage() { + return 0; + } + + @Override + public int relationshipsPerPage() { + return 0; + } + }; + + InMemoryStorageEngineImpl( + DatabaseLayout databaseLayout, + TokenHolders tokenHolders + ) { + this.databaseLayout = databaseLayout; + this.graphStore = getGraphStoreFromCatalog(databaseLayout.getDatabaseName()); + this.txStateVisitor = new InMemoryTransactionStateVisitor(graphStore, tokenHolders); + this.commandCreationContext = new InMemoryCommandCreationContextImpl(); + this.tokenManager = new TokenManager( + tokenHolders, + InMemoryStorageEngineImpl.this.txStateVisitor, + InMemoryStorageEngineImpl.this.graphStore, + commandCreationContext + ); + InMemoryStorageEngineImpl.this.graphStore.initialize(tokenHolders); + this.countsStore = new InMemoryCountsStoreImpl(graphStore, tokenHolders); + this.metadataProvider = new InMemoryMetaDataProviderImpl(); + } + + private static CypherGraphStore getGraphStoreFromCatalog(String databaseName) { + var graphName = InMemoryDatabaseCreationCatalog.getRegisteredDbCreationGraphName(databaseName); + return (CypherGraphStore) GraphStoreCatalog.getAllGraphStores() + .filter(graphStoreWithUserNameAndConfig -> graphStoreWithUserNameAndConfig + .config() + .graphName() + .equals(graphName)) + .findFirst() + .orElseThrow(() -> new IllegalArgumentException(formatWithLocale( + "No graph with name `%s` was found in GraphStoreCatalog. Available graph names are %s", + graphName, + GraphStoreCatalog.getAllGraphStores() + .map(GraphStoreCatalog.GraphStoreWithUserNameAndConfig::config) + .map(GraphProjectConfig::graphName) + .collect(Collectors.toList()) + ))) + .graphStore(); + } + + @Override + public StoreEntityCounters storeEntityCounters() { + return new StoreEntityCounters() { + @Override + public long nodes() { + return graphStore.nodeCount(); + } + + @Override + public long relationships() { + return graphStore.relationshipCount(); + } + + @Override + public long properties() { + return graphStore.nodePropertyKeys().size() + graphStore.relationshipPropertyKeys().size(); + } + + @Override + public long relationshipTypes() { + return graphStore.relationshipTypes().size(); + } + + @Override + public long allNodesCountStore(CursorContext cursorContext) { + return graphStore.nodeCount(); + } + + @Override + public long allRelationshipsCountStore(CursorContext cursorContext) { + return graphStore.relationshipCount(); + } + }; + } + + @Override + public StoreCursors createStorageCursors(CursorContext initialContext) { + return StoreCursors.NULL; + } + + @Override + public StorageLocks createStorageLocks(ResourceLocker locker) { + return new InMemoryStorageLocksImpl(locker); + } + + @Override + public List createCommands( + ReadableTransactionState state, + StorageReader storageReader, + CommandCreationContext creationContext, + LockTracer lockTracer, + TxStateVisitor.Decorator additionalTxStateVisitor, + CursorContext cursorContext, + StoreCursors storeCursors, + MemoryTracker memoryTracker + ) throws KernelException { + state.accept(txStateVisitor); + return List.of(); + } + + @Override + public void dumpDiagnostics(InternalLog internalLog, DiagnosticsLogger diagnosticsLogger) { + } + + @Override + public List createUpgradeCommands( + KernelVersion versionToUpgradeFrom, + KernelVersion versionToUpgradeTo + ) { + return List.of(); + } + + @Override + public StoreId retrieveStoreId() { + return metadataProvider.getStoreId(); + } + + @Override + public StorageEngineIndexingBehaviour indexingBehaviour() { + return INDEXING_BEHAVIOUR; + } + + @Override + public StorageReader newReader() { + return new InMemoryStorageReader513(graphStore, tokenManager.tokenHolders(), countsStore); + } + + @Override + public void addIndexUpdateListener(IndexUpdateListener listener) { + + } + + @Override + public void apply(CommandBatchToApply batch, TransactionApplicationMode mode) { + } + + @Override + public void init() { + } + + @Override + public void start() { + + } + + @Override + public void stop() { + shutdown(); + } + + @Override + public void shutdown() { + InMemoryDatabaseCreationCatalog.removeDatabaseEntry(databaseLayout.getDatabaseName()); + } + + @Override + public void listStorageFiles( + Collection atomic, Collection replayable + ) { + + } + + @Override + public Lifecycle schemaAndTokensLifecycle() { + return new LifecycleAdapter() { + @Override + public void init() { + + } + }; + } + + @Override + public CountsStore countsAccessor() { + return countsStore; + } + + @Override + public MetadataProvider metadataProvider() { + return metadataProvider; + } + + @Override + public String name() { + return "gds in-memory storage engine"; + } + + @Override + public byte id() { + return ID; + } + + @Override + public CommandCreationContext newCommandCreationContext(boolean multiVersioned) { + return commandCreationContext; + } + + @Override + public TransactionValidatorFactory createTransactionValidatorFactory( + StorageEngineFactory storageEngineFactory, + Config config, + SystemNanoClock systemNanoClock + ) { + return TransactionValidatorFactory.EMPTY_VALIDATOR_FACTORY; + } + + @Override + public void lockRecoveryCommands( + CommandStream commands, LockService lockService, LockGroup lockGroup, TransactionApplicationMode mode + ) { + + } + + @Override + public void rollback(ReadableTransactionState txState, CursorContext cursorContext) { + // rollback is not supported but it is also called when we fail for something else + // that we do not support, such as removing node properties + // TODO: do we want to inspect the txState to infer if rollback was called explicitly or not? + } + + @Override + public void checkpoint(DatabaseFlushEvent flushEvent, CursorContext cursorContext) { + // checkpoint is not supported but it is also called when we fail for something else + // that we do not support, such as removing node properties + } + + @Override + public void preAllocateStoreFilesForCommands(CommandBatchToApply batch, TransactionApplicationMode mode) { + // GDS has its own mechanism of memory allocation, so we don't need this + } + + @Override + public EnrichmentCommand createEnrichmentCommand(KernelVersion kernelVersion, Enrichment enrichment) { + return new EnrichmentCommand() { + + @Override + public Enrichment enrichment() { + return null; + } + + @Override + public void serialize(WritableChannel channel) { + + } + + @Override + public KernelVersion kernelVersion() { + return kernelVersion; + } + }; + } + + @Override + public InternalErrorTracer internalErrorTracer() { + return InternalErrorTracer.NO_TRACER; + } +} diff --git a/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/InMemoryStorageLocksImpl.java b/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/InMemoryStorageLocksImpl.java new file mode 100644 index 0000000000..e09fd7303b --- /dev/null +++ b/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/InMemoryStorageLocksImpl.java @@ -0,0 +1,86 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._513; + +import org.neo4j.lock.LockTracer; +import org.neo4j.lock.ResourceLocker; +import org.neo4j.storageengine.api.StorageLocks; +import org.neo4j.storageengine.api.txstate.ReadableTransactionState; + +public class InMemoryStorageLocksImpl implements StorageLocks { + + InMemoryStorageLocksImpl(ResourceLocker locker) {} + + @Override + public void acquireExclusiveNodeLock(LockTracer lockTracer, long... ids) {} + + @Override + public void releaseExclusiveNodeLock(long... ids) {} + + @Override + public void acquireSharedNodeLock(LockTracer lockTracer, long... ids) {} + + @Override + public void releaseSharedNodeLock(long... ids) {} + + @Override + public void acquireExclusiveRelationshipLock(LockTracer lockTracer, long... ids) {} + + @Override + public void releaseExclusiveRelationshipLock(long... ids) {} + + @Override + public void acquireSharedRelationshipLock(LockTracer lockTracer, long... ids) {} + + @Override + public void releaseSharedRelationshipLock(long... ids) {} + + @Override + public void acquireRelationshipCreationLock( + LockTracer lockTracer, + long sourceNode, + long targetNode, + boolean sourceNodeAddedInTx, + boolean targetNodeAddedInTx + ) { + } + + @Override + public void acquireRelationshipDeletionLock( + LockTracer lockTracer, + long sourceNode, + long targetNode, + long relationship, + boolean relationshipAddedInTx, + boolean sourceNodeAddedInTx, + boolean targetNodeAddedInTx + ) { + } + + @Override + public void acquireNodeDeletionLock( + ReadableTransactionState readableTransactionState, + LockTracer lockTracer, + long node + ) {} + + @Override + public void acquireNodeLabelChangeLock(LockTracer lockTracer, long node, int labelId) {} +} diff --git a/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/InMemoryStoreVersion.java b/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/InMemoryStoreVersion.java new file mode 100644 index 0000000000..1c1234b2de --- /dev/null +++ b/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/InMemoryStoreVersion.java @@ -0,0 +1,70 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._513; + +import org.neo4j.configuration.Config; +import org.neo4j.storageengine.api.StoreVersion; +import org.neo4j.storageengine.api.format.Capability; +import org.neo4j.storageengine.api.format.CapabilityType; + +import java.util.Optional; + +public class InMemoryStoreVersion implements StoreVersion { + + public static final String STORE_VERSION = "gds-experimental"; + + @Override + public String getStoreVersionUserString() { + return "Unknown"; + } + + @Override + public Optional successorStoreVersion(Config config) { + return Optional.empty(); + } + + @Override + public String formatName() { + return getClass().getSimpleName(); + } + + @Override + public boolean onlyForMigration() { + return false; + } + + @Override + public boolean hasCapability(Capability capability) { + return false; + } + + @Override + public boolean hasCompatibleCapabilities( + StoreVersion otherVersion, CapabilityType type + ) { + return false; + } + + @Override + public String introductionNeo4jVersion() { + return "foo"; + } + +} diff --git a/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/InMemoryTransactionIdStoreImpl.java b/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/InMemoryTransactionIdStoreImpl.java new file mode 100644 index 0000000000..61f5ecac5a --- /dev/null +++ b/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/InMemoryTransactionIdStoreImpl.java @@ -0,0 +1,117 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._513; + +import org.neo4j.internal.recordstorage.AbstractTransactionIdStore; +import org.neo4j.io.pagecache.context.TransactionIdSnapshot; +import org.neo4j.kernel.impl.transaction.log.LogPosition; +import org.neo4j.storageengine.api.ClosedTransactionMetadata; +import org.neo4j.storageengine.api.TransactionId; +import org.neo4j.storageengine.api.TransactionIdStore; + +public class InMemoryTransactionIdStoreImpl extends AbstractTransactionIdStore { + + @Override + protected void initLastCommittedAndClosedTransactionId( + long previouslyCommittedTxId, + int checksum, + long previouslyCommittedTxCommitTimestamp, + long previouslyCommittedTxLogByteOffset, + long previouslyCommittedTxLogVersion + ) { + this.setLastCommittedAndClosedTransactionId( + previouslyCommittedTxId, + checksum, + previouslyCommittedTxCommitTimestamp, + TransactionIdStore.UNKNOWN_CONSENSUS_INDEX, + previouslyCommittedTxLogByteOffset, + previouslyCommittedTxLogVersion + ); + } + + @Override + public ClosedTransactionMetadata getLastClosedTransaction() { + long[] metaData = this.closedTransactionId.get(); + return new ClosedTransactionMetadata( + metaData[0], + new LogPosition(metaData[1], metaData[2]), + (int) metaData[3], + metaData[4], + metaData[5] + ); + } + + @Override + public TransactionIdSnapshot getClosedTransactionSnapshot() { + return new TransactionIdSnapshot(this.getLastClosedTransactionId()); + } + + @Override + protected TransactionId transactionId(long transactionId, int checksum, long commitTimestamp) { + return new TransactionId(transactionId, checksum, commitTimestamp, TransactionIdStore.UNKNOWN_CONSENSUS_INDEX); + } + + @Override + public void transactionCommitted(long transactionId, int checksum, long commitTimestamp, long consensusIndex) { + + } + + @Override + public void setLastCommittedAndClosedTransactionId( + long transactionId, + int checksum, + long commitTimestamp, + long consensusIndex, + long byteOffset, + long logVersion + ) { + + } + + @Override + public void transactionClosed( + long transactionId, + long logVersion, + long byteOffset, + int checksum, + long commitTimestamp, + long consensusIndex + ) { + this.closedTransactionId.offer( + transactionId, + new long[]{logVersion, byteOffset, checksum, commitTimestamp, consensusIndex} + ); + } + + @Override + public void resetLastClosedTransaction( + long transactionId, + long logVersion, + long byteOffset, + int checksum, + long commitTimestamp, + long consensusIndex + ) { + this.closedTransactionId.set( + transactionId, + new long[]{logVersion, byteOffset, checksum, commitTimestamp, consensusIndex} + ); + } +} diff --git a/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/InMemoryVersionCheck.java b/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/InMemoryVersionCheck.java new file mode 100644 index 0000000000..212a8717b2 --- /dev/null +++ b/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/InMemoryVersionCheck.java @@ -0,0 +1,72 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._513; + +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.kernel.impl.store.format.FormatFamily; +import org.neo4j.storageengine.api.StoreVersionCheck; +import org.neo4j.storageengine.api.StoreVersionIdentifier; + +import static org.neo4j.gds.compat._513.InMemoryStoreVersion.STORE_VERSION; + +public class InMemoryVersionCheck implements StoreVersionCheck { + + private static final StoreVersionIdentifier STORE_IDENTIFIER = new StoreVersionIdentifier( + STORE_VERSION, + FormatFamily.STANDARD.name(), + 0, + 0 + ); + + @Override + public StoreVersionIdentifier getCurrentVersion(CursorContext cursorContext) throws + IllegalArgumentException, IllegalStateException { + return STORE_IDENTIFIER; + } + + @Override + public boolean isCurrentStoreVersionFullySupported(CursorContext cursorContext) { + return true; + } + + @Override + public boolean isStoreVersionFullySupported(StoreVersionIdentifier storeVersion, CursorContext cursorContext) { + return true; // because we don't want to block upgrades + } + + @Override + public MigrationCheckResult getAndCheckMigrationTargetVersion(String formatFamily, CursorContext cursorContext) { + return new StoreVersionCheck.MigrationCheckResult(MigrationOutcome.NO_OP, STORE_IDENTIFIER, null, null); + } + + @Override + public UpgradeCheckResult getAndCheckUpgradeTargetVersion(CursorContext cursorContext) { + return new StoreVersionCheck.UpgradeCheckResult(UpgradeOutcome.NO_OP, STORE_IDENTIFIER, null, null); + } + + @Override + public String getIntroductionVersionFromVersion(StoreVersionIdentifier storeVersionIdentifier) { + return STORE_VERSION; + } + + public StoreVersionIdentifier findLatestVersion(String s) { + return STORE_IDENTIFIER; + } +} diff --git a/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/StorageEngineProxyFactoryImpl.java b/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/StorageEngineProxyFactoryImpl.java new file mode 100644 index 0000000000..f3c7c98d3d --- /dev/null +++ b/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/StorageEngineProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._513; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jVersion; +import org.neo4j.gds.compat.StorageEngineProxyApi; +import org.neo4j.gds.compat.StorageEngineProxyFactory; + +@ServiceProvider +public class StorageEngineProxyFactoryImpl implements StorageEngineProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return version == Neo4jVersion.V_5_13; + } + + @Override + public StorageEngineProxyApi load() { + return new StorageEngineProxyImpl(); + } + + @Override + public String description() { + return "Storage Engine 5.13"; + } +} diff --git a/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/StorageEngineProxyImpl.java b/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/StorageEngineProxyImpl.java new file mode 100644 index 0000000000..6b342c1263 --- /dev/null +++ b/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/StorageEngineProxyImpl.java @@ -0,0 +1,140 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._513; + +import org.neo4j.common.Edition; +import org.neo4j.configuration.Config; +import org.neo4j.configuration.GraphDatabaseInternalSettings; +import org.neo4j.dbms.api.DatabaseManagementService; +import org.neo4j.gds.compat.AbstractInMemoryNodeCursor; +import org.neo4j.gds.compat.AbstractInMemoryNodePropertyCursor; +import org.neo4j.gds.compat.AbstractInMemoryRelationshipPropertyCursor; +import org.neo4j.gds.compat.AbstractInMemoryRelationshipTraversalCursor; +import org.neo4j.gds.compat.GdsDatabaseManagementServiceBuilder; +import org.neo4j.gds.compat.GraphDatabaseApiProxy; +import org.neo4j.gds.compat.StorageEngineProxyApi; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.graphdb.Direction; +import org.neo4j.graphdb.GraphDatabaseService; +import org.neo4j.internal.recordstorage.AbstractInMemoryRelationshipScanCursor; +import org.neo4j.io.layout.DatabaseLayout; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.RelationshipSelection; +import org.neo4j.storageengine.api.StorageEngine; +import org.neo4j.storageengine.api.StorageEntityCursor; +import org.neo4j.storageengine.api.StoragePropertyCursor; +import org.neo4j.storageengine.api.StorageRelationshipTraversalCursor; +import org.neo4j.token.TokenHolders; + +import static org.neo4j.configuration.GraphDatabaseSettings.db_format; + +public class StorageEngineProxyImpl implements StorageEngineProxyApi { + + @Override + public void initRelationshipTraversalCursorForRelType( + StorageRelationshipTraversalCursor cursor, + long sourceNodeId, + int relTypeToken + ) { + var relationshipSelection = RelationshipSelection.selection( + relTypeToken, + Direction.OUTGOING + ); + cursor.init(sourceNodeId, -1, relationshipSelection); + } + + @Override + public StorageEngine createInMemoryStorageEngine(DatabaseLayout databaseLayout, TokenHolders tokenHolders) { + return new InMemoryStorageEngineImpl(databaseLayout, tokenHolders); + } + + @Override + public void createInMemoryDatabase( + DatabaseManagementService dbms, + String dbName, + Config config + ) { + config.set(db_format, InMemoryStorageEngineFactory.IN_MEMORY_STORAGE_ENGINE_NAME); + dbms.createDatabase(dbName, config); + } + + @Override + public GraphDatabaseService startAndGetInMemoryDatabase(DatabaseManagementService dbms, String dbName) { + dbms.startDatabase(dbName); + return dbms.database(dbName); + } + + @Override + public GdsDatabaseManagementServiceBuilder setSkipDefaultIndexesOnCreationSetting(GdsDatabaseManagementServiceBuilder dbmsBuilder) { + return dbmsBuilder.setConfig(GraphDatabaseInternalSettings.skip_default_indexes_on_creation, true); + } + + @Override + public AbstractInMemoryNodeCursor inMemoryNodeCursor(CypherGraphStore graphStore, TokenHolders tokenHolders) { + return new InMemoryNodeCursor(graphStore, tokenHolders); + } + + @Override + public AbstractInMemoryNodePropertyCursor inMemoryNodePropertyCursor( + CypherGraphStore graphStore, + TokenHolders tokenHolders + ) { + return new InMemoryNodePropertyCursor(graphStore, tokenHolders); + } + + @Override + public AbstractInMemoryRelationshipTraversalCursor inMemoryRelationshipTraversalCursor( + CypherGraphStore graphStore, TokenHolders tokenHolders + ) { + return new InMemoryRelationshipTraversalCursor(graphStore, tokenHolders); + } + + @Override + public AbstractInMemoryRelationshipScanCursor inMemoryRelationshipScanCursor( + CypherGraphStore graphStore, TokenHolders tokenHolders + ) { + return new InMemoryRelationshipScanCursor(graphStore, tokenHolders); + } + + @Override + public AbstractInMemoryRelationshipPropertyCursor inMemoryRelationshipPropertyCursor( + CypherGraphStore graphStore, TokenHolders tokenHolders + ) { + return new InMemoryRelationshipPropertyCursor(graphStore, tokenHolders); + } + + @Override + public void properties( + StorageEntityCursor storageCursor, StoragePropertyCursor propertyCursor, int[] propertySelection + ) { + PropertySelection selection; + if (propertySelection.length == 0) { + selection = PropertySelection.ALL_PROPERTIES; + } else { + selection = PropertySelection.selection(propertySelection); + } + storageCursor.properties(propertyCursor, selection); + } + + @Override + public Edition dbmsEdition(GraphDatabaseService databaseService) { + return GraphDatabaseApiProxy.dbmsInfo(databaseService).edition; + } +} diff --git a/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository513.java b/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository513.java new file mode 100644 index 0000000000..91a0e77549 --- /dev/null +++ b/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository513.java @@ -0,0 +1,71 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.internal.recordstorage; + +import org.neo4j.storageengine.api.LogVersionRepository; + +import java.util.concurrent.atomic.AtomicLong; + +public class InMemoryLogVersionRepository513 implements LogVersionRepository { + + private final AtomicLong logVersion; + private final AtomicLong checkpointLogVersion; + + public InMemoryLogVersionRepository513() { + this(0, 0); + } + + private InMemoryLogVersionRepository513(long initialLogVersion, long initialCheckpointLogVersion) { + this.logVersion = new AtomicLong(); + this.checkpointLogVersion = new AtomicLong(); + this.logVersion.set(initialLogVersion); + this.checkpointLogVersion.set(initialCheckpointLogVersion); + } + + @Override + public void setCurrentLogVersion(long version) { + this.logVersion.set(version); + } + + @Override + public long incrementAndGetVersion() { + return this.logVersion.incrementAndGet(); + } + + @Override + public void setCheckpointLogVersion(long version) { + this.checkpointLogVersion.set(version); + } + + @Override + public long incrementAndGetCheckpointLogVersion() { + return this.checkpointLogVersion.incrementAndGet(); + } + + @Override + public long getCurrentLogVersion() { + return this.logVersion.get(); + } + + @Override + public long getCheckpointLogVersion() { + return this.checkpointLogVersion.get(); + } +} diff --git a/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory513.java b/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory513.java new file mode 100644 index 0000000000..7e238b01fe --- /dev/null +++ b/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory513.java @@ -0,0 +1,43 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.internal.recordstorage; + +import org.neo4j.kernel.KernelVersion; +import org.neo4j.storageengine.api.CommandReader; +import org.neo4j.storageengine.api.CommandReaderFactory; + +public class InMemoryStorageCommandReaderFactory513 implements CommandReaderFactory { + + public static final CommandReaderFactory INSTANCE = new InMemoryStorageCommandReaderFactory513(); + + @Override + public CommandReader get(KernelVersion kernelVersion) { + switch (kernelVersion) { + case V4_2: + return LogCommandSerializationV4_2.INSTANCE; + case V4_3_D4: + return LogCommandSerializationV4_3_D3.INSTANCE; + case V5_0: + return LogCommandSerializationV5_0.INSTANCE; + default: + throw new IllegalArgumentException("Unsupported kernel version " + kernelVersion); + } + } +} diff --git a/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageReader513.java b/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageReader513.java new file mode 100644 index 0000000000..602069e87b --- /dev/null +++ b/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageReader513.java @@ -0,0 +1,333 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.internal.recordstorage; + +import org.eclipse.collections.api.set.primitive.IntSet; +import org.eclipse.collections.impl.set.immutable.primitive.ImmutableIntSetFactoryImpl; +import org.neo4j.common.EntityType; +import org.neo4j.common.TokenNameLookup; +import org.neo4j.counts.CountsStore; +import org.neo4j.gds.compat._513.InMemoryNodeCursor; +import org.neo4j.gds.compat._513.InMemoryPropertyCursor; +import org.neo4j.gds.compat._513.InMemoryRelationshipScanCursor; +import org.neo4j.gds.compat._513.InMemoryRelationshipTraversalCursor; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.internal.schema.ConstraintDescriptor; +import org.neo4j.internal.schema.IndexDescriptor; +import org.neo4j.internal.schema.IndexType; +import org.neo4j.internal.schema.SchemaDescriptor; +import org.neo4j.internal.schema.constraints.IndexBackedConstraintDescriptor; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.memory.MemoryTracker; +import org.neo4j.storageengine.api.AllNodeScan; +import org.neo4j.storageengine.api.AllRelationshipsScan; +import org.neo4j.storageengine.api.StorageNodeCursor; +import org.neo4j.storageengine.api.StoragePropertyCursor; +import org.neo4j.storageengine.api.StorageReader; +import org.neo4j.storageengine.api.StorageRelationshipScanCursor; +import org.neo4j.storageengine.api.StorageRelationshipTraversalCursor; +import org.neo4j.storageengine.api.StorageSchemaReader; +import org.neo4j.storageengine.api.cursor.StoreCursors; +import org.neo4j.token.TokenHolders; + +import java.util.Collection; +import java.util.Collections; +import java.util.Iterator; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Function; + +public class InMemoryStorageReader513 implements StorageReader { + + protected final CypherGraphStore graphStore; + protected final TokenHolders tokenHolders; + protected final CountsStore counts; + private final Map, Object> dependantState; + private boolean closed; + + public InMemoryStorageReader513( + CypherGraphStore graphStore, + TokenHolders tokenHolders, + CountsStore counts + ) { + this.graphStore = graphStore; + + this.tokenHolders = tokenHolders; + this.counts = counts; + this.dependantState = new ConcurrentHashMap<>(); + } + + @Override + public Collection uniquenessConstraintsGetRelated( + long[] changedLabels, + long[] unchangedLabels, + int[] propertyKeyIds, + boolean propertyKeyListIsComplete, + EntityType entityType + ) { + return Collections.emptyList(); + } + + @Override + public long relationshipsGetCount(CursorContext cursorTracer) { + return graphStore.relationshipCount(); + } + + @Override + public boolean nodeExists(long id, StoreCursors storeCursors) { + var originalId = graphStore.nodes().toOriginalNodeId(id); + return graphStore.nodes().containsOriginalId(originalId); + } + + @Override + public boolean relationshipExists(long id, StoreCursors storeCursors) { + return true; + } + + @Override + public StorageNodeCursor allocateNodeCursor( + CursorContext cursorContext, StoreCursors storeCursors + ) { + return new InMemoryNodeCursor(graphStore, tokenHolders); + } + + @Override + public StoragePropertyCursor allocatePropertyCursor( + CursorContext cursorContext, StoreCursors storeCursors, MemoryTracker memoryTracker + ) { + return new InMemoryPropertyCursor(graphStore, tokenHolders); + } + + @Override + public StorageRelationshipTraversalCursor allocateRelationshipTraversalCursor( + CursorContext cursorContext, StoreCursors storeCursors + ) { + return new InMemoryRelationshipTraversalCursor(graphStore, tokenHolders); + } + + @Override + public StorageRelationshipScanCursor allocateRelationshipScanCursor( + CursorContext cursorContext, StoreCursors storeCursors + ) { + return new InMemoryRelationshipScanCursor(graphStore, tokenHolders); + } + + @Override + public IndexDescriptor indexGetForSchemaAndType( + SchemaDescriptor descriptor, IndexType type + ) { + return null; + } + + @Override + public AllRelationshipsScan allRelationshipScan() { + return new AbstractInMemoryAllRelationshipScan() { + @Override + boolean scanRange(AbstractInMemoryRelationshipScanCursor cursor, long start, long stopInclusive) { + return cursor.scanRange(start, stopInclusive); + } + + @Override + public boolean scanBatch(long sizeHint, AbstractInMemoryRelationshipScanCursor cursor) { + return super.scanBatch(sizeHint, cursor); + } + }; + } + + @Override + public Iterator indexGetForSchema(SchemaDescriptor descriptor) { + return Collections.emptyIterator(); + } + + @Override + public Iterator indexesGetForLabel(int labelId) { + return Collections.emptyIterator(); + } + + @Override + public Iterator indexesGetForRelationshipType(int relationshipType) { + return Collections.emptyIterator(); + } + + @Override + public IndexDescriptor indexGetForName(String name) { + return null; + } + + @Override + public ConstraintDescriptor constraintGetForName(String name) { + return null; + } + + @Override + public boolean indexExists(IndexDescriptor index) { + return false; + } + + @Override + public Iterator indexesGetAll() { + return Collections.emptyIterator(); + } + + @Override + public Collection valueIndexesGetRelated( + long[] tokens, int propertyKeyId, EntityType entityType + ) { + return valueIndexesGetRelated(tokens, new int[]{propertyKeyId}, entityType); + } + + @Override + public Collection valueIndexesGetRelated( + long[] tokens, int[] propertyKeyIds, EntityType entityType + ) { + return Collections.emptyList(); + } + + @Override + public Collection uniquenessConstraintsGetRelated( + long[] labels, + int propertyKeyId, + EntityType entityType + ) { + return Collections.emptyList(); + } + + @Override + public Collection uniquenessConstraintsGetRelated( + long[] tokens, + int[] propertyKeyIds, + EntityType entityType + ) { + return Collections.emptyList(); + } + + @Override + public boolean hasRelatedSchema(long[] labels, int propertyKey, EntityType entityType) { + return false; + } + + @Override + public boolean hasRelatedSchema(int label, EntityType entityType) { + return false; + } + + @Override + public Iterator constraintsGetForSchema(SchemaDescriptor descriptor) { + return Collections.emptyIterator(); + } + + @Override + public boolean constraintExists(ConstraintDescriptor descriptor) { + return false; + } + + @Override + public Iterator constraintsGetForLabel(int labelId) { + return Collections.emptyIterator(); + } + + @Override + public Iterator constraintsGetForRelationshipType(int typeId) { + return Collections.emptyIterator(); + } + + @Override + public Iterator constraintsGetAll() { + return Collections.emptyIterator(); + } + + @Override + public IntSet constraintsGetPropertyTokensForLogicalKey(int token, EntityType entityType) { + return ImmutableIntSetFactoryImpl.INSTANCE.empty(); + } + + @Override + public Long indexGetOwningUniquenessConstraintId(IndexDescriptor index) { + return null; + } + + @Override + public long countsForNode(int labelId, CursorContext cursorContext) { + return counts.nodeCount(labelId, cursorContext); + } + + @Override + public long countsForRelationship(int startLabelId, int typeId, int endLabelId, CursorContext cursorContext) { + return counts.relationshipCount(startLabelId, typeId, endLabelId, cursorContext); + } + + @Override + public long nodesGetCount(CursorContext cursorContext) { + return graphStore.nodeCount(); + } + + @Override + public int labelCount() { + return graphStore.nodes().availableNodeLabels().size(); + } + + @Override + public int propertyKeyCount() { + int nodePropertyCount = graphStore + .schema() + .nodeSchema() + .unionProperties() + .size(); + int relPropertyCount = graphStore + .schema() + .relationshipSchema() + .unionProperties() + .size(); + + return nodePropertyCount + relPropertyCount; + } + + @Override + public int relationshipTypeCount() { + return graphStore.schema().relationshipSchema().entries().size(); + } + + @Override + public T getOrCreateSchemaDependantState(Class type, Function factory) { + return type.cast(dependantState.computeIfAbsent(type, key -> factory.apply(this))); + } + + @Override + public AllNodeScan allNodeScan() { + return new InMemoryNodeScan(); + } + + @Override + public void close() { + assert !closed; + closed = true; + } + + @Override + public StorageSchemaReader schemaSnapshot() { + return this; + } + + @Override + public TokenNameLookup tokenNameLookup() { + return tokenHolders; + } + +} diff --git a/gradle/dependencies.gradle b/gradle/dependencies.gradle index fbf7d90509..a58201f833 100644 --- a/gradle/dependencies.gradle +++ b/gradle/dependencies.gradle @@ -14,6 +14,7 @@ ext { '5.11': properties.getOrDefault('neo4jVersion511', '5.11.0'), '5.12': properties.getOrDefault('neo4jVersion512', '5.12.0'), '5.13': properties.getOrDefault('neo4jVersion513', '5.13.0'), + '5.13': properties.getOrDefault('neo4jVersion513', '5.13.0'), ] neo4jDefault = neos.'4.4' diff --git a/neo4j-adapter/src/main/java/org/neo4j/gds/compat/Neo4jVersion.java b/neo4j-adapter/src/main/java/org/neo4j/gds/compat/Neo4jVersion.java index 0eee2223b3..69a6f098a1 100644 --- a/neo4j-adapter/src/main/java/org/neo4j/gds/compat/Neo4jVersion.java +++ b/neo4j-adapter/src/main/java/org/neo4j/gds/compat/Neo4jVersion.java @@ -40,6 +40,7 @@ public enum Neo4jVersion { V_5_10, V_5_11, V_5_12, + V_5_13, V_RC; @Override @@ -71,6 +72,8 @@ public String toString() { return "5.11"; case V_5_12: return "5.12"; + case V_5_13: + return "5.13"; case V_RC: return "rc"; default: @@ -168,6 +171,8 @@ static Neo4jVersion parse(String version) { } else if (minorVersion == 12) { return Neo4jVersion.V_5_12; } else if (minorVersion == 13) { + return Neo4jVersion.V_5_13; + } else if (minorVersion == 14) { return Neo4jVersion.V_RC; } } diff --git a/neo4j-adapter/src/test/java/org/neo4j/gds/compat/Neo4jVersionTest.java b/neo4j-adapter/src/test/java/org/neo4j/gds/compat/Neo4jVersionTest.java index 6c78c65129..bdf5f742b3 100644 --- a/neo4j-adapter/src/test/java/org/neo4j/gds/compat/Neo4jVersionTest.java +++ b/neo4j-adapter/src/test/java/org/neo4j/gds/compat/Neo4jVersionTest.java @@ -53,7 +53,8 @@ class Neo4jVersionTest { "5.10.0, V_5_10", "5.11.0, V_5_11", "5.12.0, V_5_12", - "5.13.0, V_RC", + "5.13.0, V_5_13", + "5.14.0, V_RC", }) void testParse(String input, Neo4jVersion expected) { assertEquals(expected.name(), Neo4jVersion.parse(input).name()); @@ -94,6 +95,11 @@ void shouldNotRespectVersionOverride() { "5.6.0, 5, 6", "5.7.0, 5, 7", "5.8.0, 5, 8", + "5.9.0, 5, 9", + "5.10.0, 5, 10", + "5.11.0, 5, 11", + "5.12.0, 5, 12", + "5.13.0, 5, 13", }) void semanticVersion(String input, int expectedMajor, int expectedMinor) { Neo4jVersion version = Neo4jVersion.parse(input); diff --git a/proc/sysinfo/src/test/java/org/neo4j/gds/SysInfoProcTest.java b/proc/sysinfo/src/test/java/org/neo4j/gds/SysInfoProcTest.java index 79744260a9..4149749982 100644 --- a/proc/sysinfo/src/test/java/org/neo4j/gds/SysInfoProcTest.java +++ b/proc/sysinfo/src/test/java/org/neo4j/gds/SysInfoProcTest.java @@ -112,6 +112,11 @@ class SysInfoProcTest extends BaseProcTest { "Neo4j Settings 5.12", "Neo4j Settings 5.12 (placeholder)", + "Neo4j 5.13", + "Neo4j 5.13 (placeholder)", + "Neo4j Settings 5.13", + "Neo4j Settings 5.13 (placeholder)", + "Neo4j DEV", "Neo4j DEV (placeholder)", "Neo4j Settings DEV", @@ -268,6 +273,14 @@ void testSysInfoProc() throws IOException { "Neo4j 5.12" ); break; + case V_5_13: + expectedCompatibilities = Set.of( + "Neo4j Settings 5.13 (placeholder)", + "Neo4j Settings 5.13", + "Neo4j 5.13 (placeholder)", + "Neo4j 5.13" + ); + break; case V_RC: expectedCompatibilities = Set.of( "Neo4j Settings RC", From 20555da45b59c8fd4135f6da2f6d96791000f68c Mon Sep 17 00:00:00 2001 From: Nicola Vitucci Date: Wed, 27 Sep 2023 11:21:13 +0100 Subject: [PATCH 255/273] Remove reference to dev phase in Cypher projection --- .../projections/graph-project-cypher-legacy.adoc | 3 +-- .../projections/graph-project-cypher-projection.adoc | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/doc/modules/ROOT/pages/management-ops/projections/graph-project-cypher-legacy.adoc b/doc/modules/ROOT/pages/management-ops/projections/graph-project-cypher-legacy.adoc index bc9d603409..ab21a1d636 100644 --- a/doc/modules/ROOT/pages/management-ops/projections/graph-project-cypher-legacy.adoc +++ b/doc/modules/ROOT/pages/management-ops/projections/graph-project-cypher-legacy.adoc @@ -10,8 +10,7 @@ The replacement is to use the new Cypher projection, which is described in xref: A migration guide is available at xref:migration-lcp-to-cpv2/index.adoc[Appendix C, Migration from Legacy to new Cypher projection]. -- -Using Legacy Cypher projections is a more flexible and expressive approach with diminished focus on performance compared to the xref:management-ops/projections/graph-project.adoc[native projections]. -Legacy Cypher projections are primarily recommended for the development phase (see xref:common-usage/index.adoc[Common usage]). +Using Legacy Cypher projections is a more flexible and expressive approach with diminished focus on performance compared to the xref:management-ops/graph-creation/graph-project.adoc[native projections]. == Considerations diff --git a/doc/modules/ROOT/pages/management-ops/projections/graph-project-cypher-projection.adoc b/doc/modules/ROOT/pages/management-ops/projections/graph-project-cypher-projection.adoc index 03b2ef762b..47d7158367 100644 --- a/doc/modules/ROOT/pages/management-ops/projections/graph-project-cypher-projection.adoc +++ b/doc/modules/ROOT/pages/management-ops/projections/graph-project-cypher-projection.adoc @@ -4,8 +4,7 @@ :page-aliases: management-ops/projections/graph-project-cypher-aggregation -Using Cypher projection is a more flexible and expressive approach with diminished focus on performance compared to the xref:management-ops/projections/graph-project.adoc[native projections]. -Cypher projections are primarily recommended for the development phase (see xref:common-usage/index.adoc[Common usage]). +Using Cypher projection is a more flexible and expressive approach with diminished focus on performance compared to the xref:management-ops/graph-creation/graph-project.adoc[native projections]. == Considerations From b17f0216947369a8085708e9a7bb75a8c5bc64d0 Mon Sep 17 00:00:00 2001 From: Nicola Vitucci Date: Wed, 27 Sep 2023 11:30:48 +0100 Subject: [PATCH 256/273] Add explanation to Cypher projections --- .../projections/graph-project-cypher-legacy.adoc | 4 +++- .../projections/graph-project-cypher-projection.adoc | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/doc/modules/ROOT/pages/management-ops/projections/graph-project-cypher-legacy.adoc b/doc/modules/ROOT/pages/management-ops/projections/graph-project-cypher-legacy.adoc index ab21a1d636..29be95d4d5 100644 --- a/doc/modules/ROOT/pages/management-ops/projections/graph-project-cypher-legacy.adoc +++ b/doc/modules/ROOT/pages/management-ops/projections/graph-project-cypher-legacy.adoc @@ -10,7 +10,9 @@ The replacement is to use the new Cypher projection, which is described in xref: A migration guide is available at xref:migration-lcp-to-cpv2/index.adoc[Appendix C, Migration from Legacy to new Cypher projection]. -- -Using Legacy Cypher projections is a more flexible and expressive approach with diminished focus on performance compared to the xref:management-ops/graph-creation/graph-project.adoc[native projections]. +A Cypher projection uses Cypher to create (_project_) an in-memory graph from the Neo4j database. + +Legacy Cypher projections are a more flexible and expressive approach with diminished focus on performance compared to the xref:management-ops/graph-creation/graph-project.adoc[native projections]. == Considerations diff --git a/doc/modules/ROOT/pages/management-ops/projections/graph-project-cypher-projection.adoc b/doc/modules/ROOT/pages/management-ops/projections/graph-project-cypher-projection.adoc index 47d7158367..ac24847b98 100644 --- a/doc/modules/ROOT/pages/management-ops/projections/graph-project-cypher-projection.adoc +++ b/doc/modules/ROOT/pages/management-ops/projections/graph-project-cypher-projection.adoc @@ -4,7 +4,9 @@ :page-aliases: management-ops/projections/graph-project-cypher-aggregation -Using Cypher projection is a more flexible and expressive approach with diminished focus on performance compared to the xref:management-ops/graph-creation/graph-project.adoc[native projections]. +A Cypher projection uses Cypher to create (_project_) an in-memory graph from the Neo4j database. + +Cypher projections are a more flexible and expressive approach with diminished focus on performance compared to the xref:management-ops/graph-creation/graph-project.adoc[native projections]. == Considerations From 41ad419f2e061bf3f8f4c11fb40779da3f417b41 Mon Sep 17 00:00:00 2001 From: Nicola Vitucci Date: Wed, 27 Sep 2023 15:57:27 +0100 Subject: [PATCH 257/273] Fix wording --- .../management-ops/projections/graph-project-cypher-legacy.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/modules/ROOT/pages/management-ops/projections/graph-project-cypher-legacy.adoc b/doc/modules/ROOT/pages/management-ops/projections/graph-project-cypher-legacy.adoc index 29be95d4d5..ebfda69cdb 100644 --- a/doc/modules/ROOT/pages/management-ops/projections/graph-project-cypher-legacy.adoc +++ b/doc/modules/ROOT/pages/management-ops/projections/graph-project-cypher-legacy.adoc @@ -10,7 +10,7 @@ The replacement is to use the new Cypher projection, which is described in xref: A migration guide is available at xref:migration-lcp-to-cpv2/index.adoc[Appendix C, Migration from Legacy to new Cypher projection]. -- -A Cypher projection uses Cypher to create (_project_) an in-memory graph from the Neo4j database. +A Legacy Cypher projection uses Cypher to create (_project_) an in-memory graph from the Neo4j database. Legacy Cypher projections are a more flexible and expressive approach with diminished focus on performance compared to the xref:management-ops/graph-creation/graph-project.adoc[native projections]. From 610e9a1d2beb25712317a2b07ca40651d8e441a5 Mon Sep 17 00:00:00 2001 From: Nicola Vitucci Date: Thu, 28 Sep 2023 15:18:02 +0100 Subject: [PATCH 258/273] Change wording in projection pages --- .../projections/graph-project-cypher-legacy.adoc | 3 +-- .../projections/graph-project-cypher-projection.adoc | 3 +-- .../pages/management-ops/projections/graph-project.adoc | 8 ++++++-- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/doc/modules/ROOT/pages/management-ops/projections/graph-project-cypher-legacy.adoc b/doc/modules/ROOT/pages/management-ops/projections/graph-project-cypher-legacy.adoc index ebfda69cdb..ea9adac4e8 100644 --- a/doc/modules/ROOT/pages/management-ops/projections/graph-project-cypher-legacy.adoc +++ b/doc/modules/ROOT/pages/management-ops/projections/graph-project-cypher-legacy.adoc @@ -10,10 +10,9 @@ The replacement is to use the new Cypher projection, which is described in xref: A migration guide is available at xref:migration-lcp-to-cpv2/index.adoc[Appendix C, Migration from Legacy to new Cypher projection]. -- +Legacy Cypher projections are a more flexible and expressive approach compared to xref:management-ops/graph-creation/graph-project.adoc[native projections]. A Legacy Cypher projection uses Cypher to create (_project_) an in-memory graph from the Neo4j database. -Legacy Cypher projections are a more flexible and expressive approach with diminished focus on performance compared to the xref:management-ops/graph-creation/graph-project.adoc[native projections]. - == Considerations diff --git a/doc/modules/ROOT/pages/management-ops/projections/graph-project-cypher-projection.adoc b/doc/modules/ROOT/pages/management-ops/projections/graph-project-cypher-projection.adoc index ac24847b98..810ab10887 100644 --- a/doc/modules/ROOT/pages/management-ops/projections/graph-project-cypher-projection.adoc +++ b/doc/modules/ROOT/pages/management-ops/projections/graph-project-cypher-projection.adoc @@ -4,10 +4,9 @@ :page-aliases: management-ops/projections/graph-project-cypher-aggregation +Cypher projections are a more flexible and expressive approach compared to xref:management-ops/graph-creation/graph-project.adoc[native projections]. A Cypher projection uses Cypher to create (_project_) an in-memory graph from the Neo4j database. -Cypher projections are a more flexible and expressive approach with diminished focus on performance compared to the xref:management-ops/graph-creation/graph-project.adoc[native projections]. - == Considerations diff --git a/doc/modules/ROOT/pages/management-ops/projections/graph-project.adoc b/doc/modules/ROOT/pages/management-ops/projections/graph-project.adoc index a41eceb4df..15867c18f6 100644 --- a/doc/modules/ROOT/pages/management-ops/projections/graph-project.adoc +++ b/doc/modules/ROOT/pages/management-ops/projections/graph-project.adoc @@ -3,8 +3,12 @@ :description: This section details projecting GDS graphs using `native` projections. -A native projection is the fastest and most scalable way to project a graph from a Neo4j database into the xref:management-ops/graph-catalog-ops.adoc[GDS Graph Catalog]. -Native projections are recommended for any use case, and for both the development and the production phase (see xref:common-usage/index.adoc[Common usage]). +Native projections are the easiest way to create a xref:management-ops/index.adoc[GDS graph] from a Neo4j database. +A native projection is entirely described by configuration parameters. + +_Node projections_ and _relationship projections_ describe the way nodes and relationships are loaded (_projected_) from the database into the in-memory graph. +While node projections are based on node labels, relationship projections are based on relationship types. +Both can include properties. == Considerations From 4bf854f94af18af4f7c448b19aa9c22efe6c77bf Mon Sep 17 00:00:00 2001 From: Nicola Vitucci Date: Thu, 5 Oct 2023 11:06:33 +0100 Subject: [PATCH 259/273] Fix links --- .../management-ops/projections/graph-project-cypher-legacy.adoc | 2 +- .../projections/graph-project-cypher-projection.adoc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/modules/ROOT/pages/management-ops/projections/graph-project-cypher-legacy.adoc b/doc/modules/ROOT/pages/management-ops/projections/graph-project-cypher-legacy.adoc index ea9adac4e8..a89a7c234d 100644 --- a/doc/modules/ROOT/pages/management-ops/projections/graph-project-cypher-legacy.adoc +++ b/doc/modules/ROOT/pages/management-ops/projections/graph-project-cypher-legacy.adoc @@ -10,7 +10,7 @@ The replacement is to use the new Cypher projection, which is described in xref: A migration guide is available at xref:migration-lcp-to-cpv2/index.adoc[Appendix C, Migration from Legacy to new Cypher projection]. -- -Legacy Cypher projections are a more flexible and expressive approach compared to xref:management-ops/graph-creation/graph-project.adoc[native projections]. +Legacy Cypher projections are a more flexible and expressive approach compared to xref:management-ops/projections/graph-project.adoc[native projections]. A Legacy Cypher projection uses Cypher to create (_project_) an in-memory graph from the Neo4j database. diff --git a/doc/modules/ROOT/pages/management-ops/projections/graph-project-cypher-projection.adoc b/doc/modules/ROOT/pages/management-ops/projections/graph-project-cypher-projection.adoc index 810ab10887..d4160a5883 100644 --- a/doc/modules/ROOT/pages/management-ops/projections/graph-project-cypher-projection.adoc +++ b/doc/modules/ROOT/pages/management-ops/projections/graph-project-cypher-projection.adoc @@ -4,7 +4,7 @@ :page-aliases: management-ops/projections/graph-project-cypher-aggregation -Cypher projections are a more flexible and expressive approach compared to xref:management-ops/graph-creation/graph-project.adoc[native projections]. +Cypher projections are a more flexible and expressive approach compared to xref:management-ops/projections/graph-project.adoc[native projections]. A Cypher projection uses Cypher to create (_project_) an in-memory graph from the Neo4j database. From ab48ba1f840c5de9d0be772e3962bb29b78fafb6 Mon Sep 17 00:00:00 2001 From: yuval Date: Mon, 9 Oct 2023 16:56:28 +0200 Subject: [PATCH 260/273] Bumped Aura version to 37 --- gradle/version.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle/version.gradle b/gradle/version.gradle index d9dfb7419a..8ea3203fba 100644 --- a/gradle/version.gradle +++ b/gradle/version.gradle @@ -1,6 +1,6 @@ ext { gdsBaseVersion = '2.4.7' - gdsAuraVersion = '36' + gdsAuraVersion = '37' gdsVersion = gdsBaseVersion + (rootProject.hasProperty('aurads') ? "+${gdsAuraVersion}" : "") } From 1beea00002c2d0bc7cc375eb3abe722ba9f8aa5c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Mon, 9 Oct 2023 15:15:02 +0200 Subject: [PATCH 261/273] Fix inconsistency between rel-types and schema Which would eventually lead to restore failures. --- .../main/java/org/neo4j/gds/beta/filter/GraphStoreFilter.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/subgraph-filtering/src/main/java/org/neo4j/gds/beta/filter/GraphStoreFilter.java b/subgraph-filtering/src/main/java/org/neo4j/gds/beta/filter/GraphStoreFilter.java index 0dafb6afcb..726ac5a9c7 100644 --- a/subgraph-filtering/src/main/java/org/neo4j/gds/beta/filter/GraphStoreFilter.java +++ b/subgraph-filtering/src/main/java/org/neo4j/gds/beta/filter/GraphStoreFilter.java @@ -25,7 +25,6 @@ import org.neo4j.gds.RelationshipType; import org.neo4j.gds.annotation.ValueClass; import org.neo4j.gds.api.GraphStore; -import org.neo4j.gds.api.schema.Direction; import org.neo4j.gds.api.schema.GraphSchema; import org.neo4j.gds.api.schema.MutableGraphSchema; import org.neo4j.gds.api.schema.MutableNodeSchema; @@ -186,9 +185,6 @@ public static MutableGraphSchema filterSchema(GraphSchema inputGraphSchema, Node .relationshipSchema() .filter(filteredRelationshipTypes) ); - if (relationshipSchema.availableTypes().isEmpty()) { - relationshipSchema.addRelationshipType(RelationshipType.ALL_RELATIONSHIPS, Direction.DIRECTED); - } return MutableGraphSchema.of(nodeSchema, relationshipSchema, Map.of()); } From 11745882c0cfbdd5950078af0fa4d7653cbdb199 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Tue, 10 Oct 2023 09:00:05 +0200 Subject: [PATCH 262/273] Verify consistency between relTypes in projections --- .../java/org/neo4j/gds/projection/CypherAggregationTest.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cypher-aggregation/src/test/java/org/neo4j/gds/projection/CypherAggregationTest.java b/cypher-aggregation/src/test/java/org/neo4j/gds/projection/CypherAggregationTest.java index 7c17947675..c9e784b96d 100644 --- a/cypher-aggregation/src/test/java/org/neo4j/gds/projection/CypherAggregationTest.java +++ b/cypher-aggregation/src/test/java/org/neo4j/gds/projection/CypherAggregationTest.java @@ -380,6 +380,9 @@ void testNodeLabels(String labels) { "RETURN gds.graph.project('g', s, null, { sourceNodeLabels: " + labels + ", targetNodeLabels: NULL })"); var graphStore = GraphStoreCatalog.get("", db.databaseName(), "g").graphStore(); + + assertThat(graphStore.relationshipTypes()).containsExactlyInAnyOrderElementsOf(graphStore.schema().relationshipSchema().availableTypes()); + assertThat(graphStore.nodeLabels()).extracting(NodeLabel::name).containsExactly("A", "B"); } From db1cfab311e9255125091603b65a544d7f721d92 Mon Sep 17 00:00:00 2001 From: ioannispan Date: Thu, 5 Oct 2023 10:23:58 +0200 Subject: [PATCH 263/273] Fix existing defaults-and-limits bug --- .../neo4j/gds/executor/AlgoConfigParser.java | 80 +++++++++---------- .../gds/executor/AlgoConfigParserTest.java | 25 ++++++ 2 files changed, 61 insertions(+), 44 deletions(-) diff --git a/executor/src/main/java/org/neo4j/gds/executor/AlgoConfigParser.java b/executor/src/main/java/org/neo4j/gds/executor/AlgoConfigParser.java index aa7681fa50..6cb6d25a78 100644 --- a/executor/src/main/java/org/neo4j/gds/executor/AlgoConfigParser.java +++ b/executor/src/main/java/org/neo4j/gds/executor/AlgoConfigParser.java @@ -26,11 +26,11 @@ import org.neo4j.gds.configuration.LimitsConfiguration; import org.neo4j.gds.core.CypherMapWrapper; +import java.util.Collection; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedList; import java.util.Map; -import java.util.Set; public class AlgoConfigParser implements ProcConfigParser { private final NewConfigFunction newConfigFunction; @@ -53,28 +53,13 @@ public AlgoConfigParser( @Override public CONFIG processInput(Map configuration) { // apply defaults - var configurationWithDefaultsApplied = defaults.apply(configuration, username); - + var inputWithDefaults = applyDefaults(configuration, username); // parse configuration - CONFIG algorithmConfiguration = produceConfig(configurationWithDefaultsApplied); - - // handle limits - var allowedKeys = new HashSet<>(algorithmConfiguration.configKeys()); - var irrelevantInputtedKeys = getIrrelevantInputtedKeys(configuration, allowedKeys); - var configurationButWithIrrelevantInputtedKeysRemoved = getConfigurationForLimitValidation( - configuration, - irrelevantInputtedKeys - ); - validateLimits(configurationButWithIrrelevantInputtedKeysRemoved); - - // validate configuration - var addedKeys = findAddedKeys(configurationWithDefaultsApplied, configuration.keySet()); - var addedButIrrelevantKeys = findIrrelevantKeys(addedKeys, allowedKeys); - var configurationWithDefaultsAppliedButIrrelevantKeysRemoved = weedOutIrrelevantKeys( - configurationWithDefaultsApplied, - addedButIrrelevantKeys - ); - validateConfig(allowedKeys, configurationWithDefaultsAppliedButIrrelevantKeysRemoved); + CONFIG algorithmConfiguration = produceConfig(inputWithDefaults); + //validate the original config for extra-added parameters + validateOriginalConfig(configuration, algorithmConfiguration.configKeys()); + //ensure that limits are fine + validateLimits(algorithmConfiguration, username, inputWithDefaults); return algorithmConfiguration; } @@ -96,6 +81,25 @@ private HashMap getConfigurationForLimitValidation( return configurationButWithIrrelevantInputtedKeysRemoved; } + void validateLimits( + CONFIG algorithmConfiguration, + String username, + Map userInputWithDefaults + ) throws IllegalArgumentException { + + // handle limits + var allowedKeys = new HashSet<>(algorithmConfiguration.configKeys()); + var irrelevantInputtedKeys = getIrrelevantInputtedKeys(userInputWithDefaults, allowedKeys); + var configurationButWithIrrelevantInputtedKeysRemoved = getConfigurationForLimitValidation( + userInputWithDefaults, + irrelevantInputtedKeys + ); //remove any useless configuration parameters e.g., sourceNode for Wcc + + validateLimits(configurationButWithIrrelevantInputtedKeysRemoved); + + } + + private void validateLimits(HashMap configurationButWithIrrelevantInputtedKeysRemoved) { var violations = limits.validate(configurationButWithIrrelevantInputtedKeysRemoved, username); @@ -121,31 +125,19 @@ private CONFIG produceConfig(Map configurationWithDefaultsApplie return newConfigFunction.apply(username, cypherMapWrapper); } - @NotNull - private Set findAddedKeys(Map configurationWithDefaultsApplied, Set inputtedKeys) { - Set addedKeys = new HashSet<>(configurationWithDefaultsApplied.keySet()); - addedKeys.removeAll(inputtedKeys); - return addedKeys; - } - @NotNull - private Set findIrrelevantKeys(Set addedKeys, Set allowedKeys) { - Set irrelevantKeys = new HashSet<>(addedKeys); - irrelevantKeys.removeAll(allowedKeys); - return irrelevantKeys; - } + void validateOriginalConfig( + Map configuration, + Collection allowedConfigKeys + ) throws IllegalArgumentException { + Map newConfiguration = new HashMap<>(configuration); - @NotNull - private HashMap weedOutIrrelevantKeys( - Map configurationWithDefaultsApplied, - Set irrelevantKeys - ) { - var configurationWithDefaultsAppliedButIrrelevantKeysRemoved = new HashMap<>(configurationWithDefaultsApplied); - configurationWithDefaultsAppliedButIrrelevantKeysRemoved.keySet().removeAll(irrelevantKeys); - return configurationWithDefaultsAppliedButIrrelevantKeysRemoved; + CypherMapWrapper.create(newConfiguration) + .requireOnlyKeysFrom(allowedConfigKeys); //ensure user has not included any incorrect params } - private void validateConfig(Set allowedKeys, Map configuration) { - CypherMapWrapper.create(configuration).requireOnlyKeysFrom(allowedKeys); + Map applyDefaults(Map configuration, String username) { + // apply defaults + return defaults.apply(configuration, username); } } diff --git a/executor/src/test/java/org/neo4j/gds/executor/AlgoConfigParserTest.java b/executor/src/test/java/org/neo4j/gds/executor/AlgoConfigParserTest.java index 0506484296..4024ed5c70 100644 --- a/executor/src/test/java/org/neo4j/gds/executor/AlgoConfigParserTest.java +++ b/executor/src/test/java/org/neo4j/gds/executor/AlgoConfigParserTest.java @@ -29,6 +29,7 @@ import java.util.Map; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatException; import static org.junit.jupiter.api.Assertions.fail; class AlgoConfigParserTest { @@ -196,4 +197,28 @@ void shouldReportAllLimitViolationsInOneGo() { " - Configuration parameter 'qux' with value '117' exceeds it's limit of '87'"); } } + + + @Test + void shouldComplainWhenAddedDefaultsDoNotAdhereToLimits() { + var configurationParser = new AlgoConfigParser<>( + "Miltos Tentoglou", + (NewConfigFunction) (username, config) -> new FooConfigImpl(config), + new DefaultsConfiguration( + Map.of("bar", new Default(16L), "sudo", new Default(false)), + Collections.emptyMap() + ), + new LimitsConfiguration( + Map.of("bar", LimitFactory.create(8L), "sudo", LimitFactory.create(true)), + Collections.emptyMap() + ) + ); + + assertThatException().isThrownBy(() -> configurationParser.processInput(Map.of())).withMessage( + "Configuration exceeded multiple limits:\n" + + " - Configuration parameter 'bar' with value '16' exceeds it's limit of '8'\n" + + " - Configuration parameter 'sudo' with value 'false' is in violation of it's set limit"); + + + } } From f09d905f7bfff1682e222505f4ff6bba2c39a7bf Mon Sep 17 00:00:00 2001 From: Veselin Nikolov Date: Wed, 4 Oct 2023 15:35:19 +0100 Subject: [PATCH 264/273] Use correct Euclidean formula Co-authored-by: Ioannis Panagiotas --- .../gds/similarity/knn/metrics/Euclidean.java | 28 +++++++---- .../similarity/knn/metrics/EuclideanTest.java | 48 +++++++++++++++++++ doc/modules/ROOT/pages/algorithms/knn.adoc | 10 ++-- 3 files changed, 71 insertions(+), 15 deletions(-) create mode 100644 algo/src/test/java/org/neo4j/gds/similarity/knn/metrics/EuclideanTest.java diff --git a/algo/src/main/java/org/neo4j/gds/similarity/knn/metrics/Euclidean.java b/algo/src/main/java/org/neo4j/gds/similarity/knn/metrics/Euclidean.java index 39c85fbaae..48524b2632 100644 --- a/algo/src/main/java/org/neo4j/gds/similarity/knn/metrics/Euclidean.java +++ b/algo/src/main/java/org/neo4j/gds/similarity/knn/metrics/Euclidean.java @@ -19,6 +19,8 @@ */ package org.neo4j.gds.similarity.knn.metrics; +import java.util.function.IntToDoubleFunction; + /** * Here we calculate Euclidean similarity metrics using Euclidean dictance as described in e.g. * https://en.wikipedia.org/wiki/Euclidean_distance @@ -33,22 +35,28 @@ public final class Euclidean { private Euclidean() {} public static double floatMetric(float[] left, float[] right) { - var len = Math.min(left.length, right.length); - var result = 0D; - for (int i = 0; i < len; i++) { - double delta = left[i] - right[i]; - result += delta * delta; - } - return 1.0 / (1.0 + result); + return compute( + Math.min(left.length, right.length), + i -> left[i], + i -> right[i] + ); } public static double doubleMetric(double[] left, double[] right) { - var len = Math.min(left.length, right.length); + return compute( + Math.min(left.length, right.length), + i -> left[i], + i -> right[i] + ); + } + + private static double compute(int len, IntToDoubleFunction left, IntToDoubleFunction right) { var result = 0D; for (int i = 0; i < len; i++) { - double delta = left[i] - right[i]; + double delta = left.applyAsDouble(i) - right.applyAsDouble(i); result += delta * delta; } - return 1.0 / (1.0 + result); + + return 1.0 / (1.0 + Math.sqrt(result)); } } diff --git a/algo/src/test/java/org/neo4j/gds/similarity/knn/metrics/EuclideanTest.java b/algo/src/test/java/org/neo4j/gds/similarity/knn/metrics/EuclideanTest.java new file mode 100644 index 0000000000..6769db1727 --- /dev/null +++ b/algo/src/test/java/org/neo4j/gds/similarity/knn/metrics/EuclideanTest.java @@ -0,0 +1,48 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.similarity.knn.metrics; + +import org.assertj.core.data.Offset; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +class EuclideanTest { + + @Test + void doubleArrays() { + var left = new double[] {0.008706967509313894d, 0.0d, 0.004919643078839634d, 0.007029592654453866d, 0.0d, 0.0d, 0.1257682851778286d, 0.016530303738513153d, 0.012745441170181648d, 0.0d}; + var right = new double[] {0.008706967509313894d, 0.0d, 0.0041533258820686736d, 0.003698167604532474d, 0.0d, 0.0d, 0.14438303839785882d, 0.0017742178454893931d, 0.008013536500447603d, 0.0d}; + + var metric = Euclidean.doubleMetric(left, right); + + assertThat(metric).isCloseTo(0.976123304363789d, Offset.offset(1e-5)); + } + + @Test + void floatArrays() { + var left = new float[] {0.008706967509313894f, 0.0f, 0.004919643078839634f, 0.007029592654453866f, 0.0f, 0.0f, 0.1257682851778286f, 0.016530303738513153f, 0.012745441170181648f, 0.0f}; + var right = new float[] {0.008706967509313894f, 0.0f, 0.0041533258820686736f, 0.003698167604532474f, 0.0f, 0.0f, 0.14438303839785882f, 0.0017742178454893931f, 0.008013536500447603f, 0.0f}; + + var metric = Euclidean.floatMetric(left, right); + + assertThat(metric).isCloseTo(0.976123304363789d, Offset.offset(1e-5)); + } +} diff --git a/doc/modules/ROOT/pages/algorithms/knn.adoc b/doc/modules/ROOT/pages/algorithms/knn.adoc index 5ef0c002e5..a193ff5f2d 100644 --- a/doc/modules/ROOT/pages/algorithms/knn.adoc +++ b/doc/modules/ROOT/pages/algorithms/knn.adoc @@ -680,11 +680,11 @@ ORDER BY similarity DESCENDING, Person1, Person2 [opts="header"] |=== | Person1 | Person2 | similarity -| "Alice" | "Carol" | 0.931216931216931 -| "Carol" | "Alice" | 0.931216931216931 -| "Bob" | "Carol" | 0.432336103416436 -| "Eve" | "Alice" | 0.366920651602733 -| "Dave" | "Bob" | 0.243466706038683 +| "Alice" | "Carol" | 0.8874315534 +| "Carol" | "Alice" | 0.8874315534 +| "Bob" | "Carol" | 0.4674429487 +| "Eve" | "Bob" | 0.3700361866 +| "Dave" | "Bob" | 0.2887113179 |=== Note that the two distinct maps in the query could be merged to a single one. From 5233909bd38c0ab8fedc54defdbe0064df36fbe9 Mon Sep 17 00:00:00 2001 From: ioannispan Date: Mon, 9 Oct 2023 14:17:14 +0200 Subject: [PATCH 265/273] Fix max degree bug MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Jonatan Jäderberg --- .../triangle/intersect/GraphIntersect.java | 5 +- .../triangle/TriangleCountMaxDegreeTest.java | 50 +++++++++++++++++++ 2 files changed, 53 insertions(+), 2 deletions(-) create mode 100644 algo/src/test/java/org/neo4j/gds/triangle/TriangleCountMaxDegreeTest.java diff --git a/algo/src/main/java/org/neo4j/gds/triangle/intersect/GraphIntersect.java b/algo/src/main/java/org/neo4j/gds/triangle/intersect/GraphIntersect.java index a4d2b460f4..59b15d4f41 100644 --- a/algo/src/main/java/org/neo4j/gds/triangle/intersect/GraphIntersect.java +++ b/algo/src/main/java/org/neo4j/gds/triangle/intersect/GraphIntersect.java @@ -68,8 +68,7 @@ public void intersectAll(long nodeA, IntersectionConsumer consumer) { // iterates over neighbours of A CURSOR neighboursA = cacheA; - // current neighbour of A - long nodeCFromA = NOT_FOUND; + // iterates over neighbours of B CURSOR neighboursB = cacheB; // current neighbour of B @@ -100,6 +99,8 @@ public void intersectAll(long nodeA, IntersectionConsumer consumer) { // copy the state of A's cursor neighboursA = copyCursor(neighboursAMain, neighboursA); + var nodeCFromA = NOT_FOUND; //current neighbor from a + if (degreeFilter.test(degree(nodeCFromB))) { // find the first neighbour Ca of A with id >= Cb nodeCFromA = neighboursA.advance(nodeCFromB); diff --git a/algo/src/test/java/org/neo4j/gds/triangle/TriangleCountMaxDegreeTest.java b/algo/src/test/java/org/neo4j/gds/triangle/TriangleCountMaxDegreeTest.java new file mode 100644 index 0000000000..e311c8a9ab --- /dev/null +++ b/algo/src/test/java/org/neo4j/gds/triangle/TriangleCountMaxDegreeTest.java @@ -0,0 +1,50 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.triangle; + +import org.junit.jupiter.api.Test; +import org.neo4j.gds.api.schema.Direction; +import org.neo4j.gds.beta.generator.RandomGraphGenerator; +import org.neo4j.gds.beta.generator.RelationshipDistribution; +import org.neo4j.gds.core.concurrency.DefaultPool; + +import static org.assertj.core.api.Assertions.assertThat; + +class TriangleCountMaxDegreeTest { + + @Test + void shouldWorkWithMaxDegree(){ + var graph= RandomGraphGenerator.builder().nodeCount(24).averageDegree(23).seed(101) + .relationshipDistribution( + RelationshipDistribution.RANDOM) + .direction(Direction.UNDIRECTED) + .build().generate(); + + TriangleCountBaseConfig config = ImmutableTriangleCountBaseConfig + .builder() + .maxDegree(100) + .build(); + + var tc = IntersectingTriangleCount.create(graph, config, DefaultPool.INSTANCE).compute(); + assertThat(tc.globalTriangles()).isEqualTo(1262L); + + } + +} From 78257081cd313b6d849bd79ba3892b2219fb8795 Mon Sep 17 00:00:00 2001 From: ioannispan Date: Wed, 11 Oct 2023 08:59:49 +0200 Subject: [PATCH 266/273] Exhibit issue and propose fix --- .../core/huge/CompositeAdjacencyCursor.java | 5 ++-- .../gds/core/huge/CompositeAdjacencyList.java | 10 ++++---- .../core/huge/CompositeAdjacencyListTest.java | 25 ++++++++++++++++++- 3 files changed, 32 insertions(+), 8 deletions(-) diff --git a/core/src/main/java/org/neo4j/gds/core/huge/CompositeAdjacencyCursor.java b/core/src/main/java/org/neo4j/gds/core/huge/CompositeAdjacencyCursor.java index 427b80dfae..b55d835f9c 100644 --- a/core/src/main/java/org/neo4j/gds/core/huge/CompositeAdjacencyCursor.java +++ b/core/src/main/java/org/neo4j/gds/core/huge/CompositeAdjacencyCursor.java @@ -51,9 +51,10 @@ public List cursors() { return cursors; } - void updateCursorsQueue() { + void reinitializeCursors() { cursorQueue.clear(); - cursorQueue.addAll(cursors); + initializeQueue(); + } @Override public int size() { diff --git a/core/src/main/java/org/neo4j/gds/core/huge/CompositeAdjacencyList.java b/core/src/main/java/org/neo4j/gds/core/huge/CompositeAdjacencyList.java index 14658b4c70..9f08471554 100644 --- a/core/src/main/java/org/neo4j/gds/core/huge/CompositeAdjacencyList.java +++ b/core/src/main/java/org/neo4j/gds/core/huge/CompositeAdjacencyList.java @@ -125,19 +125,19 @@ public CompositeAdjacencyCursor adjacencyCursor(@Nullable AdjacencyCursor reuse, if (reuse instanceof CompositeAdjacencyCursor) { var compositeReuse = (CompositeAdjacencyCursor) reuse; var iter = compositeReuse.cursors().listIterator(); - boolean cursorsAreUpdated = false; + while (iter.hasNext()) { var index = iter.nextIndex(); var cursor = iter.next(); var newCursor = adjacencyLists.get(index).adjacencyCursor(cursor, node, fallbackValue); if (newCursor != cursor) { iter.set(adjacencyCursorWrapperFactory.create(newCursor)); - cursorsAreUpdated = true; } } - if (cursorsAreUpdated) { - compositeReuse.updateCursorsQueue(); - } + //we must update all the cursors all the time because old cursor might be further ahead instead of start + //then, object equality can be then the same, but in reality not so much + compositeReuse.reinitializeCursors(); + return compositeReuse; } return adjacencyCursor(node, fallbackValue); diff --git a/core/src/test/java/org/neo4j/gds/core/huge/CompositeAdjacencyListTest.java b/core/src/test/java/org/neo4j/gds/core/huge/CompositeAdjacencyListTest.java index 2b2751cbcf..cd3abfed39 100644 --- a/core/src/test/java/org/neo4j/gds/core/huge/CompositeAdjacencyListTest.java +++ b/core/src/test/java/org/neo4j/gds/core/huge/CompositeAdjacencyListTest.java @@ -20,11 +20,12 @@ package org.neo4j.gds.core.huge; import org.junit.jupiter.api.Test; +import org.neo4j.gds.api.Graph; import org.neo4j.gds.extension.GdlExtension; import org.neo4j.gds.extension.GdlGraph; import org.neo4j.gds.extension.Inject; -import org.neo4j.gds.api.Graph; +import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; @GdlExtension @@ -47,4 +48,26 @@ void shouldIgnoreInputDegreeForCursor() { var cursor = adjacencyList.adjacencyCursor(0); assertEquals(2, cursor.remaining()); } + + @Test + void shouldReuseCursor() { + + var adjacencyList = ((UnionGraph) graph).relationshipTopology(); + var cursor = adjacencyList.adjacencyCursor(null, 0); + assertThat(cursor.peekVLong()).isEqualTo(1L); + cursor.nextVLong(); + + cursor = adjacencyList.adjacencyCursor(cursor, 0); + assertThat(cursor.peekVLong()).isEqualTo(1L); + + + cursor.nextVLong(); + cursor.nextVLong(); + + cursor = adjacencyList.adjacencyCursor(cursor, 0); + assertThat(cursor.hasNextVLong()).isTrue(); + + + } + } From 7f1fec326bb1359d6d8c20caf88048d5af5f8357 Mon Sep 17 00:00:00 2001 From: ioannispan Date: Wed, 11 Oct 2023 10:17:48 +0200 Subject: [PATCH 267/273] precision issue not appearing on master but appearing on 2.4 :shrug: --- doc/modules/ROOT/pages/algorithms/knn.adoc | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/modules/ROOT/pages/algorithms/knn.adoc b/doc/modules/ROOT/pages/algorithms/knn.adoc index a193ff5f2d..a3915f638e 100644 --- a/doc/modules/ROOT/pages/algorithms/knn.adoc +++ b/doc/modules/ROOT/pages/algorithms/knn.adoc @@ -680,11 +680,11 @@ ORDER BY similarity DESCENDING, Person1, Person2 [opts="header"] |=== | Person1 | Person2 | similarity -| "Alice" | "Carol" | 0.8874315534 -| "Carol" | "Alice" | 0.8874315534 -| "Bob" | "Carol" | 0.4674429487 -| "Eve" | "Bob" | 0.3700361866 -| "Dave" | "Bob" | 0.2887113179 +| "Alice" | "Carol" | 0.887431553441766 +| "Carol" | "Alice" | 0.887431553441766 +| "Bob" | "Carol" | 0.467442948683302 +| "Eve" | "Bob" | 0.37003618661809 +| "Dave" | "Bob" | 0.288711317895105 |=== Note that the two distinct maps in the query could be merged to a single one. From 1e3d2cd9a557857ee26e25a9352eb8b252632d5d Mon Sep 17 00:00:00 2001 From: Martin Junghanns Date: Wed, 25 Oct 2023 10:29:19 +0200 Subject: [PATCH 268/273] Skip ServiceAvailablity Agent attach on intel+mac+azul Co-Authored-By: Paul Horn --- .../java/org/neo4j/gds/mem/MemoryUsage.java | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/memory-usage/src/main/java/org/neo4j/gds/mem/MemoryUsage.java b/memory-usage/src/main/java/org/neo4j/gds/mem/MemoryUsage.java index f0014becdd..3fdac652ab 100644 --- a/memory-usage/src/main/java/org/neo4j/gds/mem/MemoryUsage.java +++ b/memory-usage/src/main/java/org/neo4j/gds/mem/MemoryUsage.java @@ -364,6 +364,8 @@ private static final class VmInfoHolder { */ @SuppressForbidden(reason = "we want to use system.out here") private static boolean isVmInfoAvailable() { + intelMacWorkaround(); + var sysOut = System.out; try { var swallowSysOut = new PrintStream(NullOutputStream.NULL_OUTPUT_STREAM, true, StandardCharsets.UTF_8); @@ -378,6 +380,24 @@ private static boolean isVmInfoAvailable() { } } + /** + * JOL currently gets stuck on Azul Zulu 17.44.53-ca-jdk17.0.8.1 on Intel Macs. + *

+ * We can work around this by skipping the Hotspot SA attach. + * See OpenJDK issue + *

+ */ + @Deprecated(forRemoval = true) + private static void intelMacWorkaround() { + var isAzul = System.getProperty("java.vendor").contains("Azul Systems"); + var isIntel = System.getProperty("os.arch").contains("x86_64"); + var isMac = System.getProperty("os.name").contains("Mac"); + + if (isAzul && isIntel && isMac) { + System.setProperty("jol.skipHotspotSAAttach", "true"); + } + } + private static final class NullOutputStream extends OutputStream { static final OutputStream NULL_OUTPUT_STREAM = new NullOutputStream(); From d7301396ea92b4fa03db248b3c084c9672251801 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B6ren=20Reichardt?= Date: Tue, 10 Oct 2023 12:34:20 +0200 Subject: [PATCH 269/273] Fix Neo4jProxyImpl::accessMode Co-Authored-By: Paul Horn --- .../gds/compat/_511/CompatAccessModeImpl.java | 2 +- .../org/neo4j/gds/compat/_511/Neo4jProxyImpl.java | 15 +++++++++++++-- .../gds/compat/_512/CompatAccessModeImpl.java | 2 +- .../org/neo4j/gds/compat/_512/Neo4jProxyImpl.java | 15 +++++++++++++-- .../gds/compat/_513/CompatAccessModeImpl.java | 2 +- .../org/neo4j/gds/compat/_513/Neo4jProxyImpl.java | 15 +++++++++++++-- 6 files changed, 42 insertions(+), 9 deletions(-) diff --git a/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/CompatAccessModeImpl.java b/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/CompatAccessModeImpl.java index 4af537fe9c..226f56cd1b 100644 --- a/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/CompatAccessModeImpl.java +++ b/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/CompatAccessModeImpl.java @@ -26,7 +26,7 @@ import java.util.function.Supplier; -public final class CompatAccessModeImpl extends CompatAccessMode { +public class CompatAccessModeImpl extends CompatAccessMode { CompatAccessModeImpl(CustomAccessMode custom) { super(custom); diff --git a/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/Neo4jProxyImpl.java b/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/Neo4jProxyImpl.java index a360f37af7..05a406402a 100644 --- a/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/Neo4jProxyImpl.java +++ b/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/Neo4jProxyImpl.java @@ -87,6 +87,7 @@ import org.neo4j.internal.kernel.api.RelationshipScanCursor; import org.neo4j.internal.kernel.api.Scan; import org.neo4j.internal.kernel.api.TokenPredicate; +import org.neo4j.internal.kernel.api.TokenSet; import org.neo4j.internal.kernel.api.connectioninfo.ClientConnectionInfo; import org.neo4j.internal.kernel.api.exceptions.ProcedureException; import org.neo4j.internal.kernel.api.procs.FieldSignature; @@ -96,6 +97,7 @@ import org.neo4j.internal.kernel.api.procs.UserFunctionSignature; import org.neo4j.internal.kernel.api.security.AccessMode; import org.neo4j.internal.kernel.api.security.AuthSubject; +import org.neo4j.internal.kernel.api.security.ReadSecurityPropertyProvider; import org.neo4j.internal.kernel.api.security.SecurityContext; import org.neo4j.internal.recordstorage.RecordIdType; import org.neo4j.internal.schema.IndexCapability; @@ -154,6 +156,7 @@ import java.util.Optional; import java.util.Set; import java.util.concurrent.TimeUnit; +import java.util.function.Supplier; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -177,9 +180,17 @@ public String validateExternalDatabaseName(String databaseName) { @Override public AccessMode accessMode(CustomAccessMode customAccessMode) { - return new CompatAccessModeImpl(customAccessMode); + return new CompatAccessModeImpl(customAccessMode) { + @Override + public boolean allowsReadNodeProperty( + Supplier labels, + int propertyKey, + ReadSecurityPropertyProvider propertyProvider + ) { + return custom.allowsReadNodeProperty(propertyKey); + } + }; } - @Override public String username(AuthSubject subject) { return subject.executingUser(); diff --git a/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/CompatAccessModeImpl.java b/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/CompatAccessModeImpl.java index 7ff84d9939..f11614a613 100644 --- a/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/CompatAccessModeImpl.java +++ b/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/CompatAccessModeImpl.java @@ -26,7 +26,7 @@ import java.util.function.Supplier; -public final class CompatAccessModeImpl extends CompatAccessMode { +public class CompatAccessModeImpl extends CompatAccessMode { CompatAccessModeImpl(CustomAccessMode custom) { super(custom); diff --git a/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/Neo4jProxyImpl.java b/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/Neo4jProxyImpl.java index 644e639d52..73752765f0 100644 --- a/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/Neo4jProxyImpl.java +++ b/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/Neo4jProxyImpl.java @@ -87,6 +87,7 @@ import org.neo4j.internal.kernel.api.RelationshipScanCursor; import org.neo4j.internal.kernel.api.Scan; import org.neo4j.internal.kernel.api.TokenPredicate; +import org.neo4j.internal.kernel.api.TokenSet; import org.neo4j.internal.kernel.api.connectioninfo.ClientConnectionInfo; import org.neo4j.internal.kernel.api.exceptions.ProcedureException; import org.neo4j.internal.kernel.api.procs.FieldSignature; @@ -96,6 +97,7 @@ import org.neo4j.internal.kernel.api.procs.UserFunctionSignature; import org.neo4j.internal.kernel.api.security.AccessMode; import org.neo4j.internal.kernel.api.security.AuthSubject; +import org.neo4j.internal.kernel.api.security.ReadSecurityPropertyProvider; import org.neo4j.internal.kernel.api.security.SecurityContext; import org.neo4j.internal.recordstorage.RecordIdType; import org.neo4j.internal.schema.IndexCapability; @@ -154,6 +156,7 @@ import java.util.Optional; import java.util.Set; import java.util.concurrent.TimeUnit; +import java.util.function.Supplier; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -176,9 +179,17 @@ public String validateExternalDatabaseName(String databaseName) { @Override public AccessMode accessMode(CustomAccessMode customAccessMode) { - return new CompatAccessModeImpl(customAccessMode); + return new CompatAccessModeImpl(customAccessMode) { + @Override + public boolean allowsReadNodeProperty( + Supplier labels, + int propertyKey, + ReadSecurityPropertyProvider propertyProvider + ) { + return custom.allowsReadNodeProperty(propertyKey); + } + }; } - @Override public String username(AuthSubject subject) { return subject.executingUser(); diff --git a/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/CompatAccessModeImpl.java b/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/CompatAccessModeImpl.java index d5b41cf544..568f3c7d0c 100644 --- a/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/CompatAccessModeImpl.java +++ b/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/CompatAccessModeImpl.java @@ -26,7 +26,7 @@ import java.util.function.Supplier; -public final class CompatAccessModeImpl extends CompatAccessMode { +public class CompatAccessModeImpl extends CompatAccessMode { CompatAccessModeImpl(CustomAccessMode custom) { super(custom); diff --git a/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/Neo4jProxyImpl.java b/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/Neo4jProxyImpl.java index d3b2d662ea..c3a0eee1a3 100644 --- a/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/Neo4jProxyImpl.java +++ b/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/Neo4jProxyImpl.java @@ -87,6 +87,7 @@ import org.neo4j.internal.kernel.api.RelationshipScanCursor; import org.neo4j.internal.kernel.api.Scan; import org.neo4j.internal.kernel.api.TokenPredicate; +import org.neo4j.internal.kernel.api.TokenSet; import org.neo4j.internal.kernel.api.connectioninfo.ClientConnectionInfo; import org.neo4j.internal.kernel.api.exceptions.ProcedureException; import org.neo4j.internal.kernel.api.procs.FieldSignature; @@ -96,6 +97,7 @@ import org.neo4j.internal.kernel.api.procs.UserFunctionSignature; import org.neo4j.internal.kernel.api.security.AccessMode; import org.neo4j.internal.kernel.api.security.AuthSubject; +import org.neo4j.internal.kernel.api.security.ReadSecurityPropertyProvider; import org.neo4j.internal.kernel.api.security.SecurityContext; import org.neo4j.internal.recordstorage.RecordIdType; import org.neo4j.internal.schema.IndexCapability; @@ -154,6 +156,7 @@ import java.util.Optional; import java.util.Set; import java.util.concurrent.TimeUnit; +import java.util.function.Supplier; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -176,9 +179,17 @@ public String validateExternalDatabaseName(String databaseName) { @Override public AccessMode accessMode(CustomAccessMode customAccessMode) { - return new CompatAccessModeImpl(customAccessMode); + return new CompatAccessModeImpl(customAccessMode) { + @Override + public boolean allowsReadNodeProperty( + Supplier labels, + int propertyKey, + ReadSecurityPropertyProvider propertyProvider + ) { + return custom.allowsReadNodeProperty(propertyKey); + } + }; } - @Override public String username(AuthSubject subject) { return subject.executingUser(); From f4a5a5343319ac9b25cbf0938dc9d0c0ab864b3d Mon Sep 17 00:00:00 2001 From: Martin Junghanns Date: Wed, 25 Oct 2023 10:10:54 +0200 Subject: [PATCH 270/273] Document workaround for MacOS intel bug in Desktop Co-Authored-By: Paul Horn --- doc/modules/ROOT/pages/installation/neo4j-desktop.adoc | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/doc/modules/ROOT/pages/installation/neo4j-desktop.adoc b/doc/modules/ROOT/pages/installation/neo4j-desktop.adoc index 9d45b14d9e..d688482abc 100644 --- a/doc/modules/ROOT/pages/installation/neo4j-desktop.adoc +++ b/doc/modules/ROOT/pages/installation/neo4j-desktop.adoc @@ -28,4 +28,12 @@ If the procedure allowlist is configured, make sure to also include procedures f ---- dbms.security.procedures.allowlist=gds.* ----- \ No newline at end of file +---- + +If you are running MacOS on an intel CPU, it is required to add the following configuration entry to the configuration: + +--- +server.jvm.additional=-Djol.skipHotspotSAAttach=true +--- + +This is a workaround for a bug in the Java runtime that is used by Neo4j Desktop. From d20cbca1fbf7edbc192a9470143c2090a85cf704 Mon Sep 17 00:00:00 2001 From: Martin Junghanns Date: Wed, 25 Oct 2023 15:03:57 +0200 Subject: [PATCH 271/273] Fix doc code block Co-Authored-By: Paul Horn --- doc/modules/ROOT/pages/installation/neo4j-desktop.adoc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/modules/ROOT/pages/installation/neo4j-desktop.adoc b/doc/modules/ROOT/pages/installation/neo4j-desktop.adoc index d688482abc..8b24e04958 100644 --- a/doc/modules/ROOT/pages/installation/neo4j-desktop.adoc +++ b/doc/modules/ROOT/pages/installation/neo4j-desktop.adoc @@ -32,8 +32,8 @@ dbms.security.procedures.allowlist=gds.* If you are running MacOS on an intel CPU, it is required to add the following configuration entry to the configuration: ---- +---- server.jvm.additional=-Djol.skipHotspotSAAttach=true ---- +---- This is a workaround for a bug in the Java runtime that is used by Neo4j Desktop. From 654b8ac73a11f3c817d45b91a149ee6338e1d9af Mon Sep 17 00:00:00 2001 From: Martin Junghanns Date: Thu, 26 Oct 2023 14:28:18 +0200 Subject: [PATCH 272/273] Expand intel Mac workaround to Mac generally Co-Authored-By: Paul Horn --- .../main/java/org/neo4j/gds/mem/MemoryUsage.java | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/memory-usage/src/main/java/org/neo4j/gds/mem/MemoryUsage.java b/memory-usage/src/main/java/org/neo4j/gds/mem/MemoryUsage.java index 3fdac652ab..9b3fc1bf90 100644 --- a/memory-usage/src/main/java/org/neo4j/gds/mem/MemoryUsage.java +++ b/memory-usage/src/main/java/org/neo4j/gds/mem/MemoryUsage.java @@ -364,7 +364,7 @@ private static final class VmInfoHolder { */ @SuppressForbidden(reason = "we want to use system.out here") private static boolean isVmInfoAvailable() { - intelMacWorkaround(); + macWorkaround(); var sysOut = System.out; try { @@ -381,19 +381,16 @@ private static boolean isVmInfoAvailable() { } /** - * JOL currently gets stuck on Azul Zulu 17.44.53-ca-jdk17.0.8.1 on Intel Macs. + * JOL currently kills the JVM on Mac OS X when trying to attach to the Hotspot SA. + * This happens with several JVMs (Java.net, Azul) and on both platforms (x86, ARM). *

* We can work around this by skipping the Hotspot SA attach. * See OpenJDK issue *

*/ @Deprecated(forRemoval = true) - private static void intelMacWorkaround() { - var isAzul = System.getProperty("java.vendor").contains("Azul Systems"); - var isIntel = System.getProperty("os.arch").contains("x86_64"); - var isMac = System.getProperty("os.name").contains("Mac"); - - if (isAzul && isIntel && isMac) { + private static void macWorkaround() { + if (System.getProperty("os.name").contains("Mac")) { System.setProperty("jol.skipHotspotSAAttach", "true"); } } From 0a1362564896ff89fbbe3767e47a081d7dccfdbf Mon Sep 17 00:00:00 2001 From: Martin Junghanns Date: Thu, 26 Oct 2023 14:37:30 +0200 Subject: [PATCH 273/273] Add MacOS-specific config hint to installation page Co-Authored-By: Paul Horn --- doc/modules/ROOT/pages/installation/index.adoc | 9 +++++++++ doc/modules/ROOT/pages/installation/neo4j-desktop.adoc | 4 +--- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/doc/modules/ROOT/pages/installation/index.adoc b/doc/modules/ROOT/pages/installation/index.adoc index d7f9ec926d..cd1d691554 100644 --- a/doc/modules/ROOT/pages/installation/index.adoc +++ b/doc/modules/ROOT/pages/installation/index.adoc @@ -68,6 +68,15 @@ For example, exporting graphs xref:graph-catalog-export-ops.adoc#catalog-graph-e You can find the list of all the configuration options xref:production-deployment/configuration-settings.adoc[here]. Refer to the <<_installation_methods,installation methods>> for details on how to edit a Neo4j database configuration depending on the Neo4j deployment. +== Graph Data Science on MacOS + +If you are running MacOS (x86 or ARM), it is currently required to add the following configuration entry to `neo4j.conf`: + +---- +server.jvm.additional=-Djol.skipHotspotSAAttach=true +---- + + == Reference * xref:installation/supported-neo4j-versions.adoc[Supported Neo4j versions] diff --git a/doc/modules/ROOT/pages/installation/neo4j-desktop.adoc b/doc/modules/ROOT/pages/installation/neo4j-desktop.adoc index 8b24e04958..524ae107c1 100644 --- a/doc/modules/ROOT/pages/installation/neo4j-desktop.adoc +++ b/doc/modules/ROOT/pages/installation/neo4j-desktop.adoc @@ -30,10 +30,8 @@ If the procedure allowlist is configured, make sure to also include procedures f dbms.security.procedures.allowlist=gds.* ---- -If you are running MacOS on an intel CPU, it is required to add the following configuration entry to the configuration: +If you are running MacOS (x86 or ARM), it is required to add the following configuration entry to the configuration: ---- server.jvm.additional=-Djol.skipHotspotSAAttach=true ---- - -This is a workaround for a bug in the Java runtime that is used by Neo4j Desktop.