Update to use adiff file

Author: hanbyul-here

pipeline/README.md

@@ -10,6 +10,6 @@ Note:
 ```sh
 
 docker build -t osm-gradient .
-docker run -it -v ./data:/tmp osm-gradient sh -c "./cli.js process-hour 2024-05-18 02"
-docker run -it -v ./data:/tmp osm-gradient sh -c "ogr2ogr -f FlatGeobuf /tmp/2024-05-18T02/2024-05-18T02:00.fgb /tmp/2024-05-18T02/2024-05-18T02:00.geojson -skipfailures"
-```
+docker run -it -v ./data:/tmp osm-gradient sh -c "./cli.js process-hour 2025-05-18 02"
+docker run -it -v ./data:/tmp osm-gradient sh -c "ogr2ogr -f FlatGeobuf /tmp/2025-05-18T02/2025-05-18T02:00.fgb /tmp/2025-05-18T02/2025-05-18T02:00.geojson -skipfailures"
+```

pipeline/package.json

@@ -8,6 +8,7 @@
   "author": "",
   "license": "ISC",
   "dependencies": {
+    "@osmcha/osm-adiff-parser": "^3.1.0",
     "axios": "^1.5.0",
     "commander": "12.0.0",
     "real-changesets-parser": "https://github.com/developmentseed/real-changesets-parser.git",

pipeline/src/process.js

@@ -1,5 +1,6 @@
 const axios = require('axios');
-const changesetParser = require("real-changesets-parser");
+const adiffParser = require("@osmcha/osm-adiff-parser");
+const changesetParser =  require("real-changesets-parser");
 const fs = require('fs');
 const readline = require('readline');
 
@@ -44,15 +45,24 @@ async function processChangesets(changesets, date, hour) {
  * @returns {Promise<string>} A promise that resolves to the file path of the processed changeset.
  */
 async function processChangeset(changeset, dataPath) {
-    // Process the changeset asynchronously and return the result
-    const url = `https://real-changesets.s3.amazonaws.com/${changeset}.json`;
+    const url = `https://adiffs.osmcha.org/changesets/${changeset}.adiff`;
     // console.log(`Processing changeset ${changeset}`);
     try {
-        const response = await axios.get(url);
+        const response = await axios.get(url, { responseType: "text" });
         const data = response.data;
-        const geojson = changesetParser(data);
-        // console.log(`geojson: ${JSON.stringify(geojson)}`);
-        const features = geojson.features;
+        const diffData = await adiffParser(data);
+
+        // Convert adiff changesets to the shape of real changesets to minimize the code change
+        const features  = diffData.actions
+        .map((a) => {
+          return {
+            ...a.new,
+            ...(a.old && { old: a.old }),
+            action: a.type,
+          };
+        })
+        .map(changesetParser.elementParser)
+        .flat();
 
         if (features.length === 0) {
             console.log(`No features found in changeset ${changeset}`);

pipeline/yarn.lock

@@ -2,6 +2,13 @@
 # yarn lockfile v1
 
 
+"@osmcha/osm-adiff-parser@^3.1.0":
+  version "3.1.0"
+  resolved "https://registry.npmjs.org/@osmcha/osm-adiff-parser/-/osm-adiff-parser-3.1.0.tgz#bbd6a4a4052145ba19fe04a1039a28a164cfe1e6"
+  integrity sha512-TX+cCDkBE3bDMqn4ice5R7UzQY+cX+4h16DGRTvZfzq5QwRxluPcnD7PlQdc2JoJ7L2ViACLMkc7VxRqnf2noA==
+  dependencies:
+    sax "^1.4.1"
+
 "@turf/bbox-polygon@^6.0.1":
   version "6.5.0"
   resolved "https://registry.yarnpkg.com/@turf/bbox-polygon/-/bbox-polygon-6.5.0.tgz#f18128b012eedfa860a521d8f2b3779cc0801032"
@@ -120,3 +127,8 @@ sax@^1.3.0:
   version "1.3.0"
   resolved "https://registry.yarnpkg.com/sax/-/sax-1.3.0.tgz#a5dbe77db3be05c9d1ee7785dbd3ea9de51593d0"
   integrity sha512-0s+oAmw9zLl1V1cS9BtZN7JAd0cW5e0QH4W3LWEK6a4LaLEA2OTpGYWDY+6XasBLtz6wkm3u1xRw95mRuJ59WA==
+
+sax@^1.4.1:
+  version "1.4.1"
+  resolved "https://registry.npmjs.org/sax/-/sax-1.4.1.tgz#44cc8988377f126304d3b3fc1010c733b929ef0f"
+  integrity sha512-+aWOz7yVScEGoKNd4PA10LZ8sk0A/z5+nXQG5giUO5rprX9jgYsTdov9qCchZiPIZezbZH+jRut8nPodFAX4Jg==