Move queries to API

api/endpoint/analyzeSentiment.ts

@@ -0,0 +1,52 @@
+import {
+  VFiniteNumber,
+  VHumanString,
+  VInteger,
+  VStruct,
+  Valid,
+} from "@wzlin/valid";
+import assertInstanceOf from "@xtjs/lib/assertInstanceOf";
+import { QueryGroupByOutput, makeQuery } from "../query";
+
+const input = new VStruct({
+  query: new VHumanString(1, 512),
+  simMinHundredths: new VInteger(80, 100),
+});
+
+export const endpointAnalyzeSentiment = {
+  input,
+  handler: async ({ query, simMinHundredths }: Valid<typeof input>) => {
+    const res = await makeQuery({
+      dataset: "comment",
+      queries: [query],
+      thresholds: {
+        sim: simMinHundredths / 100,
+        sent_pos: 0.5,
+        sent_neg: 0.5,
+      },
+      post_filter_clip: {
+        sim_thresh: { min: 1.0, max: 1.0 },
+      },
+      outputs: [
+        {
+          group_by: {
+            by: "ts_day",
+            bucket: 7,
+            cols: [
+              ["sent_pos_thresh", "sum"],
+              ["sent_neg_thresh", "sum"],
+            ],
+          },
+        },
+      ],
+    });
+    const data = assertInstanceOf(res[0], QueryGroupByOutput);
+    return {
+      timestamps: [...data.groups(new VInteger())].map(
+        (d) => new Date(d * 7 * 24 * 60 * 60 * 1000),
+      ),
+      positives: [...data.column("sent_pos_thresh", new VFiniteNumber())],
+      negatives: [...data.column("sent_neg_thresh", new VFiniteNumber())],
+    };
+  },
+};

api/endpoint/heatmap.ts

@@ -0,0 +1,55 @@
+import {
+  VHumanString,
+  VInteger,
+  VMember,
+  VStruct,
+  VTuple,
+  Valid,
+} from "@wzlin/valid";
+import assertInstanceOf from "@xtjs/lib/assertInstanceOf";
+import { QueryHeatmapOutput, makeQuery } from "../query";
+
+const input = new VStruct({
+  query: new VHumanString(1, 512),
+  dataset: new VMember(["post", "toppost"] as const),
+  color: new VTuple([
+    new VInteger(0, 255),
+    new VInteger(0, 255),
+    new VInteger(0, 255),
+  ] as const),
+});
+
+export const endpointHeatmap = {
+  input,
+  handler: async ({ dataset, query, color }: Valid<typeof input>) => {
+    const res = await makeQuery({
+      dataset,
+      queries: [query],
+      scales: {
+        sim: {
+          post: { min: 0.7, max: 1 },
+          toppost: { min: 0.55, max: 1 },
+        }[dataset],
+      },
+      post_filter_clip: {
+        scaled: { min: 0.01, max: 1 },
+      },
+      weights: {
+        sim_scaled: 1,
+      },
+      outputs: [
+        {
+          heatmap: {
+            alpha_scale: 2, // TODO This is really a hack, investigate distribution of scores.
+            density: 25,
+            color,
+            upscale: 2,
+            sigma: 4,
+          },
+        },
+      ],
+    });
+    const data = assertInstanceOf(res[0], QueryHeatmapOutput);
+    return new Uint8Array(data.raw);
+  },
+};

api/endpoint/search.ts

@@ -6,12 +6,13 @@ import {
   VStruct,
   Valid,
 } from "@wzlin/valid";
-import serialiseToQueryString from "@xtjs/lib/serialiseToQueryString";
+import assertInstanceOf from "@xtjs/lib/assertInstanceOf";
+import { QueryItemsOutput, makeQuery } from "../query";
 
 const input = new VStruct({
   query: new VHumanString(1, 512),
   limit: new VInteger(1, 128),
-  dataset: new VMember(["posts", "posts_bgem3", "comments"]),
+  dataset: new VMember(["post", "toppost"] as const),
   weightSimilarity: new VFiniteNumber(),
   weightScore: new VFiniteNumber(),
   weightTimestamp: new VFiniteNumber(),
@@ -29,17 +30,41 @@ export const endpointSearch = {
     weightSimilarity,
     weightTimestamp,
   }: Valid<typeof input>) => {
-    const results = await fetch(
-      `http://127.0.0.1:7001/${serialiseToQueryString({
-        query,
-        limit: 128,
-        dataset,
-        w_sim: weightSimilarity,
-        w_score: weightScore,
-        w_ts: weightTimestamp,
-        decay_ts: decayTimestamp,
-      })}`,
-    ).then((r) => r.json());
-    return { results: results.slice(0, limit) };
+    const res = await makeQuery({
+      dataset,
+      queries: [query],
+      ts_decay: decayTimestamp,
+      scales: {
+        sim: {
+          post: { min: 0.7, max: 1 },
+          toppost: { min: 0.55, max: 1 },
+        }[dataset],
+      },
+      weights: {
+        sim_scaled: weightSimilarity,
+        ts_norm: weightTimestamp,
+        votes_norm: weightScore,
+      },
+      outputs: [
+        {
+          items: {
+            cols: ["id", "x", "y", "sim", "final_score"],
+            limit,
+          },
+        },
+      ],
+    });
+    const data = assertInstanceOf(res[0], QueryItemsOutput);
+    return {
+      items: Array.from(
+        data.items({
+          id: new VInteger(1),
+          x: new VFiniteNumber(),
+          y: new VFiniteNumber(),
+          sim: new VFiniteNumber(),
+          final_score: new VFiniteNumber(),
+        }),
+      ),
+    };
   },
 };

api/main.ts

@@ -7,11 +7,13 @@ import readBufferStream from "@xtjs/lib/readBufferStream";
 import uint8ArrayToBuffer from "@xtjs/lib/uint8ArrayToBuffer";
 import { createSecureServer } from "http2";
 import { lg } from "../common/res";
+import { endpointAnalyzeSentiment } from "./endpoint/analyzeSentiment";
+import { endpointHeatmap } from "./endpoint/heatmap";
 import { endpointSearch } from "./endpoint/search";
 
 const getPemEnv = (name: string) =>
   uint8ArrayToBuffer(
-    decodeBase64(assertExists(process.env[`API_SSL_${name}_BASE64`])),
+    decodeBase64(assertExists(process.env[`SSL_${name}_BASE64`])),
   );
 
 type Endpoint = {
@@ -20,6 +22,8 @@ type Endpoint = {
 };
 
 const ENDPOINTS: Record<string, Endpoint> = {
+  analyzeSentiment: endpointAnalyzeSentiment,
+  heatmap: endpointHeatmap,
   search: endpointSearch,
 };
 
@@ -66,7 +70,19 @@ createSecureServer(
     try {
       resBody = await endpoint.handler(reqBody);
     } catch (error) {
-      lg.error({ error, endpoint: endpointName }, "endpoint error");
+      lg.error(
+        {
+          error: {
+            trace: error.stack,
+            message: error.message,
+            type: error.constructor?.name,
+            name: error.name,
+            data: { ...error },
+          },
+          endpoint: endpointName,
+        },
+        "endpoint error",
+      );
       return res.writeHead(500).end();
     }
     return res