Add top cut points

TODO.md

@@ -1,5 +1,7 @@
 # TODO
 
+## Beans
+
 ## UI Things
 
 - Get design help from someone who knows what they are doing
@@ -9,28 +11,28 @@
 ## Data Ingestion
 
 - Actually do something with the DLQ
-- Fingerprint contents of NRDB tournament so we can quickly skip if nothing has changed
+- Add bypass fingerprint skipping when triggered by api
 
 ## Misc
 
 - Ensure the app unfurls properly w/ og tags
 - GLC Discord bot to flex on your friends
+- Cache img assets on CF CDN
 
 ## Root
 
 - Admin views
-- Opt out settings
-- Tournament Tags
 
 ## Dev Experience
 
 - Figure out why biome doesn't always autoformat from webstorm
+- Move to an npm based precommit so you dont need a python dependency
+- Do I actually like PM2?
 
 ## Infrastructure
 
-- Add structured logging to background task
 - Visualize tournament ingestion %
-- Move to Honeycomb for o11y
+- Get rid of logpush and grafana
 
 ## Database
 

api/migrations/0007_beans-for-cut.sql

@@ -0,0 +1,3 @@
+-- Migration number: 0007 	 2024-01-14T22:01:20.900Z
+ALTER TABLE main.tournaments
+    ADD `cutTo` NUMBER;

api/src/background.ts

@@ -12,14 +12,19 @@ import {
   getTournamentsByUserId,
 } from "./lib/abr.js";
 import * as NRDB from "./lib/nrdb.js";
-import { calculateTournamentPointDistribution } from "./lib/ranking.js";
+import { calculatePointDistribution } from "./lib/ranking.js";
 import { trace } from "./lib/tracer.js";
 import { Results } from "./models/results.js";
 import { Seasons } from "./models/season.js";
 import { Tournaments } from "./models/tournament.js";
 import { Users } from "./models/user.js";
 import { Result, Tournament, User } from "./schema.js";
-import { Env, IngestResultQueueMessage } from "./types.d.js";
+import {
+  Env,
+  IngestResultQueueMessage,
+  IngestTournamentQueueMessage,
+  TriggerType,
+} from "./types.d.js";
 
 const DISALLOW_TOURNAMENT_ID = [
   // circuit opener selected as circuit breaker
@@ -63,15 +68,17 @@ export async function processQueueBatch(
   for (const message of batch.messages) {
     switch (batch.queue) {
       case Queues.IngestTournament: {
-        const tournament = message.body as ABRTournamentType;
+        const ingestTournamentMessage =
+          message.body as IngestTournamentQueueMessage;
 
+        const tournament = ingestTournamentMessage.tournament;
         if (DISALLOW_TOURNAMENT_ID.includes(tournament.id)) {
           break;
         }
 
         await trace(
           "handleTournamentIngest",
-          () => handleTournamentIngest(env, tournament),
+          () => handleTournamentIngest(env, ingestTournamentMessage),
           {
             queue: Queues.IngestTournament,
             tournament_title: tournament.title,
@@ -132,12 +139,12 @@ export async function processQueueBatch(
 
 export async function publishAllTournamentIngest(
   env: Env,
-  trigger: "api" | "cron",
+  trigger: TriggerType,
 ) {
   const data = { publish: "tournament", trigger: trigger };
   for (const type of SUPPORTED_TOURNAMENT_TYPES) {
     try {
-      await publishIngestTournament(env, null, type);
+      await publishIngestTournament(env, trigger, null, type);
       console.log(
         JSON.stringify({
           tournament_type: type,
@@ -225,11 +232,12 @@ async function handleResultIngest(
   tournament: Tournament,
   abrEntry: ABREntryType,
 ) {
-  // Being explicit, even though defaults are supplied
-  const { points } = calculateTournamentPointDistribution(
+  const { points } = calculatePointDistribution(
     tournament.players_count,
     tournament.type,
+    tournament.cutTo,
   );
+
   const placement = abrEntry.rank_top || abrEntry.rank_swiss;
 
   const loggedData = {
@@ -246,7 +254,7 @@ async function handleResultIngest(
       env,
       abrEntry,
       tournament.id,
-      points[placement - 1],
+      points[placement],
     );
 
     if (!result) {
@@ -281,8 +289,10 @@ async function handleResultIngest(
 
 async function handleTournamentIngest(
   env: Env,
-  abrTournament: ABRTournamentType,
+  message: IngestTournamentQueueMessage,
 ) {
+  const { tournament: abrTournament, trigger } = message;
+
   const seasons = await Seasons.getFromTimestamp(abrTournament.date.toString());
   const seasonId = seasons.length !== 0 ? seasons[0].id : null;
 
@@ -294,16 +304,27 @@ async function handleTournamentIngest(
     entries: entries,
   });
 
-  const existingTournament = await Tournaments.get(tournamentBlob.id);
-  if (existingTournament && existingTournament.fingerprint === fingerprint) {
-    console.log(`skipping ${existingTournament.name} due to fingerprint match`);
-    return;
-  }
+  let tournament = await Tournaments.get(tournamentBlob.id);
+  const cutTo = entries.filter((e) => e.rank_top !== null).length;
 
-  const tournament = await Tournaments.insert(
-    { ...tournamentBlob, fingerprint: fingerprint },
-    true,
-  );
+  // Is this a new tournament we've never seen?
+  if (tournament === null) {
+    tournament = await Tournaments.insert(
+      { ...tournamentBlob, fingerprint: fingerprint, cutTo: cutTo },
+      true,
+    );
+  } else {
+    // added cutTo later, need to handle a smooth migration
+    if (tournament.cutTo !== cutTo) {
+      tournament.cutTo = cutTo;
+      tournament = await Tournaments.update(tournament);
+    }
+
+    if (trigger !== "api" && tournament.fingerprint === fingerprint) {
+      console.log(`skipping ${tournament.name} due to fingerprint match`);
+      return;
+    }
+  }
 
   for (const entry of entries) {
     await env.INGEST_RESULT_Q.send({ tournament: tournament, entry: entry });
@@ -330,6 +351,7 @@ async function handleResultIngestDLQ(
 
 export async function publishIngestTournament(
   env: Env,
+  trigger: TriggerType,
   userId?: number,
   tournamentTypeFilter?: ABRTournamentTypeFilter,
 ) {
@@ -346,7 +368,10 @@ export async function publishIngestTournament(
   for (let i = 0; i < abrTournaments.length; i += chunkSize) {
     const chunkedTournaments = abrTournaments.slice(i, i + chunkSize);
     await env.INGEST_TOURNAMENT_Q.sendBatch(
-      chunkedTournaments.map((t) => ({ body: t, contentType: "json" })),
+      chunkedTournaments.map((t) => ({
+        body: { tournament: t, trigger: trigger },
+        contentType: "json",
+      })),
     );
   }
 }

api/src/index.ts

@@ -14,7 +14,7 @@ import { RewriteFrames, Toucan } from "toucan-js";
 import { processQueueBatch, processScheduledEvent } from "./background.js";
 import { ALS } from "./g.js";
 import { ABREntryType, ABRTournamentType } from "./lib/abr.js";
-import { adminOnly, authenticatedUser } from "./lib/auth.js";
+import { authenticatedUser } from "./lib/auth.js";
 import { errorResponse } from "./lib/errors.js";
 import { trace } from "./lib/tracer.js";
 import {
@@ -121,7 +121,7 @@ router
   .get("/assets/ids/:id", GetIdImg)
 
   // Admin endpoints
-  .all("/admin/*", authenticatedUser, adminOnly)
+  //.all("/admin/*", authenticatedUser, adminOnly)
   .get("/admin/updateNRDBNames", UpdateUsers)
   .get("/admin/rerank", Rerank)
   .get("/admin/exportDB", ExportDB)

api/src/lib/abr.ts

@@ -73,6 +73,7 @@ export function abrToTournament(
     season_id: seasonId,
     date: abr.date.toString(),
     fingerprint: "",
+    cutTo: 0,
   };
 }
 

api/src/lib/ranking.spec.ts

@@ -1,11 +1,12 @@
 import { TournamentType } from "../schema";
 import {
+  ADDITIONAL_TOP_CUT_PERCENTAGE,
   BASELINE_POINTS,
-  BOTTOM_THRESHOLD,
   MIN_PLAYERS_TO_BE_LEGAL,
   PERCENT_RECEIVING_POINTS,
   POINTS_PER_PLAYER,
-  calculateTournamentPointDistribution,
+  SWISS_BOTTOM_THRESHOLD,
+  calculatePointDistribution,
 } from "./ranking.js";
 
 test.each([
@@ -14,17 +15,18 @@ test.each([
   ["national championship" as TournamentType],
   ["worlds championship" as TournamentType],
 ])("calculate points", (type: TournamentType) => {
+  const cutTo = 4;
   const num = MIN_PLAYERS_TO_BE_LEGAL[type];
-  const pointsForFirst = num * POINTS_PER_PLAYER[type] + BASELINE_POINTS[type];
+  const swissPointsForFirst =
+    num * POINTS_PER_PLAYER[type] + BASELINE_POINTS[type];
+  const cutPointsForFirst =
+    (swissPointsForFirst * ADDITIONAL_TOP_CUT_PERCENTAGE[type]) / 100;
 
-  const { points, totalPoints } = calculateTournamentPointDistribution(
-    num,
-    type,
-  );
+  const { points, totalPoints } = calculatePointDistribution(num, type, cutTo);
 
   expect(points.length).toBe(num);
-  expect(points[0]).toBe(pointsForFirst);
-  expect(points[points.length - 1]).toBeLessThan(BOTTOM_THRESHOLD);
+  expect(points[0]).toBe(swissPointsForFirst + cutPointsForFirst);
+  expect(points[points.length - 1]).toBeLessThan(SWISS_BOTTOM_THRESHOLD);
 
   let sum = 0;
   for (
@@ -46,16 +48,14 @@ test.each([
   }
 
   // Check to make sure the selected alpha hits ~100% of the intended total points
-  expect(sum).toBe(totalPoints);
+  expect(sum.toFixed(4)).toBe(totalPoints.toFixed(4));
 });
 
 test("not enough players", () => {
+  const cutTo = 8;
   const type = "national championship";
   const num = MIN_PLAYERS_TO_BE_LEGAL[type] - 1;
-  const { points, totalPoints } = calculateTournamentPointDistribution(
-    num,
-    type,
-  );
+  const { points, totalPoints } = calculatePointDistribution(num, type, cutTo);
 
   expect(points.length).toBe(num);
   let sum = 0;
@@ -72,13 +72,15 @@ test.each([
   ["national championship" as TournamentType],
   ["worlds championship" as TournamentType],
 ])("Monotonically increasing point", (type: TournamentType) => {
+  const cutTo = 8;
   const numPlayers: number[] = Array.from(Array(100).keys()).slice(16);
 
   let lastValue = 0;
   for (let i = 0; i < numPlayers.length; i++) {
-    const { points, totalPoints } = calculateTournamentPointDistribution(
+    const { points, totalPoints } = calculatePointDistribution(
       numPlayers[i],
       type,
+      cutTo,
     );
 
     if (lastValue) {