fix: UTC time mishap; switch to bigint (Haidra-Org#421)

* fix: bigger data type for ps_month/count_total * fix: use UTC time for stats; fix more column types
Jensinjames · Jun 10, 2024 · 16b1ec2 · 16b1ec2
1 parent 4072a48
commit 16b1ec2
Show file tree

Hide file tree

Showing 7 changed files with 38 additions and 32 deletions.
diff --git a/horde/classes/stable/genstats.py b/horde/classes/stable/genstats.py
@@ -173,8 +173,8 @@ class CompiledImageGenStatsTotals(db.Model):
     day_images = db.Column(db.Integer, nullable=False)
     day_pixels = db.Column(db.Integer, nullable=False)
     month_images = db.Column(db.Integer, nullable=False)
-    month_pixels = db.Column(db.Integer, nullable=False)
-    total_images = db.Column(db.Integer, nullable=False)
+    month_pixels = db.Column(db.BigInteger, nullable=False)
+    total_images = db.Column(db.BigInteger, nullable=False)
     total_pixels = db.Column(db.BigInteger, nullable=False)
 
 

diff --git a/horde/consts.py b/horde/consts.py
@@ -1,4 +1,4 @@
-HORDE_VERSION = "4.36.0"
+HORDE_VERSION = "4.37.1"
 
 WHITELISTED_SERVICE_IPS = {
     "212.227.227.178",  # Turing Bot

diff --git a/sql_statements/4.37.1.txt b/sql_statements/4.37.1.txt
@@ -0,0 +1,5 @@
+ALTER TABLE compiled_image_gen_stats_totals ALTER COLUMN minute_pixels TYPE bigint;
+ALTER TABLE compiled_image_gen_stats_totals ALTER COLUMN hour_pixels TYPE bigint;
+ALTER TABLE compiled_image_gen_stats_totals ALTER COLUMN day_pixels TYPE bigint;
+ALTER TABLE compiled_image_gen_stats_totals ALTER COLUMN month_pixels TYPE bigint;
+ALTER TABLE compiled_image_gen_stats_totals ALTER COLUMN total_pixels TYPE bigint;
diff --git a/sql_statements/stored_procedures/compile_imagegen_stats_models.sql b/sql_statements/stored_procedures/compile_imagegen_stats_models.sql
@@ -10,8 +10,8 @@ BEGIN
                 WHEN kim.id IS NOT NULL THEN 'known'
                 ELSE 'custom'
             END as model_state,
-            COUNT(*) FILTER (WHERE igs.finished >= NOW() - INTERVAL '1 day') as day_images,
-            COUNT(*) FILTER (WHERE igs.finished >= NOW() - INTERVAL '30 days') as month_images,
+            COUNT(*) FILTER (WHERE igs.finished >= (NOW() at time zone 'utc') - INTERVAL '1 day') as day_images,
+            COUNT(*) FILTER (WHERE igs.finished >= (NOW() at time zone 'utc') - INTERVAL '30 days') as month_images,
             COUNT(*) as total_images
         FROM 
             image_gen_stats as igs
@@ -21,7 +21,7 @@ BEGIN
     )
     INSERT INTO compiled_image_gen_stats_models (created, model_id, model_name, model_state, day_images, month_images, total_images)
     SELECT 
-            NOW(),
+            (NOW() at time zone 'utc'),
             model_id,
             model_name,
             model_state,

diff --git a/sql_statements/stored_procedures/compile_imagegen_stats_totals.sql b/sql_statements/stored_procedures/compile_imagegen_stats_totals.sql
@@ -6,34 +6,35 @@ DECLARE
     count_hour INTEGER;
     count_day INTEGER;
     count_month INTEGER;
-    count_total INTEGER;
-    ps_minute INTEGER;
-    ps_hour INTEGER;
-    ps_day INTEGER;
-    ps_month INTEGER;
+    count_total BIGINT;
+    ps_minute BIGINT;
+    ps_hour BIGINT;
+    ps_day BIGINT;
+    ps_month BIGINT;
     ps_total BIGINT;
 BEGIN
     -- Calculate image counts
-    SELECT COUNT(*) INTO count_minute FROM image_gen_stats WHERE finished >= NOW() - INTERVAL '1 minute';
-    SELECT COUNT(*) INTO count_hour FROM image_gen_stats WHERE finished >= NOW() - INTERVAL '1 hour';
-    SELECT COUNT(*) INTO count_day FROM image_gen_stats WHERE finished >= NOW() - INTERVAL '1 day';
-    SELECT COUNT(*) INTO count_month FROM image_gen_stats WHERE finished >= NOW() - INTERVAL '30 days';
+    SELECT COUNT(*) INTO count_minute FROM image_gen_stats WHERE finished >= (NOW() at time zone 'utc') - INTERVAL '1 minute';
+    SELECT COUNT(*) INTO count_hour FROM image_gen_stats WHERE finished >= (NOW() at time zone 'utc') - INTERVAL '1 hour';
+    SELECT COUNT(*) INTO count_day FROM image_gen_stats WHERE finished >= (NOW() at time zone 'utc') - INTERVAL '1 day';
+    SELECT COUNT(*) INTO count_month FROM image_gen_stats WHERE finished >= (NOW() at time zone 'utc') - INTERVAL '30 days';
     SELECT COUNT(*) INTO count_total FROM image_gen_stats;
 
     -- Calculate pixel sums
-    SELECT COALESCE(SUM(width * height * steps), 0) INTO ps_minute FROM image_gen_stats WHERE finished >= NOW() - INTERVAL '1 minute';
-    SELECT COALESCE(SUM(width * height * steps), 0) INTO ps_hour FROM image_gen_stats WHERE finished >= NOW() - INTERVAL '1 hour';
-    SELECT COALESCE(SUM(width * height * steps), 0) INTO ps_day FROM image_gen_stats WHERE finished >= NOW() - INTERVAL '1 day';
-    SELECT COALESCE(SUM(width * height * steps), 0) INTO ps_month FROM image_gen_stats WHERE finished >= NOW() - INTERVAL '30 days';
+    SELECT COALESCE(SUM(width * height * steps), 0) INTO ps_minute FROM image_gen_stats WHERE finished >= (NOW() at time zone 'utc') - INTERVAL '1 minute';
+    SELECT COALESCE(SUM(width * height * steps), 0) INTO ps_hour FROM image_gen_stats WHERE finished >= (NOW() at time zone 'utc') - INTERVAL '1 hour';
+    SELECT COALESCE(SUM(width * height * steps), 0) INTO ps_day FROM image_gen_stats WHERE finished >= (NOW() at time zone 'utc') - INTERVAL '1 day';
+    SELECT COALESCE(SUM(width * height * steps), 0) INTO ps_month FROM image_gen_stats WHERE finished >= (NOW() at time zone 'utc') - INTERVAL '30 days';
     SELECT COALESCE(SUM(width * height * steps), 0) INTO ps_total FROM image_gen_stats;
 
     -- Insert compiled statistics into compiled_image_gen_stats_totals
     INSERT INTO compiled_image_gen_stats_totals (
         created, minute_images, minute_pixels, hour_images, hour_pixels, 
         day_images, day_pixels, month_images, month_pixels, total_images, total_pixels
     ) VALUES (
-        NOW(), count_minute, ps_minute, count_hour, ps_hour, 
+        (NOW() at time zone 'utc'), count_minute, ps_minute, count_hour, ps_hour, 
         count_day, ps_day, count_month, ps_month, count_total, ps_total
     );
 END;
 $$;
+
diff --git a/sql_statements/stored_procedures/compile_textgen_stats_models.sql b/sql_statements/stored_procedures/compile_textgen_stats_models.sql
@@ -5,8 +5,8 @@ BEGIN
     WITH model_stats AS (
         SELECT 
             tgs.model as model_name,
-            COUNT(*) FILTER (WHERE tgs.finished >= NOW() - INTERVAL '1 day') as day_requests,
-            COUNT(*) FILTER (WHERE tgs.finished >= NOW() - INTERVAL '30 days') as month_requests,
+            COUNT(*) FILTER (WHERE tgs.finished >= (NOW() at time zone 'utc') - INTERVAL '1 day') as day_requests,
+            COUNT(*) FILTER (WHERE tgs.finished >= (NOW() at time zone 'utc') - INTERVAL '30 days') as month_requests,
             COUNT(*) as total_requests
         FROM 
             text_gen_stats as tgs
@@ -15,7 +15,7 @@ BEGIN
     )
     INSERT INTO compiled_text_gen_stats_models (created, model, day_requests, month_requests, total_requests)
     SELECT 
-        NOW(),
+        (NOW() at time zone 'utc'),
         model_name,
         day_requests,
         month_requests,

diff --git a/sql_statements/stored_procedures/compile_textgen_stats_totals.sql b/sql_statements/stored_procedures/compile_textgen_stats_totals.sql
@@ -14,25 +14,25 @@ DECLARE
     tokens_total BIGINT;
 BEGIN
     -- Calculate request counts
-    SELECT COUNT(*) INTO count_minute FROM text_gen_stats WHERE finished >= NOW() - INTERVAL '1 minute';
-    SELECT COUNT(*) INTO count_hour FROM text_gen_stats WHERE finished >= NOW() - INTERVAL '1 hour';
-    SELECT COUNT(*) INTO count_day FROM text_gen_stats WHERE finished >= NOW() - INTERVAL '1 day';
-    SELECT COUNT(*) INTO count_month FROM text_gen_stats WHERE finished >= NOW() - INTERVAL '30 days';
+    SELECT COUNT(*) INTO count_minute FROM text_gen_stats WHERE finished >= (NOW() at time zone 'utc') - INTERVAL '1 minute';
+    SELECT COUNT(*) INTO count_hour FROM text_gen_stats WHERE finished >= (NOW() at time zone 'utc') - INTERVAL '1 hour';
+    SELECT COUNT(*) INTO count_day FROM text_gen_stats WHERE finished >= (NOW() at time zone 'utc') - INTERVAL '1 day';
+    SELECT COUNT(*) INTO count_month FROM text_gen_stats WHERE finished >= (NOW() at time zone 'utc') - INTERVAL '30 days';
     SELECT COUNT(*) INTO count_total FROM text_gen_stats;
 
     -- Calculate token sums
-    SELECT COALESCE(SUM(max_length), 0) INTO tokens_minute FROM text_gen_stats WHERE finished >= NOW() - INTERVAL '1 minute';
-    SELECT COALESCE(SUM(max_length), 0) INTO tokens_hour FROM text_gen_stats WHERE finished >= NOW() - INTERVAL '1 hour';
-    SELECT COALESCE(SUM(max_length), 0) INTO tokens_day FROM text_gen_stats WHERE finished >= NOW() - INTERVAL '1 day';
-    SELECT COALESCE(SUM(max_length), 0) INTO tokens_month FROM text_gen_stats WHERE finished >= NOW() - INTERVAL '30 days';
+    SELECT COALESCE(SUM(max_length), 0) INTO tokens_minute FROM text_gen_stats WHERE finished >= (NOW() at time zone 'utc') - INTERVAL '1 minute';
+    SELECT COALESCE(SUM(max_length), 0) INTO tokens_hour FROM text_gen_stats WHERE finished >= (NOW() at time zone 'utc') - INTERVAL '1 hour';
+    SELECT COALESCE(SUM(max_length), 0) INTO tokens_day FROM text_gen_stats WHERE finished >= (NOW() at time zone 'utc') - INTERVAL '1 day';
+    SELECT COALESCE(SUM(max_length), 0) INTO tokens_month FROM text_gen_stats WHERE finished >= (NOW() at time zone 'utc') - INTERVAL '30 days';
     SELECT COALESCE(SUM(max_length), 0) INTO tokens_total FROM text_gen_stats;
 
     -- Insert compiled statistics into compiled_text_gen_stats_totals
     INSERT INTO compiled_text_gen_stats_totals (
         created, minute_requests, minute_tokens, hour_requests, hour_tokens, 
         day_requests, day_tokens, month_requests, month_tokens, total_requests, total_tokens
     ) VALUES (
-        NOW(), count_minute, tokens_minute, count_hour, tokens_hour, 
+        (NOW() at time zone 'utc'), count_minute, tokens_minute, count_hour, tokens_hour, 
         count_day, tokens_day, count_month, tokens_month, count_total, tokens_total
     );
 END;