Uptime calculation improvement and 1-year uptime (louislam#2750)

bstef · Aug 31, 2023 · 076331b · 076331b
1 parent eec2212
commit 076331b
Show file tree

Hide file tree

Showing 22 changed files with 1,306 additions and 264 deletions.
diff --git a/.eslintrc.js b/.eslintrc.js
@@ -1,6 +1,7 @@
 module.exports = {
     ignorePatterns: [
-        "test/*",
+        "test/*.js",
+        "test/cypress",
         "server/modules/apicache/*",
         "src/util.js"
     ],

diff --git a/db/knex_migrations/2023-08-16-0000-create-uptime.js b/db/knex_migrations/2023-08-16-0000-create-uptime.js
@@ -0,0 +1,41 @@
+exports.up = function (knex) {
+    return knex.schema
+        .createTable("stat_minutely", function (table) {
+            table.increments("id");
+            table.comment("This table contains the minutely aggregate statistics for each monitor");
+            table.integer("monitor_id").unsigned().notNullable()
+                .references("id").inTable("monitor")
+                .onDelete("CASCADE")
+                .onUpdate("CASCADE");
+            table.integer("timestamp")
+                .notNullable()
+                .comment("Unix timestamp rounded down to the nearest minute");
+            table.float("ping").notNullable().comment("Average ping in milliseconds");
+            table.smallint("up").notNullable();
+            table.smallint("down").notNullable();
+
+            table.unique([ "monitor_id", "timestamp" ]);
+        })
+        .createTable("stat_daily", function (table) {
+            table.increments("id");
+            table.comment("This table contains the daily aggregate statistics for each monitor");
+            table.integer("monitor_id").unsigned().notNullable()
+                .references("id").inTable("monitor")
+                .onDelete("CASCADE")
+                .onUpdate("CASCADE");
+            table.integer("timestamp")
+                .notNullable()
+                .comment("Unix timestamp rounded down to the nearest day");
+            table.float("ping").notNullable().comment("Average ping in milliseconds");
+            table.smallint("up").notNullable();
+            table.smallint("down").notNullable();
+
+            table.unique([ "monitor_id", "timestamp" ]);
+        });
+};
+
+exports.down = function (knex) {
+    return knex.schema
+        .dropTable("stat_minutely")
+        .dropTable("stat_daily");
+};
diff --git a/db/knex_migrations/2023-08-18-0301-heartbeat.js b/db/knex_migrations/2023-08-18-0301-heartbeat.js
@@ -0,0 +1,16 @@
+exports.up = function (knex) {
+    // Add new column heartbeat.end_time
+    return knex.schema
+        .alterTable("heartbeat", function (table) {
+            table.datetime("end_time").nullable().defaultTo(null);
+        });
+
+};
+
+exports.down = function (knex) {
+    // Rename heartbeat.start_time to heartbeat.time
+    return knex.schema
+        .alterTable("heartbeat", function (table) {
+            table.dropColumn("end_time");
+        });
+};
diff --git a/db/knex_migrations/README.md b/db/knex_migrations/README.md
@@ -4,13 +4,11 @@ https://knexjs.org/guide/migrations.html#knexfile-in-other-languages
 
 ## Basic rules
 - All tables must have a primary key named `id`
-- Filename format: `YYYY-MM-DD-HHMM-patch-name.js`
-- Avoid native SQL syntax, use knex methods, because Uptime Kuma supports multiple databases
+- Filename format: `YYYY-MM-DD-HHMM-patch-name.js` 
+- Avoid native SQL syntax, use knex methods, because Uptime Kuma supports SQLite and MariaDB.
 
 ## Template
 
-Filename: YYYYMMDDHHMMSS_name.js
-
 ```js
 exports.up = function(knex) {
 

diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -24,8 +24,11 @@
         "start-server": "node server/server.js",
         "start-server-dev": "cross-env NODE_ENV=development node server/server.js",
         "build": "vite build --config ./config/vite.config.js",
-        "test": "node test/prepare-test-server.js && npm run jest-backend",
+        "test": "node test/prepare-test-server.js && npm run test-backend",
         "test-with-build": "npm run build && npm test",
+        "test-backend": "node test/backend-test-entry.js && npm run jest-backend",
+        "test-backend:14": "cross-env TEST_BACKEND=1 NODE_OPTIONS=\"--experimental-abortcontroller --no-warnings\" node--test test/backend-test",
+        "test-backend:18": "cross-env TEST_BACKEND=1 node --test test/backend-test",
         "jest-backend": "cross-env TEST_BACKEND=1 jest --runInBand --detectOpenHandles --forceExit --config=./config/jest-backend.config.js",
         "tsc": "tsc",
         "vite-preview-dist": "vite preview --host --config ./config/vite.config.js",
@@ -181,6 +184,7 @@
         "stylelint": "^15.10.1",
         "stylelint-config-standard": "~25.0.0",
         "terser": "~5.15.0",
+        "test": "~3.3.0",
         "timezones-list": "~3.0.1",
         "typescript": "~4.4.4",
         "v-pagination-3": "~0.1.7",

diff --git a/server/client.js b/server/client.js
@@ -45,8 +45,6 @@ async function sendNotificationList(socket) {
  * @returns {Promise<void>}
  */
 async function sendHeartbeatList(socket, monitorID, toUser = false, overwrite = false) {
-    const timeLogger = new TimeLogger();
-
     let list = await R.getAll(`
         SELECT * FROM heartbeat
         WHERE monitor_id = ?
@@ -63,8 +61,6 @@ async function sendHeartbeatList(socket, monitorID, toUser = false, overwrite =
     } else {
         socket.emit("heartbeatList", monitorID, result, overwrite);
     }
-
-    timeLogger.print(`[Monitor: ${monitorID}] sendHeartbeatList`);
 }
 
 /**

diff --git a/server/database.js b/server/database.js
@@ -183,6 +183,12 @@ class Database {
 
         let config = {};
 
+        let mariadbPoolConfig = {
+            afterCreate: function (conn, done) {
+
+            }
+        };
+
         log.info("db", `Database Type: ${dbConfig.type}`);
 
         if (dbConfig.type === "sqlite") {
@@ -233,7 +239,9 @@ class Database {
                     user: dbConfig.username,
                     password: dbConfig.password,
                     database: dbConfig.dbName,
-                }
+                    timezone: "UTC",
+                },
+                pool: mariadbPoolConfig,
             };
         } else if (dbConfig.type === "embedded-mariadb") {
             let embeddedMariaDB = EmbeddedMariaDB.getInstance();
@@ -245,7 +253,8 @@ class Database {
                     socketPath: embeddedMariaDB.socketPath,
                     user: "node",
                     database: "kuma",
-                }
+                },
+                pool: mariadbPoolConfig,
             };
         } else {
             throw new Error("Unknown Database type: " + dbConfig.type);
@@ -350,6 +359,7 @@ class Database {
     }
 
     /**
+     * TODO
      * @returns {Promise<void>}
      */
     static async rollbackLatestPatch() {
@@ -582,14 +592,6 @@ class Database {
         }
     }
 
-    /**
-     * Aquire a direct connection to database
-     * @returns {any} Database connection
-     */
-    static getBetterSQLite3Database() {
-        return R.knex.client.acquireConnection();
-    }
-
     /**
      * Special handle, because tarn.js throw a promise reject that cannot be caught
      * @returns {Promise<void>}
@@ -603,7 +605,9 @@ class Database {
         log.info("db", "Closing the database");
 
         // Flush WAL to main database
-        await R.exec("PRAGMA wal_checkpoint(TRUNCATE)");
+        if (Database.dbConfig.type === "sqlite") {
+            await R.exec("PRAGMA wal_checkpoint(TRUNCATE)");
+        }
 
         while (true) {
             Database.noReject = true;
@@ -616,28 +620,33 @@ class Database {
                 log.info("db", "Waiting to close the database");
             }
         }
-        log.info("db", "SQLite closed");
+        log.info("db", "Database closed");
 
         process.removeListener("unhandledRejection", listener);
     }
 
     /**
-     * Get the size of the database
+     * Get the size of the database (SQLite only)
      * @returns {number} Size of database
      */
     static getSize() {
-        log.debug("db", "Database.getSize()");
-        let stats = fs.statSync(Database.sqlitePath);
-        log.debug("db", stats);
-        return stats.size;
+        if (Database.dbConfig.type === "sqlite") {
+            log.debug("db", "Database.getSize()");
+            let stats = fs.statSync(Database.sqlitePath);
+            log.debug("db", stats);
+            return stats.size;
+        }
+        return 0;
     }
 
     /**
      * Shrink the database
      * @returns {Promise<void>}
      */
     static async shrink() {
-        await R.exec("VACUUM");
+        if (Database.dbConfig.type === "sqlite") {
+            await R.exec("VACUUM");
+        }
     }
 
     /**

diff --git a/server/jobs/clear-old-data.js b/server/jobs/clear-old-data.js
@@ -43,7 +43,9 @@ const clearOldData = async () => {
                 [ parsedPeriod * -24 ]
             );
 
-            await R.exec("PRAGMA optimize;");
+            if (Database.dbConfig.type === "sqlite") {
+                await R.exec("PRAGMA optimize;");
+            }
         } catch (e) {
             log.error("clearOldData", `Failed to clear old data: ${e.message}`);
         }