Merge pull request umami-software#3061 from umami-software/dev

v2.14.0
AkashRajpurohit · Nov 21, 2024 · d76603b · d76603b
2 parents d324305 + 620a838
commit d76603b
Show file tree

Hide file tree

Showing 128 changed files with 16,222 additions and 2,374 deletions.
diff --git a/.github/workflows/cd-cloud.yml b/.github/workflows/cd-cloud.yml
@@ -0,0 +1,28 @@
+name: Create docker images
+
+on:
+  push:
+    branches:
+      - analytics
+
+jobs:
+  build:
+    name: Build, push, and deploy
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Generate random hash
+        id: random_hash
+        run: echo "hash=$(openssl rand -hex 4)" >> $GITHUB_OUTPUT
+
+      - uses: mr-smithers-excellent/docker-build-push@v6
+        name: Build & push Docker image to docker.io
+        with:
+          image: umamisoftware/umami
+          tags: cloud-${{ steps.random_hash.outputs.hash }}, cloud-latest
+          buildArgs: DATABASE_TYPE=postgresql
+          registry: docker.io
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
diff --git a/.github/workflows/cd-manual.yml b/.github/workflows/cd-manual.yml
@@ -20,11 +20,26 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
+      - name: Extract version parts from input
+        id: extract_version
+        run: |
+          echo "version=$(echo ${{ github.event.inputs.version }})" >> $GITHUB_ENV
+          echo "major=$(echo ${{ github.event.inputs.version }} | cut -d. -f1)" >> $GITHUB_ENV
+          echo "minor=$(echo ${{ github.event.inputs.version }} | cut -d. -f2)" >> $GITHUB_ENV
+
+      - name: Generate tags
+        id: generate_tags
+        run: |
+          echo "tag_major=$(echo ${{ matrix.db-type }}-${{ env.major }})" >> $GITHUB_ENV
+          echo "tag_minor=$(echo ${{ matrix.db-type }}-${{ env.major }}.${{ env.minor }})" >> $GITHUB_ENV
+          echo "tag_patch=$(echo ${{ matrix.db-type }}-${{ env.version }})" >> $GITHUB_ENV
+          echo "tag_latest=$(echo ${{ matrix.db-type }}-latest)" >> $GITHUB_ENV
+
       - uses: mr-smithers-excellent/docker-build-push@v6
         name: Build & push Docker image to ghcr.io for ${{ matrix.db-type }}
         with:
           image: umami
-          tags: ${{ matrix.db-type }}-${{ inputs.version }}, ${{ matrix.db-type }}-latest
+          tags: ${{ env.tag_major }}, ${{ env.tag_minor }}, ${{ env.tag_patch }}, ${{ env.tag_latest }}
           buildArgs: DATABASE_TYPE=${{ matrix.db-type }}
           registry: ghcr.io
           multiPlatform: true
@@ -36,7 +51,7 @@ jobs:
         name: Build & push Docker image to docker.io for ${{ matrix.db-type }}
         with:
           image: umamisoftware/umami
-          tags: ${{ matrix.db-type }}-${{ inputs.version }}, ${{ matrix.db-type }}-latest
+          tags: ${{ env.tag_major }}, ${{ env.tag_minor }}, ${{ env.tag_patch }}, ${{ env.tag_latest }}
           buildArgs: DATABASE_TYPE=${{ matrix.db-type }}
           registry: docker.io
           username: ${{ secrets.DOCKER_USERNAME }}

diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml
@@ -17,27 +17,33 @@ jobs:
 
       - name: Set env
         run: |
-          echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
           echo "NOW=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> $GITHUB_ENV
+          
+      - name: Generate tags
+        id: generate_tags
+        run: |
+          echo "tag_patch=$(echo ${{ matrix.db-type }})-${GITHUB_REF#refs/tags/}" >> $GITHUB_ENV
+          echo "tag_minor=$(echo ${{ matrix.db-type }})-$(echo ${GITHUB_REF#refs/tags/} | cut -d. -f1,2)" >> $GITHUB_ENV
+          echo "tag_major=$(echo ${{ matrix.db-type }})-$(echo ${GITHUB_REF#refs/tags/} | cut -d. -f1)" >> $GITHUB_ENV
+          echo "tag_latest=$(echo ${{ matrix.db-type }})-latest" >> $GITHUB_ENV
 
       - uses: mr-smithers-excellent/docker-build-push@v6
         name: Build & push Docker image to ghcr.io for ${{ matrix.db-type }}
         with:
           image: umami
-          tags: ${{ matrix.db-type }}-${{ env.RELEASE_VERSION }}, ${{ matrix.db-type }}-latest
+          tags: ${{ env.tag_major }}, ${{ env.tag_minor }}, ${{ env.tag_patch }}, ${{ env.tag_latest }}
           buildArgs: DATABASE_TYPE=${{ matrix.db-type }}
           registry: ghcr.io
           multiPlatform: true
           platform: linux/amd64,linux/arm64
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
-
       - uses: mr-smithers-excellent/docker-build-push@v6
         name: Build & push Docker image to docker.io for ${{ matrix.db-type }}
         with:
           image: umamisoftware/umami
-          tags: ${{ matrix.db-type }}-${{ env.RELEASE_VERSION }}, ${{ matrix.db-type }}-latest
+          tags: ${{ env.tag_major }}, ${{ env.tag_minor }}, ${{ env.tag_patch }}, ${{ env.tag_latest }}
           buildArgs: DATABASE_TYPE=${{ matrix.db-type }}
           registry: docker.io
           username: ${{ secrets.DOCKER_USERNAME }}

diff --git a/db/clickhouse/migrations/04_add_tag.sql b/db/clickhouse/migrations/04_add_tag.sql
@@ -0,0 +1,77 @@
+-- add tag column
+ALTER TABLE umami.website_event ADD COLUMN "tag" String AFTER "event_name";
+ALTER TABLE umami.website_event_stats_hourly ADD COLUMN "tag" SimpleAggregateFunction(groupArrayArray, Array(String)) AFTER "max_time";
+
+-- update materialized view
+DROP TABLE umami.website_event_stats_hourly_mv;
+
+CREATE MATERIALIZED VIEW umami.website_event_stats_hourly_mv
+TO umami.website_event_stats_hourly
+AS
+SELECT
+    website_id,
+    session_id,
+    visit_id,
+    hostname,
+    browser,
+    os,
+    device,
+    screen,
+    language,
+    country,
+    subdivision1,
+    city,
+    entry_url,
+    exit_url,
+    url_paths as url_path,
+    url_query,
+    referrer_domain,
+    page_title,
+    event_type,
+    event_name,
+    views,
+    min_time,
+    max_time,
+    tag,
+    timestamp as created_at
+FROM (SELECT
+    website_id,
+    session_id,
+    visit_id,
+    hostname,
+    browser,
+    os,
+    device,
+    screen,
+    language,
+    country,
+    subdivision1,
+    city,
+    argMinState(url_path, created_at) entry_url,
+    argMaxState(url_path, created_at) exit_url,
+    arrayFilter(x -> x != '', groupArray(url_path)) as url_paths,
+    arrayFilter(x -> x != '', groupArray(url_query)) url_query,
+    arrayFilter(x -> x != '', groupArray(referrer_domain)) referrer_domain,
+    arrayFilter(x -> x != '', groupArray(page_title)) page_title,
+    event_type,
+    if(event_type = 2, groupArray(event_name), []) event_name,
+    sumIf(1, event_type = 1) views,
+    min(created_at) min_time,
+    max(created_at) max_time,
+    arrayFilter(x -> x != '', groupArray(tag)) tag,
+    toStartOfHour(created_at) timestamp
+FROM umami.website_event
+GROUP BY website_id,
+    session_id,
+    visit_id,
+    hostname,
+    browser,
+    os,
+    device,
+    screen,
+    language,
+    country,
+    subdivision1,
+    city,
+    event_type,
+    timestamp);
diff --git a/db/clickhouse/schema.sql b/db/clickhouse/schema.sql
@@ -26,6 +26,7 @@ CREATE TABLE umami.website_event
     --events
     event_type UInt32,
     event_name String,
+    tag String,
     created_at DateTime('UTC'),
     job_id Nullable(UUID)
 )
@@ -96,6 +97,7 @@ CREATE TABLE umami.website_event_stats_hourly
     views SimpleAggregateFunction(sum, UInt64),
     min_time SimpleAggregateFunction(min, DateTime('UTC')),
     max_time SimpleAggregateFunction(max, DateTime('UTC')),
+    tag SimpleAggregateFunction(groupArrayArray, Array(String)),
     created_at Datetime('UTC')
 )
 ENGINE = AggregatingMergeTree
@@ -136,6 +138,7 @@ SELECT
     views,
     min_time,
     max_time,
+    tag,
     timestamp as created_at
 FROM (SELECT
     website_id,
@@ -161,6 +164,7 @@ FROM (SELECT
     sumIf(1, event_type = 1) views,
     min(created_at) min_time,
     max(created_at) max_time,
+    arrayFilter(x -> x != '', groupArray(tag)) tag,
     toStartOfHour(created_at) timestamp
 FROM umami.website_event
 GROUP BY website_id,

diff --git a/db/mysql/migrations/01_init/migration.sql b/db/mysql/migrations/01_init/migration.sql
@@ -11,7 +11,7 @@ CREATE TABLE `user` (
     UNIQUE INDEX `user_user_id_key`(`user_id`),
     UNIQUE INDEX `user_username_key`(`username`),
     PRIMARY KEY (`user_id`)
-) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+) ENGINE=InnoDB DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
 
 -- CreateTable
 CREATE TABLE `session` (
@@ -33,7 +33,7 @@ CREATE TABLE `session` (
     INDEX `session_created_at_idx`(`created_at`),
     INDEX `session_website_id_idx`(`website_id`),
     PRIMARY KEY (`session_id`)
-) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+) ENGINE=InnoDB DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
 
 -- CreateTable
 CREATE TABLE `website` (
@@ -53,7 +53,7 @@ CREATE TABLE `website` (
     INDEX `website_created_at_idx`(`created_at`),
     INDEX `website_share_id_idx`(`share_id`),
     PRIMARY KEY (`website_id`)
-) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+) ENGINE=InnoDB DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
 
 -- CreateTable
 CREATE TABLE `website_event` (
@@ -76,7 +76,7 @@ CREATE TABLE `website_event` (
     INDEX `website_event_website_id_created_at_idx`(`website_id`, `created_at`),
     INDEX `website_event_website_id_session_id_created_at_idx`(`website_id`, `session_id`, `created_at`),
     PRIMARY KEY (`event_id`)
-) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+) ENGINE=InnoDB DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
 
 -- CreateTable
 CREATE TABLE `event_data` (
@@ -95,7 +95,7 @@ CREATE TABLE `event_data` (
     INDEX `event_data_website_event_id_idx`(`website_event_id`),
     INDEX `event_data_website_id_website_event_id_created_at_idx`(`website_id`, `website_event_id`, `created_at`),
     PRIMARY KEY (`event_id`)
-) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+) ENGINE=InnoDB DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
 
 -- CreateTable
 CREATE TABLE `team` (
@@ -109,7 +109,7 @@ CREATE TABLE `team` (
     UNIQUE INDEX `team_access_code_key`(`access_code`),
     INDEX `team_access_code_idx`(`access_code`),
     PRIMARY KEY (`team_id`)
-) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+) ENGINE=InnoDB DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
 
 -- CreateTable
 CREATE TABLE `team_user` (
@@ -124,7 +124,7 @@ CREATE TABLE `team_user` (
     INDEX `team_user_team_id_idx`(`team_id`),
     INDEX `team_user_user_id_idx`(`user_id`),
     PRIMARY KEY (`team_user_id`)
-) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+) ENGINE=InnoDB DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
 
 -- CreateTable
 CREATE TABLE `team_website` (
@@ -137,7 +137,7 @@ CREATE TABLE `team_website` (
     INDEX `team_website_team_id_idx`(`team_id`),
     INDEX `team_website_website_id_idx`(`website_id`),
     PRIMARY KEY (`team_website_id`)
-) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+) ENGINE=InnoDB DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
 
 -- AddSystemUser
 INSERT INTO user (user_id, username, role, password) VALUES ('41e2b680-648e-4b09-bcd7-3e2b10c06264' , 'admin', 'admin', '$2b$10$BUli0c.muyCW1ErNJc3jL.vFRFtFJWrT8/GcR4A.sUdCznaXiqFXa');
diff --git a/db/mysql/migrations/02_report_schema_session_data/migration.sql b/db/mysql/migrations/02_report_schema_session_data/migration.sql
@@ -21,7 +21,7 @@ CREATE TABLE `session_data` (
     INDEX `session_data_website_id_idx`(`website_id`),
     INDEX `session_data_session_id_idx`(`session_id`),
     PRIMARY KEY (`session_data_id`)
-) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+) ENGINE=InnoDB DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
 
 -- CreateTable
 CREATE TABLE `report` (
@@ -41,7 +41,7 @@ CREATE TABLE `report` (
     INDEX `report_type_idx`(`type`),
     INDEX `report_name_idx`(`name`),
     PRIMARY KEY (`report_id`)
-) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+) ENGINE=InnoDB DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
 
 -- EventData migration
 UPDATE event_data

diff --git a/db/mysql/migrations/07_add_tag/migration.sql b/db/mysql/migrations/07_add_tag/migration.sql
@@ -0,0 +1,5 @@
+-- AlterTable
+ALTER TABLE `website_event` ADD COLUMN `tag` VARCHAR(50) NULL;
+
+-- CreateIndex
+CREATE INDEX `website_event_website_id_created_at_tag_idx` ON `website_event`(`website_id`, `created_at`, `tag`);
diff --git a/db/mysql/schema.prisma b/db/mysql/schema.prisma
@@ -102,6 +102,7 @@ model WebsiteEvent {
   pageTitle      String?   @map("page_title") @db.VarChar(500)
   eventType      Int       @default(1) @map("event_type") @db.UnsignedInt
   eventName      String?   @map("event_name") @db.VarChar(50)
+  tag            String?   @db.VarChar(50)
 
   eventData EventData[]
   session   Session     @relation(fields: [sessionId], references: [id])
@@ -116,6 +117,7 @@ model WebsiteEvent {
   @@index([websiteId, createdAt, referrerDomain])
   @@index([websiteId, createdAt, pageTitle])
   @@index([websiteId, createdAt, eventName])
+  @@index([websiteId, createdAt, tag])
   @@index([websiteId, sessionId, createdAt])
   @@index([websiteId, visitId, createdAt])
   @@map("website_event")

diff --git a/db/postgresql/migrations/07_add_tag/migration.sql b/db/postgresql/migrations/07_add_tag/migration.sql
@@ -0,0 +1,5 @@
+-- AlterTable
+ALTER TABLE "website_event" ADD COLUMN     "tag" VARCHAR(50);
+
+-- CreateIndex
+CREATE INDEX "website_event_website_id_created_at_tag_idx" ON "website_event"("website_id", "created_at", "tag");
diff --git a/db/postgresql/schema.prisma b/db/postgresql/schema.prisma
@@ -102,6 +102,7 @@ model WebsiteEvent {
   pageTitle      String?   @map("page_title") @db.VarChar(500)
   eventType      Int       @default(1) @map("event_type") @db.Integer
   eventName      String?   @map("event_name") @db.VarChar(50)
+  tag            String?   @db.VarChar(50)
 
   eventData EventData[]
   session   Session     @relation(fields: [sessionId], references: [id])
@@ -111,11 +112,13 @@ model WebsiteEvent {
   @@index([visitId])
   @@index([websiteId])
   @@index([websiteId, createdAt])
+
   @@index([websiteId, createdAt, urlPath])
   @@index([websiteId, createdAt, urlQuery])
   @@index([websiteId, createdAt, referrerDomain])
   @@index([websiteId, createdAt, pageTitle])
   @@index([websiteId, createdAt, eventName])
+  @@index([websiteId, createdAt, tag])
   @@index([websiteId, sessionId, createdAt])
   @@index([websiteId, visitId, createdAt])
   @@map("website_event")

diff --git a/next-env.d.ts b/next-env.d.ts
@@ -3,4 +3,4 @@
 /// <reference types="next/navigation-types/compat/navigation" />
 
 // NOTE: This file should not be edited
-// see https://nextjs.org/docs/basic-features/typescript for more information.
+// see https://nextjs.org/docs/app/building-your-application/configuring/typescript for more information.