Updates to insights, event data, telemetry.

components/pages/event-data/EventDataTable.js

@@ -7,25 +7,21 @@ export function EventDataTable({ data = [] }) {
   const { formatMessage, labels } = useMessages();
   const { resolveUrl } = usePageQuery();
 
-  function linkToView(row, cell) {
-    return (
-      <Link href={resolveUrl({ view: row.field, event: row.event })} shallow={true}>
-        {cell}
-      </Link>
-    );
-  }
-
   if (data.length === 0) {
     return <Empty />;
   }
 
   return (
     <GridTable data={data}>
       <GridColumn name="event" label={formatMessage(labels.event)}>
-        {row => linkToView(row, row.event)}
+        {row => (
+          <Link href={resolveUrl({ event: row.event })} shallow={true}>
+            {row.event}
+          </Link>
+        )}
       </GridColumn>
       <GridColumn name="field" label={formatMessage(labels.field)}>
-        {row => linkToView(row, row.field)}
+        {row => row.field}
       </GridColumn>
       <GridColumn name="total" label={formatMessage(labels.totalRecords)}>
         {({ total }) => total.toLocaleString()}

components/pages/event-data/EventDataValueTable.js

@@ -5,24 +5,22 @@ import Icons from 'components/icons';
 import PageHeader from 'components/layout/PageHeader';
 import Empty from 'components/common/Empty';
 
-export function EventDataTable({ data = [], field, event }) {
+export function EventDataValueTable({ data = [], event }) {
   const { formatMessage, labels } = useMessages();
   const { resolveUrl } = usePageQuery();
 
   const Title = () => {
     return (
       <>
-        <Link href={resolveUrl({ view: undefined })}>
+        <Link href={resolveUrl({ event: undefined })}>
           <Button>
             <Icon rotate={180}>
               <Icons.ArrowRight />
             </Icon>
             <Text>{formatMessage(labels.back)}</Text>
           </Button>
         </Link>
-        <Text>
-          {event} - {field}
-        </Text>
+        <Text>{event}</Text>
       </>
     );
   };
@@ -33,6 +31,7 @@ export function EventDataTable({ data = [], field, event }) {
       {data.length <= 0 && <Empty />}
       {data.length > 0 && (
         <GridTable data={data}>
+          <GridColumn name="field" label={formatMessage(labels.field)} />
           <GridColumn name="value" label={formatMessage(labels.value)} />
           <GridColumn name="total" label={formatMessage(labels.totalRecords)} width="200px">
             {({ total }) => total.toLocaleString()}
@@ -43,4 +42,4 @@ export function EventDataTable({ data = [], field, event }) {
   );
 }
 
-export default EventDataTable;
+export default EventDataValueTable;

components/pages/websites/WebsiteEventData.js

@@ -5,20 +5,18 @@ import { EventDataMetricsBar } from 'components/pages/event-data/EventDataMetric
 import { useDateRange, useApi, usePageQuery } from 'hooks';
 import styles from './WebsiteEventData.module.css';
 
-function useFields(websiteId, field, event) {
+function useData(websiteId, event) {
   const [dateRange] = useDateRange(websiteId);
   const { startDate, endDate } = dateRange;
   const { get, useQuery } = useApi();
   const { data, error, isLoading } = useQuery(
-    ['event-data:fields', { websiteId, startDate, endDate, field }],
+    ['event-data:events', { websiteId, startDate, endDate, event }],
     () =>
-      get('/event-data/fields', {
+      get('/event-data/events', {
         websiteId,
         startAt: +startDate,
         endAt: +endDate,
-        field,
         event,
-        withEventNames: true,
       }),
     { enabled: !!(websiteId && startDate && endDate) },
   );
@@ -28,15 +26,15 @@ function useFields(websiteId, field, event) {
 
 export default function WebsiteEventData({ websiteId }) {
   const {
-    query: { view, event },
+    query: { event },
   } = usePageQuery();
-  const { data } = useFields(websiteId, view, event);
+  const { data } = useData(websiteId, event);
 
   return (
     <Flexbox className={styles.container} direction="column" gap={20}>
       <EventDataMetricsBar websiteId={websiteId} />
-      {!view && <EventDataTable data={data} />}
-      {view && <EventDataValueTable field={view} event={event} data={data} />}
+      {!event && <EventDataTable data={data} />}
+      {event && <EventDataValueTable event={event} data={data} />}
     </Flexbox>
   );
 }

db/clickhouse/schema.sql

@@ -6,7 +6,7 @@ CREATE TABLE umami.website_event
     website_id UUID,
     session_id UUID,
     event_id UUID,
-    --session
+    --sessions
     hostname LowCardinality(String),
     browser LowCardinality(String),
     os LowCardinality(String),
@@ -17,14 +17,14 @@ CREATE TABLE umami.website_event
     subdivision1 LowCardinality(String),
     subdivision2 LowCardinality(String),
     city String,
-    --pageview
+    --pageviews
     url_path String,
     url_query String,
     referrer_path String,
     referrer_query String,
     referrer_domain String,
     page_title String,
-    --event
+    --events
     event_type UInt32,
     event_name String,
     created_at DateTime('UTC'),
@@ -38,7 +38,7 @@ CREATE TABLE umami.website_event_queue (
     website_id UUID,
     session_id UUID,
     event_id UUID,
-    --session
+    --sessions
     hostname LowCardinality(String),
     browser LowCardinality(String),
     os LowCardinality(String),
@@ -49,14 +49,14 @@ CREATE TABLE umami.website_event_queue (
     subdivision1 LowCardinality(String),
     subdivision2 LowCardinality(String),
     city String,
-    --pageview
+    --pageviews
     url_path String,
     url_query String,
     referrer_path String,
     referrer_query String,
     referrer_domain String,
     page_title String,
-    --event
+    --events
     event_type UInt32,
     event_name String,
     created_at DateTime('UTC'),
@@ -66,7 +66,7 @@ CREATE TABLE umami.website_event_queue (
 )
 ENGINE = Kafka
 SETTINGS kafka_broker_list = 'domain:9092,domain:9093,domain:9094', -- input broker list
-       kafka_topic_list = 'event',
+       kafka_topic_list = 'events',
        kafka_group_name = 'event_consumer_group',
        kafka_format = 'JSONEachRow',
        kafka_max_block_size = 1048576,

lib/clickhouse.ts

@@ -62,10 +62,6 @@ function getDateFormat(date) {
   return `'${dateFormat(date, 'UTC:yyyy-mm-dd HH:MM:ss')}'`;
 }
 
-function getBetweenDates(field, startAt, endAt) {
-  return `${field} between ${getDateFormat(startAt)} and ${getDateFormat(endAt)}`;
-}
-
 function getEventDataFilterQuery(
   filters: {
     eventKey?: string;
@@ -150,7 +146,22 @@ function parseFilters(filters: WebsiteMetricFilter = {}, params: any = {}) {
   };
 }
 
-async function rawQuery<T>(query, params = {}): Promise<T> {
+function formatField(field, type, value) {
+  switch (type) {
+    case 'date':
+      return getDateFormat(value);
+    default:
+      return field;
+  }
+}
+
+async function rawQuery<T>(sql, params = {}): Promise<T> {
+  const query = sql.replaceAll(/\{\{\w+:\w+}}/g, token => {
+    const [, field, type] = token.match(/\{\{(\w+):(\w+)}}/);
+
+    return formatField(field, type, params[field]);
+  });
+
   if (process.env.LOG_QUERY) {
     log('QUERY:\n', query);
     log('PARAMETERS:\n', params);
@@ -189,7 +200,6 @@ export default {
   getDateStringQuery,
   getDateQuery,
   getDateFormat,
-  getBetweenDates,
   getFilterQuery,
   getFunnelQuery,
   getEventDataFilterQuery,

lib/constants.ts

@@ -18,7 +18,7 @@ export const DEFAULT_THEME = 'light';
 export const DEFAULT_ANIMATION_DURATION = 300;
 export const DEFAULT_DATE_RANGE = '24hour';
 export const DEFAULT_WEBSITE_LIMIT = 10;
-export const DEFAULT_CREATED_AT = '2000-01-01';
+export const DEFAULT_RESET_DATE = '2000-01-01';
 
 export const REALTIME_RANGE = 30;
 export const REALTIME_INTERVAL = 5000;

lib/crypto.js

@@ -1,5 +1,3 @@
-import crypto from 'crypto';
-import { v4, v5 } from 'uuid';
 import { startOfMonth } from 'date-fns';
 import { hash } from 'next-basics';
 
@@ -12,13 +10,3 @@ export function salt() {
 
   return hash(secret(), ROTATING_SALT);
 }
-
-export function uuid(...args) {
-  if (!args.length) return v4();
-
-  return v5(hash(...args, salt()), v5.DNS);
-}
-
-export function md5(...args) {
-  return crypto.createHash('md5').update(args.join('')).digest('hex');
-}

lib/detect.ts

@@ -1,5 +1,5 @@
 import path from 'path';
-import requestIp from 'request-ip';
+import { getClientIp } from 'request-ip';
 import { browserName, detectOS } from 'detect-browser';
 import isLocalhost from 'is-localhost-ip';
 import maxmind from 'maxmind';
@@ -25,7 +25,7 @@ export function getIpAddress(req) {
     return req.headers['cf-connecting-ip'];
   }
 
-  return requestIp.getClientIp(req);
+  return getClientIp(req);
 }
 
 export function getDevice(screen, os) {

lib/session.ts

@@ -1,6 +1,6 @@
-import { secret, uuid } from 'lib/crypto';
+import { secret } from 'lib/crypto';
 import { getClientInfo, getJsonBody } from 'lib/detect';
-import { parseToken } from 'next-basics';
+import { parseToken, uuid } from 'next-basics';
 import { CollectRequestBody, NextApiRequestCollect } from 'pages/api/send';
 import { createSession } from 'queries';
 import { validate } from 'uuid';
@@ -30,7 +30,6 @@ export async function findSession(req: NextApiRequestCollect) {
   // Verify payload
   const { website: websiteId, hostname, screen, language } = payload;
 
-
   // Check the hostname value for legality to eliminate dirty data
   const validHostnameRegex = /^[\w-.]+$/;
   if (!validHostnameRegex.test(hostname)) {

lib/sql.ts

@@ -0,0 +1,9 @@
+export function buildSql(query: string, parameters: object) {
+  const params = { ...parameters };
+
+  const sql = query.replaceAll(/\$[\w_]+/g, name => {
+    return name;
+  });
+
+  return { sql, params };
+}

package.json

@@ -90,7 +90,7 @@
     "maxmind": "^4.3.6",
     "moment-timezone": "^0.5.35",
     "next": "13.3.1",
-    "next-basics": "^0.31.0",
+    "next-basics": "^0.33.0",
     "node-fetch": "^3.2.8",
     "npm-run-all": "^4.1.5",
     "react": "^18.2.0",

pages/api/event-data/events.ts

@@ -0,0 +1,39 @@
+import { canViewWebsite } from 'lib/auth';
+import { useCors, useAuth } from 'lib/middleware';
+import { NextApiRequestQueryBody } from 'lib/types';
+import { NextApiResponse } from 'next';
+import { ok, methodNotAllowed, unauthorized } from 'next-basics';
+import { getEventDataEvents } from 'queries';
+
+export interface EventDataFieldsRequestBody {
+  websiteId: string;
+  dateRange: {
+    startDate: string;
+    endDate: string;
+  };
+}
+
+export default async (
+  req: NextApiRequestQueryBody<any, EventDataFieldsRequestBody>,
+  res: NextApiResponse<any>,
+) => {
+  await useCors(req, res);
+  await useAuth(req, res);
+
+  if (req.method === 'GET') {
+    const { websiteId, startAt, endAt, field, event } = req.query;
+
+    if (!(await canViewWebsite(req.auth, websiteId))) {
+      return unauthorized(res);
+    }
+
+    const data = await getEventDataEvents(websiteId, new Date(+startAt), new Date(+endAt), {
+      field,
+      event,
+    });
+
+    return ok(res, data);
+  }
+
+  return methodNotAllowed(res);
+};

pages/api/event-data/fields.ts

@@ -21,20 +21,13 @@ export default async (
   await useAuth(req, res);
 
   if (req.method === 'GET') {
-    const { websiteId, startAt, endAt, field, event, withEventNames } = req.query;
+    const { websiteId, startAt, endAt, field } = req.query;
 
     if (!(await canViewWebsite(req.auth, websiteId))) {
       return unauthorized(res);
     }
 
-    const data = await getEventDataFields(
-      websiteId,
-      new Date(+startAt),
-      new Date(+endAt),
-      field,
-      event,
-      withEventNames,
-    );
+    const data = await getEventDataFields(websiteId, new Date(+startAt), new Date(+endAt), field);
 
     return ok(res, data);
   }

pages/api/reports/funnel.ts

@@ -3,7 +3,7 @@ import { useCors, useAuth } from 'lib/middleware';
 import { NextApiRequestQueryBody } from 'lib/types';
 import { NextApiResponse } from 'next';
 import { ok, methodNotAllowed, unauthorized } from 'next-basics';
-import { getPageviewFunnel } from 'queries';
+import { getFunnel } from 'queries';
 
 export interface FunnelRequestBody {
   websiteId: string;
@@ -41,7 +41,7 @@ export default async (
       return unauthorized(res);
     }
 
-    const data = await getPageviewFunnel(websiteId, {
+    const data = await getFunnel(websiteId, {
       startDate: new Date(startDate),
       endDate: new Date(endDate),
       urls,

pages/api/reports/index.ts

@@ -1,8 +1,7 @@
-import { uuid } from 'lib/crypto';
 import { useAuth, useCors } from 'lib/middleware';
 import { NextApiRequestQueryBody } from 'lib/types';
 import { NextApiResponse } from 'next';
-import { methodNotAllowed, ok, unauthorized } from 'next-basics';
+import { methodNotAllowed, ok, unauthorized, uuid } from 'next-basics';
 import { createReport, getReports } from 'queries';
 import { canViewWebsite } from 'lib/auth';