feat: in product metrics (NangoHQ#2541)

## Describe your changes

Fixes
https://linear.app/nango/issue/NAN-1466/empty-state-when-elasticsearch-is-not-present
Fixes
https://linear.app/nango/issue/NAN-1467/empty-state-when-there-is-no-data
Fixes
https://linear.app/nango/issue/NAN-1465/endpointquery-to-get-metrics
Fixes https://linear.app/nango/issue/NAN-1464/code-the-ui

- New endpoint `POST /api/v1/logs/insights` 
Retrieve insights by operations type, depending on the performance I
might add some Redis cache.

- Dashboard homepage
The UI now displays the homepage by default, if you have finished your
interactive demo.


## Test

> Currently deployed in staging 

<img width="1512" alt="Screenshot 2024-07-25 at 10 25 11"
src="https://github.com/user-attachments/assets/44cde003-dc9c-4f3b-957d-199f5d877587">

packages/logs/lib/index.ts

@@ -3,4 +3,5 @@ export * from './client.js';
 export * from './models/helpers.js';
 export * from './models/logContextGetter.js';
 export * as model from './models/messages.js';
+export * as modelOperations from './models/insights.js';
 export { envs, defaultOperationExpiration } from './env.js';

packages/logs/lib/models/insights.ts

@@ -0,0 +1,50 @@
+import type { estypes } from '@elastic/elasticsearch';
+import { indexMessages } from '../es/schema.js';
+import { client } from '../es/client.js';
+import type { InsightsHistogramEntry } from '@nangohq/types';
+
+export async function retrieveInsights(opts: { accountId: number; environmentId: number; type: string }) {
+    const query: estypes.QueryDslQueryContainer = {
+        bool: {
+            must: [{ term: { accountId: opts.accountId } }, { term: { environmentId: opts.environmentId } }, { term: { 'operation.type': opts.type } }],
+            must_not: { exists: { field: 'parentId' } },
+            should: []
+        }
+    };
+
+    const res = await client.search<
+        never,
+        {
+            histogram: estypes.AggregationsDateHistogramAggregate;
+        }
+    >({
+        index: indexMessages.index,
+        size: 0,
+        sort: [{ createdAt: 'desc' }, 'id'],
+        track_total_hits: true,
+        aggs: {
+            histogram: {
+                date_histogram: { field: 'createdAt', calendar_interval: '1d', format: 'yyyy-MM-dd' },
+                aggs: {
+                    state_agg: {
+                        terms: { field: 'state' }
+                    }
+                }
+            }
+        },
+        query
+    });
+
+    const agg = res.aggregations!['histogram'];
+
+    const computed: InsightsHistogramEntry[] = [];
+    for (const item of agg.buckets as any[]) {
+        const success = (item.state_agg.buckets as { key: string; doc_count: number }[]).find((i) => i.key === 'success');
+        const failure = (item.state_agg.buckets as { key: string; doc_count: number }[]).find((i) => i.key === 'failed');
+        computed.push({ key: item.key_as_string, total: item.doc_count, success: success?.doc_count || 0, failure: failure?.doc_count || 0 });
+    }
+
+    return {
+        items: computed
+    };
+}

packages/server/lib/controllers/v1/logs/postInsights.integration.test.ts

@@ -0,0 +1,88 @@
+import { migrateLogsMapping } from '@nangohq/logs';
+import { multipleMigrations } from '@nangohq/database';
+import { seeders } from '@nangohq/shared';
+import { afterAll, beforeAll, describe, expect, it } from 'vitest';
+import { isSuccess, runServer, shouldBeProtected, shouldRequireQueryEnv } from '../../../utils/tests.js';
+
+let api: Awaited<ReturnType<typeof runServer>>;
+describe('POST /logs/insights', () => {
+    beforeAll(async () => {
+        await multipleMigrations();
+        await migrateLogsMapping();
+
+        api = await runServer();
+    });
+    afterAll(() => {
+        api.server.close();
+    });
+
+    it('should be protected', async () => {
+        const res = await api.fetch('/api/v1/logs/insights', { method: 'POST', query: { env: 'dev' }, body: { type: 'action' } });
+
+        shouldBeProtected(res);
+    });
+
+    it('should enforce env query params', async () => {
+        const { env } = await seeders.seedAccountEnvAndUser();
+        const res = await api.fetch(
+            '/api/v1/logs/insights',
+            // @ts-expect-error missing query on purpose
+            {
+                method: 'POST',
+                token: env.secret_key,
+                body: { type: 'action' }
+            }
+        );
+
+        shouldRequireQueryEnv(res);
+    });
+
+    it('should validate body', async () => {
+        const { env } = await seeders.seedAccountEnvAndUser();
+        const res = await api.fetch('/api/v1/logs/insights', {
+            method: 'POST',
+            query: {
+                env: 'dev',
+                // @ts-expect-error on purpose
+                foo: 'bar'
+            },
+            token: env.secret_key,
+            body: {
+                // @ts-expect-error on purpose
+                type: 'foobar'
+            }
+        });
+
+        expect(res.json).toStrictEqual<typeof res.json>({
+            error: {
+                code: 'invalid_query_params',
+                errors: [
+                    {
+                        code: 'unrecognized_keys',
+                        message: "Unrecognized key(s) in object: 'foo'",
+                        path: []
+                    }
+                ]
+            }
+        });
+        expect(res.res.status).toBe(400);
+    });
+
+    it('should get empty result', async () => {
+        const { env } = await seeders.seedAccountEnvAndUser();
+        const res = await api.fetch('/api/v1/logs/insights', {
+            method: 'POST',
+            query: { env: 'dev' },
+            token: env.secret_key,
+            body: { type: 'sync' }
+        });
+
+        isSuccess(res.json);
+        expect(res.res.status).toBe(200);
+        expect(res.json).toStrictEqual<typeof res.json>({
+            data: {
+                histogram: []
+            }
+        });
+    });
+});

packages/server/lib/controllers/v1/logs/postInsights.ts

@@ -0,0 +1,46 @@
+import { z } from 'zod';
+import { asyncWrapper } from '../../../utils/asyncWrapper.js';
+import { requireEmptyQuery, zodErrorToHTTP } from '@nangohq/utils';
+import type { PostInsights } from '@nangohq/types';
+import { envs, modelOperations } from '@nangohq/logs';
+
+const validation = z
+    .object({
+        type: z.enum(['sync', 'action', 'proxy', 'webhook_external'])
+    })
+    .strict();
+
+export const postInsights = asyncWrapper<PostInsights>(async (req, res) => {
+    if (!envs.NANGO_LOGS_ENABLED) {
+        res.status(404).send({ error: { code: 'feature_disabled' } });
+        return;
+    }
+
+    const emptyQuery = requireEmptyQuery(req, { withEnv: true });
+    if (emptyQuery) {
+        res.status(400).send({ error: { code: 'invalid_query_params', errors: zodErrorToHTTP(emptyQuery.error) } });
+        return;
+    }
+
+    const val = validation.safeParse(req.body);
+    if (!val.success) {
+        res.status(400).send({
+            error: { code: 'invalid_body', errors: zodErrorToHTTP(val.error) }
+        });
+        return;
+    }
+
+    const env = res.locals['environment'];
+    const body: PostInsights['Body'] = val.data;
+    const insights = await modelOperations.retrieveInsights({
+        accountId: env.account_id,
+        environmentId: env.id,
+        type: body.type
+    });
+
+    res.status(200).send({
+        data: {
+            histogram: insights.items
+        }
+    });
+});

packages/server/lib/routes.ts

@@ -63,11 +63,12 @@ import { deleteInvite } from './controllers/v1/invite/deleteInvite.js';
 import { deleteTeamUser } from './controllers/v1/team/users/deleteTeamUser.js';
 import { getUser } from './controllers/v1/user/getUser.js';
 import { patchUser } from './controllers/v1/user/patchUser.js';
+import { postInsights } from './controllers/v1/logs/postInsights.js';
 import { getInvite } from './controllers/v1/invite/getInvite.js';
 import { declineInvite } from './controllers/v1/invite/declineInvite.js';
 import { acceptInvite } from './controllers/v1/invite/acceptInvite.js';
-import { securityMiddlewares } from './middleware/security.js';
 import { getMeta } from './controllers/v1/meta/getMeta.js';
+import { securityMiddlewares } from './middleware/security.js';
 import { postManagedSignup } from './controllers/v1/account/managed/postSignup.js';
 import { getManagedCallback } from './controllers/v1/account/managed/getCallback.js';
 import { getEnvJs } from './controllers/v1/getEnvJs.js';
@@ -281,6 +282,7 @@ web.route('/api/v1/logs/operations').post(webAuth, searchOperations);
 web.route('/api/v1/logs/messages').post(webAuth, searchMessages);
 web.route('/api/v1/logs/filters').post(webAuth, searchFilters);
 web.route('/api/v1/logs/operations/:operationId').get(webAuth, getOperation);
+web.route('/api/v1/logs/insights').post(webAuth, postInsights);
 
 // Hosted signin
 if (!isCloud && !isEnterprise) {

packages/types/lib/api.endpoints.ts

@@ -1,5 +1,5 @@
 import type { EndpointMethod } from './api';
-import type { GetOperation, SearchFilters, SearchMessages, SearchOperations } from './logs/api';
+import type { GetOperation, PostInsights, SearchFilters, SearchMessages, SearchOperations } from './logs/api';
 import type { GetOnboardingStatus } from './onboarding/api';
 import type { SetMetadata, UpdateMetadata } from './connection/api/metadata';
 import type { PostDeploy, PostDeployConfirmation } from './deploy/api';
@@ -18,6 +18,7 @@ export type APIEndpoints =
     | PostInvite
     | DeleteInvite
     | DeleteTeamUser
+    | PostInsights
     | PostForgotPassword
     | PutResetPassword
     | SearchOperations

packages/webapp/package.json

@@ -72,6 +72,7 @@
         "react-scripts": "5.0.1",
         "react-toastify": "9.1.1",
         "react-use": "17.5.0",
+        "recharts": "2.12.7",
         "swr": "2.2.5",
         "tailwind-merge": "2.3.0",
         "tailwindcss": "3.4.3",

packages/webapp/src/App.tsx

@@ -27,13 +27,14 @@ import { VerifyEmail } from './pages/Account/VerifyEmail';
 import { VerifyEmailByExpiredToken } from './pages/Account/VerifyEmailByExpiredToken';
 import { EmailVerified } from './pages/Account/EmailVerified';
 import AuthLink from './pages/AuthLink';
-import { Homepage } from './pages/Homepage';
+import { Homepage } from './pages/Homepage/Show';
 import { NotFound } from './pages/NotFound';
 import { LogsSearch } from './pages/Logs/Search';
 import { TooltipProvider } from '@radix-ui/react-tooltip';
 import { SentryRoutes } from './utils/sentry';
 import { TeamSettings } from './pages/Team/Settings';
 import { UserSettings } from './pages/User/Settings';
+import { Root } from './pages/Root';
 import { globalEnv } from './utils/env';
 
 const theme = createTheme({
@@ -69,8 +70,9 @@ const App = () => {
                     }}
                 >
                     <SentryRoutes>
-                        <Route path="/" element={<Homepage />} />
+                        <Route path="/" element={<Root />} />
                         <Route element={<PrivateRoute />} key={env}>
+                            <Route path="/:env" element={<Homepage />} />
                             {showInteractiveDemo && (
                                 <Route path="/dev/interactive-demo" element={<PrivateRoute />}>
                                     <Route path="/dev/interactive-demo" element={<InteractiveDemo />} />