From 0626f2c80eb1b93d1f34882ce30317fb80d75d1a Mon Sep 17 00:00:00 2001 From: Rui Jie <80191727+rjkoh@users.noreply.github.com> Date: Wed, 18 Sep 2024 22:19:33 +0800 Subject: [PATCH 01/18] Create Code Analysis Job: get repo, create projects, pull/clone, scan --- backend/app.ts | 6 +- backend/jobs/codeAnalysisJob.ts | 288 ++++++++++++++++++++++++++++++++ 2 files changed, 292 insertions(+), 2 deletions(-) create mode 100644 backend/jobs/codeAnalysisJob.ts diff --git a/backend/app.ts b/backend/app.ts index cb57c472..27efea20 100644 --- a/backend/app.ts +++ b/backend/app.ts @@ -13,14 +13,16 @@ import teamRoutes from './routes/teamRoutes'; import teamSetRoutes from './routes/teamSetRoutes'; import userRoutes from './routes/userRoutes'; import { connectToDatabase } from './utils/database'; +import setupCodeAnalysisJob from 'jobs/githubBuildToolJob'; const env = process.env.NODE_ENV ?? 'development'; config({ path: `.env.${env}` }); const setupApp = async () => { await connectToDatabase(); - setupGitHubJob(); - setupJiraJob(); + // setupGitHubJob(); + // setupJiraJob(); + setupCodeAnalysisJob(); }; setupApp(); diff --git a/backend/jobs/codeAnalysisJob.ts b/backend/jobs/codeAnalysisJob.ts new file mode 100644 index 00000000..208d842a --- /dev/null +++ b/backend/jobs/codeAnalysisJob.ts @@ -0,0 +1,288 @@ +import CourseModel from '@models/Course'; +import cron from 'node-cron'; +import { App, Octokit } from 'octokit'; +import { getGitHubApp } from '../utils/github'; +import * as fs from 'fs'; +import * as path from 'path'; + +const { exec } = require('child_process'); + +const fetchAndSaveCodeAnalysisData = async () => { + const app: App = getGitHubApp(); + const octokit = app.octokit; + + const response = await octokit.rest.apps.listInstallations(); + const installationIds = response.data.map(installation => installation.id); + + const courses = await CourseModel.find({ + installationId: { $in: installationIds }, + }); + + await Promise.all( + courses.map(async course => { + if (course && course.installationId) { + const installationOctokit = await app.getInstallationOctokit( + course.installationId + ); + await getCourseCodeData(installationOctokit, course); + } + }) + ); + + console.log('Code analysis data fetched'); +}; + +const getCourseCodeData = async (octokit: Octokit, course: any) => { + if (!course.gitHubOrgName) return; + const gitHubOrgName = course.gitHubOrgName; + + const repos = await octokit.rest.repos.listForOrg({ + org: course.gitHubOrgName, + sort: 'updated', + per_page: 300, + direction: 'desc', + }); + let allRepos = repos.data; + + if (course.repoNameFilter) { + allRepos = allRepos.filter(repo => + repo.name.includes(course.repoNameFilter as string) + ); + } + + for (const repo of allRepos) { + const buildTool = await getRepoBuildTool(octokit, gitHubOrgName, repo.name); + if (buildTool === '.NET' || buildTool === 'Error') continue; + + await createProjectIfNotExists(gitHubOrgName + '_' + repo.name); + + await getLatestCommit(gitHubOrgName, repo.name); + + await scanRepo(gitHubOrgName, repo.name, buildTool); + } +}; + +const getRepoBuildTool = async ( + octokit: Octokit, + owner: string, + repo: string +) => { + try { + // Fetch repository root directory content + const contents = await octokit.rest.repos.getContent({ + owner, + repo, + path: '', + }); + + if (Array.isArray(contents.data)) { + const files = contents.data.map(file => file.name); + + // Check for Maven + if (files.includes('pom.xml')) { + return 'Maven'; + } + + // Check for Gradle + if ( + files.includes('build.gradle') || + files.includes('build.gradle.kts') + ) { + return 'Gradle'; + } + + // Check for .NET-related project files + const dotnetProjectFiles = files.filter( + file => + file.endsWith('.csproj') || + file.endsWith('.fsproj') || + file.endsWith('.vbproj') || + file.endsWith('.sln') + ); + + if (dotnetProjectFiles.length > 0) return '.NET'; + + // If none of the above, return 'Others' + return 'Others'; + } + + return 'Others'; + } catch (error) { + if (error instanceof Error) { + console.error( + `Error fetching build tool for repository ${repo}: ${error.message}` + ); + } else { + console.error( + `Unknown error fetching build tool for repository ${repo}:`, + error + ); + } + return 'Error'; + } +}; + +const createProjectIfNotExists = async (repo: string) => { + try { + const sonarUri = process.env.SONAR_URI; + const sonarToken = process.env.SONAR_TOKEN; + + const searchResponse = await fetch( + `${sonarUri}/api/projects/search?projects=${repo}`, + { + method: 'GET', + headers: { + Authorization: `Basic ${Buffer.from(`${sonarToken}:`).toString('base64')}`, + }, + } + ); + + const searchResult = await searchResponse.json(); + + if (searchResult.components.length > 0) { + return; + } + + const formData = `name=${encodeURIComponent(repo)}&project=${encodeURIComponent(repo)}`; + + const createResponse = await fetch(`${sonarUri}/api/projects/create`, { + method: 'POST', + headers: { + Authorization: `Basic ${Buffer.from(`${sonarToken}:`).toString('base64')}`, + 'Content-Type': 'application/x-www-form-urlencoded', + }, + body: formData, + }); + + if (!createResponse.ok) { + const errorResponse = await createResponse.json(); + console.error(`Failed to create project: ${errorResponse.errors[0].msg}`); + } + } catch (error) { + if (error instanceof Error) { + console.error( + `Error checking / creating project for repository ${repo}: ${error.message}` + ); + } else { + console.error( + `Unknown error checking / creating project for repository ${repo}:`, + error + ); + } + } +}; + +const getLatestCommit = async (gitHubOrgName: string, repoName: string) => { + try { + const repoPath = path.join( + process.env.REPO_PATH || '', + gitHubOrgName, + repoName + ); + + if (!fs.existsSync(repoPath)) { + console.log(`Cloning repository ${repoName}...`); + await execShellCommand( + `git clone https://github.com/${gitHubOrgName}/${repoName}.git ${repoPath}` + ); + } else { + await execShellCommand(`git -C ${repoPath} pull`); + } + } catch (error) { + console.error( + `Error updating repository ${repoName}: ${(error as Error).message}` + ); + } +}; + +const execShellCommand = (cmd: string): Promise => { + return new Promise((resolve, reject) => { + exec(cmd, (error: Error | null, stdout: string, stderr: string) => { + if (error) { + reject( + `Command failed: ${cmd}\nError: ${error.message}\nStderr: ${stderr}` + ); + return; + } + resolve(stdout); + }); + }); +}; + +const scanRepo = async ( + gitHubOrgName: string, + repo: string, + buildTool: string +) => { + try { + const repoPath = path.join( + process.env.REPO_PATH || '', + gitHubOrgName, + repo + ); + + const projectKey = gitHubOrgName + '_' + repo; + + if (!fs.existsSync(repoPath)) { + console.error(`Repository path not found: ${repoPath}`); + return; + } + + process.chdir(repoPath); + + if (buildTool === 'Maven') { + const mavenCommand = `mvn clean verify sonar:sonar \ + -Dsonar.projectKey=${projectKey} \ + -Dsonar.projectName=${projectKey} \ + -Dsonar.host.url=${process.env.SONAR_URI} \ + -Dsonar.token=${process.env.SONAR_TOKEN}`; + + await execShellCommand(mavenCommand); + } else if (buildTool === 'Gradle') { + const gradleCommand = `./gradlew sonar \ + -Dsonar.projectKey=${projectKey} \ + -Dsonar.projectName=${projectKey} \ + -Dsonar.host.url=${process.env.SONAR_URI} \ + -Dsonar.token=${process.env.SONAR_TOKEN}`; + + await execShellCommand(gradleCommand); + } else if (buildTool === 'Others') { + // Handle other build tools + const othersCommand = `${process.env.SONAR_PATH} \ + -Dsonar.projectKey=${projectKey} \ + -Dsonar.sources=. \ + -Dsonar.host.url=${process.env.SONAR_URI} \ + -Dsonar.token=${process.env.SONAR_TOKEN}`; + + await execShellCommand(othersCommand); + } + } catch (error) { + if (error instanceof Error) { + console.error(`Error scanning repository ${repo}: ${error.message}`); + } else { + console.error(`Unknown error scanning repository ${repo}:`, error); + } + } +}; + +export const setupCodeAnalysisJob = () => { + // Schedule the job to run every day at midnight + cron.schedule('0 0 * * *', async () => { + console.log('Running fetchAndSaveCodeAnalysisData job:', new Date().toString()); + try { + await fetchAndSaveCodeAnalysisData(); + } catch (err) { + console.error('Error in cron job fetchAndSaveCodeAnalysisData:', err); + } + }); + + // To run the job immediately for testing + if (process.env.RUN_JOB_NOW === 'true') { + console.log('Running fetchAndSaveCodeAnalysisData job:', new Date().toString()); + fetchAndSaveCodeAnalysisData().catch(err => { + console.error('Error running job manually:', err); + }); + } +}; + +export default setupCodeAnalysisJob; From fd25876566eb791337ffe0ca5b7e81ddebba5d26 Mon Sep 17 00:00:00 2001 From: Rui Jie <80191727+rjkoh@users.noreply.github.com> Date: Thu, 19 Sep 2024 18:09:00 +0800 Subject: [PATCH 02/18] Code Analysis Model, get metrics and save --- backend/app.ts | 6 +- backend/jobs/codeAnalysisJob.ts | 88 +++++++++++++++++++++++++++++- backend/models/CodeAnalysisData.ts | 21 +++++++ shared/types/CodeAnalysisData.ts | 11 ++++ 4 files changed, 121 insertions(+), 5 deletions(-) create mode 100644 backend/models/CodeAnalysisData.ts create mode 100644 shared/types/CodeAnalysisData.ts diff --git a/backend/app.ts b/backend/app.ts index 27efea20..d11b5d7b 100644 --- a/backend/app.ts +++ b/backend/app.ts @@ -13,15 +13,15 @@ import teamRoutes from './routes/teamRoutes'; import teamSetRoutes from './routes/teamSetRoutes'; import userRoutes from './routes/userRoutes'; import { connectToDatabase } from './utils/database'; -import setupCodeAnalysisJob from 'jobs/githubBuildToolJob'; +import setupCodeAnalysisJob from 'jobs/codeAnalysisJob'; const env = process.env.NODE_ENV ?? 'development'; config({ path: `.env.${env}` }); const setupApp = async () => { await connectToDatabase(); - // setupGitHubJob(); - // setupJiraJob(); + setupGitHubJob(); + setupJiraJob(); setupCodeAnalysisJob(); }; setupApp(); diff --git a/backend/jobs/codeAnalysisJob.ts b/backend/jobs/codeAnalysisJob.ts index 208d842a..cc5a73ad 100644 --- a/backend/jobs/codeAnalysisJob.ts +++ b/backend/jobs/codeAnalysisJob.ts @@ -4,6 +4,7 @@ import { App, Octokit } from 'octokit'; import { getGitHubApp } from '../utils/github'; import * as fs from 'fs'; import * as path from 'path'; +import codeAnalysisDataModel from '../models/CodeAnalysisData' const { exec } = require('child_process'); @@ -59,6 +60,8 @@ const getCourseCodeData = async (octokit: Octokit, course: any) => { await getLatestCommit(gitHubOrgName, repo.name); await scanRepo(gitHubOrgName, repo.name, buildTool); + + await getAndSaveCodeData(gitHubOrgName, repo); } }; @@ -265,10 +268,89 @@ const scanRepo = async ( } }; +const getAndSaveCodeData = async (gitHubOrgName: string, repo: any) => { + try { + const sonarUri = process.env.SONAR_URI; + const sonarToken = process.env.SONAR_TOKEN; + const projectKey = gitHubOrgName + '_' + repo.name; + const metricKeys = + 'complexity, cognitive_complexity, branch_coverage, coverage, line_coverage, tests, uncovered_conditions, uncovered_lines, test_execution_time, test_errors, test_failures, test_success_density, skipped_tests, duplicated_blocks, duplicated_files, duplicated_lines, duplicated_lines_density, code_smells, sqale_index, sqale_debt_ratio, sqale_rating, alert_status, quality_gate_details, bugs, reliability_rating, reliability_remediation_effort, vulnerabilities, security_rating, security_remediation_effort, security_hotspots, classes, comment_lines, comment_lines_density, files, lines, ncloc, functions, statements'; + + const codeAnalysisResponse = await fetch( + `${sonarUri}/api/measures/component?component=${projectKey}&metricKeys=${metricKeys}&additionalFields=metrics`, + { + method: 'GET', + headers: { + Authorization: `Basic ${Buffer.from(`${sonarToken}:`).toString('base64')}`, + }, + } + ); + + if (!codeAnalysisResponse.ok) { + throw new Error( + `Failed to fetch data from Sonar API: ${codeAnalysisResponse.statusText}` + ); + } + + const responseData = await codeAnalysisResponse.json(); + const { component, metrics } = responseData; + + if (!component || !component.measures) { + throw new Error('Invalid response structure from Sonar API'); + } + + const metricsArray: string[] = []; + const valuesArray: string[] = []; + const typesArray: string[] = []; + const domainsArray: string[] = []; + + const metricMap = new Map(); + + metrics.forEach((metric: any) => { + metricMap.set(metric.key, { + type: metric.type, + domain: metric.domain, + }); + }); + + component.measures.forEach((measure: any) => { + const metricKey = measure.metric; + const metricInfo = metricMap.get(metricKey); + + if (metricInfo) { + metricsArray.push(metricKey); + valuesArray.push(measure.value || ''); + typesArray.push(metricInfo.type || ''); + domainsArray.push(metricInfo.domain || ''); + } + }); + + const codeAnalysisData = new codeAnalysisDataModel({ + executionTime: new Date(), + gitHubOrgName, + teamId: repo.id, + repoName: repo.name, + metrics: metricsArray, + values: valuesArray, + types: typesArray, + domains: domainsArray, + }); + + // console.log('Saving code analysis data:', codeAnalysisData); + + await codeAnalysisData.save(); + } catch (error) { + console.error(`Error fetching or saving data for: ${repo.name}`, error); + } +}; + export const setupCodeAnalysisJob = () => { // Schedule the job to run every day at midnight cron.schedule('0 0 * * *', async () => { - console.log('Running fetchAndSaveCodeAnalysisData job:', new Date().toString()); + console.log( + 'Running fetchAndSaveCodeAnalysisData job:', + new Date().toString() + ); try { await fetchAndSaveCodeAnalysisData(); } catch (err) { @@ -278,7 +360,9 @@ export const setupCodeAnalysisJob = () => { // To run the job immediately for testing if (process.env.RUN_JOB_NOW === 'true') { - console.log('Running fetchAndSaveCodeAnalysisData job:', new Date().toString()); + console.log( + 'Running fetchAndSaveCodeAnalysisData job:', + ); fetchAndSaveCodeAnalysisData().catch(err => { console.error('Error running job manually:', err); }); diff --git a/backend/models/CodeAnalysisData.ts b/backend/models/CodeAnalysisData.ts new file mode 100644 index 00000000..d4630bc6 --- /dev/null +++ b/backend/models/CodeAnalysisData.ts @@ -0,0 +1,21 @@ +import { CodeAnalysisData as SharedCodeAnalysisData } from '@shared/types/CodeAnalysisData'; +import mongoose, { Schema, Types } from 'mongoose'; + +export interface CodeAnalysisData extends Omit, Document { + _id: Types.ObjectId; +} + +const codeAnalysisDataSchema: Schema = new Schema({ + executionTime: { type : Date, required : true}, + gitHubOrgName: { type : String, required : true}, + teamId: { type : Number, required : true}, + repoName: { type : String, required : true}, + metrics: { type : [String], required : true}, + values: { type : [String], required : true}, + types: { type : [String], required : true}, + domains: { type : [String], required : true}, +}); + +const codeAnalysisDataModel = mongoose.model('codeAnalysis', codeAnalysisDataSchema); + +export default codeAnalysisDataModel; diff --git a/shared/types/CodeAnalysisData.ts b/shared/types/CodeAnalysisData.ts new file mode 100644 index 00000000..ebf925a8 --- /dev/null +++ b/shared/types/CodeAnalysisData.ts @@ -0,0 +1,11 @@ +export interface CodeAnalysisData { + _id: string; + executionTime: Date; + gitHubOrgName: string; + teamId: number; + repoName: string; + metrics: string[]; + values: string[]; + types: string[]; + domains: string[]; +} From 8345eb5746c40d46a57aa917144a9badb14b3291 Mon Sep 17 00:00:00 2001 From: Rui Jie <80191727+rjkoh@users.noreply.github.com> Date: Thu, 19 Sep 2024 18:29:09 +0800 Subject: [PATCH 03/18] style fixing --- backend/jobs/codeAnalysisJob.ts | 6 ++---- backend/models/CodeAnalysisData.ts | 25 +++++++++++++++---------- 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/backend/jobs/codeAnalysisJob.ts b/backend/jobs/codeAnalysisJob.ts index cc5a73ad..5c32bba8 100644 --- a/backend/jobs/codeAnalysisJob.ts +++ b/backend/jobs/codeAnalysisJob.ts @@ -4,7 +4,7 @@ import { App, Octokit } from 'octokit'; import { getGitHubApp } from '../utils/github'; import * as fs from 'fs'; import * as path from 'path'; -import codeAnalysisDataModel from '../models/CodeAnalysisData' +import codeAnalysisDataModel from '../models/CodeAnalysisData'; const { exec } = require('child_process'); @@ -360,9 +360,7 @@ export const setupCodeAnalysisJob = () => { // To run the job immediately for testing if (process.env.RUN_JOB_NOW === 'true') { - console.log( - 'Running fetchAndSaveCodeAnalysisData job:', - ); + console.log('Running fetchAndSaveCodeAnalysisData job'); fetchAndSaveCodeAnalysisData().catch(err => { console.error('Error running job manually:', err); }); diff --git a/backend/models/CodeAnalysisData.ts b/backend/models/CodeAnalysisData.ts index d4630bc6..d5cc204f 100644 --- a/backend/models/CodeAnalysisData.ts +++ b/backend/models/CodeAnalysisData.ts @@ -1,21 +1,26 @@ import { CodeAnalysisData as SharedCodeAnalysisData } from '@shared/types/CodeAnalysisData'; import mongoose, { Schema, Types } from 'mongoose'; -export interface CodeAnalysisData extends Omit, Document { +export interface CodeAnalysisData + extends Omit, + Document { _id: Types.ObjectId; } const codeAnalysisDataSchema: Schema = new Schema({ - executionTime: { type : Date, required : true}, - gitHubOrgName: { type : String, required : true}, - teamId: { type : Number, required : true}, - repoName: { type : String, required : true}, - metrics: { type : [String], required : true}, - values: { type : [String], required : true}, - types: { type : [String], required : true}, - domains: { type : [String], required : true}, + executionTime: { type: Date, required: true }, + gitHubOrgName: { type: String, required: true }, + teamId: { type: Number, required: true }, + repoName: { type: String, required: true }, + metrics: { type: [String], required: true }, + values: { type: [String], required: true }, + types: { type: [String], required: true }, + domains: { type: [String], required: true }, }); -const codeAnalysisDataModel = mongoose.model('codeAnalysis', codeAnalysisDataSchema); +const codeAnalysisDataModel = mongoose.model( + 'codeAnalysis', + codeAnalysisDataSchema +); export default codeAnalysisDataModel; From 4bcfa89400561416f44f4f7e3a5afb097da5dfec Mon Sep 17 00:00:00 2001 From: Rui Jie <80191727+rjkoh@users.noreply.github.com> Date: Sat, 21 Sep 2024 17:50:52 +0800 Subject: [PATCH 04/18] update imports and cron schedule --- backend/app.ts | 2 +- backend/jobs/codeAnalysisJob.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/app.ts b/backend/app.ts index d11b5d7b..d0a3b4f4 100644 --- a/backend/app.ts +++ b/backend/app.ts @@ -13,7 +13,7 @@ import teamRoutes from './routes/teamRoutes'; import teamSetRoutes from './routes/teamSetRoutes'; import userRoutes from './routes/userRoutes'; import { connectToDatabase } from './utils/database'; -import setupCodeAnalysisJob from 'jobs/codeAnalysisJob'; +import setupCodeAnalysisJob from './jobs/codeAnalysisJob'; const env = process.env.NODE_ENV ?? 'development'; config({ path: `.env.${env}` }); diff --git a/backend/jobs/codeAnalysisJob.ts b/backend/jobs/codeAnalysisJob.ts index 5c32bba8..71799ac7 100644 --- a/backend/jobs/codeAnalysisJob.ts +++ b/backend/jobs/codeAnalysisJob.ts @@ -346,7 +346,7 @@ const getAndSaveCodeData = async (gitHubOrgName: string, repo: any) => { export const setupCodeAnalysisJob = () => { // Schedule the job to run every day at midnight - cron.schedule('0 0 * * *', async () => { + cron.schedule('0 2 * * *', async () => { console.log( 'Running fetchAndSaveCodeAnalysisData job:', new Date().toString() From 07a32e6b533bc1ba71891faf1a729666736ae54e Mon Sep 17 00:00:00 2001 From: Dexter Date: Sun, 22 Sep 2024 16:15:20 +0800 Subject: [PATCH 05/18] Update course schema with Trofos REST API fields --- backend/models/Course.ts | 5 +++++ shared/types/Course.ts | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/backend/models/Course.ts b/backend/models/Course.ts index c6516936..cf3b0989 100644 --- a/backend/models/Course.ts +++ b/backend/models/Course.ts @@ -52,6 +52,11 @@ export const courseSchema = new Schema({ accessToken: { type: String }, refreshToken: { type: String }, }, + trofos: { + isRegistered: { type: Boolean, required: true, default: false }, + apiKey: { type: String }, + courseId: { type: Number }, + }, repoNameFilter: String, installationId: Number, }); diff --git a/shared/types/Course.ts b/shared/types/Course.ts index e83b268e..b57041ee 100644 --- a/shared/types/Course.ts +++ b/shared/types/Course.ts @@ -48,4 +48,9 @@ export interface Course { accessToken: string; refreshToken: string; }; + trofos: { + isRegistered: boolean; + apiKey: string; + courseId: number; + } } From d0bcc2cff9755ce6a3b65b1900f3117bde299a53 Mon Sep 17 00:00:00 2001 From: Dexter Date: Sun, 22 Sep 2024 16:52:43 +0800 Subject: [PATCH 06/18] Add front end button and modal for Trofos integration --- .../components/forms/ConnectTrofosForm.tsx | 57 +++++++++++++++++++ .../views/ProjectManagementInfo.tsx | 23 +++++++- 2 files changed, 77 insertions(+), 3 deletions(-) create mode 100644 multi-git-dashboard/src/components/forms/ConnectTrofosForm.tsx diff --git a/multi-git-dashboard/src/components/forms/ConnectTrofosForm.tsx b/multi-git-dashboard/src/components/forms/ConnectTrofosForm.tsx new file mode 100644 index 00000000..09e0f4fc --- /dev/null +++ b/multi-git-dashboard/src/components/forms/ConnectTrofosForm.tsx @@ -0,0 +1,57 @@ +import { TextInput, Button, Space, NumberInput } from '@mantine/core'; +import { useForm } from '@mantine/form'; + +interface ConnectTrofosFormProps { + closeModal: () => void; // Define closeModal as a prop +} + +const ConnectTrofosForm = ({ closeModal }: ConnectTrofosFormProps) => { + // Create a form with two fields: apiKey and courseId + const form = useForm({ + initialValues: { + apiKey: '', + courseId: 0, + }, + + // Optional: Add validation for the fields + validate: { + apiKey: value => (value.length === 0 ? 'API key is required' : null), + courseId: value => (value < 0 ? 'This is not a valid Course ID' : null), + }, + }); + + // Function to handle form submission + const handleSubmit = (values: typeof form.values) => { + console.log('Form submitted with values:', values); + + // Handle the form submission logic here (e.g., API call) + + // Close the modal after submission + closeModal(); + }; + + return ( +
handleSubmit(values))}> + + + + + + + + ); +}; + +export default ConnectTrofosForm; diff --git a/multi-git-dashboard/src/components/views/ProjectManagementInfo.tsx b/multi-git-dashboard/src/components/views/ProjectManagementInfo.tsx index a73fea27..4ad1b243 100644 --- a/multi-git-dashboard/src/components/views/ProjectManagementInfo.tsx +++ b/multi-git-dashboard/src/components/views/ProjectManagementInfo.tsx @@ -3,13 +3,16 @@ import { Button, Container, Group, + Modal, Notification, ScrollArea, Tabs, } from '@mantine/core'; +import { useDisclosure } from '@mantine/hooks'; import { TeamSet } from '@shared/types/TeamSet'; import { useEffect, useState } from 'react'; import ProjectManagementJiraCard from '../cards/ProjectManagementJiraCard'; +import ConnectTrofosForm from '../../components/forms/ConnectTrofosForm'; interface ProjectManagementProps { courseId: string; @@ -31,6 +34,8 @@ const ProjectManagementInfo: React.FC = ({ ); const [error, setError] = useState(null); + const [opened, { open, close }] = useDisclosure(false); + const setActiveTabAndSave = (tabName: string) => { onUpdate(); setActiveTab(tabName); @@ -112,12 +117,24 @@ const ProjectManagementInfo: React.FC = ({ )} {hasFacultyPermission && ( - + + + + )} + {hasFacultyPermission && ( + + + + )} {teamSets.map(teamSet => ( From ca614e5e0b76ae537d5d97d354762c734211db94 Mon Sep 17 00:00:00 2001 From: Dexter Date: Sun, 22 Sep 2024 19:47:02 +0800 Subject: [PATCH 07/18] Add course update API call logic for Trofos integration --- .../components/forms/ConnectTrofosForm.tsx | 56 ++++++++++++++++--- .../views/ProjectManagementInfo.tsx | 2 +- 2 files changed, 50 insertions(+), 8 deletions(-) diff --git a/multi-git-dashboard/src/components/forms/ConnectTrofosForm.tsx b/multi-git-dashboard/src/components/forms/ConnectTrofosForm.tsx index 09e0f4fc..b115f470 100644 --- a/multi-git-dashboard/src/components/forms/ConnectTrofosForm.tsx +++ b/multi-git-dashboard/src/components/forms/ConnectTrofosForm.tsx @@ -1,30 +1,66 @@ -import { TextInput, Button, Space, NumberInput } from '@mantine/core'; +import { + TextInput, + Button, + Space, + NumberInput, + Notification, +} from '@mantine/core'; import { useForm } from '@mantine/form'; +import { useState } from 'react'; interface ConnectTrofosFormProps { + courseId: string; closeModal: () => void; // Define closeModal as a prop } -const ConnectTrofosForm = ({ closeModal }: ConnectTrofosFormProps) => { +const ConnectTrofosForm = ({ + courseId, + closeModal, +}: ConnectTrofosFormProps) => { // Create a form with two fields: apiKey and courseId + const [error, setError] = useState(null); + const form = useForm({ initialValues: { apiKey: '', - courseId: 0, + trofosCourseId: 0, }, // Optional: Add validation for the fields validate: { apiKey: value => (value.length === 0 ? 'API key is required' : null), - courseId: value => (value < 0 ? 'This is not a valid Course ID' : null), + trofosCourseId: value => + value < 0 ? 'This is not a valid Course ID' : null, }, }); // Function to handle form submission - const handleSubmit = (values: typeof form.values) => { - console.log('Form submitted with values:', values); - + const handleSubmit = async (values: typeof form.values) => { // Handle the form submission logic here (e.g., API call) + const apiRoute = `/api/courses/${courseId}`; + + try { + const response = await fetch(apiRoute, { + method: 'PUT', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + trofos: values, + }), + }); + + if (!response.ok) { + console.error('Error connecting with Trofos:', response.statusText); + setError('Error connecting with Trofos. Please try again.'); + return; + } + + await response.json(); + } catch (error) { + console.error('Error connecting with Trofos:', error); + setError('Error connecting with Trofos. Please try again.'); + } // Close the modal after submission closeModal(); @@ -32,6 +68,12 @@ const ConnectTrofosForm = ({ closeModal }: ConnectTrofosFormProps) => { return (
handleSubmit(values))}> + {error && ( + setError(null)}> + {error} + + )} + = ({ onClose={close} title="Connect With Trofos" > - + )} From 7dc6d093ac8088c820c24adaf35595e46e96b61e Mon Sep 17 00:00:00 2001 From: Dexter Date: Mon, 23 Sep 2024 02:49:07 +0800 Subject: [PATCH 08/18] Change import ordering --- .../src/components/forms/ConnectTrofosForm.tsx | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/multi-git-dashboard/src/components/forms/ConnectTrofosForm.tsx b/multi-git-dashboard/src/components/forms/ConnectTrofosForm.tsx index b115f470..0ff2a021 100644 --- a/multi-git-dashboard/src/components/forms/ConnectTrofosForm.tsx +++ b/multi-git-dashboard/src/components/forms/ConnectTrofosForm.tsx @@ -1,9 +1,9 @@ import { - TextInput, Button, - Space, - NumberInput, Notification, + NumberInput, + Space, + TextInput, } from '@mantine/core'; import { useForm } from '@mantine/form'; import { useState } from 'react'; From cc8809828d6f0efcb2d8d2b06def8d67077fdbec Mon Sep 17 00:00:00 2001 From: Dexter Date: Sat, 28 Sep 2024 02:42:25 +0800 Subject: [PATCH 09/18] Add pagination to Octokit REST API calls --- backend/jobs/githubJob.ts | 253 ++++++++++++++++++++++++++------------ 1 file changed, 172 insertions(+), 81 deletions(-) diff --git a/backend/jobs/githubJob.ts b/backend/jobs/githubJob.ts index e2f9c2fa..efa15a8f 100644 --- a/backend/jobs/githubJob.ts +++ b/backend/jobs/githubJob.ts @@ -40,14 +40,17 @@ const getCourseData = async (octokit: Octokit, course: any) => { if (!course.gitHubOrgName) return; const gitHubOrgName = course.gitHubOrgName; - // TODO: Add pagination - const repos = await octokit.rest.repos.listForOrg({ - org: course.gitHubOrgName, - sort: 'updated', - per_page: 300, - direction: 'desc', - }); - let allRepos = repos.data; + // Fetch all repositories with pagination and backoff + const repos = await fetchAllPagesWithBackoff(page => + octokit.rest.repos.listForOrg({ + org: course.gitHubOrgName, + sort: 'updated', + per_page: 100, // GitHub API max is 100 per page + page, + }) + ); + + let allRepos = repos; await fetchGitHubProjectData(octokit, course, gitHubOrgName); @@ -61,50 +64,56 @@ const getCourseData = async (octokit: Octokit, course: any) => { const teamContributions: Record = {}; const [commits, issues, prs, contributors, milestones] = await Promise.all([ - octokit.rest.repos.listCommits({ - owner: gitHubOrgName, - repo: repo.name, - }), - octokit.rest.issues.listForRepo({ - owner: gitHubOrgName, - repo: repo.name, - }), - octokit.rest.pulls.list({ - owner: gitHubOrgName, - repo: repo.name, - state: 'all', - }), - octokit.rest.repos.listContributors({ - owner: gitHubOrgName, - repo: repo.name, - }), - octokit.rest.issues.listMilestones({ - owner: gitHubOrgName, - repo: repo.name, - state: 'all', - }), + fetchAllPagesWithBackoff(page => + octokit.rest.repos.listCommits({ + owner: gitHubOrgName, + repo: repo.name, + since: getFourMonthsAgo(), + per_page: 100, + page, + }) + ), + fetchAllPagesWithBackoff(page => + octokit.rest.issues.listForRepo({ + owner: gitHubOrgName, + repo: repo.name, + since: getFourMonthsAgo(), + per_page: 100, + page, + }) + ), + fetchAllPagesWithBackoff(page => + octokit.rest.pulls.list({ + owner: gitHubOrgName, + repo: repo.name, + since: getFourMonthsAgo(), + state: 'all', + per_page: 100, + page, + }) + ), + fetchAllPagesWithBackoff(page => + octokit.rest.repos.listContributors({ + owner: gitHubOrgName, + repo: repo.name, + since: getFourMonthsAgo(), + per_page: 100, + page, + }) + ), + fetchAllPagesWithBackoff(page => + octokit.rest.issues.listMilestones({ + owner: gitHubOrgName, + repo: repo.name, + since: getFourMonthsAgo(), + state: 'all', + per_page: 100, + page, + }) + ), ]); - // Filter out non team members - // TODO: Enable only after gitHandle mapping is done - if (process.env.NEW_FILTER === 'true') { - const teamMembers = await getTeamMembers(repo.id); - if (teamMembers) { - commits.data = commits.data.filter( - commit => commit.author && teamMembers.has(commit.author.login) - ); - issues.data = issues.data.filter( - issue => issue.user && teamMembers.has(issue.user.login) - ); - prs.data = prs.data.filter( - pr => pr.user && teamMembers.has(pr.user.login) - ); - contributors.data = contributors.data.filter( - contributor => contributor.login && teamMembers.has(contributor.login) - ); - } - } - + // Filter and process contributions (code remains the same) let codeFrequencyStats: number[][] = []; try { codeFrequencyStats = await fetchCodeFrequencyStats( @@ -117,28 +126,36 @@ const getCourseData = async (octokit: Octokit, course: any) => { } const teamPRs: TeamPR[] = await Promise.all( - prs.data.map(async pr => { - const reviews = await octokit.rest.pulls.listReviews({ - owner: gitHubOrgName, - repo: repo.name, - pull_number: pr.number, - }); + prs.map(async pr => { + const reviews = await fetchAllPagesWithBackoff(page => + octokit.rest.pulls.listReviews({ + owner: gitHubOrgName, + repo: repo.name, + pull_number: pr.number, + per_page: 100, + page, + }) + ); const reviewDetails: Review[] = await Promise.all( - reviews.data.map(async review => ({ + reviews.map(async review => ({ id: review.id, user: review.user?.login, body: review.body, state: review.state, submittedAt: review.submitted_at, comments: ( - await octokit.rest.pulls.listReviewComments({ - owner: gitHubOrgName, - repo: repo.name, - pull_number: pr.number, - review_id: review.id, - }) - ).data.map(comment => ({ + await fetchAllPagesWithBackoff(page => + octokit.rest.pulls.listReviewComments({ + owner: gitHubOrgName, + repo: repo.name, + pull_number: pr.number, + review_id: review.id, + per_page: 100, + page, + }) + ) + ).map(comment => ({ id: comment.id, body: comment.body, user: comment.user.login, @@ -172,30 +189,31 @@ const getCourseData = async (octokit: Octokit, course: any) => { }) ); - for (const contributor of contributors.data) { + // Process team contributions (as in the original code) + for (const contributor of contributors) { if (!contributor.login) continue; - const contributorCommits = commits.data.filter( + const contributorCommits = commits.filter( commit => commit.author?.login === contributor.login ).length; - const createdIssues = issues.data.filter( + const createdIssues = issues.filter( issue => issue.user && issue.user.login === contributor.login ).length; - const openIssues = issues.data.filter( + const openIssues = issues.filter( issue => issue.user && issue.user.login === contributor.login && issue.state === 'open' ).length; - const closedIssues = issues.data.filter( + const closedIssues = issues.filter( issue => issue.user && issue.user.login === contributor.login && issue.state === 'closed' ).length; - const contributorPRs = prs.data.filter( + const contributorPRs = prs.filter( pr => pr.user && pr.user.login === contributor.login ).length; @@ -210,6 +228,7 @@ const getCourseData = async (octokit: Octokit, course: any) => { }; } + // Continue with team PR reviews and save data for (const teamPR of teamPRs) { for (const review of teamPR.reviews) { if (!review.user || !(review.user in teamContributions)) continue; @@ -218,24 +237,18 @@ const getCourseData = async (octokit: Octokit, course: any) => { } } - if (!process.env.NEW_FILTER) { - if ('github-classroom[bot]' in teamContributions) { - delete teamContributions['github-classroom[bot]']; - } - } - const teamData = { gitHubOrgName: gitHubOrgName.toLowerCase(), teamId: repo.id, repoName: repo.name, - commits: commits.data.length, + commits: commits.length, weeklyCommits: codeFrequencyStats, - issues: issues.data.length, - pullRequests: prs.data.length, - updatedIssues: issues.data.map(issue => issue.updated_at), + issues: issues.length, + pullRequests: prs.length, + updatedIssues: issues.map(issue => issue.updated_at), teamContributions, teamPRs, - milestones: milestones.data, + milestones: milestones, }; console.log('Saving team data:', teamData); @@ -575,6 +588,84 @@ const fetchCodeFrequencyStats = async ( throw new Error('Max attempts reached. Data may not be available yet.'); }; +const fetchWithExponentialBackoff = async ( + request: () => Promise, + retries = 5, + delay = 1000 +): Promise => { + for (let i = 0; i < retries; i++) { + try { + const response = await request(); + return response; + } catch (err: any) { + if ( + err.status === 403 && + err.response?.headers['x-ratelimit-remaining'] === '0' + ) { + // Rate limit hit, perform backoff + const resetTime = + parseInt(err.response.headers['x-ratelimit-reset']) * 1000; + const waitTime = resetTime - Date.now(); + console.warn(`Rate limit exceeded, waiting for ${waitTime} ms`); + await new Promise(resolve => setTimeout(resolve, waitTime)); + } else if ( + err.status === 403 && + err.message.includes('SecondaryRateLimit') + ) { + // Handle secondary rate limit with exponential backoff + const backoffTime = delay * Math.pow(2, i); // Exponential backoff + console.warn( + `SecondaryRateLimit detected, retrying in ${backoffTime} ms...` + ); + await new Promise(resolve => setTimeout(resolve, backoffTime)); + } else { + throw err; + } + } + } + throw new Error('Max retries reached. Unable to complete the request.'); +}; + +const fetchAllPagesWithBackoff = async ( + request: (page: number) => Promise, + retries = 5, + delay = 1000, + perPage = 100 +): Promise => { + let page = 1; + let allData: any[] = []; + let hasMore = true; + + while (hasMore) { + try { + const response = await fetchWithExponentialBackoff( + () => request(page), + retries, + delay + ); + allData = allData.concat(response.data); + + // Check if there are more pages + if (response.data.length < perPage) { + hasMore = false; // No more pages + } else { + page++; + } + } catch (error) { + console.error('Error fetching paginated data:', error); + throw error; + } + } + + return allData; +}; + +const getFourMonthsAgo = (): string => { + const date = new Date(); + date.setMonth(date.getMonth() - 4); + return date.toISOString(); // Format the date in ISO string (required by GitHub API) +}; + export const setupGitHubJob = () => { // Schedule the job to run every day at midnight cron.schedule('0 0 * * *', async () => { From 0b5825f8a4adb0b7b623a4c400f05a41b79b9f99 Mon Sep 17 00:00:00 2001 From: Dexter Date: Tue, 1 Oct 2024 00:03:43 +0800 Subject: [PATCH 10/18] Create boilerplate code for Trofos cron job --- backend/app.ts | 2 ++ backend/jobs/trofosJob.ts | 37 ++++++++++++++++++++++++++++++++++++ trofos-project.json | 40 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 79 insertions(+) create mode 100644 backend/jobs/trofosJob.ts create mode 100644 trofos-project.json diff --git a/backend/app.ts b/backend/app.ts index cb57c472..93085bc9 100644 --- a/backend/app.ts +++ b/backend/app.ts @@ -3,6 +3,7 @@ import { config } from 'dotenv'; import express, { Express } from 'express'; import setupGitHubJob from './jobs/githubJob'; import setupJiraJob from './jobs/jiraJob'; +import setupTrofosJob from './jobs/trofosJob'; import accountRoutes from './routes/accountRoutes'; import assessmentRoutes from './routes/assessmentRoutes'; import courseRoutes from './routes/courseRoutes'; @@ -21,6 +22,7 @@ const setupApp = async () => { await connectToDatabase(); setupGitHubJob(); setupJiraJob(); + setupTrofosJob(); }; setupApp(); diff --git a/backend/jobs/trofosJob.ts b/backend/jobs/trofosJob.ts new file mode 100644 index 00000000..5c2ecd9f --- /dev/null +++ b/backend/jobs/trofosJob.ts @@ -0,0 +1,37 @@ +import cron from 'node-cron'; +import { URLSearchParams } from 'url'; +import { Course } from '@models/Course'; +import CourseModel from '@models/Course'; + +const fetchAndSaveTrofosData = async () => { + const trofosCourseUri = 'https://trofos.comp.nus.edu.sg/api/external/v1/course'; + const trofosProjectUri = 'https://trofos.comp.nus.edu.sg/api/external/v1/project'; + + const courses: Course[] = await CourseModel.find(); + + for (const course of courses) { + const apiKey = course.trofos.apiKey; + + } +} + +const setupTrofosJob = () => { + // Schedule the job to run every day at 01:00 hours + cron.schedule('0 3 * * *', async () => { + console.log('Running fetchAndSaveTrofosData job:', new Date().toString()); + try { + await fetchAndSaveTrofosData(); + } catch (err) { + console.error('Error in cron job fetchAndSaveTrofosData:', err); + } + }); + + // To run the job immediately for testing + if (process.env.RUN_JOB_NOW === 'true') { + fetchAndSaveTrofosData().catch(err => { + console.error('Error running job manually:', err); + }); + } +}; + +export default setupTrofosJob; diff --git a/trofos-project.json b/trofos-project.json new file mode 100644 index 00000000..ababdca3 --- /dev/null +++ b/trofos-project.json @@ -0,0 +1,40 @@ +[ + { + "id": 10, + "pname": "crisp-project-1", + "pkey": "c1", + "description": null, + "course_id": 18, + "public": false, + "created_at": "2024-09-15T12:54:06.748Z", + "backlog_counter": 1, + "telegramChannelLink": null, + "is_archive": null, + "course": { + "id": 18, + "code": "crisp", + "startYear": 2024, + "startSem": 1, + "endYear": 2024, + "endSem": 1, + "cname": "crisp course", + "description": null, + "public": false, + "created_at": "2024-09-15T12:53:21.666Z", + "shadow_course": false, + "milestones": [] + }, + "users": [ + { + "user": { + "user_id": 21, + "user_email": "faculty@crisp.com", + "user_display_name": "crisp_faculty", + "courses": [ + { "id": 40, "user_id": 21, "role_id": 1, "course_id": 18 } + ] + } + } + ] + } +] From 6703a71d301ca5004d9134504a1524a6371d321c Mon Sep 17 00:00:00 2001 From: Dexter Date: Tue, 1 Oct 2024 01:59:43 +0800 Subject: [PATCH 11/18] Create fetch jobs for Trofos course and projects --- backend/jobs/trofosJob.ts | 58 +++++++++++++++++++++++++++++++++++---- 1 file changed, 53 insertions(+), 5 deletions(-) diff --git a/backend/jobs/trofosJob.ts b/backend/jobs/trofosJob.ts index 5c2ecd9f..6da132f0 100644 --- a/backend/jobs/trofosJob.ts +++ b/backend/jobs/trofosJob.ts @@ -1,19 +1,67 @@ import cron from 'node-cron'; -import { URLSearchParams } from 'url'; import { Course } from '@models/Course'; import CourseModel from '@models/Course'; const fetchAndSaveTrofosData = async () => { - const trofosCourseUri = 'https://trofos.comp.nus.edu.sg/api/external/v1/course'; - const trofosProjectUri = 'https://trofos.comp.nus.edu.sg/api/external/v1/project'; + const trofosCourseUri = + 'https://trofos.comp.nus.edu.sg/api/external/v1/course'; + const trofosProjectUri = + 'https://trofos.comp.nus.edu.sg/api/external/v1/project'; const courses: Course[] = await CourseModel.find(); for (const course of courses) { - const apiKey = course.trofos.apiKey; + const { + trofos: { apiKey }, + } = course; + if (!apiKey) { + continue; + } + + try { + const trofosCourseResponse = await fetch(trofosCourseUri, { + method: 'GET', + headers: { + 'x-api-key': apiKey, + Accept: 'application/json', + 'Content-Type': 'application/json', + }, + }); + + if (!trofosCourseResponse.ok) { + throw new Error('Network response was not ok'); + } + + const trofosCourseData = await trofosCourseResponse.json(); + console.log(trofosCourseData); + } catch (error) { + console.error('Error in fetching Trofos course:', error); + } + + try { + const trofosProjectResponse = await fetch(trofosProjectUri, { + method: 'GET', + headers: { + 'x-api-key': apiKey, + Accept: 'application/json', + 'Content-Type': 'application/json', + }, + }); + + if (!trofosProjectResponse.ok) { + throw new Error('Network response was not ok'); + } + + const trofosProjectData = await trofosProjectResponse.json(); + console.log(trofosProjectData); + } catch (error) { + console.error('Error in fetching Trofos project:', error); + } } -} + + console.log('fetchAndSaveTrofosData job done'); +}; const setupTrofosJob = () => { // Schedule the job to run every day at 01:00 hours From 53aadcc0c3e8683f8001b069e0e8a7ba801a03b8 Mon Sep 17 00:00:00 2001 From: Dexter Date: Tue, 1 Oct 2024 02:10:10 +0800 Subject: [PATCH 12/18] Add Trofos registration check in backend cron job --- backend/jobs/trofosJob.ts | 4 ++-- .../src/components/forms/ConnectTrofosForm.tsx | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/backend/jobs/trofosJob.ts b/backend/jobs/trofosJob.ts index 6da132f0..6e9aed93 100644 --- a/backend/jobs/trofosJob.ts +++ b/backend/jobs/trofosJob.ts @@ -12,10 +12,10 @@ const fetchAndSaveTrofosData = async () => { for (const course of courses) { const { - trofos: { apiKey }, + trofos: { isRegistered, apiKey }, } = course; - if (!apiKey) { + if (!isRegistered) { continue; } diff --git a/multi-git-dashboard/src/components/forms/ConnectTrofosForm.tsx b/multi-git-dashboard/src/components/forms/ConnectTrofosForm.tsx index 0ff2a021..9f456b96 100644 --- a/multi-git-dashboard/src/components/forms/ConnectTrofosForm.tsx +++ b/multi-git-dashboard/src/components/forms/ConnectTrofosForm.tsx @@ -22,6 +22,7 @@ const ConnectTrofosForm = ({ const form = useForm({ initialValues: { + isRegistered: true, apiKey: '', trofosCourseId: 0, }, From a303119b4800f9f6d51837eacdb81612da6307bc Mon Sep 17 00:00:00 2001 From: Dexter Date: Wed, 2 Oct 2024 01:48:48 +0800 Subject: [PATCH 13/18] Create a function to fetch a single Trofos project --- backend/jobs/trofosJob.ts | 42 +++++++++++++++++++++++++++++++------- backend/utils/endpoints.ts | 6 ++++++ trofos-project.json | 40 ------------------------------------ 3 files changed, 41 insertions(+), 47 deletions(-) delete mode 100644 trofos-project.json diff --git a/backend/jobs/trofosJob.ts b/backend/jobs/trofosJob.ts index 6e9aed93..6b116f41 100644 --- a/backend/jobs/trofosJob.ts +++ b/backend/jobs/trofosJob.ts @@ -1,13 +1,9 @@ import cron from 'node-cron'; import { Course } from '@models/Course'; import CourseModel from '@models/Course'; +import { TROFOS_COURSE_URI, TROFOS_PROJECT_URI } from '../utils/endpoints'; const fetchAndSaveTrofosData = async () => { - const trofosCourseUri = - 'https://trofos.comp.nus.edu.sg/api/external/v1/course'; - const trofosProjectUri = - 'https://trofos.comp.nus.edu.sg/api/external/v1/project'; - const courses: Course[] = await CourseModel.find(); for (const course of courses) { @@ -20,7 +16,7 @@ const fetchAndSaveTrofosData = async () => { } try { - const trofosCourseResponse = await fetch(trofosCourseUri, { + const trofosCourseResponse = await fetch(TROFOS_COURSE_URI, { method: 'GET', headers: { 'x-api-key': apiKey, @@ -40,7 +36,7 @@ const fetchAndSaveTrofosData = async () => { } try { - const trofosProjectResponse = await fetch(trofosProjectUri, { + const trofosProjectResponse = await fetch(TROFOS_PROJECT_URI, { method: 'GET', headers: { 'x-api-key': apiKey, @@ -55,6 +51,11 @@ const fetchAndSaveTrofosData = async () => { const trofosProjectData = await trofosProjectResponse.json(); console.log(trofosProjectData); + + for (const trofosProject of trofosProjectData) { + const trofosProjectId = trofosProject.id; + await fetchSingleTrofosProject(trofosProjectId, apiKey); + } } catch (error) { console.error('Error in fetching Trofos project:', error); } @@ -63,6 +64,33 @@ const fetchAndSaveTrofosData = async () => { console.log('fetchAndSaveTrofosData job done'); }; +const fetchSingleTrofosProject = async ( + trofosProjectId: number, + apiKey: string +) => { + const singleTrofosProjectUri = `${TROFOS_PROJECT_URI}/${trofosProjectId}`; + + try { + const singleTrofosProjectResponse = await fetch(singleTrofosProjectUri, { + method: 'GET', + headers: { + 'x-api-key': apiKey, + Accept: 'application/json', + 'Content-Type': 'application/json', + }, + }); + + if (!singleTrofosProjectResponse.ok) { + throw new Error('Network response was not ok'); + } + + const singleTrofosProjectData = await singleTrofosProjectResponse.json(); + console.log(singleTrofosProjectData); + } catch (error) { + console.error('Error in fetching single Trofos project:', error); + } +}; + const setupTrofosJob = () => { // Schedule the job to run every day at 01:00 hours cron.schedule('0 3 * * *', async () => { diff --git a/backend/utils/endpoints.ts b/backend/utils/endpoints.ts index 57ea0902..e87c5b7c 100644 --- a/backend/utils/endpoints.ts +++ b/backend/utils/endpoints.ts @@ -9,3 +9,9 @@ export const REDIRECT_URI_PATH = '/api/jira/callback'; export const JIRA_API_BASE_URL = 'https://api.atlassian.com/ex/jira'; export const BOARD_API_PATH = '/rest/agile/1.0/board'; export const ISSUE_API_PATH = '/rest/agile/1.0/issue'; + +/*---------------------------------------Trofos---------------------------------------*/ +export const TROFOS_COURSE_URI = + 'https://trofos.comp.nus.edu.sg/api/external/v1/course'; +export const TROFOS_PROJECT_URI = + 'https://trofos.comp.nus.edu.sg/api/external/v1/project'; diff --git a/trofos-project.json b/trofos-project.json deleted file mode 100644 index ababdca3..00000000 --- a/trofos-project.json +++ /dev/null @@ -1,40 +0,0 @@ -[ - { - "id": 10, - "pname": "crisp-project-1", - "pkey": "c1", - "description": null, - "course_id": 18, - "public": false, - "created_at": "2024-09-15T12:54:06.748Z", - "backlog_counter": 1, - "telegramChannelLink": null, - "is_archive": null, - "course": { - "id": 18, - "code": "crisp", - "startYear": 2024, - "startSem": 1, - "endYear": 2024, - "endSem": 1, - "cname": "crisp course", - "description": null, - "public": false, - "created_at": "2024-09-15T12:53:21.666Z", - "shadow_course": false, - "milestones": [] - }, - "users": [ - { - "user": { - "user_id": 21, - "user_email": "faculty@crisp.com", - "user_display_name": "crisp_faculty", - "courses": [ - { "id": 40, "user_id": 21, "role_id": 1, "course_id": 18 } - ] - } - } - ] - } -] From 91c7b914827f7319e62846e8cf14248046ba46dd Mon Sep 17 00:00:00 2001 From: Dexter Date: Sun, 6 Oct 2024 17:49:13 +0800 Subject: [PATCH 14/18] Update Trofos API endpoints to Trofos production server --- backend/jobs/trofosJob.ts | 37 +++++++++++++++++++++++++++++++++++-- backend/utils/endpoints.ts | 5 +++-- 2 files changed, 38 insertions(+), 4 deletions(-) diff --git a/backend/jobs/trofosJob.ts b/backend/jobs/trofosJob.ts index 6b116f41..7ae336d8 100644 --- a/backend/jobs/trofosJob.ts +++ b/backend/jobs/trofosJob.ts @@ -1,7 +1,11 @@ import cron from 'node-cron'; import { Course } from '@models/Course'; import CourseModel from '@models/Course'; -import { TROFOS_COURSE_URI, TROFOS_PROJECT_URI } from '../utils/endpoints'; +import { + TROFOS_COURSE_URI, + TROFOS_PROJECT_URI, + TROFOS_SPRINT_PATH, +} from '../utils/endpoints'; const fetchAndSaveTrofosData = async () => { const courses: Course[] = await CourseModel.find(); @@ -86,13 +90,42 @@ const fetchSingleTrofosProject = async ( const singleTrofosProjectData = await singleTrofosProjectResponse.json(); console.log(singleTrofosProjectData); + + await fetchSprintsFromSingleTrofosProject(trofosProjectId, apiKey); } catch (error) { console.error('Error in fetching single Trofos project:', error); } }; +const fetchSprintsFromSingleTrofosProject = async ( + trofosProjectId: number, + apiKey: string +) => { + const trofosSprintUri = `${TROFOS_PROJECT_URI}/${trofosProjectId}${TROFOS_SPRINT_PATH}`; + + try { + const trofosSprintResponse = await fetch(trofosSprintUri, { + method: 'GET', + headers: { + 'x-api-key': apiKey, + Accept: 'application/json', + 'Content-Type': 'application/json', + }, + }); + + if (!trofosSprintResponse.ok) { + throw new Error('Network response was not ok'); + } + + const trofosSprintData = await trofosSprintResponse.json(); + console.log(trofosSprintData); + } catch (error) { + console.error('Error in fetching sprints from a Trofos project:', error); + } +}; + const setupTrofosJob = () => { - // Schedule the job to run every day at 01:00 hours + // Schedule the job to run every day at 03:00 hours cron.schedule('0 3 * * *', async () => { console.log('Running fetchAndSaveTrofosData job:', new Date().toString()); try { diff --git a/backend/utils/endpoints.ts b/backend/utils/endpoints.ts index e87c5b7c..a1a3890e 100644 --- a/backend/utils/endpoints.ts +++ b/backend/utils/endpoints.ts @@ -12,6 +12,7 @@ export const ISSUE_API_PATH = '/rest/agile/1.0/issue'; /*---------------------------------------Trofos---------------------------------------*/ export const TROFOS_COURSE_URI = - 'https://trofos.comp.nus.edu.sg/api/external/v1/course'; + 'https://trofos-production.comp.nus.edu.sg/api/external/v1/course'; export const TROFOS_PROJECT_URI = - 'https://trofos.comp.nus.edu.sg/api/external/v1/project'; + 'https://trofos-production.comp.nus.edu.sg/api/external/v1/project'; +export const TROFOS_SPRINT_PATH = '/sprint'; From 68cec5280e5008ee3268c15896c845725d6bce54 Mon Sep 17 00:00:00 2001 From: Dexter Date: Mon, 7 Oct 2024 01:32:25 +0800 Subject: [PATCH 15/18] Add fetch Trofos sprint function --- backend/jobs/trofosJob.ts | 89 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 86 insertions(+), 3 deletions(-) diff --git a/backend/jobs/trofosJob.ts b/backend/jobs/trofosJob.ts index 7ae336d8..d5bd28da 100644 --- a/backend/jobs/trofosJob.ts +++ b/backend/jobs/trofosJob.ts @@ -1,6 +1,12 @@ import cron from 'node-cron'; import { Course } from '@models/Course'; import CourseModel from '@models/Course'; +import { + JiraBoardModel, + JiraIssueModel, + JiraSprintModel, +} from '@models/JiraData'; +import { JiraBoard, JiraIssue, JiraSprint } from '@shared/types/JiraData'; import { TROFOS_COURSE_URI, TROFOS_PROJECT_URI, @@ -58,7 +64,7 @@ const fetchAndSaveTrofosData = async () => { for (const trofosProject of trofosProjectData) { const trofosProjectId = trofosProject.id; - await fetchSingleTrofosProject(trofosProjectId, apiKey); + await fetchSingleTrofosProject(course, trofosProjectId, apiKey); } } catch (error) { console.error('Error in fetching Trofos project:', error); @@ -69,6 +75,7 @@ const fetchAndSaveTrofosData = async () => { }; const fetchSingleTrofosProject = async ( + course: any, trofosProjectId: number, apiKey: string ) => { @@ -89,7 +96,38 @@ const fetchSingleTrofosProject = async ( } const singleTrofosProjectData = await singleTrofosProjectResponse.json(); - console.log(singleTrofosProjectData); + + // Transform the trofos project data to fit the JiraBoard interface + const transformedJiraBoard: Omit = { + id: singleTrofosProjectData.id, + self: singleTrofosProjectUri, + name: singleTrofosProjectData.pname, + type: 'Trofos', + jiraLocation: { + projectId: singleTrofosProjectData.id, + displayName: singleTrofosProjectData.pname, + projectName: singleTrofosProjectData.pname, + projectKey: singleTrofosProjectData.pkey, + projectTypeKey: undefined, // Optional field, set according to your logic + avatarURI: undefined, // Optional field, set if available + name: singleTrofosProjectData.pname, + }, + columns: singleTrofosProjectData.backlogStatuses.map((status: { name: string; }) => ({ + name: status.name, + })), + jiraSprints: [], + jiraIssues: [], + course: course._id + }; + + await JiraBoardModel.findOneAndUpdate( + { self: singleTrofosProjectUri }, + transformedJiraBoard, + { + upsert: true, + new: true, + } + ); await fetchSprintsFromSingleTrofosProject(trofosProjectId, apiKey); } catch (error) { @@ -118,12 +156,57 @@ const fetchSprintsFromSingleTrofosProject = async ( } const trofosSprintData = await trofosSprintResponse.json(); - console.log(trofosSprintData); + + const transformedSprints = trofosSprintData.sprints.map((sprint: any) => ({ + id: sprint.id, + self: `${trofosSprintUri}/${sprint.id}`, // Assuming `trofosSprintUri` is defined elsewhere + state: sprint.status === 'current' ? 'active' : 'future', // Map status + name: sprint.name, + startDate: new Date(sprint.start_date), + endDate: new Date(sprint.end_date), + createdDate: new Date(sprint.start_date), + originBoardId: sprint.project_id, // Relating it to the board ID + goal: sprint.goals || '', // Default to empty string if no goals + jiraIssues: [] // You can populate this later or leave it empty for now + })); + + // Iterate over each transformed sprint and save to the database + for (const sprintData of transformedSprints) { + try { + const sprint = await JiraSprintModel.findOneAndUpdate( + { self: sprintData.self }, + sprintData, + { + upsert: true, + new: true, + } + ); + + const boardSelfUri = `${TROFOS_PROJECT_URI}/${trofosProjectId}`; + await JiraBoardModel.findOneAndUpdate( + { self: boardSelfUri }, + { $push: { jiraSprints: sprint._id } }, + {} + ); + + console.log(`Saved sprint with ID: ${sprintData.id}`); + } catch (error) { + console.error(`Error saving sprint with ID: ${sprintData.id}`, error); + } + } + + await saveBacklogToDatabase(trofosSprintData); + } catch (error) { console.error('Error in fetching sprints from a Trofos project:', error); } }; +const saveBacklogToDatabase = async (trofosSprintData: any[]) => { + +} + + const setupTrofosJob = () => { // Schedule the job to run every day at 03:00 hours cron.schedule('0 3 * * *', async () => { From 9fe561ecbddfc401243a216835dd74b61ab4d5ef Mon Sep 17 00:00:00 2001 From: Dexter Date: Mon, 7 Oct 2024 02:31:32 +0800 Subject: [PATCH 16/18] Add fetch Trofos backlog function --- backend/jobs/trofosJob.ts | 92 ++++++++++++++++++++++++++++++++++----- 1 file changed, 80 insertions(+), 12 deletions(-) diff --git a/backend/jobs/trofosJob.ts b/backend/jobs/trofosJob.ts index d5bd28da..72faa96f 100644 --- a/backend/jobs/trofosJob.ts +++ b/backend/jobs/trofosJob.ts @@ -112,12 +112,14 @@ const fetchSingleTrofosProject = async ( avatarURI: undefined, // Optional field, set if available name: singleTrofosProjectData.pname, }, - columns: singleTrofosProjectData.backlogStatuses.map((status: { name: string; }) => ({ - name: status.name, - })), + columns: singleTrofosProjectData.backlogStatuses.map( + (status: { name: string }) => ({ + name: status.name, + }) + ), jiraSprints: [], jiraIssues: [], - course: course._id + course: course._id, }; await JiraBoardModel.findOneAndUpdate( @@ -129,6 +131,8 @@ const fetchSingleTrofosProject = async ( } ); + console.log(`Saved Trofos project with ID: ${transformedJiraBoard.id}`); + await fetchSprintsFromSingleTrofosProject(trofosProjectId, apiKey); } catch (error) { console.error('Error in fetching single Trofos project:', error); @@ -157,17 +161,22 @@ const fetchSprintsFromSingleTrofosProject = async ( const trofosSprintData = await trofosSprintResponse.json(); - const transformedSprints = trofosSprintData.sprints.map((sprint: any) => ({ + const transformedSprints: Omit[] = trofosSprintData.sprints.map((sprint: any) => ({ id: sprint.id, self: `${trofosSprintUri}/${sprint.id}`, // Assuming `trofosSprintUri` is defined elsewhere - state: sprint.status === 'current' ? 'active' : 'future', // Map status + state: + sprint.status === 'current' + ? 'active' + : sprint.status === 'upcoming' + ? 'future' + : 'closed', name: sprint.name, startDate: new Date(sprint.start_date), endDate: new Date(sprint.end_date), createdDate: new Date(sprint.start_date), originBoardId: sprint.project_id, // Relating it to the board ID goal: sprint.goals || '', // Default to empty string if no goals - jiraIssues: [] // You can populate this later or leave it empty for now + jiraIssues: [], })); // Iterate over each transformed sprint and save to the database @@ -189,23 +198,82 @@ const fetchSprintsFromSingleTrofosProject = async ( {} ); - console.log(`Saved sprint with ID: ${sprintData.id}`); + console.log(`Saved Trofos sprint with ID: ${sprintData.id}`); } catch (error) { - console.error(`Error saving sprint with ID: ${sprintData.id}`, error); + console.error( + `Error saving Trofos sprint with ID: ${sprintData.id}`, + error + ); } } await saveBacklogToDatabase(trofosSprintData); - } catch (error) { console.error('Error in fetching sprints from a Trofos project:', error); } }; -const saveBacklogToDatabase = async (trofosSprintData: any[]) => { +const saveBacklogToDatabase = async (trofosSprintData: any) => { + for (const sprint of trofosSprintData.sprints) { + const backlogItems = sprint.backlogs; + + // Iterate through each backlog item in the sprint + for (const backlog of backlogItems) { + const trofosSprintUri = `${TROFOS_PROJECT_URI}/${sprint.project_id}${TROFOS_SPRINT_PATH}/${sprint.id}`; + + const transformedBacklog: Omit = { + id: backlog.backlog_id, // Assuming 'backlog_id' is the equivalent of 'id' + self: `${trofosSprintUri}/${backlog.backlog_id}`, + key: `${trofosSprintUri}/${backlog.backlog_id}`, + storyPoints: backlog.points || 0, // Default to 0 if no points are provided + fields: { + summary: backlog.summary, + issuetype: { + name: backlog.type, // Type of issue, e.g., "story", "bug" + subtask: false, // Assuming no subtasks, adjust if needed + }, + status: { + name: backlog.status, // Status of the backlog item + }, + assignee: backlog.assignee + ? { displayName: backlog.assignee.user.user_display_name } + : undefined, // If there's an assignee, map it + }, + }; -} + try { + const issue = await JiraIssueModel.findOneAndUpdate( + { self: transformedBacklog.self }, + transformedBacklog, + { + upsert: true, + new: true, + } + ); + const boardSelfUri = `${TROFOS_PROJECT_URI}/${backlog.project_id}`; + await JiraBoardModel.findOneAndUpdate( + { self: boardSelfUri }, + { $push: { jiraIssues: issue._id } }, + {} + ); + + await JiraSprintModel.findOneAndUpdate( + { self: trofosSprintUri }, + { $push: { jiraIssues: issue._id } }, + {} + ); + + console.log(`Saved Trofos backlog item with ID: ${backlog.backlog_id}`); + } catch (error) { + console.error( + `Error saving Trofos backlog item with ID: ${backlog.backlog_id}`, + error + ); + } + } + } +}; const setupTrofosJob = () => { // Schedule the job to run every day at 03:00 hours From 3e77cf41cb186f0f470389b3956f89bac1aa2a57 Mon Sep 17 00:00:00 2001 From: Dexter Date: Mon, 7 Oct 2024 02:45:25 +0800 Subject: [PATCH 17/18] Fix formatting --- backend/jobs/trofosJob.ts | 35 ++++++++++++++++++----------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/backend/jobs/trofosJob.ts b/backend/jobs/trofosJob.ts index 72faa96f..4f0415a6 100644 --- a/backend/jobs/trofosJob.ts +++ b/backend/jobs/trofosJob.ts @@ -161,23 +161,24 @@ const fetchSprintsFromSingleTrofosProject = async ( const trofosSprintData = await trofosSprintResponse.json(); - const transformedSprints: Omit[] = trofosSprintData.sprints.map((sprint: any) => ({ - id: sprint.id, - self: `${trofosSprintUri}/${sprint.id}`, // Assuming `trofosSprintUri` is defined elsewhere - state: - sprint.status === 'current' - ? 'active' - : sprint.status === 'upcoming' - ? 'future' - : 'closed', - name: sprint.name, - startDate: new Date(sprint.start_date), - endDate: new Date(sprint.end_date), - createdDate: new Date(sprint.start_date), - originBoardId: sprint.project_id, // Relating it to the board ID - goal: sprint.goals || '', // Default to empty string if no goals - jiraIssues: [], - })); + const transformedSprints: Omit[] = + trofosSprintData.sprints.map((sprint: any) => ({ + id: sprint.id, + self: `${trofosSprintUri}/${sprint.id}`, // Assuming `trofosSprintUri` is defined elsewhere + state: + sprint.status === 'current' + ? 'active' + : sprint.status === 'upcoming' + ? 'future' + : 'closed', + name: sprint.name, + startDate: new Date(sprint.start_date), + endDate: new Date(sprint.end_date), + createdDate: new Date(sprint.start_date), + originBoardId: sprint.project_id, // Relating it to the board ID + goal: sprint.goals || '', // Default to empty string if no goals + jiraIssues: [], + })); // Iterate over each transformed sprint and save to the database for (const sprintData of transformedSprints) { From bcbdb7b066f58a304da5f78eb4cf91fac674fb86 Mon Sep 17 00:00:00 2001 From: Dexter Date: Thu, 10 Oct 2024 00:39:55 +0800 Subject: [PATCH 18/18] Update project management visualizations --- .../cards/ProjectManagementJiraCard.tsx | 90 ++++++++++++------- 1 file changed, 59 insertions(+), 31 deletions(-) diff --git a/multi-git-dashboard/src/components/cards/ProjectManagementJiraCard.tsx b/multi-git-dashboard/src/components/cards/ProjectManagementJiraCard.tsx index bf247393..cfb2a0fe 100644 --- a/multi-git-dashboard/src/components/cards/ProjectManagementJiraCard.tsx +++ b/multi-git-dashboard/src/components/cards/ProjectManagementJiraCard.tsx @@ -3,6 +3,7 @@ import { BarChart } from '@mantine/charts'; import { Card, Group, + MultiSelect, Select, SimpleGrid, Stack, @@ -46,7 +47,9 @@ const ProjectManagementJiraCard: React.FC = ({ const [embla, setEmbla] = useState(null); const [selectedVelocityChart, setSelectedVelocityChart] = - useState(VelocityChartType.Issues); // Default to 'issues' + useState(VelocityChartType.StoryPoints); // Default to 'story points' + + const [selectedAssignees, setSelectedAssignees] = useState([]); // const [storyPointsEstimate, setStoryPointsEstimate] = useState(4); @@ -66,18 +69,16 @@ const ProjectManagementJiraCard: React.FC = ({ return ( jiraSprint && columns && ( - - - {columns.map((column, index) => ( - - - {column.name} - - {getJiraBoardColumn(jiraSprint, column.name)} - - ))} - - + + {columns.map((column, index) => ( + + + {column.name} + + {getJiraBoardColumn(jiraSprint, column.name)} + + ))} + ) ); }; @@ -85,32 +86,39 @@ const ProjectManagementJiraCard: React.FC = ({ const getJiraBoardColumn = (jiraSprint: JiraSprint, status: string) => { return ( jiraSprint.jiraIssues && - jiraSprint.jiraIssues.map( - issue => - issue.fields.status?.name.toLowerCase() === status.toLowerCase() && - getJiraBoardCard(issue) - ) + jiraSprint.jiraIssues + .filter(issue => { + // Filter by status and selected assignees + const issueAssignee = + issue.fields.assignee?.displayName ?? 'Unassigned'; + return ( + issue.fields.status?.name.toLowerCase() === status.toLowerCase() && + (selectedAssignees.length === 0 || + (issueAssignee && selectedAssignees.includes(issueAssignee))) + ); + }) + .map(issue => getJiraBoardCard(issue)) ); }; const getJiraBoardCard = (issue: JiraIssue) => ( - + - + {issue.fields.summary || '-'} - Issue Type: - {issue.fields.issuetype?.name || '-'} + Issue Type: + {issue.fields.issuetype?.name || '-'} - Story Points: - {issue.storyPoints || '-'} + Story Points: + {issue.storyPoints || '-'} - Assignee: - {issue.fields.assignee?.displayName || '-'} + Assignee: + {issue.fields.assignee?.displayName || '-'} ); @@ -455,11 +463,31 @@ const ProjectManagementJiraCard: React.FC = ({ End Date: {getActiveSprintEndDate(jiraBoard)} - {jiraBoard.jiraSprints && - getActiveSprintBoard( - jiraBoard.jiraSprints.find(sprint => sprint.state === 'active'), - jiraBoard.columns - )} + {jiraBoard.jiraSprints && ( + + sprint.state === 'active') + ?.jiraIssues.map( + issue => + issue.fields.assignee?.displayName || 'Unassigned' + ) + ), + ]} + value={selectedAssignees} + onChange={setSelectedAssignees} + placeholder="Select Assignees" + label="Filter by Assignee" + /> + {getActiveSprintBoard( + jiraBoard.jiraSprints.find(sprint => sprint.state === 'active'), + jiraBoard.columns + )} + + )} + {jiraBoard.jiraSprints && getStatsTable(jiraBoard.jiraSprints)} {jiraBoard.jiraSprints && getVelocityChart(jiraBoard.jiraSprints)}