Skip to content

Commit

Permalink
ui - show slow responses on admin UI
Browse files Browse the repository at this point in the history
  • Loading branch information
@ishaan-jaff
ishaan-jaff committed May 2, 2024
1 parent 26eda88 commit 5041b92
Show file tree
Hide file tree
Showing 2 changed files with 93 additions and 43 deletions.
96 changes: 53 additions & 43 deletions ui/litellm-dashboard/src/components/model_dashboard.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ import {
} from "@tremor/react";
import { TabPanel, TabPanels, TabGroup, TabList, Tab, TextInput, Icon } from "@tremor/react";
import { Select, SelectItem, MultiSelect, MultiSelectItem } from "@tremor/react";
import { modelInfoCall, userGetRequesedtModelsCall, modelCreateCall, Model, modelCostMap, modelDeleteCall, healthCheckCall, modelUpdateCall, modelMetricsCall, modelExceptionsCall } from "./networking";
import { modelInfoCall, userGetRequesedtModelsCall, modelCreateCall, Model, modelCostMap, modelDeleteCall, healthCheckCall, modelUpdateCall, modelMetricsCall, modelExceptionsCall, modelMetricsSlowResponsesCall } from "./networking";
import { BarChart, AreaChart } from "@tremor/react";
import {
Button as Button2,
Expand Down Expand Up @@ -205,6 +205,7 @@ const ModelDashboard: React.FC<ModelDashboardProps> = ({
const [modelExceptions, setModelExceptions] = useState<any[]>([]);
const [allExceptions, setAllExceptions] = useState<any[]>([]);
const [failureTableData, setFailureTableData] = useState<any[]>([]);
const [slowResponsesData, setSlowResponsesData] = useState<any[]>([]);

const EditModelModal: React.FC<EditModelModalProps> = ({ visible, onCancel, model, onSubmit }) => {
const [form] = Form.useForm();
Expand Down Expand Up @@ -479,39 +480,51 @@ const handleEditSubmit = async (formValues: Record<string, any>) => {
setAllExceptions(modelExceptionsResponse.exception_types);


let modelMetricsData = modelMetricsResponse.data;
let successdeploymentToSuccess: Record<string, number> = {};
for (let i = 0; i < modelMetricsData.length; i++) {
let element = modelMetricsData[i];
let _model_name = element.model;
let _num_requests = element.num_requests;
successdeploymentToSuccess[_model_name] = _num_requests
}
console.log("successdeploymentToSuccess:", successdeploymentToSuccess)
const slowResponses = await modelMetricsSlowResponsesCall(
accessToken,
userID,
userRole,
null
)

console.log("slowResponses:", slowResponses)

setSlowResponsesData(slowResponses);


// let modelMetricsData = modelMetricsResponse.data;
// let successdeploymentToSuccess: Record<string, number> = {};
// for (let i = 0; i < modelMetricsData.length; i++) {
// let element = modelMetricsData[i];
// let _model_name = element.model;
// let _num_requests = element.num_requests;
// successdeploymentToSuccess[_model_name] = _num_requests
// }
// console.log("successdeploymentToSuccess:", successdeploymentToSuccess)

let failureTableData = [];
let _failureData = modelExceptionsResponse.data;
for (let i = 0; i < _failureData.length; i++) {
const model = _failureData[i];
let _model_name = model.model;
let total_exceptions = model.total_exceptions;
let total_Requests = successdeploymentToSuccess[_model_name];
if (total_Requests == null) {
total_Requests = 0
}
let _data = {
model: _model_name,
total_exceptions: total_exceptions,
total_Requests: total_Requests,
failure_rate: total_Requests / total_exceptions
}
failureTableData.push(_data);
// sort failureTableData by failure_rate
failureTableData.sort((a, b) => b.failure_rate - a.failure_rate);
// let failureTableData = [];
// let _failureData = modelExceptionsResponse.data;
// for (let i = 0; i < _failureData.length; i++) {
// const model = _failureData[i];
// let _model_name = model.model;
// let total_exceptions = model.total_exceptions;
// let total_Requests = successdeploymentToSuccess[_model_name];
// if (total_Requests == null) {
// total_Requests = 0
// }
// let _data = {
// model: _model_name,
// total_exceptions: total_exceptions,
// total_Requests: total_Requests,
// failure_rate: total_Requests / total_exceptions
// }
// failureTableData.push(_data);
// // sort failureTableData by failure_rate
// failureTableData.sort((a, b) => b.failure_rate - a.failure_rate);

setFailureTableData(failureTableData);
console.log("failureTableData:", failureTableData);
}
// setFailureTableData(failureTableData);
// console.log("failureTableData:", failureTableData);
// }

} catch (error) {
console.error("There was an error fetching the model data", error);
Expand Down Expand Up @@ -1126,29 +1139,26 @@ const handleEditSubmit = async (formValues: Record<string, any>) => {
/>
)}



</Card>
</Col>
<Col>
<Card className="ml-2">
<Table>
<TableHead>
<TableRow>
<TableHeaderCell>Model</TableHeaderCell>
<TableHeaderCell>Success Requests</TableHeaderCell>
<TableHeaderCell>Error Requests</TableHeaderCell>
<TableHeaderCell>Failure %</TableHeaderCell>
<TableHeaderCell>Deployment</TableHeaderCell>
<TableHeaderCell>Success Responses</TableHeaderCell>
<TableHeaderCell>Slow Responses</TableHeaderCell>

</TableRow>
</TableHead>
<TableBody>
{failureTableData.map((metric, idx) => (
{slowResponsesData.map((metric, idx) => (
<TableRow key={idx}>
<TableCell>{metric.model}</TableCell>
<TableCell>{metric.total_Requests}</TableCell>
<TableCell>{metric.total_exceptions}</TableCell>
<TableCell>{metric.failure_rate}%</TableCell>
<TableCell>{metric.api_base}</TableCell>
<TableCell>{metric.total_count}</TableCell>
<TableCell>{metric.slow_count}</TableCell>

</TableRow>

))}
Expand Down
40 changes: 40 additions & 0 deletions ui/litellm-dashboard/src/components/networking.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -475,6 +475,46 @@ export const modelMetricsCall = async (
};



export const modelMetricsSlowResponsesCall = async (
accessToken: String,
userID: String,
userRole: String,
modelGroup: String | null,
) => {
/**
* Get all models on proxy
*/
try {
let url = proxyBaseUrl ? `${proxyBaseUrl}/model/metrics/slow_responses` : `/model/metrics/slow_responses`;
if (modelGroup) {
url = `${url}?_selected_model_group=${modelGroup}`
}
// message.info("Requesting model data");
const response = await fetch(url, {
method: "GET",
headers: {
Authorization: `Bearer ${accessToken}`,
"Content-Type": "application/json",
},
});

if (!response.ok) {
const errorData = await response.text();
message.error(errorData, 20);
throw new Error("Network response was not ok");
}
const data = await response.json();
// message.info("Received model data");
return data;
// Handle success - you might want to update some state or UI based on the created key
} catch (error) {
console.error("Failed to create key:", error);
throw error;
}
};


export const modelExceptionsCall = async (
accessToken: String,
userID: String,
Expand Down

0 comments on commit 5041b92

Please sign in to comment.