Skip to content

Commit

Permalink
Show the curve for other keys in metric dict in the "default metric g…
Browse files Browse the repository at this point in the history 
…raph", currently only support the default key (microsoft#4973)
  • Loading branch information
@Lijiaoa
Lijiaoa authored Jul 20, 2022
1 parent 6bf06a2 commit 2f69f28
Show file tree
Hide file tree
Showing 4 changed files with 102 additions and 23 deletions.
5 changes: 2 additions & 3 deletions ts/webui/src/components/experiment/trialdetail/TrialsDetail.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,8 +48,7 @@ class TrialsDetail extends React.Component<{}, TrialDetailState> {
render(): React.ReactNode {
const { whichChart } = this.state;
const source = TRIALS.toArray();
const trialIds = TRIALS.toArray().map(trial => trial.id);

const succeededTrialIds = TRIALS.succeededTrials().map(trial => trial.id);
return (
<AppContext.Consumer>
{(_value): React.ReactNode => (
Expand All @@ -65,7 +64,7 @@ class TrialsDetail extends React.Component<{}, TrialDetailState> {
<PivotItem headerText='Default metric' itemIcon='HomeGroup' key='Default metric'>
<Stack className='graph'>
<DefaultPoint
trialIds={trialIds}
trialIds={succeededTrialIds}
hasBestCurve={true}
chartHeight={402}
changeExpandRowIDs={_value.changeExpandRowIDs}
Expand Down
96 changes: 83 additions & 13 deletions ts/webui/src/components/experiment/trialdetail/chart/DefaultMetricPoint.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,8 @@ import 'echarts/lib/chart/scatter';
import 'echarts/lib/component/tooltip';
import 'echarts/lib/component/title';

// this file is for overview page and detail page's Default metric graph

const EmptyGraph = {
grid: {
left: '8%'
Expand All @@ -37,6 +39,7 @@ interface DefaultPointState {
startY: number; // dataZoomY
endY: number;
userSelectOptimizeMode: string;
userSelectAccuracyNumberKey: string;
}

class DefaultPoint extends React.Component<DefaultPointProps, DefaultPointState> {
Expand All @@ -46,7 +49,8 @@ class DefaultPoint extends React.Component<DefaultPointProps, DefaultPointState>
bestCurveEnabled: false,
startY: 0, // dataZoomY
endY: 100,
userSelectOptimizeMode: optimizeModeValue(EXPERIMENT.optimizeMode)
userSelectOptimizeMode: optimizeModeValue(EXPERIMENT.optimizeMode),
userSelectAccuracyNumberKey: 'default'
};
}

Expand All @@ -72,7 +76,7 @@ class DefaultPoint extends React.Component<DefaultPointProps, DefaultPointState>
};

generateGraphConfig(_maxSequenceId: number): any {
const { startY, endY } = this.state;
const { startY, endY, userSelectAccuracyNumberKey } = this.state;
const { hasBestCurve } = this.props;
return {
grid: {
Expand All @@ -86,7 +90,7 @@ class DefaultPoint extends React.Component<DefaultPointProps, DefaultPointState>
<div class="tooldetailAccuracy">
<div>Trial No.: ${data.data[0]}</div>
<div>Trial ID: ${data.data[2]}</div>
<div>Default metric: ${data.data[1]}</div>
<div>${userSelectAccuracyNumberKey}: ${data.data[1]}</div>
<div>Parameters: <pre>${JSON.stringify(
reformatRetiariiParameter(data.data[3]),
null,
Expand Down Expand Up @@ -118,8 +122,32 @@ class DefaultPoint extends React.Component<DefaultPointProps, DefaultPointState>
};
}

private formatAccuracy(accuracy: number | undefined): number {
if (accuracy === undefined || isNaN(accuracy) || !isFinite(accuracy)) {
return 0;
}
return accuracy;
}

generateScatterSeries(trials: Trial[]): any {
const data = trials.map(trial => [trial.sequenceId, trial.accuracy, trial.id, trial.description.parameters]);
const { userSelectAccuracyNumberKey } = this.state;
let data;
if (trials[0].accuracyNumberTypeDictKeys.length > 1) {
// dict type final results
data = trials.map(trial => [
trial.sequenceId,
trial.acc === undefined ? 0 : this.formatAccuracy(trial.acc[userSelectAccuracyNumberKey]),
trial.id,
trial.description.parameters
]);
} else {
data = trials.map(trial => [
trial.sequenceId,
this.formatAccuracy(trial.accuracy),
trial.id,
trial.description.parameters
]);
}
return {
symbolSize: 6,
type: 'scatter',
Expand All @@ -128,20 +156,36 @@ class DefaultPoint extends React.Component<DefaultPointProps, DefaultPointState>
}

generateBestCurveSeries(trials: Trial[]): any {
const { userSelectOptimizeMode } = this.state;
const { userSelectOptimizeMode, userSelectAccuracyNumberKey } = this.state;
let best = trials[0];
const data = [[best.sequenceId, best.accuracy, best.id, best.description.parameters]];

const data = [
[
best.sequenceId,
best.acc === undefined ? 0 : this.formatAccuracy(best.acc[userSelectAccuracyNumberKey]),
best.id,
best.description.parameters
]
];
for (let i = 1; i < trials.length; i++) {
const trial = trials[i];
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
const delta = trial.accuracy! - best.accuracy!;
const delta = trial.acc![userSelectAccuracyNumberKey] - best.acc![userSelectAccuracyNumberKey];
const better = userSelectOptimizeMode === 'minimize' ? delta < 0 : delta > 0;
if (better) {
data.push([trial.sequenceId, trial.accuracy, best.id, trial.description.parameters]);
data.push([
trial.sequenceId,
trial.acc === undefined ? 0 : this.formatAccuracy(trial.acc[userSelectAccuracyNumberKey]),
best.id,
trial.description.parameters
]);
best = trial;
} else {
data.push([trial.sequenceId, best.accuracy, best.id, trial.description.parameters]);
data.push([
trial.sequenceId,
best.acc === undefined ? 0 : this.formatAccuracy(best.acc[userSelectAccuracyNumberKey]),
best.id,
trial.description.parameters
]);
}
}

Expand All @@ -152,20 +196,37 @@ class DefaultPoint extends React.Component<DefaultPointProps, DefaultPointState>
};
}

// get user mode number 'max' or 'min'
updateUserOptimizeMode = (event: React.FormEvent<HTMLDivElement>, item?: IDropdownOption): void => {
if (item !== undefined) {
this.setState({ userSelectOptimizeMode: item.key.toString() });
}
};

// final result keys dropdown click event
updateTrialfinalResultKeys = (event: React.FormEvent<HTMLDivElement>, item?: IDropdownOption): void => {
if (item !== undefined) {
this.setState(
{
userSelectAccuracyNumberKey: item.key.toString()
},
() => {
this.generateGraph();
}
);
}
};

render(): React.ReactNode {
const { hasBestCurve, chartHeight } = this.props;
const { userSelectOptimizeMode } = this.state;
const { userSelectOptimizeMode, userSelectAccuracyNumberKey } = this.state;
const trials = TRIALS.getTrials(this.props.trialIds).filter(trial => trial.sortable);
const graph = this.generateGraph();
const accNodata = graph === EmptyGraph ? 'No data' : '';
const onEvents = { dataZoom: this.metricDataZoom, click: this.pointClick };

let dictDropdown: string[] = [];
if (trials.length > 0) {
dictDropdown = trials[0].accuracyNumberTypeDictKeys;
}
return (
<div>
{hasBestCurve && (
Expand All @@ -181,6 +242,15 @@ class DefaultPoint extends React.Component<DefaultPointProps, DefaultPointState>
styles={{ dropdown: { width: 100 } }}
className='para-filter-percent'
/>
{dictDropdown.length > 1 && (
<Dropdown
selectedKey={userSelectAccuracyNumberKey}
onChange={this.updateTrialfinalResultKeys}
options={dictDropdown.map(item => ({ key: item, text: item }))}
styles={{ dropdown: { width: 100 } }}
className='para-filter-percent'
/>
)}
</Stack>
)}
<div className='default-metric-graph graph'>
Expand Down
1 change: 0 additions & 1 deletion ts/webui/src/static/interface.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,6 @@ interface TableRecord {
accuracy?: number | any;
latestAccuracy: number | undefined;
formattedLatestAccuracy: string; // format (LATEST/FINAL),
accDictionary: FinalType | undefined;
}

interface SearchSpace {
Expand Down
23 changes: 17 additions & 6 deletions ts/webui/src/static/model/trial.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -159,8 +159,7 @@ class Trial implements TableObj {
intermediateCount: this.intermediates.length,
accuracy: accuracy,
latestAccuracy: this.latestAccuracy,
formattedLatestAccuracy: this.formatLatestAccuracy(),
accDictionary: this.acc
formattedLatestAccuracy: this.formatLatestAccuracy()
};
}

Expand Down Expand Up @@ -193,6 +192,22 @@ class Trial implements TableObj {
return getFinal(this.info.finalMetricData);
}

get accuracyNumberTypeDictKeys(): string[] {
let accuracyTypeList: string[] = [];

if (this.acc !== undefined) {
for (const [item, value] of Object.entries(this.acc)) {
if (typeof value === 'number') {
accuracyTypeList.push(item);
}
}
} else {
accuracyTypeList = ['default'];
}

return accuracyTypeList;
}

get description(): Parameters {
const ret: Parameters = {
parameters: {},
Expand Down Expand Up @@ -272,10 +287,6 @@ class Trial implements TableObj {
return ret;
}

get color(): string | undefined {
return undefined;
}

public finalKeys(): string[] {
if (this.acc !== undefined) {
return Object.keys(this.acc);
Expand Down

0 comments on commit 2f69f28

Please sign in to comment.