Skip to content

Commit

Permalink
raftstore: flush waterfall metrics when using synchronous write (tikv…
Browse files Browse the repository at this point in the history 
…#11451)

Signed-off-by: gengliqi <[email protected]>

Co-authored-by: Ti Chi Robot <[email protected]>
  • Loading branch information
@gengliqi @ti-chi-bot
gengliqi and ti-chi-bot authored Nov 24, 2021
1 parent 4f337d0 commit 6a0792a
Show file tree
Hide file tree
Showing 2 changed files with 39 additions and 139 deletions.
14 changes: 7 additions & 7 deletions components/raftstore/src/store/async_io/write.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -423,14 +423,7 @@ where

self.write_to_db(true);

self.metrics.flush();

self.clear_latency_inspect();
// update config
if let Some(incoming) = self.cfg_tracker.any_new() {
self.raft_write_size_limit = incoming.raft_write_size_limit.0 as usize;
self.metrics.waterfall_metrics = incoming.waterfall_metrics;
}

STORE_WRITE_LOOP_DURATION_HISTOGRAM
.observe(duration_to_sec(handle_begin.saturating_elapsed()));
Expand Down Expand Up @@ -618,6 +611,13 @@ where
);

self.batch.clear();
self.metrics.flush();

// update config
if let Some(incoming) = self.cfg_tracker.any_new() {
self.raft_write_size_limit = incoming.raft_write_size_limit.0 as usize;
self.metrics.waterfall_metrics = incoming.waterfall_metrics;
}
}

pub fn is_empty(&self) -> bool {
Expand Down
164 changes: 32 additions & 132 deletions metrics/grafana/tikv_details.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@
"gnetId": null,
"graphTooltip": 1,
"id": null,
"iteration": 1637575565298,
"iteration": 1637648133817,
"links": [],
"panels": [
{
Expand Down Expand Up @@ -11137,7 +11137,7 @@
"targets": [
{
"exemplar": true,
"expr": "histogram_quantile(0.999, sum(rate(tikv_raftstore_store_wf_commit_log_duration_seconds_bucket{instance=~\"$instance\"}[30s])) by (le))",
"expr": "histogram_quantile(0.999, sum(rate(tikv_raftstore_store_wf_commit_not_persist_log_duration_seconds_bucket{instance=~\"$instance\"}[30s])) by (le))",
"hide": false,
"interval": "",
"legendFormat": "999%",
Expand Down Expand Up @@ -12154,7 +12154,7 @@
"dashLength": 10,
"dashes": false,
"datasource": "${DS_TEST-CLUSTER}",
"description": "The time consumed when store write raft db on each TiKV instance",
"description": "",
"fieldConfig": {
"defaults": {},
"overrides": []
Expand All @@ -12168,7 +12168,7 @@
"y": 45
},
"hiddenSeries": false,
"id": 13378,
"id": 13382,
"legend": {
"alignAsTable": true,
"avg": false,
Expand Down Expand Up @@ -12199,19 +12199,27 @@
"targets": [
{
"exemplar": true,
"expr": "histogram_quantile(0.99, sum(rate(tikv_raftstore_store_write_raftdb_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le, instance))",
"expr": "sum(tikv_raftstore_io_reschedule_region_total{instance=~\"$instance\"}) by (instance)",
"format": "time_series",
"interval": "",
"intervalFactor": 2,
"legendFormat": "{{instance}}",
"intervalFactor": 1,
"legendFormat": "rechedule-{{instance}}",
"refId": "A"
},
{
"exemplar": true,
"expr": "sum(tikv_raftstore_io_reschedule_pending_task_total{instance=~\"$instance\"}) by (instance)",
"hide": false,
"interval": "",
"legendFormat": "pending-task-{{instance}}",
"refId": "B"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "99% Write raft db duration per server",
"title": "Store io task reschedule",
"tooltip": {
"shared": true,
"sort": 0,
Expand All @@ -12227,7 +12235,7 @@
},
"yaxes": [
{
"format": "s",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
Expand Down Expand Up @@ -12347,114 +12355,6 @@
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "${DS_TEST-CLUSTER}",
"description": "",
"fieldConfig": {
"defaults": {},
"overrides": []
},
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 7,
"w": 12,
"x": 0,
"y": 52
},
"hiddenSeries": false,
"id": 13382,
"legend": {
"alignAsTable": true,
"avg": false,
"current": true,
"max": true,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": true
},
"lines": true,
"linewidth": 1,
"links": [],
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.5.7",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"exemplar": true,
"expr": "sum(tikv_raftstore_io_reschedule_region_total{instance=~\"$instance\"}) by (instance)",
"format": "time_series",
"interval": "",
"intervalFactor": 1,
"legendFormat": "rechedule-{{instance}}",
"refId": "A"
},
{
"exemplar": true,
"expr": "sum(tikv_raftstore_io_reschedule_pending_task_total{instance=~\"$instance\"}) by (instance)",
"hide": false,
"interval": "",
"legendFormat": "pending-task-{{instance}}",
"refId": "B"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Store io task reschedule",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
}
],
"repeat": null,
Expand Down Expand Up @@ -19324,7 +19224,7 @@
"h": 1,
"w": 24,
"x": 0,
"y": 29
"y": 30
},
"id": 2755,
"panels": [
Expand Down Expand Up @@ -19780,7 +19680,7 @@
"h": 1,
"w": 24,
"x": 0,
"y": 30
"y": 31
},
"id": 2758,
"panels": [
Expand Down Expand Up @@ -21106,7 +21006,7 @@
"h": 1,
"w": 24,
"x": 0,
"y": 31
"y": 32
},
"id": 2759,
"panels": [
Expand Down Expand Up @@ -21723,7 +21623,7 @@
"h": 1,
"w": 24,
"x": 0,
"y": 32
"y": 33
},
"id": 2760,
"panels": [
Expand Down Expand Up @@ -22135,7 +22035,7 @@
"h": 1,
"w": 24,
"x": 0,
"y": 33
"y": 34
},
"id": 2757,
"panels": [
Expand Down Expand Up @@ -22935,7 +22835,7 @@
"h": 1,
"w": 24,
"x": 0,
"y": 34
"y": 35
},
"id": 3197,
"panels": [
Expand Down Expand Up @@ -24029,7 +23929,7 @@
"h": 1,
"w": 24,
"x": 0,
"y": 35
"y": 36
},
"id": 2761,
"panels": [
Expand Down Expand Up @@ -24426,7 +24326,7 @@
"h": 1,
"w": 24,
"x": 0,
"y": 36
"y": 37
},
"id": 2762,
"panels": [
Expand Down Expand Up @@ -29166,7 +29066,7 @@
"h": 1,
"w": 24,
"x": 0,
"y": 38
"y": 39
},
"id": 12802,
"panels": [
Expand Down Expand Up @@ -30218,7 +30118,7 @@
"h": 1,
"w": 24,
"x": 0,
"y": 39
"y": 40
},
"id": 3301,
"panels": [
Expand Down Expand Up @@ -33227,7 +33127,7 @@
"h": 1,
"w": 24,
"x": 0,
"y": 40
"y": 41
},
"id": 2820,
"panels": [
Expand Down Expand Up @@ -33956,7 +33856,7 @@
"h": 1,
"w": 24,
"x": 0,
"y": 41
"y": 42
},
"id": 8389,
"panels": [
Expand Down Expand Up @@ -35137,7 +35037,7 @@
"h": 1,
"w": 24,
"x": 0,
"y": 42
"y": 43
},
"id": 2763,
"panels": [
Expand Down Expand Up @@ -35242,7 +35142,7 @@
"h": 1,
"w": 24,
"x": 0,
"y": 43
"y": 44
},
"id": 3922,
"panels": [
Expand Down Expand Up @@ -37963,7 +37863,7 @@
"h": 1,
"w": 24,
"x": 0,
"y": 44
"y": 45
},
"id": 4466,
"panels": [
Expand Down

0 comments on commit 6a0792a

Please sign in to comment.