diff --git a/components/raftstore/src/store/async_io/write.rs b/components/raftstore/src/store/async_io/write.rs index adba60a54ef..3cad2922311 100644 --- a/components/raftstore/src/store/async_io/write.rs +++ b/components/raftstore/src/store/async_io/write.rs @@ -423,14 +423,7 @@ where self.write_to_db(true); - self.metrics.flush(); - self.clear_latency_inspect(); - // update config - if let Some(incoming) = self.cfg_tracker.any_new() { - self.raft_write_size_limit = incoming.raft_write_size_limit.0 as usize; - self.metrics.waterfall_metrics = incoming.waterfall_metrics; - } STORE_WRITE_LOOP_DURATION_HISTOGRAM .observe(duration_to_sec(handle_begin.saturating_elapsed())); @@ -618,6 +611,13 @@ where ); self.batch.clear(); + self.metrics.flush(); + + // update config + if let Some(incoming) = self.cfg_tracker.any_new() { + self.raft_write_size_limit = incoming.raft_write_size_limit.0 as usize; + self.metrics.waterfall_metrics = incoming.waterfall_metrics; + } } pub fn is_empty(&self) -> bool { diff --git a/metrics/grafana/tikv_details.json b/metrics/grafana/tikv_details.json index 50ce649ad05..a65a7b4769b 100644 --- a/metrics/grafana/tikv_details.json +++ b/metrics/grafana/tikv_details.json @@ -64,7 +64,7 @@ "gnetId": null, "graphTooltip": 1, "id": null, - "iteration": 1637575565298, + "iteration": 1637648133817, "links": [], "panels": [ { @@ -11137,7 +11137,7 @@ "targets": [ { "exemplar": true, - "expr": "histogram_quantile(0.999, sum(rate(tikv_raftstore_store_wf_commit_log_duration_seconds_bucket{instance=~\"$instance\"}[30s])) by (le))", + "expr": "histogram_quantile(0.999, sum(rate(tikv_raftstore_store_wf_commit_not_persist_log_duration_seconds_bucket{instance=~\"$instance\"}[30s])) by (le))", "hide": false, "interval": "", "legendFormat": "999%", @@ -12154,7 +12154,7 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "The time consumed when store write raft db on each TiKV instance", + "description": "", "fieldConfig": { "defaults": {}, "overrides": [] @@ -12168,7 +12168,7 @@ "y": 45 }, "hiddenSeries": false, - "id": 13378, + "id": 13382, "legend": { "alignAsTable": true, "avg": false, @@ -12199,19 +12199,27 @@ "targets": [ { "exemplar": true, - "expr": "histogram_quantile(0.99, sum(rate(tikv_raftstore_store_write_raftdb_duration_seconds_bucket{instance=~\"$instance\"}[1m])) by (le, instance))", + "expr": "sum(tikv_raftstore_io_reschedule_region_total{instance=~\"$instance\"}) by (instance)", "format": "time_series", "interval": "", - "intervalFactor": 2, - "legendFormat": "{{instance}}", + "intervalFactor": 1, + "legendFormat": "rechedule-{{instance}}", "refId": "A" + }, + { + "exemplar": true, + "expr": "sum(tikv_raftstore_io_reschedule_pending_task_total{instance=~\"$instance\"}) by (instance)", + "hide": false, + "interval": "", + "legendFormat": "pending-task-{{instance}}", + "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "99% Write raft db duration per server", + "title": "Store io task reschedule", "tooltip": { "shared": true, "sort": 0, @@ -12227,7 +12235,7 @@ }, "yaxes": [ { - "format": "s", + "format": "short", "label": null, "logBase": 1, "max": null, @@ -12347,114 +12355,6 @@ "align": false, "alignLevel": null } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", - "description": "", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 52 - }, - "hiddenSeries": false, - "id": 13382, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "exemplar": true, - "expr": "sum(tikv_raftstore_io_reschedule_region_total{instance=~\"$instance\"}) by (instance)", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "rechedule-{{instance}}", - "refId": "A" - }, - { - "exemplar": true, - "expr": "sum(tikv_raftstore_io_reschedule_pending_task_total{instance=~\"$instance\"}) by (instance)", - "hide": false, - "interval": "", - "legendFormat": "pending-task-{{instance}}", - "refId": "B" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Store io task reschedule", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } } ], "repeat": null, @@ -19324,7 +19224,7 @@ "h": 1, "w": 24, "x": 0, - "y": 29 + "y": 30 }, "id": 2755, "panels": [ @@ -19780,7 +19680,7 @@ "h": 1, "w": 24, "x": 0, - "y": 30 + "y": 31 }, "id": 2758, "panels": [ @@ -21106,7 +21006,7 @@ "h": 1, "w": 24, "x": 0, - "y": 31 + "y": 32 }, "id": 2759, "panels": [ @@ -21723,7 +21623,7 @@ "h": 1, "w": 24, "x": 0, - "y": 32 + "y": 33 }, "id": 2760, "panels": [ @@ -22135,7 +22035,7 @@ "h": 1, "w": 24, "x": 0, - "y": 33 + "y": 34 }, "id": 2757, "panels": [ @@ -22935,7 +22835,7 @@ "h": 1, "w": 24, "x": 0, - "y": 34 + "y": 35 }, "id": 3197, "panels": [ @@ -24029,7 +23929,7 @@ "h": 1, "w": 24, "x": 0, - "y": 35 + "y": 36 }, "id": 2761, "panels": [ @@ -24426,7 +24326,7 @@ "h": 1, "w": 24, "x": 0, - "y": 36 + "y": 37 }, "id": 2762, "panels": [ @@ -29166,7 +29066,7 @@ "h": 1, "w": 24, "x": 0, - "y": 38 + "y": 39 }, "id": 12802, "panels": [ @@ -30218,7 +30118,7 @@ "h": 1, "w": 24, "x": 0, - "y": 39 + "y": 40 }, "id": 3301, "panels": [ @@ -33227,7 +33127,7 @@ "h": 1, "w": 24, "x": 0, - "y": 40 + "y": 41 }, "id": 2820, "panels": [ @@ -33956,7 +33856,7 @@ "h": 1, "w": 24, "x": 0, - "y": 41 + "y": 42 }, "id": 8389, "panels": [ @@ -35137,7 +35037,7 @@ "h": 1, "w": 24, "x": 0, - "y": 42 + "y": 43 }, "id": 2763, "panels": [ @@ -35242,7 +35142,7 @@ "h": 1, "w": 24, "x": 0, - "y": 43 + "y": 44 }, "id": 3922, "panels": [ @@ -37963,7 +37863,7 @@ "h": 1, "w": 24, "x": 0, - "y": 44 + "y": 45 }, "id": 4466, "panels": [