Skip to content
This repository was archived by the owner on Jul 6, 2021. It is now read-only.

Commit bf1fc16

Browse files
dmiusdmius
committed
Build list of indexes reworked, generation json data for indexes fixed
In some cases some partial indexes and indexes with conditions we have recommended to remove as redundant. Incorrect conditions was removed from code, so now it should detect redundant indexes more correctly. Also in cases when schema where placed table of index is null such indexes can be skipped in json code. It fixed.
2 parents 87c4cfa + c480173 commit bf1fc16

File tree

10 files changed

+279
-252
lines changed

10 files changed

+279
-252
lines changed

‎.ci/test_db_dump.sql‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,17 @@ create index concurrently i_unused on t_with_unused_index(i);
1919
create table t_with_redundant_index as select i from generate_series(1, 1000000) _(i);
2020
create index concurrently i_redundant_1 on t_with_redundant_index(i);
2121
create index concurrently i_redundant_2 on t_with_redundant_index(i);
22+
23+
-- redundant for functional and uniq keys
24+
create table ctnr as select id, id as pnum, id as m_id, id as type_id, id as b, id as todel from generate_series(1, 1000000) _(id);
25+
alter table ctnr add primary key (id);
26+
alter table ctnr add constraint ctnr_uk01 unique (pnum, m_id);
27+
create index ctnr_idx01 on ctnr using btree(pnum);
28+
create index ctnr_idx02 on ctnr using btree(type_id);
29+
create index ctnr_idx03 on ctnr using btree(b);
30+
create index ctnr_idx04 on ctnr using btree(b) where pnum > 0;
31+
create index ctnr_idx06 on ctnr using btree(todel);
32+
2233
-- redundant for uniq, primary keys
2334
create table t_with_redundant_idx as select id, id as f1, id as f2, id as f3, id as f4 from generate_series(1, 1000000) _(id);
2435
alter table t_with_redundant_idx add primary key (id);

‎.gitlab-ci.yml‎

Lines changed: 16 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -37,17 +37,19 @@ variables:
3737
- ./checkup collect -c .ci/test.yml --file ./resources/checks/H003_non_indexed_fks.sh
3838
- data_dir=$(cat ./artifacts/test/nodes.json | jq -r '.last_check | .dir') && result=$(cat ./artifacts/test/json_reports/$data_dir/H003_non_indexed_fks.json | jq '.results .postgres .data .indexes') && echo "$result" && cat ./artifacts/test/json_reports/$data_dir/H003_non_indexed_fks.json && ([[ ! "$result" == "[]" ]] && [[ ! "$result" == "null" ]]) && exit 302
3939
- echo "H003 passed"
40-
- echo "Test H002 redundant indexes"
40+
- echo "Test H004 redundant indexes"
4141
- psql -h postgres -d dbname -U test_user -f .ci/test_db_dump.sql
42-
- ./checkup collect -c .ci/test.yml --file ./resources/checks/H002_unused_indexes.sh
43-
- data_dir=$(cat ./artifacts/test/nodes.json | jq -r '.last_check | .dir') && result=$(cat ./artifacts/test/json_reports/$data_dir/H002_unused_indexes.json | jq '.results .postgres .data .redundant_indexes ."public.t_with_redundant_idx_id"') && ( [[ "$result" == "" ]] || [[ "$result" == "null" ]]) && exit 201
44-
- data_dir=$(cat ./artifacts/test/nodes.json | jq -r '.last_check | .dir') && result=$(cat ./artifacts/test/json_reports/$data_dir/H002_unused_indexes.json | jq '.results .postgres .data .redundant_indexes ."public.t_with_redundant_idx_f1"') && ( [[ "$result" == "" ]] || [[ "$result" == "null" ]]) && exit 202
45-
- data_dir=$(cat ./artifacts/test/nodes.json | jq -r '.last_check | .dir') && result=$(cat ./artifacts/test/json_reports/$data_dir/H002_unused_indexes.json | jq '.results .postgres .data .redundant_indexes ."public.t_with_redundant_idx_f1_uniq"') && ([[ ! "$result" == "[]" ]] && [[ ! "$result" == "null" ]]) && exit 203
46-
- data_dir=$(cat ./artifacts/test/nodes.json | jq -r '.last_check | .dir') && result=$(cat ./artifacts/test/json_reports/$data_dir/H002_unused_indexes.json | jq '.results .postgres .data .redundant_indexes ."public.t_with_redundant_idx_pkey"') && ([[ ! "$result" == "[]" ]] && [[ ! "$result" == "null" ]]) && exit 204
47-
- data_dir=$(cat ./artifacts/test/nodes.json | jq -r '.last_check | .dir') && result=$(cat ./artifacts/test/json_reports/$data_dir/H002_unused_indexes.json | jq '.results .postgres .data .redundant_indexes ."public.t_with_redundant_ref_idx_1"') && ([[ ! "$result" == "[]" ]] && [[ ! "$result" == "null" ]]) && exit 205
48-
- data_dir=$(cat ./artifacts/test/nodes.json | jq -r '.last_check | .dir') && result=$(cat ./artifacts/test/json_reports/$data_dir/H002_unused_indexes.json | jq '.results .postgres .data .redundant_indexes ."public.t_with_redundant_ref_idx_2"') && ( [[ "$result" == "" ]] || [[ "$result" == "null" ]]) && exit 206
49-
- data_dir=$(cat ./artifacts/test/nodes.json | jq -r '.last_check | .dir') && result=$(cat ./artifacts/test/json_reports/$data_dir/H002_unused_indexes.json | jq '.results .postgres .data .redundant_indexes ."exp_redundant.t_with_redundant_ref_idx_1"') && ([[ ! "$result" == "[]" ]] && [[ ! "$result" == "null" ]]) && exit 207
50-
- data_dir=$(cat ./artifacts/test/nodes.json | jq -r '.last_check | .dir') && result=$(cat ./artifacts/test/json_reports/$data_dir/H002_unused_indexes.json | jq '.results .postgres .data .redundant_indexes ."exp_redundant.t_with_redundant_ref_idx_2"') && ( [[ "$result" == "" ]] || [[ "$result" == "null" ]]) && exit 208
42+
- ./checkup collect -h postgres --username test_user --project test --dbname dbname -e 1 --file ./resources/checks/H004_redundant_indexes.sh
43+
- data_dir=$(cat ./artifacts/test/nodes.json | jq -r '.last_check | .dir') && result=$(cat ./artifacts/test/json_reports/$data_dir/H004_redundant_indexes.json | jq '.results .postgres .data .redundant_indexes ."public.t_with_redundant_idx_id"') && ( [[ "$result" == "" ]] || [[ "$result" == "null" ]]) && exit 201
44+
- data_dir=$(cat ./artifacts/test/nodes.json | jq -r '.last_check | .dir') && result=$(cat ./artifacts/test/json_reports/$data_dir/H004_redundant_indexes.json | jq '.results .postgres .data .redundant_indexes ."public.t_with_redundant_idx_f1"') && ( [[ "$result" == "" ]] || [[ "$result" == "null" ]]) && exit 202
45+
- data_dir=$(cat ./artifacts/test/nodes.json | jq -r '.last_check | .dir') && result=$(cat ./artifacts/test/json_reports/$data_dir/H004_redundant_indexes.json | jq '.results .postgres .data .redundant_indexes ."public.t_with_redundant_idx_f1_uniq"') && ([[ ! "$result" == "[]" ]] && [[ ! "$result" == "null" ]]) && exit 203
46+
- data_dir=$(cat ./artifacts/test/nodes.json | jq -r '.last_check | .dir') && result=$(cat ./artifacts/test/json_reports/$data_dir/H004_redundant_indexes.json | jq '.results .postgres .data .redundant_indexes ."public.t_with_redundant_idx_pkey"') && ([[ ! "$result" == "[]" ]] && [[ ! "$result" == "null" ]]) && exit 204
47+
- data_dir=$(cat ./artifacts/test/nodes.json | jq -r '.last_check | .dir') && result=$(cat ./artifacts/test/json_reports/$data_dir/H004_redundant_indexes.json | jq '.results .postgres .data .redundant_indexes ."public.t_with_redundant_ref_idx_1"') && ([[ ! "$result" == "[]" ]] && [[ ! "$result" == "null" ]]) && exit 205
48+
- data_dir=$(cat ./artifacts/test/nodes.json | jq -r '.last_check | .dir') && result=$(cat ./artifacts/test/json_reports/$data_dir/H004_redundant_indexes.json | jq '.results .postgres .data .redundant_indexes ."public.t_with_redundant_ref_idx_2"') && ( [[ "$result" == "" ]] || [[ "$result" == "null" ]]) && exit 206
49+
- data_dir=$(cat ./artifacts/test/nodes.json | jq -r '.last_check | .dir') && result=$(cat ./artifacts/test/json_reports/$data_dir/H004_redundant_indexes.json | jq '.results .postgres .data .redundant_indexes ."exp_redundant.t_with_redundant_ref_idx_1"') && ([[ ! "$result" == "[]" ]] && [[ ! "$result" == "null" ]]) && exit 207
50+
- data_dir=$(cat ./artifacts/test/nodes.json | jq -r '.last_check | .dir') && result=$(cat ./artifacts/test/json_reports/$data_dir/H004_redundant_indexes.json | jq '.results .postgres .data .redundant_indexes ."exp_redundant.t_with_redundant_ref_idx_2"') && ( [[ "$result" == "" ]] || [[ "$result" == "null" ]]) && exit 208
51+
- data_dir=$(cat ./artifacts/test/nodes.json | jq -r '.last_check | .dir') && result=$(cat ./artifacts/test/json_reports/$data_dir/H004_redundant_indexes.json | jq '.results .postgres .data .redundant_indexes ."public.ctnr_idx01"') && ( [[ "$result" == "" ]] || [[ "$result" == "null" ]]) && exit 209
52+
- data_dir=$(cat ./artifacts/test/nodes.json | jq -r '.last_check | .dir') && result=$(cat "./artifacts/test/json_reports/$data_dir/H004_redundant_indexes.json" | jq '.results .postgres .data .redundant_indexes ."public.ctnr_idx04"') && ([[ ! "$result" == "[]" ]] && [[ ! "$result" == "null" ]]) && exit 210
5153
- echo "H002 passed"
5254
- ./checkup -c .ci/test.yml --file ./resources/checks/F005_index_bloat.sh
5355
- ./checkup -c .ci/test.yml --file ./resources/checks/L001_table_sizes.sh
@@ -62,16 +64,16 @@ variables:
6264
- .ci/prepare_test_db.sh postgres
6365
- ./checkup -c .ci/test.yml --file ./resources/checks/H002_unused_indexes.sh
6466
## unused
65-
- data_dir=$(cat ./artifacts/test/nodes.json | jq -r '.last_check | .dir') && result=$(cat "./artifacts/test/json_reports/$data_dir/H002_unused_indexes.json" | jq '.results .postgres .data .never_used_indexes ."public.i_u_12"') && ( [[ "$result" == "" ]] || [[ "$result" == "null" ]]) && exit 209
67+
- data_dir=$(cat ./artifacts/test/nodes.json | jq -r '.last_check | .dir') && result=$(cat "./artifacts/test/json_reports/$data_dir/H002_unused_indexes.json" | jq '.results .postgres .data .never_used_indexes ."public.i_u_12"') && ( [[ "$result" == "" ]] || [[ "$result" == "null" ]]) && exit 210
6668
## redundant
67-
- data_dir=$(cat ./artifacts/test/nodes.json | jq -r '.last_check | .dir') && result=$(cat "./artifacts/test/json_reports/$data_dir/H002_unused_indexes.json" | jq '.results .postgres .data .redundant_indexes ."public.i_r_12"') && ( [[ "$result" == "" ]] || [[ "$result" == "null" ]]) && exit 210
69+
- data_dir=$(cat ./artifacts/test/nodes.json | jq -r '.last_check | .dir') && result=$(cat "./artifacts/test/json_reports/$data_dir/H004_redundant_indexes.json" | jq '.results .postgres .data .redundant_indexes ."public.i_r_12"') && ( [[ "$result" == "" ]] || [[ "$result" == "null" ]]) && exit 211
6870
- echo "Check small indexes in small db mode passed"
6971
- .ci/prepare_large_db.sh postgres
7072
- ./checkup -c .ci/test.yml --file ./resources/checks/H002_unused_indexes.sh
7173
## unused
72-
- data_dir=$(cat ./artifacts/test/nodes.json | jq -r '.last_check | .dir') && result=$(cat "./artifacts/test/json_reports/$data_dir/H002_unused_indexes.json" | jq '.results .postgres .data .never_used_indexes ."public.i_u_12"') && ([[ ! "$result" == "[]" ]] && [[ ! "$result" == "null" ]]) && exit 211
74+
- data_dir=$(cat ./artifacts/test/nodes.json | jq -r '.last_check | .dir') && result=$(cat "./artifacts/test/json_reports/$data_dir/H002_unused_indexes.json" | jq '.results .postgres .data .never_used_indexes ."public.i_u_12"') && ([[ ! "$result" == "[]" ]] && [[ ! "$result" == "null" ]]) && exit 212
7375
## redundant
74-
- data_dir=$(cat ./artifacts/test/nodes.json | jq -r '.last_check | .dir') && result=$(cat "./artifacts/test/json_reports/$data_dir/H002_unused_indexes.json" | jq '.results .postgres .data .redundant_indexes ."public.i_r_12"') && ([[ ! "$result" == "[]" ]] && [[ ! "$result" == "null" ]]) && exit 212
76+
- data_dir=$(cat ./artifacts/test/nodes.json | jq -r '.last_check | .dir') && result=$(cat "./artifacts/test/json_reports/$data_dir/H004_redundant_indexes.json" | jq '.results .postgres .data .redundant_indexes ."public.i_r_12"') && ([[ ! "$result" == "[]" ]] && [[ ! "$result" == "null" ]]) && exit 213
7577
- echo "Check small indexes in large db mode passed"
7678

7779
test-general:

‎HELP.md‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -214,7 +214,7 @@ The list of broken indexes (invalid state) to be removed or reindexed.
214214

215215
### H002 Unused and Redundant Indexes
216216

217-
Shows the list of never used, rarely used and redundant indexes.
217+
Shows the list of never used and rarely used indexes.
218218
Helps to understand how much space they occupy.
219219

220220
> Insights:
@@ -227,6 +227,12 @@ Helps to understand how much space they occupy.
227227

228228
Checks if all foreign keys have indexes in referencing tables.
229229

230+
### H004 Redundant Indexes
231+
232+
Shows the list of redundant indexes.
233+
Helps to understand how much space they occupy.
234+
235+
230236
# K. SQL Query Analysis
231237

232238
This is important group of reports providing deep SQL query analysis, based on pg_stat_statements and (optional) pg_stat_kcache.

‎README.md‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -364,8 +364,9 @@ Docker support implemented by [Ivan Muratov](https://gitlab.com/binakot).
364364
## H. Index Analysis
365365

366366
- [x] H001 Invalid indexes #192, #51
367-
- [x] H002 Unused and redundant indexes #51, #180, #170, #168, #322
367+
- [x] H002 Unused indexes #51, #180, #170, #168, #322
368368
- [x] H003 Non-indexed foreign keys #52, #142, #173
369+
- [x] H004 Redundant indexes
369370

370371
## J. Capacity Planning
371372

‎checkup‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -875,14 +875,15 @@ check_bin_deps() {
875875
glue_md_reports() {
876876
# final report path and name
877877
local out_fname="${MD_REPORTS_DIR}/${FULL_REPORT_FNAME}"
878-
local database=$(jq '.last_check.database' ${PROJECT_DIR}/nodes.json)
878+
local epoch=$(jq -r '.last_check.epoch' ${PROJECT_DIR}/nodes.json)
879+
local database=$(jq -r '.last_check.database' ${PROJECT_DIR}/nodes.json)
879880

880881
# do not re-generate full report if '--file' is given
881882
[[ "${FILE}" != "None" ]] && return 0
882883

883884
# make header
884885
echo "# PostgreSQL Checkup. Project: '${PROJECT}'. Database: '${database}'" > "${out_fname}"
885-
echo "## Epoch number: '${EPOCH}'" >> "${out_fname}"
886+
echo "## Epoch number: '${epoch}'" >> "${out_fname}"
886887
echo "NOTICE: while most reports describe the “current database”, some of them may contain cluster-wide information describing all databases in the cluster." >> "${out_fname}"
887888
echo >> "${out_fname}"
888889
echo "Last modified at: " $(date +'%Y-%m-%d %H:%M:%S %z') >> "${out_fname}"

‎pghrep/templates/H002.tpl‎

Lines changed: 5 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,12 @@
1-
# {{ .checkId }} Unused and Redundant Indexes #
1+
# {{ .checkId }} Unused Indexes #
22
## Observations ##
33
Data collected: {{ DtFormat .timestamptz }}
44
Current database: {{ .database }}
55

6-
{{- if (index .results .reorderedHosts.master)}}
6+
{{ if (index .results .reorderedHosts.master)}}
77
{{- if (index (index .results .reorderedHosts.master) "data") }}
88
Stats reset: {{ (index (index (index .results .reorderedHosts.master) "data") "database_stat").stats_age }} ago ({{ DtFormat (index (index (index .results .reorderedHosts.master) "data") "database_stat").stats_reset }})
9-
{{- if le (Int (index (index (index .results .reorderedHosts.master) "data") "database_stat").days) 30 }}
9+
{{ if le (Int (index (index (index .results .reorderedHosts.master) "data") "database_stat").days) 30 }}
1010
:warning: Statistics age is less than 30 days. Make decisions on index cleanup with caution!
1111
{{- end }}
1212
{{ if gt (Int (index (index (index .results .reorderedHosts.master) "data") "min_index_size_bytes")) 0 }}NOTICE: only indexes larger than {{ ByteFormat (index (index (index .results .reorderedHosts.master) "data") "min_index_size_bytes") 0 }} are analyzed. {{end}}
@@ -33,7 +33,7 @@ Nothing found.
3333

3434
### Rarely Used Indexes ###
3535
{{ if (index (index (index .results .reorderedHosts.master) "data") "rarely_used_indexes") }}
36-
{{ if ge (len (index (index (index .results .reorderedHosts.master) "data") "rarely_used_indexes")) .LISTLIMIT }}The list is limited to {{.LISTLIMIT}} items. Total: {{ Sub (len (index (index (index .results .reorderedHosts.master) "data") "rarely_used_indexes")) 1 }}.{{ end }}
36+
{{ if ge (len (index (index (index .results .reorderedHosts.master) "data") "rarely_used_indexes")) .LISTLIMIT }}The list is limited to {{.LISTLIMIT}} items. Total: {{ Sub (len (index (index (index .results .reorderedHosts.master) "data") "rarely_used_indexes")) 1 }}.{{ end }}
3737

3838
|\#| Table | Index | {{.reorderedHosts.master}} usage {{ range $skey, $host := .reorderedHosts.replicas }}| {{ $host }} usage {{ end }}| ▼ Index size | Table size | Comment | Supports FK|
3939
|--|-------|-------|-----{{ range $skey, $host := .reorderedHosts.replicas }}|--------{{ end }}|-----|-----|----|-----|
@@ -53,27 +53,6 @@ Nothing found.
5353
Nothing found.
5454
{{ end }}{{/* rarely used indexes found */}}
5555

56-
### Redundant Indexes ###
57-
{{ if (index (index (index .results .reorderedHosts.master) "data") "redundant_indexes") }}
58-
{{ if ge (len (index (index (index .results .reorderedHosts.master) "data") "redundant_indexes")) .LISTLIMIT }}The list is limited to {{.LISTLIMIT}} items. Total: {{ Sub (len (index (index (index .results .reorderedHosts.master) "data") "redundant_indexes")) 1 }}.{{ end }}
59-
60-
|\#| Table | Index | Redundant to |{{.reorderedHosts.master}} usage {{ range $skey, $host := .reorderedHosts.replicas }}| {{ $host }} usage {{ end }}| ▼ Index size | Table size | Supports FK |
61-
|--|-------|-------|--------------|--{{ range $skey, $host := .reorderedHosts.replicas }}|--------{{ end }}|-----|-----|-----|
62-
{{ if (index (index (index .results .reorderedHosts.master) "data") "redundant_indexes_total") }}| |=====TOTAL=====|||{{ range $skey, $host := .reorderedHosts.replicas }}|{{ end }}|{{ ByteFormat ((index (index (index .results .reorderedHosts.master) "data") "redundant_indexes_total").index_size_bytes_sum) 2 }}|{{ ByteFormat ((index (index (index .results .reorderedHosts.master) "data") "redundant_indexes_total").table_size_bytes_sum) 2 }}||{{ end }}
63-
{{ range $i, $key := (index (index (index (index .results .reorderedHosts.master) "data") "redundant_indexes") "_keys") }}
64-
{{- if lt $i $.LISTLIMIT -}}
65-
{{- $value:=(index (index (index (index $.results $.reorderedHosts.master) "data") "redundant_indexes") $key) -}}
66-
|{{- $value.num}}|`{{ $value.formated_relation_name}}`|`{{- $value.formated_index_name}}`|
67-
{{- $rinexes := Split $value.reason ", " -}}{{ range $r, $rto:= $rinexes }}`{{$rto}}`<br/>{{end}}|{{- RawIntFormat $value.idx_scan }}{{ range $skey, $host := $.reorderedHosts.replicas }}|{{ if (index $.results $host) }}{{ if (index (index $.results $host) "data") }}{{ if (index (index (index $.results $host) "data") "never_used_indexes") }}{{ if (index (index (index (index $.results $host) "data") "never_used_indexes") $key) }}{{ RawIntFormat ((index (index (index (index $.results $host) "data") "redundant_indexes") $key).idx_scan) }}{{end}}{{ end }}{{ end }}{{ end }}{{end}}|
68-
{{- ByteFormat $value.index_size_bytes 2}}|
69-
{{- ByteFormat $value.table_size_bytes 2}}|
70-
{{- if $value.supports_fk }}Yes{{end}}|
71-
{{/* if limit list */}}{{ end -}}
72-
{{ end }}{{/* range */}}
73-
{{else}}
74-
Nothing found.
75-
{{ end }}{{/* redundant indexes found */}}
76-
7756
{{- else -}}{{/* end if master*/}}
7857
Nothing found
7958
{{end}}{{/* end if master*/}}
@@ -83,23 +62,5 @@ Nothing found
8362

8463
## Conclusions ##
8564

86-
8765
## Recommendations ##
88-
{{ if (index .results .reorderedHosts.master) }}
89-
{{ if (index (index .results .reorderedHosts.master) "data") }}
90-
{{ if (index (index (index .results .reorderedHosts.master) "data") "do")}}
91-
#### "DO" database migration code ####
92-
```
93-
{{ range $i, $drop_code := (index (index (index .results .reorderedHosts.master) "data") "do") }}{{ $drop_code }}
94-
{{ end }}
95-
```
96-
{{end}}
97-
{{ if (index (index (index .results .reorderedHosts.master) "data") "undo") }}
98-
#### "UNDO" database migration code ####
99-
```
100-
{{ range $i, $revert_code := (index (index (index .results .reorderedHosts.master) "data") "undo") }}{{ $revert_code }}
101-
{{ end }}
102-
```
103-
{{end}}
104-
{{ end }}{{/* data found */}}
105-
{{ end }}{{/* master */}}
66+

0 commit comments

Comments
 (0)