Skip to content
This repository was archived by the owner on Jul 6, 2021. It is now read-only.

Commit 72fcdd1

Browse files
dmiusdmius
committed
Dynamic adjustment of min relpages value for analyzed objects added.
1 parent 109cbac commit 72fcdd1

14 files changed

+210
-35
lines changed

‎.ci/prepare_large_db.sh‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
#!/bin/bash
2+
3+
if [[ ! -z "${1+x}" ]]; then
4+
hostname="$1"
5+
else
6+
hostname='localhost'
7+
fi
8+
9+
# Generate more than 30000 tables with 2 index to check that
10+
# in large db mode small tables and indexes are not analyzed.
11+
for ((i=1; i < 3400; i++))
12+
do
13+
psql -h ${hostname} -d dbname -U test_user -f -<<SQL
14+
create table t_lg1_$i as select i from generate_series(1, 10) _(i);
15+
create index concurrently i_lg1u_$i on t_lg1_$i(i);
16+
create index concurrently i_lg1r_$i on t_lg1_$i(i);
17+
18+
create table t_lg2_$i as select i from generate_series(1, 10) _(i);
19+
create index concurrently i_lg2u_$i on t_lg2_$i(i);
20+
create index concurrently i_lg2r_$i on t_lg2_$i(i);
21+
22+
create table t_lg3_$i as select i from generate_series(1, 10) _(i);
23+
create index concurrently i_lg3u_$i on t_lg3_$i(i);
24+
create index concurrently i_lg3r_$i on t_lg3_$i(i);
25+
26+
create table t_lg4_$i as select i from generate_series(1, 10) _(i);
27+
create index concurrently i_lg4u_$i on t_lg4_$i(i);
28+
create index concurrently i_lg4r_$i on t_lg4_$i(i);
29+
30+
create table t_lg5_$i as select i from generate_series(1, 10) _(i);
31+
create index concurrently i_lg5u_$i on t_lg5_$i(i);
32+
create index concurrently i_lg5r_$i on t_lg5_$i(i);
33+
34+
create table t_lg6_$i as select i from generate_series(1, 10) _(i);
35+
create index concurrently i_lg6u_$i on t_lg6_$i(i);
36+
create index concurrently i_lg6r_$i on t_lg6_$i(i);
37+
38+
create table t_lg7_$i as select i from generate_series(1, 10) _(i);
39+
create index concurrently i_lg7u_$i on t_lg7_$i(i);
40+
create index concurrently i_lg7r_$i on t_lg7_$i(i);
41+
42+
create table t_lg8_$i as select i from generate_series(1, 10) _(i);
43+
create index concurrently i_lg8u_$i on t_lg8_$i(i);
44+
create index concurrently i_lg8r_$i on t_lg8_$i(i);
45+
46+
create table t_lg9_$i as select i from generate_series(1, 10) _(i);
47+
create index concurrently i_lg9u_$i on t_lg9_$i(i);
48+
create index concurrently i_lg9r_$i on t_lg9_$i(i);
49+
50+
create table t_lg10_$i as select i from generate_series(1, 10) _(i);
51+
create index concurrently i_lg10u_$i on t_lg10_$i(i);
52+
create index concurrently i_lg10r_$i on t_lg10_$i(i);
53+
SQL
54+
done

‎.ci/prepare_test_db.sh‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,18 @@
11
#!/bin/bash
22

3+
if [[ ! -z "${1+x}" ]]; then
4+
hostname=$1
5+
else
6+
hostname='localhost'
7+
fi
8+
39
# Generate more than 100 tables with index to check that
410
# the limitation of .md reports' rows counts work as expected.
511
for ((i=1; i < 110; i++))
612
do
7-
psql -d dbname -U test_user -c "create table t_$i as select i from generate_series(1, 1000) _(i);"
8-
psql -d dbname -U test_user -c "create index concurrently i_$i on t_$i(i);"
13+
psql -h "${hostname}" -d dbname -U test_user -f -<<SQL
14+
create table t_$i as select i from generate_series(1, 1000) _(i);
15+
create index concurrently i_u_$i on t_$i(i);
16+
create index concurrently i_r_$i on t_$i(i);
17+
SQL
918
done

‎.gitlab-ci.yml‎

Lines changed: 20 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -31,12 +31,12 @@ variables:
3131
- echo "Test H003 Non indexed FKs"
3232
- psql -h postgres -d dbname -U test_user -f .ci/h003_step_1.sql
3333
- ./checkup -h postgres --username test_user --project test --dbname dbname -e 1 --file ./resources/checks/H003_non_indexed_fks.sh
34-
- data_dir=$(cat ./artifacts/test/nodes.json | jq -r '.last_check | .dir') && result=$(cat ./artifacts/test/json_reports/$data_dir/H003_non_indexed_fks.json | jq '.results .postgres .data') && ([[ "$result" == "[]" ]] || [[ "$result" == "null" ]]) && exit 301
34+
- data_dir=$(cat ./artifacts/test/nodes.json | jq -r '.last_check | .dir') && result=$(cat ./artifacts/test/json_reports/$data_dir/H003_non_indexed_fks.json | jq '.results .postgres .data .indexes') && ([[ "$result" == "[]" ]] || [[ "$result" == "null" ]]) && exit 301
3535
- psql -h postgres -d dbname -U test_user -f .ci/h003_step_2.sql
3636
- rm -Rf ./artifacts/
3737
- ./checkup -h postgres --username test_user --project test --dbname dbname -e 1 --file ./resources/checks/H003_non_indexed_fks.sh
38-
- data_dir=$(cat ./artifacts/test/nodes.json | jq -r '.last_check | .dir') && result=$(cat ./artifacts/test/json_reports/$data_dir/H003_non_indexed_fks.json | jq '.results .postgres .data') && echo "$result" && cat ./artifacts/test/json_reports/$data_dir/H003_non_indexed_fks.json && (! [[ "$result" == "[]" ]]) && exit 302
39-
- echo "H003 Passed"
38+
- data_dir=$(cat ./artifacts/test/nodes.json | jq -r '.last_check | .dir') && result=$(cat ./artifacts/test/json_reports/$data_dir/H003_non_indexed_fks.json | jq '.results .postgres .data .indexes') && echo "$result" && cat ./artifacts/test/json_reports/$data_dir/H003_non_indexed_fks.json && ([[ ! "$result" == "[]" ]] && [[ ! "$result" == "null" ]]) && exit 302
39+
- echo "H003 passed"
4040
- echo "Test H002 redundant indexes"
4141
- psql -h postgres -d dbname -U test_user -f .ci/test_db_dump.sql
4242
- ./checkup -h postgres --username test_user --project test --dbname dbname -e 1 --file ./resources/checks/H002_unused_indexes.sh
@@ -48,7 +48,23 @@ variables:
4848
- data_dir=$(cat ./artifacts/test/nodes.json | jq -r '.last_check | .dir') && result=$(cat ./artifacts/test/json_reports/$data_dir/H002_unused_indexes.json | jq '.results .postgres .data .redundant_indexes ."public.t_with_redundant_ref_idx_2"') && ( [[ "$result" == "" ]] || [[ "$result" == "null" ]]) && exit 206
4949
- data_dir=$(cat ./artifacts/test/nodes.json | jq -r '.last_check | .dir') && result=$(cat ./artifacts/test/json_reports/$data_dir/H002_unused_indexes.json | jq '.results .postgres .data .redundant_indexes ."exp_redundant.t_with_redundant_ref_idx_1"') && ([[ ! "$result" == "[]" ]] && [[ ! "$result" == "null" ]]) && exit 207
5050
- data_dir=$(cat ./artifacts/test/nodes.json | jq -r '.last_check | .dir') && result=$(cat ./artifacts/test/json_reports/$data_dir/H002_unused_indexes.json | jq '.results .postgres .data .redundant_indexes ."exp_redundant.t_with_redundant_ref_idx_2"') && ( [[ "$result" == "" ]] || [[ "$result" == "null" ]]) && exit 208
51-
- echo "H002 Passed"
51+
- echo "H002 passed"
52+
- ([[ "$CI_COMMIT_REF_NAME" != "master" ]]) && exit 0
53+
# Check small indexes
54+
- .ci/prepare_test_db.sh postgres
55+
- ./checkup -h postgres --username test_user --project test --dbname dbname -e 1 --file ./resources/checks/H002_unused_indexes.sh
56+
## unused
57+
- data_dir=$(cat ./artifacts/test/nodes.json | jq -r '.last_check | .dir') && result=$(cat "./artifacts/test/json_reports/$data_dir/H002_unused_indexes.json" | jq '.results .postgres .data .never_used_indexes ."public.i_u_12"') && ( [[ "$result" == "" ]] || [[ "$result" == "null" ]]) && exit 209
58+
## redundant
59+
- data_dir=$(cat ./artifacts/test/nodes.json | jq -r '.last_check | .dir') && result=$(cat "./artifacts/test/json_reports/$data_dir/H002_unused_indexes.json" | jq '.results .postgres .data .redundant_indexes ."public.i_r_12"') && ( [[ "$result" == "" ]] || [[ "$result" == "null" ]]) && exit 210
60+
- echo "Check small indexes in small db mode passed"
61+
- .ci/prepare_large_db.sh postgres
62+
- ./checkup -h postgres --username test_user --project test --dbname dbname -e 1 --file ./resources/checks/H002_unused_indexes.sh
63+
## unused
64+
- data_dir=$(cat ./artifacts/test/nodes.json | jq -r '.last_check | .dir') && result=$(cat "./artifacts/test/json_reports/$data_dir/H002_unused_indexes.json" | jq '.results .postgres .data .never_used_indexes ."public.i_u_12"') && ([[ ! "$result" == "[]" ]] && [[ ! "$result" == "null" ]]) && exit 211
65+
## redundant
66+
- data_dir=$(cat ./artifacts/test/nodes.json | jq -r '.last_check | .dir') && result=$(cat "./artifacts/test/json_reports/$data_dir/H002_unused_indexes.json" | jq '.results .postgres .data .redundant_indexes ."public.i_r_12"') && ([[ ! "$result" == "[]" ]] && [[ ! "$result" == "null" ]]) && exit 212
67+
- echo "Check small indexes in large db mode passed"
5268

5369
test-general:
5470
stage: test

‎checkup‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ SAFE_IFS="$IFS"
2222
ALL_ARGS="$@"
2323
OPTIONS_ERROR_EXIT="false"
2424
DEFAULT_LIST_LIMIT=50
25+
LARGE_DB_ITEMS_COUNT=100000
2526

2627
# Output styles (only BOLD is supported by default GNU screen)
2728
BOLD=`tput md 2>/dev/null` || :
@@ -662,6 +663,29 @@ is_in_recovery() {
662663
return 13
663664
}
664665

666+
#######################################
667+
# Check the number of objects in the database:
668+
# return 0 if the database has more than LARGE_DB_ITEMS_COUNT
669+
# (100000) objects, 1 otherwise.
670+
#
671+
# Do not use this function before 'host_pre_start_checks()'
672+
#
673+
# Globals:
674+
# _PSQL, LARGE_DB_ITEMS_COUNT
675+
# Arguments:
676+
# None
677+
# Returns:
678+
# Integer
679+
#######################################
680+
is_large_database() {
681+
local res="$(${CHECK_HOST_CMD} "${_PSQL} -c \"select count(*) from pg_class\"")"
682+
if [[ "$res" -gt $LARGE_DB_ITEMS_COUNT ]]; then
683+
return 0
684+
else
685+
return 1
686+
fi
687+
}
688+
665689
#######################################
666690
# Build pghrep - a .md report generator
667691
# Globals:
@@ -1049,6 +1073,14 @@ run_checks() {
10491073
"F004_heap_bloat" "F005_index_bloat" "F008_autovacuum_resource_usage" "H001_invalid_indexes" "H003_non_indexed_fks"
10501074
"L001_table_sizes" "L003_integer_in_pk")
10511075

1076+
if is_large_database; then
1077+
dbg "The database has more than ${LARGE_DB_ITEMS_COUNT} objects. To speed up the performance, small objects will not be analyzed."
1078+
export IS_LARGE_DB=1
1079+
else
1080+
dbg "The database has less than ${LARGE_DB_ITEMS_COUNT} objects."
1081+
export IS_LARGE_DB=0
1082+
fi
1083+
10521084
local output
10531085
for CURRENT_CHECK_FNAME in "${SCRIPT_DIR}"/resources/checks/*_*.sh; do
10541086
[[ -e "${CURRENT_CHECK_FNAME}" ]] || continue

‎pghrep/templates/F004.tpl‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
## Observations ##
66
Data collected: {{ DtFormat .timestamptz }}
77
Current database: {{ .database }}
8+
{{ if gt (Int (index (index (index .results .reorderedHosts.master) "data") "min_table_size_bytes")) 0 }}NOTICE: only tables larger than {{ ByteFormat (index (index (index .results .reorderedHosts.master) "data") "min_table_size_bytes") 0 }} are analyzed. {{end}}
89
{{ if .hosts.master }}
910
{{ if (index .results .hosts.master) }}
1011
### Master (`{{.hosts.master}}`) ###

‎pghrep/templates/F005.tpl‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,9 +5,11 @@
55
## Observations ##
66
Data collected: {{ DtFormat .timestamptz }}
77
Current database: {{ .database }}
8+
{{ if gt (Int (index (index (index .results .reorderedHosts.master) "data") "min_table_size_bytes")) 0 }}NOTICE: only tables larger than {{ ByteFormat (index (index (index .results .reorderedHosts.master) "data") "min_table_size_bytes") 0 }} are analyzed. {{end}}
89
{{ if .hosts.master }}
910
{{ if (index .results .hosts.master)}}
1011
{{ if (index (index .results .hosts.master) "data") }}
12+
{{ if (index (index (index .results .hosts.master) "data") "index_bloat") }}
1113
### Master (`{{.hosts.master}}`) ###
1214
{{ if ge (len (index (index (index $.results $.hosts.master) "data") "index_bloat")) .LISTLIMIT }}The list is limited to {{.LISTLIMIT}} items. Total: {{ Sub (len (index (index (index $.results $.hosts.master) "data") "index_bloat")) 1 }}. {{ end }}
1315

@@ -38,6 +40,9 @@ Current database: {{ .database }}
3840
{{- if gt (Int (index (index (index .results .hosts.master) "data") "overrided_settings_count")) 0 }}
3941
\* This table has specific autovacuum settings. See 'F001 Autovacuum: Current settings'
4042
{{- end }}
43+
{{- else -}}{{/*Index bloat*/}}
44+
Nothing found
45+
{{- end }}{{/*Index bloat*/}}
4146
{{- else -}}{{/*Master data*/}}
4247
Nothing found
4348
{{- end }}{{/*Master data*/}}

‎pghrep/templates/H002.tpl‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,8 @@ Current database: {{ .database }}
88
Stats reset: {{ (index (index (index .results .reorderedHosts.master) "data") "database_stat").stats_age }} ago ({{ DtFormat (index (index (index .results .reorderedHosts.master) "data") "database_stat").stats_reset }})
99
{{- if le (Int (index (index (index .results .reorderedHosts.master) "data") "database_stat").days) 30 }}
1010
:warning: Statistics age is less than 30 days. Make decisions on index cleanup with caution!
11-
{{- end }}
12-
NOTICE: only indexes larger than 800 kiB are analyzed.
11+
{{- end }}
12+
{{ if gt (Int (index (index (index .results .reorderedHosts.master) "data") "min_index_size_bytes")) 0 }}NOTICE: only indexes larger than {{ ByteFormat (index (index (index .results .reorderedHosts.master) "data") "min_index_size_bytes") 0 }} are analyzed. {{end}}
1313
### Never Used Indexes ###
1414
{{ if (index (index (index .results .reorderedHosts.master) "data") "never_used_indexes") }}
1515
{{ if ge (len (index (index (index .results .reorderedHosts.master) "data") "never_used_indexes")) .LISTLIMIT }}The list is limited to {{.LISTLIMIT}} items. Total: {{ Sub (len (index (index (index .results .reorderedHosts.master) "data") "never_used_indexes")) 1 }}.{{ end }}

‎pghrep/templates/H003.tpl‎

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,17 +3,19 @@
33
## Observations ##
44
Data collected: {{ DtFormat .timestamptz }}
55
Current database: {{ .database }}
6+
{{ if gt (Int (index (index (index .results .reorderedHosts.master) "data") "min_index_size_bytes")) 0 }}NOTICE: only indexes larger than {{ ByteFormat (index (index (index .results .reorderedHosts.master) "data") "min_index_size_bytes") 0 }} on tables larger than {{ ByteFormat (index (index (index .results .reorderedHosts.master) "data") "min_table_size_bytes") 0 }} are analyzed. {{end}}
67
{{ if .hosts.master }}
78
### Master (`{{.hosts.master}}`) ###
89
{{ if (index .results .hosts.master) }}
910
{{ if (index (index .results .hosts.master) "data") }}
11+
{{ if (index (index (index .results .hosts.master) "data") "indexes") }}
1012
{{ if ge (len (index (index .results .hosts.master) "data")) .LISTLIMIT }}The list is limited to {{.LISTLIMIT}} items. Total: {{ Sub (len (index (index .results .hosts.master) "data")) 1 }}.{{ end }}
1113

1214
| Num | Schema name | Table name | FK name | Issue | Table mb | writes | Table scans | Parent name | Parent mb | Parent writes | Cols list | Indexdef |
13-
|-----|-------------|------------|---------|-------|----------|--------|-------------|-------------|-----------|---------------|-----------|----------|
14-
{{ range $i, $key := (index (index (index .results .hosts.master) "data") "_keys") }}
15+
|----|-------------|------------|---------|-------|----------|--------|-------------|-------------|-----------|---------------|-----------|----------|
16+
{{ range $i, $key := ( index (index (index (index .results .hosts.master) "data") "indexes") "_keys") }}
1517
{{- if lt $i $.LISTLIMIT -}}
16-
{{- $value := (index (index (index $.results $.hosts.master) "data") $key) -}}
18+
{{- $value := (index (index (index (index $.results $.hosts.master) "data") "indexes") $key) -}}
1719
|{{ $key }} | `{{ $value.schema_name }}` | `{{ $value.table_name }}` | `{{- $value.fk_name }}` |
1820
{{- $value.issue }} |
1921
{{- $value.table_mb }} |
@@ -27,7 +29,10 @@ Current database: {{ .database }}
2729
{{/* if limit list */}}{{ end -}}
2830
{{ end }}{{/* range */}}
2931
{{ else }}
30-
No data
32+
Nothing found
33+
{{- end -}}{{/* if data.indexes */}}
34+
{{ else }}
35+
Nothing found
3136
{{- end -}}{{/* if data */}}
3237
{{- end -}}{{/* if master results */}}
3338
{{ end }}{{/* if .host.master */}}

‎pghrep/templates/L003.tpl‎

Lines changed: 12 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -3,20 +3,26 @@
33
## Observations ##
44
Data collected: {{ DtFormat .timestamptz }}
55
Current database: {{ .database }}
6-
{{ if .hosts.master }}
7-
{{ if (index .results .hosts.master)}}
8-
{{ if (index (index .results .hosts.master) "data") }}
6+
{{- if .hosts.master }}
7+
{{- if (index .results .hosts.master)}}
8+
{{- if (index (index .results .hosts.master) "data") }}
9+
{{ if gt (Int (index (index (index .results .reorderedHosts.master) "data") "min_table_size_bytes")) 0 }}NOTICE: only tables larger than {{ ByteFormat (index (index (index .results .reorderedHosts.master) "data") "min_table_size_bytes") 0 }} are analyzed.
10+
{{end}}
11+
{{- if (index (index (index .results .hosts.master) "data") "tables") }}
912
### Master (`{{.hosts.master}}`) ###
1013
{{ if ge (len (index (index .results .hosts.master) "data")) .LISTLIMIT }}The list is limited to {{.LISTLIMIT}} items. Total: {{ Sub (len (index (index .results .hosts.master) "data")) 1 }}.{{ end }}
1114

1215
| Table | PK | Type | Current max value | &#9660;&nbsp;Capacity used, % |
1316
|------|----|------|-------------------|-------------------------------|
14-
{{ range $i, $key := (index (index (index .results .hosts.master) "data") "_keys") }}
17+
{{ range $i, $key := (index (index (index (index .results .hosts.master) "data") "tables") "_keys") }}
1518
{{- if lt $i $.LISTLIMIT -}}
16-
{{- $value := (index (index (index $.results $.hosts.master) "data") $key) -}}
19+
{{- $value := (index (index (index (index $.results $.hosts.master) "data") "tables") $key) -}}
1720
|`{{ index $value "table"}}` | `{{ index $value "pk"}}` | {{ index $value "type"}} | {{- RawIntFormat (index $value "current_max_value")}} | {{ index $value "capacity_used_percent"}}|
18-
{{- end}}
1921
{{ end }}
22+
{{- end }}
23+
{{- else -}}{{/*Tables data*/}}
24+
Nothing found
25+
{{- end }}{{/*Tables data*/}}
2026
{{- else -}}{{/*Master data*/}}
2127
Nothing found
2228
{{- end }}{{/*Master data*/}}

‎resources/checks/F004_heap_bloat.sh‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,9 @@
1+
if [[ ! -z ${IS_LARGE_DB+x} ]] && [[ ${IS_LARGE_DB} == "1" ]]; then
2+
MIN_RELPAGES=100
3+
else
4+
MIN_RELPAGES=0
5+
fi
6+
17
${CHECK_HOST_CMD} "${_PSQL} -f -" <<SQL
28
with data as (
39
with overrided_tables as (
@@ -31,7 +37,7 @@ with data as (
3137
join pg_namespace as ns on ns.oid = tbl.relnamespace
3238
join pg_stats as s on s.schemaname = ns.nspname and s.tablename = tbl.relname and not s.inherited and s.attname = att.attname
3339
left join pg_class as toast on tbl.reltoastrelid = toast.oid
34-
where att.attnum > 0 and not att.attisdropped and s.schemaname not in ('pg_catalog', 'information_schema') and tbl.relpages > 10
40+
where att.attnum > 0 and not att.attisdropped and s.schemaname not in ('pg_catalog', 'information_schema') and tbl.relpages > ${MIN_RELPAGES}
3541
group by 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, tbl.relhasoids
3642
order by 2, 3
3743
), step2 as (
@@ -136,7 +142,9 @@ select
136142
'overrided_settings_count',
137143
(select count(1) from limited_data where overrided_settings = true),
138144
'database_size_bytes',
139-
(select pg_database_size(current_database()))
145+
(select pg_database_size(current_database())),
146+
'min_table_size_bytes',
147+
(select ${MIN_RELPAGES} * 8192)
140148
)
141149
SQL
142150

0 commit comments

Comments
 (0)