Skip to content
This repository was archived by the owner on Jul 6, 2021. It is now read-only.

Commit 13bb8a4

Browse files
committed
It turned out, that database names are being read from config file wrongly when database name ends with s.
Loading configuration has been reworked on Golang and the format of the config file has been changed a little bit. Now not available otpions differs from CLI options in config file.
2 parents d1d0e44 + abcd13e commit 13bb8a4

File tree

8 files changed

+103
-98
lines changed

8 files changed

+103
-98
lines changed

.ci/test.yml

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
- project: test
2+
hostname: postgres
3+
username: test_user
4+
dbname: dbname
5+
epoch: 1

.gitlab-ci.yml

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -30,16 +30,16 @@ variables:
3030
- psql -h postgres -d dbname -U test_user -c "SELECT version();"
3131
- echo "Test H003 Non indexed FKs"
3232
- psql -h postgres -d dbname -U test_user -f .ci/h003_step_1.sql
33-
- ./checkup collect -h postgres --username test_user --project test --dbname dbname -e 1 --file ./resources/checks/H003_non_indexed_fks.sh
33+
- ./checkup collect -c .ci/test.yml --file ./resources/checks/H003_non_indexed_fks.sh
3434
- data_dir=$(cat ./artifacts/test/nodes.json | jq -r '.last_check | .dir') && result=$(cat ./artifacts/test/json_reports/$data_dir/H003_non_indexed_fks.json | jq '.results .postgres .data .indexes') && ([[ "$result" == "[]" ]] || [[ "$result" == "null" ]]) && exit 301
3535
- psql -h postgres -d dbname -U test_user -f .ci/h003_step_2.sql
3636
- rm -Rf ./artifacts/
37-
- ./checkup collect -h postgres --username test_user --project test --dbname dbname -e 1 --file ./resources/checks/H003_non_indexed_fks.sh
37+
- ./checkup collect -c .ci/test.yml --file ./resources/checks/H003_non_indexed_fks.sh
3838
- data_dir=$(cat ./artifacts/test/nodes.json | jq -r '.last_check | .dir') && result=$(cat ./artifacts/test/json_reports/$data_dir/H003_non_indexed_fks.json | jq '.results .postgres .data .indexes') && echo "$result" && cat ./artifacts/test/json_reports/$data_dir/H003_non_indexed_fks.json && ([[ ! "$result" == "[]" ]] && [[ ! "$result" == "null" ]]) && exit 302
3939
- echo "H003 passed"
4040
- echo "Test H002 redundant indexes"
4141
- psql -h postgres -d dbname -U test_user -f .ci/test_db_dump.sql
42-
- ./checkup collect -h postgres --username test_user --project test --dbname dbname -e 1 --file ./resources/checks/H002_unused_indexes.sh
42+
- ./checkup collect -c .ci/test.yml --file ./resources/checks/H002_unused_indexes.sh
4343
- data_dir=$(cat ./artifacts/test/nodes.json | jq -r '.last_check | .dir') && result=$(cat ./artifacts/test/json_reports/$data_dir/H002_unused_indexes.json | jq '.results .postgres .data .redundant_indexes ."public.t_with_redundant_idx_id"') && ( [[ "$result" == "" ]] || [[ "$result" == "null" ]]) && exit 201
4444
- data_dir=$(cat ./artifacts/test/nodes.json | jq -r '.last_check | .dir') && result=$(cat ./artifacts/test/json_reports/$data_dir/H002_unused_indexes.json | jq '.results .postgres .data .redundant_indexes ."public.t_with_redundant_idx_f1"') && ( [[ "$result" == "" ]] || [[ "$result" == "null" ]]) && exit 202
4545
- data_dir=$(cat ./artifacts/test/nodes.json | jq -r '.last_check | .dir') && result=$(cat ./artifacts/test/json_reports/$data_dir/H002_unused_indexes.json | jq '.results .postgres .data .redundant_indexes ."public.t_with_redundant_idx_f1_uniq"') && ([[ ! "$result" == "[]" ]] && [[ ! "$result" == "null" ]]) && exit 203
@@ -49,25 +49,25 @@ variables:
4949
- data_dir=$(cat ./artifacts/test/nodes.json | jq -r '.last_check | .dir') && result=$(cat ./artifacts/test/json_reports/$data_dir/H002_unused_indexes.json | jq '.results .postgres .data .redundant_indexes ."exp_redundant.t_with_redundant_ref_idx_1"') && ([[ ! "$result" == "[]" ]] && [[ ! "$result" == "null" ]]) && exit 207
5050
- data_dir=$(cat ./artifacts/test/nodes.json | jq -r '.last_check | .dir') && result=$(cat ./artifacts/test/json_reports/$data_dir/H002_unused_indexes.json | jq '.results .postgres .data .redundant_indexes ."exp_redundant.t_with_redundant_ref_idx_2"') && ( [[ "$result" == "" ]] || [[ "$result" == "null" ]]) && exit 208
5151
- echo "H002 passed"
52-
- ./checkup -h postgres --username test_user --project test --dbname dbname -e 1 --file ./resources/checks/F005_index_bloat.sh
53-
- ./checkup -h postgres --username test_user --project test --dbname dbname -e 1 --file ./resources/checks/L001_table_sizes.sh
52+
- ./checkup -c .ci/test.yml --file ./resources/checks/F005_index_bloat.sh
53+
- ./checkup -c .ci/test.yml --file ./resources/checks/L001_table_sizes.sh
5454
- data_dir=$(cat ./artifacts/test/nodes.json | jq -r '.last_check | .dir') && f005_indexes_size=$(cat ./artifacts/test/json_reports/$data_dir/F005_index_bloat.json | jq '.results .postgres .data .index_bloat_total .real_size_bytes_sum') && echo $f005_indexes_size
5555
- data_dir=$(cat ./artifacts/test/nodes.json | jq -r '.last_check | .dir') && l001_indexes_size=$(cat ./artifacts/test/json_reports/$data_dir/L001_table_sizes.json | jq '.results .postgres .data .tables_data_total .indexes_size_bytes_sum') && echo $l001_indexes_size
5656
- diff=$((f005_indexes_size - l001_indexes_size)) && diff=${diff#-} && echo $diff && ratio=$((diff * 100 / l001_indexes_size)) && echo $ratio && ([[ $ratio -gt "5" ]]) && exit 303
5757
- echo "F005/L001 Total indexes size passed"
58-
- ./checkup -h postgres --username test_user --project test --dbname dbname -e 1 --file ./resources/checks/F004_heap_bloat.sh
58+
- ./checkup -c .ci/test.yml --file ./resources/checks/F004_heap_bloat.sh
5959
- data_dir=$(cat ./artifacts/test/nodes.json | jq -r '.last_check | .dir') && msg=$(grep "can be reduced 0.00 times" ./artifacts/test/md_reports/$data_dir/F004.md) && [[ ! -z "$msg" ]] && exit 304
6060
- ([[ "$CI_COMMIT_REF_NAME" != "master" ]]) && exit 0
6161
# Check small indexes
6262
- .ci/prepare_test_db.sh postgres
63-
- ./checkup -h postgres --username test_user --project test --dbname dbname -e 1 --file ./resources/checks/H002_unused_indexes.sh
63+
- ./checkup -c .ci/test.yml --file ./resources/checks/H002_unused_indexes.sh
6464
## unused
6565
- data_dir=$(cat ./artifacts/test/nodes.json | jq -r '.last_check | .dir') && result=$(cat "./artifacts/test/json_reports/$data_dir/H002_unused_indexes.json" | jq '.results .postgres .data .never_used_indexes ."public.i_u_12"') && ( [[ "$result" == "" ]] || [[ "$result" == "null" ]]) && exit 209
6666
## redundant
6767
- data_dir=$(cat ./artifacts/test/nodes.json | jq -r '.last_check | .dir') && result=$(cat "./artifacts/test/json_reports/$data_dir/H002_unused_indexes.json" | jq '.results .postgres .data .redundant_indexes ."public.i_r_12"') && ( [[ "$result" == "" ]] || [[ "$result" == "null" ]]) && exit 210
6868
- echo "Check small indexes in small db mode passed"
6969
- .ci/prepare_large_db.sh postgres
70-
- ./checkup -h postgres --username test_user --project test --dbname dbname -e 1 --file ./resources/checks/H002_unused_indexes.sh
70+
- ./checkup -c .ci/test.yml --file ./resources/checks/H002_unused_indexes.sh
7171
## unused
7272
- data_dir=$(cat ./artifacts/test/nodes.json | jq -r '.last_check | .dir') && result=$(cat "./artifacts/test/json_reports/$data_dir/H002_unused_indexes.json" | jq '.results .postgres .data .never_used_indexes ."public.i_u_12"') && ([[ ! "$result" == "[]" ]] && [[ ! "$result" == "null" ]]) && exit 211
7373
## redundant

checkup

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -193,10 +193,12 @@ load_config_params () {
193193
dbg "Load params from config file"
194194
config_filename=${1}
195195
if [[ ! -f "$config_filename" ]]; then
196-
msg "Config filename ${config_filename} not found."
196+
err "Config filename ${config_filename} not found."
197197
exit 1
198198
fi
199-
source ${BASH_SOURCE%/*}/resources/parse_yaml.sh $config_filename "CONFIG_"
199+
200+
eval $(${PGHREP_BIN} --mode loadcfg --path $config_filename 2>/dev/null)
201+
200202
for i in $(seq 0 ${CLI_ARGS_POSSIBLE}); do
201203
var_name="CONFIG__"${FULL_NAME[$i]}
202204
var_name=${var_name//-/_}
@@ -740,7 +742,7 @@ build_pghrep() {
740742
dbg "building pghrep..."
741743
local save_pwd="${PWD}"
742744
cd "${SCRIPT_DIR}/pghrep"
743-
make install buildplugins main >/dev/null
745+
make install main >/dev/null
744746
cd "${save_pwd}"
745747
export PGHREP_BIN="${SCRIPT_DIR}/pghrep/bin/pghrep"
746748
dbg "done building pghrep"
@@ -1357,6 +1359,7 @@ main() {
13571359
mode="$1"
13581360
fi
13591361

1362+
build_pghrep
13601363
read_possible_args
13611364
process_cli_args "$@"
13621365
validate_args
@@ -1368,19 +1371,16 @@ main() {
13681371
;;
13691372
"process" )
13701373
check_bin_deps
1371-
build_pghrep
13721374
run_process
13731375
;;
13741376
"upload" )
13751377
check_bin_deps
1376-
build_pghrep
13771378
run_upload
13781379
;;
13791380
"run" )
13801381
host_pre_start_checks
13811382
run_checks
13821383
check_bin_deps
1383-
build_pghrep
13841384
run_process
13851385
;;
13861386
esac

pghrep/Makefile

Lines changed: 2 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -23,18 +23,13 @@ LDFLAGS = -ldflags "-s -w \
2323
PLUGINS_SRC := $(wildcard plugins/*.go)
2424

2525
# Build the project
26-
all: clean install vet buildplugins main
26+
all: clean install vet main
2727

2828
install:
2929
go get golang.org/x/net/html
3030
go get github.com/dustin/go-humanize
3131
go get gopkg.in/yaml.v2
3232

33-
buildplugins:
34-
@for f in $(basename $(patsubst plugins/%.go,%,$(PLUGINS_SRC))); do \
35-
go build ${LDFLAGS} -buildmode=plugin -o bin/$$f.so ./plugins/$$f.go ; \
36-
done
37-
3833
main:
3934
GOARCH=${GOARCH} go build ${LDFLAGS} -o bin/${BINARY} ./src/
4035

@@ -44,11 +39,6 @@ test:
4439
go test ./src/checkup/*/
4540

4641
vet:
47-
# Command go vet examine all go files in a single scope, which isn't suitable for plugins.
48-
# We will run it separately for plugins and main sources.
49-
@for f in $(basename $(patsubst plugins/%.go,%,$(PLUGINS_SRC))); do \
50-
go vet ./plugins/$$f.go || exit 1 ; \
51-
done
5242
go vet ./src/...
5343

5444
fmt:
@@ -60,5 +50,5 @@ clean:
6050
run:
6151
go run ${LDFLAGS} ./src/*
6252

63-
.PHONY: all buildplugins main test vet fmt clean run
53+
.PHONY: all main test vet fmt clean run
6454

pghrep/src/config.go

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
package main
2+
3+
import (
4+
"fmt"
5+
"io/ioutil"
6+
"strings"
7+
8+
"gopkg.in/yaml.v2"
9+
)
10+
11+
const CONFIG_PARAM_PREFIX = "CONFIG"
12+
13+
type CheckupProjectConfig map[string]string
14+
15+
type CheckupConfig []CheckupProjectConfig
16+
17+
func loadConfig(path string) (CheckupConfig, error) {
18+
var config CheckupConfig
19+
if !FileExists(path) {
20+
return config, fmt.Errorf("Config file '%s' not found.", path)
21+
}
22+
23+
data, cerr := ioutil.ReadFile(path)
24+
if cerr != nil {
25+
return config, fmt.Errorf("Cannot read file '%s'. %s", path, cerr)
26+
}
27+
28+
uerr := yaml.Unmarshal([]byte(data), &config)
29+
if uerr != nil {
30+
return config, fmt.Errorf("Cannot parse config file: '%s'. %s", path, uerr)
31+
}
32+
33+
return config, nil
34+
}
35+
36+
func outputConfig(config CheckupConfig) {
37+
if len(config) > 0 {
38+
// Now we support only one project config
39+
for key, value := range config[0] {
40+
fmt.Printf("%s__%s=\"%s\"\n", CONFIG_PARAM_PREFIX, strings.Replace(key, "-", "_", -1), value)
41+
}
42+
}
43+
}

pghrep/src/log/log.go

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -65,21 +65,21 @@ func prepareMessage(v ...interface{}) string {
6565

6666
// Output message
6767
func Msg(v ...interface{}) {
68-
log.Println(">>> INFO: " + prepareMessage(v...))
68+
log.Println("INFO: " + prepareMessage(v...))
6969
}
7070

7171
// Output debug message
7272
func Dbg(v ...interface{}) {
7373
if DEBUG {
74-
log.Println(">>> DEBUG: " + prepareMessage(v...))
74+
log.Println("DEBUG: " + prepareMessage(v...))
7575
}
7676
}
7777

7878
// Output error message
7979
func Err(v ...interface{}) {
80-
log.New(os.Stderr, "", 0).Println(">>> ERROR: " + prepareMessage(v...))
80+
log.New(os.Stderr, "", 0).Println("ERROR: " + prepareMessage(v...))
8181
}
8282

8383
func Fatal(v ...interface{}) {
84-
log.Fatal(">>> ERROR: " + prepareMessage(v...))
84+
log.Fatal("ERROR: " + prepareMessage(v...))
8585
}

pghrep/src/main.go

Lines changed: 35 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -109,7 +109,7 @@ func LoadJsonFile(filePath string) map[string]interface{} {
109109
if FileExists(filePath) {
110110
fileContent, err := ioutil.ReadFile(GetFilePath(filePath)) // just pass the file name
111111
if err != nil {
112-
log.Err("Can't read file: ", filePath, err)
112+
log.Err("Can't read file:", filePath, err)
113113
return nil
114114
}
115115

@@ -191,7 +191,7 @@ func getRawData(data map[string]interface{}) {
191191
// for every host get data
192192
var rawData []interface{}
193193
hosts := pyraconv.ToInterfaceMap(data["hosts"])
194-
log.Dbg("Data hosts: ", hosts)
194+
log.Dbg("Data hosts:", hosts)
195195
results := pyraconv.ToInterfaceMap(data["results"])
196196
masterName := pyraconv.ToString(hosts["master"])
197197
masterResults := pyraconv.ToInterfaceMap(results[masterName])
@@ -300,7 +300,7 @@ func generateMdReport(checkId string, reportFilename string, reportData map[stri
300300
reporTpl := templates.Lookup(reportFileName)
301301
data := reportData
302302
if reporTpl == nil {
303-
log.Err("Template " + checkId + ".tpl not found.")
303+
log.Err("Template " + checkId + ".tpl not found")
304304
getRawData(data)
305305
reportFileName = "raw.tpl"
306306
reporTpl = templates.Lookup(reportFileName)
@@ -415,6 +415,7 @@ func main() {
415415
projectDataPtr := flag.String("project", "", "target project used during uploading")
416416
pathDataPtr := flag.String("path", "", "path to artifacts directory used during uploading")
417417
apiUrlDataPtr := flag.String("apiurl", "", "API URL for reports uploading")
418+
418419
flag.Parse()
419420
checkData = *checkDataPtr
420421

@@ -431,27 +432,24 @@ func main() {
431432
log.DEBUG = false
432433
}
433434

434-
if *modeDataPtr == "upload" {
435+
switch *modeDataPtr {
436+
case "upload":
435437
token := *tokenDataPtr
436438
project := *projectDataPtr
437439
path := *pathDataPtr
438440
apiUrl := *apiUrlDataPtr
439441

440442
if len(token) == 0 {
441-
log.Err("Token is not defined")
442-
return
443+
log.Fatal("Token is not defined")
443444
}
444445
if len(apiUrl) == 0 {
445-
log.Err("API URL is not defined")
446-
return
446+
log.Fatal("API URL is not defined")
447447
}
448448
if len(project) == 0 {
449-
log.Err("Project (for reports uploading) is not defined")
450-
return
449+
log.Fatal("Project (for reports uploading) is not defined")
451450
}
452451
if len(path) == 0 {
453-
log.Err("Artifacts directory is not defined")
454-
return
452+
log.Fatal("Artifacts directory is not defined")
455453
}
456454

457455
err := upload.UploadReport(apiUrl, token, project, path)
@@ -460,28 +458,47 @@ func main() {
460458
os.Exit(1)
461459
}
462460

461+
return
462+
case "loadcfg":
463+
path := *pathDataPtr
464+
if len(path) == 0 {
465+
log.Fatal("Config path is not defined")
466+
}
467+
468+
config, err := loadConfig(path)
469+
if err != nil {
470+
log.Fatal(fmt.Sprintf("Cannot load config. %s", err))
471+
os.Exit(1)
472+
}
473+
474+
if len(config) == 0 {
475+
log.Fatal(fmt.Sprintf("Config '%s' is empty", path))
476+
}
477+
478+
outputConfig(config)
479+
463480
return
464481
}
465482

466483
if FileExists(checkData) {
467484
resultData = LoadJsonFile(checkData)
468485

469486
if resultData == nil {
470-
log.Fatal("ERROR: File given by --checkdata content wrong json data.")
487+
log.Fatal("File given by --checkdata content wrong json data")
471488
return
472489
}
473490

474491
resultData["source_path_full"] = checkData
475492
resultData["source_path_parts"] = strings.Split(checkData, string(os.PathSeparator))
476493
} else {
477-
log.Err("ERROR: File given by --checkdata not found")
494+
log.Err("File given by --checkdata not found")
478495
return
479496
}
480497

481498
if resultData != nil {
482499
checkId = pyraconv.ToString(resultData["checkId"])
483500
} else {
484-
log.Fatal("ERROR: Content given by --checkdata is wrong json content.")
501+
log.Fatal("Content defined by '--checkdata' is invalid JSON")
485502
}
486503

487504
checkId = strings.ToUpper(checkId)
@@ -490,7 +507,7 @@ func main() {
490507

491508
err := reorderHosts(resultData)
492509
if err != nil {
493-
log.Err("There is no data to generate the report.")
510+
log.Err("There is no data to generate the report")
494511
}
495512

496513
config := cfg.NewConfig()
@@ -511,13 +528,13 @@ func main() {
511528

512529
reportDone := generateMdReports(checkId, resultData, outputDir)
513530
if !reportDone {
514-
log.Fatal("Cannot generate report. Data file or template is wrong.")
531+
log.Fatal("Cannot generate report. Data file or template is wrong")
515532
}
516533
}
517534

518535
func preprocessReportData(checkId string, config cfg.Config,
519536
data map[string]interface{}) error {
520-
switch checkId {
537+
switch strings.ToUpper(checkId) {
521538
case "A002":
522539
// Try to load actual Postgres versions.
523540
err := config.LoadVersions()

0 commit comments

Comments
 (0)