Skip to content
This repository was archived by the owner on Jul 6, 2021. It is now read-only.

Commit abcd13e

Browse files
committed
fix: loading config from file reworked
1 parent d1d0e44 commit abcd13e

File tree

8 files changed

+103
-98
lines changed

8 files changed

+103
-98
lines changed

.ci/test.yml

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
- project: test
2+
hostname: postgres
3+
username: test_user
4+
dbname: dbname
5+
epoch: 1

.gitlab-ci.yml

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -30,16 +30,16 @@ variables:
3030
- psql -h postgres -d dbname -U test_user -c "SELECT version();"
3131
- echo "Test H003 Non indexed FKs"
3232
- psql -h postgres -d dbname -U test_user -f .ci/h003_step_1.sql
33-
- ./checkup collect -h postgres --username test_user --project test --dbname dbname -e 1 --file ./resources/checks/H003_non_indexed_fks.sh
33+
- ./checkup collect -c .ci/test.yml --file ./resources/checks/H003_non_indexed_fks.sh
3434
- data_dir=$(cat ./artifacts/test/nodes.json | jq -r '.last_check | .dir') && result=$(cat ./artifacts/test/json_reports/$data_dir/H003_non_indexed_fks.json | jq '.results .postgres .data .indexes') && ([[ "$result" == "[]" ]] || [[ "$result" == "null" ]]) && exit 301
3535
- psql -h postgres -d dbname -U test_user -f .ci/h003_step_2.sql
3636
- rm -Rf ./artifacts/
37-
- ./checkup collect -h postgres --username test_user --project test --dbname dbname -e 1 --file ./resources/checks/H003_non_indexed_fks.sh
37+
- ./checkup collect -c .ci/test.yml --file ./resources/checks/H003_non_indexed_fks.sh
3838
- data_dir=$(cat ./artifacts/test/nodes.json | jq -r '.last_check | .dir') && result=$(cat ./artifacts/test/json_reports/$data_dir/H003_non_indexed_fks.json | jq '.results .postgres .data .indexes') && echo "$result" && cat ./artifacts/test/json_reports/$data_dir/H003_non_indexed_fks.json && ([[ ! "$result" == "[]" ]] && [[ ! "$result" == "null" ]]) && exit 302
3939
- echo "H003 passed"
4040
- echo "Test H002 redundant indexes"
4141
- psql -h postgres -d dbname -U test_user -f .ci/test_db_dump.sql
42-
- ./checkup collect -h postgres --username test_user --project test --dbname dbname -e 1 --file ./resources/checks/H002_unused_indexes.sh
42+
- ./checkup collect -c .ci/test.yml --file ./resources/checks/H002_unused_indexes.sh
4343
- data_dir=$(cat ./artifacts/test/nodes.json | jq -r '.last_check | .dir') && result=$(cat ./artifacts/test/json_reports/$data_dir/H002_unused_indexes.json | jq '.results .postgres .data .redundant_indexes ."public.t_with_redundant_idx_id"') && ( [[ "$result" == "" ]] || [[ "$result" == "null" ]]) && exit 201
4444
- data_dir=$(cat ./artifacts/test/nodes.json | jq -r '.last_check | .dir') && result=$(cat ./artifacts/test/json_reports/$data_dir/H002_unused_indexes.json | jq '.results .postgres .data .redundant_indexes ."public.t_with_redundant_idx_f1"') && ( [[ "$result" == "" ]] || [[ "$result" == "null" ]]) && exit 202
4545
- data_dir=$(cat ./artifacts/test/nodes.json | jq -r '.last_check | .dir') && result=$(cat ./artifacts/test/json_reports/$data_dir/H002_unused_indexes.json | jq '.results .postgres .data .redundant_indexes ."public.t_with_redundant_idx_f1_uniq"') && ([[ ! "$result" == "[]" ]] && [[ ! "$result" == "null" ]]) && exit 203
@@ -49,25 +49,25 @@ variables:
4949
- data_dir=$(cat ./artifacts/test/nodes.json | jq -r '.last_check | .dir') && result=$(cat ./artifacts/test/json_reports/$data_dir/H002_unused_indexes.json | jq '.results .postgres .data .redundant_indexes ."exp_redundant.t_with_redundant_ref_idx_1"') && ([[ ! "$result" == "[]" ]] && [[ ! "$result" == "null" ]]) && exit 207
5050
- data_dir=$(cat ./artifacts/test/nodes.json | jq -r '.last_check | .dir') && result=$(cat ./artifacts/test/json_reports/$data_dir/H002_unused_indexes.json | jq '.results .postgres .data .redundant_indexes ."exp_redundant.t_with_redundant_ref_idx_2"') && ( [[ "$result" == "" ]] || [[ "$result" == "null" ]]) && exit 208
5151
- echo "H002 passed"
52-
- ./checkup -h postgres --username test_user --project test --dbname dbname -e 1 --file ./resources/checks/F005_index_bloat.sh
53-
- ./checkup -h postgres --username test_user --project test --dbname dbname -e 1 --file ./resources/checks/L001_table_sizes.sh
52+
- ./checkup -c .ci/test.yml --file ./resources/checks/F005_index_bloat.sh
53+
- ./checkup -c .ci/test.yml --file ./resources/checks/L001_table_sizes.sh
5454
- data_dir=$(cat ./artifacts/test/nodes.json | jq -r '.last_check | .dir') && f005_indexes_size=$(cat ./artifacts/test/json_reports/$data_dir/F005_index_bloat.json | jq '.results .postgres .data .index_bloat_total .real_size_bytes_sum') && echo $f005_indexes_size
5555
- data_dir=$(cat ./artifacts/test/nodes.json | jq -r '.last_check | .dir') && l001_indexes_size=$(cat ./artifacts/test/json_reports/$data_dir/L001_table_sizes.json | jq '.results .postgres .data .tables_data_total .indexes_size_bytes_sum') && echo $l001_indexes_size
5656
- diff=$((f005_indexes_size - l001_indexes_size)) && diff=${diff#-} && echo $diff && ratio=$((diff * 100 / l001_indexes_size)) && echo $ratio && ([[ $ratio -gt "5" ]]) && exit 303
5757
- echo "F005/L001 Total indexes size passed"
58-
- ./checkup -h postgres --username test_user --project test --dbname dbname -e 1 --file ./resources/checks/F004_heap_bloat.sh
58+
- ./checkup -c .ci/test.yml --file ./resources/checks/F004_heap_bloat.sh
5959
- data_dir=$(cat ./artifacts/test/nodes.json | jq -r '.last_check | .dir') && msg=$(grep "can be reduced 0.00 times" ./artifacts/test/md_reports/$data_dir/F004.md) && [[ ! -z "$msg" ]] && exit 304
6060
- ([[ "$CI_COMMIT_REF_NAME" != "master" ]]) && exit 0
6161
# Check small indexes
6262
- .ci/prepare_test_db.sh postgres
63-
- ./checkup -h postgres --username test_user --project test --dbname dbname -e 1 --file ./resources/checks/H002_unused_indexes.sh
63+
- ./checkup -c .ci/test.yml --file ./resources/checks/H002_unused_indexes.sh
6464
## unused
6565
- data_dir=$(cat ./artifacts/test/nodes.json | jq -r '.last_check | .dir') && result=$(cat "./artifacts/test/json_reports/$data_dir/H002_unused_indexes.json" | jq '.results .postgres .data .never_used_indexes ."public.i_u_12"') && ( [[ "$result" == "" ]] || [[ "$result" == "null" ]]) && exit 209
6666
## redundant
6767
- data_dir=$(cat ./artifacts/test/nodes.json | jq -r '.last_check | .dir') && result=$(cat "./artifacts/test/json_reports/$data_dir/H002_unused_indexes.json" | jq '.results .postgres .data .redundant_indexes ."public.i_r_12"') && ( [[ "$result" == "" ]] || [[ "$result" == "null" ]]) && exit 210
6868
- echo "Check small indexes in small db mode passed"
6969
- .ci/prepare_large_db.sh postgres
70-
- ./checkup -h postgres --username test_user --project test --dbname dbname -e 1 --file ./resources/checks/H002_unused_indexes.sh
70+
- ./checkup -c .ci/test.yml --file ./resources/checks/H002_unused_indexes.sh
7171
## unused
7272
- data_dir=$(cat ./artifacts/test/nodes.json | jq -r '.last_check | .dir') && result=$(cat "./artifacts/test/json_reports/$data_dir/H002_unused_indexes.json" | jq '.results .postgres .data .never_used_indexes ."public.i_u_12"') && ([[ ! "$result" == "[]" ]] && [[ ! "$result" == "null" ]]) && exit 211
7373
## redundant

checkup

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -193,10 +193,12 @@ load_config_params () {
193193
dbg "Load params from config file"
194194
config_filename=${1}
195195
if [[ ! -f "$config_filename" ]]; then
196-
msg "Config filename ${config_filename} not found."
196+
err "Config filename ${config_filename} not found."
197197
exit 1
198198
fi
199-
source ${BASH_SOURCE%/*}/resources/parse_yaml.sh $config_filename "CONFIG_"
199+
200+
eval $(${PGHREP_BIN} --mode loadcfg --path $config_filename 2>/dev/null)
201+
200202
for i in $(seq 0 ${CLI_ARGS_POSSIBLE}); do
201203
var_name="CONFIG__"${FULL_NAME[$i]}
202204
var_name=${var_name//-/_}
@@ -740,7 +742,7 @@ build_pghrep() {
740742
dbg "building pghrep..."
741743
local save_pwd="${PWD}"
742744
cd "${SCRIPT_DIR}/pghrep"
743-
make install buildplugins main >/dev/null
745+
make install main >/dev/null
744746
cd "${save_pwd}"
745747
export PGHREP_BIN="${SCRIPT_DIR}/pghrep/bin/pghrep"
746748
dbg "done building pghrep"
@@ -1357,6 +1359,7 @@ main() {
13571359
mode="$1"
13581360
fi
13591361

1362+
build_pghrep
13601363
read_possible_args
13611364
process_cli_args "$@"
13621365
validate_args
@@ -1368,19 +1371,16 @@ main() {
13681371
;;
13691372
"process" )
13701373
check_bin_deps
1371-
build_pghrep
13721374
run_process
13731375
;;
13741376
"upload" )
13751377
check_bin_deps
1376-
build_pghrep
13771378
run_upload
13781379
;;
13791380
"run" )
13801381
host_pre_start_checks
13811382
run_checks
13821383
check_bin_deps
1383-
build_pghrep
13841384
run_process
13851385
;;
13861386
esac

pghrep/Makefile

Lines changed: 2 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -23,18 +23,13 @@ LDFLAGS = -ldflags "-s -w \
2323
PLUGINS_SRC := $(wildcard plugins/*.go)
2424

2525
# Build the project
26-
all: clean install vet buildplugins main
26+
all: clean install vet main
2727

2828
install:
2929
go get golang.org/x/net/html
3030
go get github.com/dustin/go-humanize
3131
go get gopkg.in/yaml.v2
3232

33-
buildplugins:
34-
@for f in $(basename $(patsubst plugins/%.go,%,$(PLUGINS_SRC))); do \
35-
go build ${LDFLAGS} -buildmode=plugin -o bin/$$f.so ./plugins/$$f.go ; \
36-
done
37-
3833
main:
3934
GOARCH=${GOARCH} go build ${LDFLAGS} -o bin/${BINARY} ./src/
4035

@@ -44,11 +39,6 @@ test:
4439
go test ./src/checkup/*/
4540

4641
vet:
47-
# Command go vet examine all go files in a single scope, which isn't suitable for plugins.
48-
# We will run it separately for plugins and main sources.
49-
@for f in $(basename $(patsubst plugins/%.go,%,$(PLUGINS_SRC))); do \
50-
go vet ./plugins/$$f.go || exit 1 ; \
51-
done
5242
go vet ./src/...
5343

5444
fmt:
@@ -60,5 +50,5 @@ clean:
6050
run:
6151
go run ${LDFLAGS} ./src/*
6252

63-
.PHONY: all buildplugins main test vet fmt clean run
53+
.PHONY: all main test vet fmt clean run
6454

pghrep/src/config.go

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
package main
2+
3+
import (
4+
"fmt"
5+
"io/ioutil"
6+
"strings"
7+
8+
"gopkg.in/yaml.v2"
9+
)
10+
11+
const CONFIG_PARAM_PREFIX = "CONFIG"
12+
13+
type CheckupProjectConfig map[string]string
14+
15+
type CheckupConfig []CheckupProjectConfig
16+
17+
func loadConfig(path string) (CheckupConfig, error) {
18+
var config CheckupConfig
19+
if !FileExists(path) {
20+
return config, fmt.Errorf("Config file '%s' not found.", path)
21+
}
22+
23+
data, cerr := ioutil.ReadFile(path)
24+
if cerr != nil {
25+
return config, fmt.Errorf("Cannot read file '%s'. %s", path, cerr)
26+
}
27+
28+
uerr := yaml.Unmarshal([]byte(data), &config)
29+
if uerr != nil {
30+
return config, fmt.Errorf("Cannot parse config file: '%s'. %s", path, uerr)
31+
}
32+
33+
return config, nil
34+
}
35+
36+
func outputConfig(config CheckupConfig) {
37+
if len(config) > 0 {
38+
// Now we support only one project config
39+
for key, value := range config[0] {
40+
fmt.Printf("%s__%s=\"%s\"\n", CONFIG_PARAM_PREFIX, strings.Replace(key, "-", "_", -1), value)
41+
}
42+
}
43+
}

pghrep/src/log/log.go

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -65,21 +65,21 @@ func prepareMessage(v ...interface{}) string {
6565

6666
// Output message
6767
func Msg(v ...interface{}) {
68-
log.Println(">>> INFO: " + prepareMessage(v...))
68+
log.Println("INFO: " + prepareMessage(v...))
6969
}
7070

7171
// Output debug message
7272
func Dbg(v ...interface{}) {
7373
if DEBUG {
74-
log.Println(">>> DEBUG: " + prepareMessage(v...))
74+
log.Println("DEBUG: " + prepareMessage(v...))
7575
}
7676
}
7777

7878
// Output error message
7979
func Err(v ...interface{}) {
80-
log.New(os.Stderr, "", 0).Println(">>> ERROR: " + prepareMessage(v...))
80+
log.New(os.Stderr, "", 0).Println("ERROR: " + prepareMessage(v...))
8181
}
8282

8383
func Fatal(v ...interface{}) {
84-
log.Fatal(">>> ERROR: " + prepareMessage(v...))
84+
log.Fatal("ERROR: " + prepareMessage(v...))
8585
}

pghrep/src/main.go

Lines changed: 35 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -109,7 +109,7 @@ func LoadJsonFile(filePath string) map[string]interface{} {
109109
if FileExists(filePath) {
110110
fileContent, err := ioutil.ReadFile(GetFilePath(filePath)) // just pass the file name
111111
if err != nil {
112-
log.Err("Can't read file: ", filePath, err)
112+
log.Err("Can't read file:", filePath, err)
113113
return nil
114114
}
115115

@@ -191,7 +191,7 @@ func getRawData(data map[string]interface{}) {
191191
// for every host get data
192192
var rawData []interface{}
193193
hosts := pyraconv.ToInterfaceMap(data["hosts"])
194-
log.Dbg("Data hosts: ", hosts)
194+
log.Dbg("Data hosts:", hosts)
195195
results := pyraconv.ToInterfaceMap(data["results"])
196196
masterName := pyraconv.ToString(hosts["master"])
197197
masterResults := pyraconv.ToInterfaceMap(results[masterName])
@@ -300,7 +300,7 @@ func generateMdReport(checkId string, reportFilename string, reportData map[stri
300300
reporTpl := templates.Lookup(reportFileName)
301301
data := reportData
302302
if reporTpl == nil {
303-
log.Err("Template " + checkId + ".tpl not found.")
303+
log.Err("Template " + checkId + ".tpl not found")
304304
getRawData(data)
305305
reportFileName = "raw.tpl"
306306
reporTpl = templates.Lookup(reportFileName)
@@ -415,6 +415,7 @@ func main() {
415415
projectDataPtr := flag.String("project", "", "target project used during uploading")
416416
pathDataPtr := flag.String("path", "", "path to artifacts directory used during uploading")
417417
apiUrlDataPtr := flag.String("apiurl", "", "API URL for reports uploading")
418+
418419
flag.Parse()
419420
checkData = *checkDataPtr
420421

@@ -431,27 +432,24 @@ func main() {
431432
log.DEBUG = false
432433
}
433434

434-
if *modeDataPtr == "upload" {
435+
switch *modeDataPtr {
436+
case "upload":
435437
token := *tokenDataPtr
436438
project := *projectDataPtr
437439
path := *pathDataPtr
438440
apiUrl := *apiUrlDataPtr
439441

440442
if len(token) == 0 {
441-
log.Err("Token is not defined")
442-
return
443+
log.Fatal("Token is not defined")
443444
}
444445
if len(apiUrl) == 0 {
445-
log.Err("API URL is not defined")
446-
return
446+
log.Fatal("API URL is not defined")
447447
}
448448
if len(project) == 0 {
449-
log.Err("Project (for reports uploading) is not defined")
450-
return
449+
log.Fatal("Project (for reports uploading) is not defined")
451450
}
452451
if len(path) == 0 {
453-
log.Err("Artifacts directory is not defined")
454-
return
452+
log.Fatal("Artifacts directory is not defined")
455453
}
456454

457455
err := upload.UploadReport(apiUrl, token, project, path)
@@ -460,28 +458,47 @@ func main() {
460458
os.Exit(1)
461459
}
462460

461+
return
462+
case "loadcfg":
463+
path := *pathDataPtr
464+
if len(path) == 0 {
465+
log.Fatal("Config path is not defined")
466+
}
467+
468+
config, err := loadConfig(path)
469+
if err != nil {
470+
log.Fatal(fmt.Sprintf("Cannot load config. %s", err))
471+
os.Exit(1)
472+
}
473+
474+
if len(config) == 0 {
475+
log.Fatal(fmt.Sprintf("Config '%s' is empty", path))
476+
}
477+
478+
outputConfig(config)
479+
463480
return
464481
}
465482

466483
if FileExists(checkData) {
467484
resultData = LoadJsonFile(checkData)
468485

469486
if resultData == nil {
470-
log.Fatal("ERROR: File given by --checkdata content wrong json data.")
487+
log.Fatal("File given by --checkdata content wrong json data")
471488
return
472489
}
473490

474491
resultData["source_path_full"] = checkData
475492
resultData["source_path_parts"] = strings.Split(checkData, string(os.PathSeparator))
476493
} else {
477-
log.Err("ERROR: File given by --checkdata not found")
494+
log.Err("File given by --checkdata not found")
478495
return
479496
}
480497

481498
if resultData != nil {
482499
checkId = pyraconv.ToString(resultData["checkId"])
483500
} else {
484-
log.Fatal("ERROR: Content given by --checkdata is wrong json content.")
501+
log.Fatal("Content defined by '--checkdata' is invalid JSON")
485502
}
486503

487504
checkId = strings.ToUpper(checkId)
@@ -490,7 +507,7 @@ func main() {
490507

491508
err := reorderHosts(resultData)
492509
if err != nil {
493-
log.Err("There is no data to generate the report.")
510+
log.Err("There is no data to generate the report")
494511
}
495512

496513
config := cfg.NewConfig()
@@ -511,13 +528,13 @@ func main() {
511528

512529
reportDone := generateMdReports(checkId, resultData, outputDir)
513530
if !reportDone {
514-
log.Fatal("Cannot generate report. Data file or template is wrong.")
531+
log.Fatal("Cannot generate report. Data file or template is wrong")
515532
}
516533
}
517534

518535
func preprocessReportData(checkId string, config cfg.Config,
519536
data map[string]interface{}) error {
520-
switch checkId {
537+
switch strings.ToUpper(checkId) {
521538
case "A002":
522539
// Try to load actual Postgres versions.
523540
err := config.LoadVersions()

0 commit comments

Comments
 (0)