Skip to content

Commit 5e6cbdb

Browse files
a.pervushinadanolivo
a.pervushina
authored and
danolivo
committed
Add tests for the 'Learn after an query interruption by timeout' feature. Fix the bug with false finished node. Add some DEBUG messages. Just for conveniency.
1 parent 38840b2 commit 5e6cbdb

File tree

6 files changed

+182
-6
lines changed

6 files changed

+182
-6
lines changed

‎Makefile

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ REGRESS = aqo_disabled \
2323
unsupported \
2424
clean_aqo_data \
2525
plancache \
26+
statement_timeout \
2627
top_queries
2728

2829
fdw_srcdir = $(top_srcdir)/contrib/postgres_fdw

‎expected/statement_timeout.out

Lines changed: 109 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,109 @@
1+
-- Check the learning-on-timeout feature
2+
-- For stabilized reproduction autovacuum must be disabled.
3+
CREATE FUNCTION check_estimated_rows(text) RETURNS TABLE (estimated int)
4+
LANGUAGE plpgsql AS $$
5+
DECLARE
6+
ln text;
7+
tmp text[];
8+
first_row bool := true;
9+
BEGIN
10+
FOR ln IN
11+
execute format('explain %s', $1)
12+
LOOP
13+
IF first_row THEN
14+
first_row := false;
15+
tmp := regexp_match(ln, 'rows=(\d*)');
16+
RETURN QUERY SELECT tmp[1]::int;
17+
END IF;
18+
END LOOP;
19+
END; $$;
20+
CREATE TABLE t AS SELECT * FROM generate_series(1,100) AS x;
21+
ANALYZE t;
22+
DELETE FROM t WHERE x > 5; -- Force optimizer to make overestimated prediction.
23+
CREATE EXTENSION IF NOT EXISTS aqo;
24+
SET aqo.mode = 'learn';
25+
SET aqo.show_details = 'off';
26+
SET aqo.learn_statement_timeout = 'on';
27+
SET statement_timeout = 800; -- [0.8s]
28+
SELECT *, pg_sleep(1) FROM t;
29+
NOTICE: [AQO] Time limit for execution of the statement was expired. AQO tried to learn on partial data.
30+
ERROR: canceling statement due to statement timeout
31+
SELECT check_estimated_rows('SELECT *, pg_sleep(1) FROM t;'); -- haven't any partial data
32+
check_estimated_rows
33+
----------------------
34+
100
35+
(1 row)
36+
37+
-- Don't learn because running node has smaller cardinality than an optimizer prediction
38+
SET statement_timeout = 3500;
39+
SELECT *, pg_sleep(1) FROM t;
40+
NOTICE: [AQO] Time limit for execution of the statement was expired. AQO tried to learn on partial data.
41+
ERROR: canceling statement due to statement timeout
42+
SELECT check_estimated_rows('SELECT *, pg_sleep(1) FROM t;');
43+
check_estimated_rows
44+
----------------------
45+
100
46+
(1 row)
47+
48+
-- We have a real learning data.
49+
SET statement_timeout = 10000;
50+
SELECT *, pg_sleep(1) FROM t;
51+
x | pg_sleep
52+
---+----------
53+
1 |
54+
2 |
55+
3 |
56+
4 |
57+
5 |
58+
(5 rows)
59+
60+
SELECT check_estimated_rows('SELECT *, pg_sleep(1) FROM t;');
61+
check_estimated_rows
62+
----------------------
63+
5
64+
(1 row)
65+
66+
-- Force to make an underestimated prediction
67+
DELETE FROM t WHERE x > 2;
68+
ANALYZE t;
69+
INSERT INTO t (x) (SELECT * FROM generate_series(3,5) AS x);
70+
TRUNCATE aqo_data;
71+
SET statement_timeout = 800;
72+
SELECT *, pg_sleep(1) FROM t; -- Not learned
73+
NOTICE: [AQO] Time limit for execution of the statement was expired. AQO tried to learn on partial data.
74+
ERROR: canceling statement due to statement timeout
75+
SELECT check_estimated_rows('SELECT *, pg_sleep(1) FROM t;');
76+
check_estimated_rows
77+
----------------------
78+
2
79+
(1 row)
80+
81+
SET statement_timeout = 3500;
82+
SELECT *, pg_sleep(1) FROM t; -- Learn!
83+
NOTICE: [AQO] Time limit for execution of the statement was expired. AQO tried to learn on partial data.
84+
ERROR: canceling statement due to statement timeout
85+
SELECT check_estimated_rows('SELECT *, pg_sleep(1) FROM t;');
86+
check_estimated_rows
87+
----------------------
88+
3
89+
(1 row)
90+
91+
SET statement_timeout = 5500;
92+
SELECT *, pg_sleep(1) FROM t; -- Get reliable data
93+
x | pg_sleep
94+
---+----------
95+
1 |
96+
2 |
97+
3 |
98+
4 |
99+
5 |
100+
(5 rows)
101+
102+
SELECT check_estimated_rows('SELECT *, pg_sleep(1) FROM t;');
103+
check_estimated_rows
104+
----------------------
105+
5
106+
(1 row)
107+
108+
DROP TABLE t;
109+
DROP EXTENSION aqo;

‎learn_cache.c

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,7 @@ lc_update_fss(uint64 fs, int fss, OkNNrdata *data, List *relids)
8686
* Collision found: the same {fs,fss}, but something different.
8787
* For simplicity - just don't update.
8888
*/
89+
elog(DEBUG5, "[AQO]: A collision found in the temporary storage.");
8990
LWLockRelease(&aqo_state->lock);
9091
return false;
9192
}
@@ -134,6 +135,8 @@ lc_update_fss(uint64 fs, int fss, OkNNrdata *data, List *relids)
134135
/* Check the invariant */
135136
Assert((uint32)(ptr - (char *) hdr) == size);
136137

138+
elog(DEBUG5, "DSM entry: %s, targets: %d.",
139+
found ? "Reused" : "New entry", hdr->rows);
137140
LWLockRelease(&aqo_state->lock);
138141
return true;
139142
}

‎machine_learning.c

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -199,8 +199,7 @@ OkNNr_learn(OkNNrdata *data, double *features, double target, double rfactor)
199199

200200
return data->rows;
201201
}
202-
203-
if (data->rows < aqo_K)
202+
else if (data->rows < aqo_K)
204203
{
205204
/* We don't reach a limit of stored neighbors */
206205

@@ -275,6 +274,5 @@ OkNNr_learn(OkNNrdata *data, double *features, double target, double rfactor)
275274
}
276275
}
277276
}
278-
279277
return data->rows;
280278
}

‎postprocessing.c

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -349,11 +349,12 @@ should_learn(PlanState *ps, AQOPlanNode *node, aqo_obj_stat *ctx,
349349
}
350350

351351
/* Has the executor finished its work? */
352-
if (TupIsNull(ps->ps_ResultTupleSlot) &&
352+
if (!ps->instrument->running && TupIsNull(ps->ps_ResultTupleSlot) &&
353353
ps->instrument->nloops > 0.) /* Node was visited by executor at least once. */
354354
{
355355
/* This is much more reliable data. So we can correct our prediction. */
356-
if (ctx->learn && aqo_show_details && fabs(*nrows - predicted) / predicted > 0.2)
356+
if (ctx->learn && aqo_show_details &&
357+
fabs(*nrows - predicted) / predicted > 0.2)
357358
elog(NOTICE,
358359
"[AQO] Learn on a finished plan node (%lu, %d), "
359360
"predicted rows: %.0lf, updated prediction: %.0lf",
@@ -693,7 +694,7 @@ aqo_timeout_handler(void)
693694
ctx.learn = query_context.learn_aqo;
694695
ctx.isTimedOut = true;
695696

696-
elog(NOTICE, "[AQO] Time limit for execution of the statement was expired. Try to learn on partial data.");
697+
elog(NOTICE, "[AQO] Time limit for execution of the statement was expired. AQO tried to learn on partial data.");
697698
learnOnPlanState(timeoutCtl.queryDesc->planstate, (void *) &ctx);
698699
}
699700

‎sql/statement_timeout.sql

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
-- Check the learning-on-timeout feature
2+
-- For stabilized reproduction autovacuum must be disabled.
3+
4+
CREATE FUNCTION check_estimated_rows(text) RETURNS TABLE (estimated int)
5+
LANGUAGE plpgsql AS $$
6+
DECLARE
7+
ln text;
8+
tmp text[];
9+
first_row bool := true;
10+
BEGIN
11+
FOR ln IN
12+
execute format('explain %s', $1)
13+
LOOP
14+
IF first_row THEN
15+
first_row := false;
16+
tmp := regexp_match(ln, 'rows=(\d*)');
17+
RETURN QUERY SELECT tmp[1]::int;
18+
END IF;
19+
END LOOP;
20+
END; $$;
21+
22+
CREATE TABLE t AS SELECT * FROM generate_series(1,100) AS x;
23+
ANALYZE t;
24+
DELETE FROM t WHERE x > 5; -- Force optimizer to make overestimated prediction.
25+
26+
CREATE EXTENSION IF NOT EXISTS aqo;
27+
SET aqo.mode = 'learn';
28+
SET aqo.show_details = 'off';
29+
SET aqo.learn_statement_timeout = 'on';
30+
31+
SET statement_timeout = 800; -- [0.8s]
32+
SELECT *, pg_sleep(1) FROM t;
33+
SELECT check_estimated_rows('SELECT *, pg_sleep(1) FROM t;'); -- haven't any partial data
34+
35+
-- Don't learn because running node has smaller cardinality than an optimizer prediction
36+
SET statement_timeout = 3500;
37+
SELECT *, pg_sleep(1) FROM t;
38+
SELECT check_estimated_rows('SELECT *, pg_sleep(1) FROM t;');
39+
40+
-- We have a real learning data.
41+
SET statement_timeout = 10000;
42+
SELECT *, pg_sleep(1) FROM t;
43+
SELECT check_estimated_rows('SELECT *, pg_sleep(1) FROM t;');
44+
45+
-- Force to make an underestimated prediction
46+
DELETE FROM t WHERE x > 2;
47+
ANALYZE t;
48+
INSERT INTO t (x) (SELECT * FROM generate_series(3,5) AS x);
49+
TRUNCATE aqo_data;
50+
51+
SET statement_timeout = 800;
52+
SELECT *, pg_sleep(1) FROM t; -- Not learned
53+
SELECT check_estimated_rows('SELECT *, pg_sleep(1) FROM t;');
54+
55+
SET statement_timeout = 3500;
56+
SELECT *, pg_sleep(1) FROM t; -- Learn!
57+
SELECT check_estimated_rows('SELECT *, pg_sleep(1) FROM t;');
58+
59+
SET statement_timeout = 5500;
60+
SELECT *, pg_sleep(1) FROM t; -- Get reliable data
61+
SELECT check_estimated_rows('SELECT *, pg_sleep(1) FROM t;');
62+
63+
DROP TABLE t;
64+
DROP EXTENSION aqo;

0 commit comments

Comments
 (0)