planner: eliminate top level Sort in subquery (pingcap#11935)

cmd/explaintest/r/select.result

@@ -458,10 +458,10 @@ drop table if exists t2;
 create table t2(a int, b int);
 explain select * from t1 where t1.a in (select t2.a as a from t2 where t2.b > t1.b order by t1.b);
 id	count	task	operator info
-Apply_11	9990.00	root	semi join, inner:TableReader_19, equal:[eq(test.t1.a, test.t2.a)]
-├─TableReader_14	9990.00	root	data:Selection_13
-│ └─Selection_13	9990.00	cop	not(isnull(test.t1.a))
-│   └─TableScan_12	10000.00	cop	table:t1, range:[-inf,+inf], keep order:false, stats:pseudo
-└─TableReader_19	7992.00	root	data:Selection_18
-  └─Selection_18	7992.00	cop	gt(test.t2.b, test.t1.b), not(isnull(test.t2.a))
-    └─TableScan_17	10000.00	cop	table:t2, range:[-inf,+inf], keep order:false, stats:pseudo
+HashLeftJoin_10	7984.01	root	semi join, inner:TableReader_16, equal:[eq(test.t1.a, test.t2.a)], other cond:gt(test.t2.b, test.t1.b)
+├─TableReader_13	9980.01	root	data:Selection_12
+│ └─Selection_12	9980.01	cop	not(isnull(test.t1.a)), not(isnull(test.t1.b))
+│   └─TableScan_11	10000.00	cop	table:t1, range:[-inf,+inf], keep order:false, stats:pseudo
+└─TableReader_16	9980.01	root	data:Selection_15
+  └─Selection_15	9980.01	cop	not(isnull(test.t2.a)), not(isnull(test.t2.b))
+    └─TableScan_14	10000.00	cop	table:t2, range:[-inf,+inf], keep order:false, stats:pseudo

planner/core/logical_plan_test.go

@@ -845,6 +845,23 @@ func (s *testPlanSuite) TestSubquery(c *C) {
 			sql:  "select t1.b from t t1 where t1.b = (select avg(t2.a) from t t2 where t1.g=t2.g and (t1.b = 4 or t2.b = 2))",
 			best: "Apply{DataScan(t1)->DataScan(t2)->Sel([eq(test.t1.g, test.t2.g) or(eq(test.t1.b, 4), eq(test.t2.b, 2))])->Aggr(avg(test.t2.a))}->Projection->Sel([eq(cast(test.t1.b), avg(t2.a))])->Projection",
 		},
+		{
+			sql:  "select t1.b from t t1 where t1.b = (select max(t2.a) from t t2 where t1.b=t2.b order by t1.a)",
+			best: "Join{DataScan(t1)->DataScan(t2)->Aggr(max(test.t2.a),firstrow(test.t2.b))}(test.t1.b,test.t2.b)->Projection->Sel([eq(test.t1.b, max(t2.a))])->Projection",
+		},
+		{
+			sql:  "select t1.b from t t1 where t1.b in (select t2.b from t t2 where t2.a = t1.a order by t2.a)",
+			best: "Join{DataScan(t1)->DataScan(t2)}(test.t1.a,test.t2.a)(test.t1.b,test.t2.b)->Projection",
+		},
+		{
+			sql:  "select t1.b from t t1 where exists(select t2.b from t t2 where t2.a = t1.a order by t2.a)",
+			best: "Join{DataScan(t1)->DataScan(t2)}(test.t1.a,test.t2.a)->Projection",
+		},
+		{
+			// `Sort` will not be eliminated, if it is not the top level operator.
+			sql:  "select t1.b from t t1 where t1.b = (select t2.b from t t2 where t2.a = t1.a order by t2.a limit 1)",
+			best: "Apply{DataScan(t1)->DataScan(t2)->Sel([eq(test.t2.a, test.t1.a)])->Projection->Sort->Limit}->Projection->Sel([eq(test.t1.b, test.t2.b)])->Projection",
+		},
 	}
 
 	ctx := context.Background()

planner/core/rule_decorrelate.go

@@ -243,6 +243,12 @@ func (s *decorrelateSolver) optimize(ctx context.Context, p LogicalPlan) (Logica
 					apply.corCols = extractCorColumnsBySchema(apply.children[1], apply.children[0].Schema())
 				}
 			}
+		} else if sort, ok := innerPlan.(*LogicalSort); ok {
+			// Since we only pull up Selection, Projection, Aggregation, MaxOneRow,
+			// the top level Sort has no effect on the subquery's result.
+			innerPlan = sort.children[0]
+			apply.SetChildren(outerPlan, innerPlan)
+			return s.optimize(ctx, p)
 		}
 	}
 	newChildren := make([]LogicalPlan, 0, len(p.Children()))