[SPARK-25092] Add RewriteExceptAll and RewriteIntersectAll in the list of nonExcludab...
authorDilip Biswal <dbiswal@us.ibm.com>
Sun, 12 Aug 2018 05:51:11 +0000 (22:51 -0700)
committerXiao Li <gatorsmile@gmail.com>
Sun, 12 Aug 2018 05:51:11 +0000 (22:51 -0700)
## What changes were proposed in this pull request?
Add RewriteExceptAll and RewriteIntersectAll in the list of nonExcludableRules as the rewrites are essential for the functioning of EXCEPT ALL and INTERSECT ALL feature.

## How was this patch tested?
Added test in OptimizerRuleExclusionSuite.

Closes #22080 from dilipbiswal/exceptall_rewrite_exclusion.

Authored-by: Dilip Biswal <dbiswal@us.ibm.com>
Signed-off-by: Xiao Li <gatorsmile@gmail.com>
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala
sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/OptimizerRuleExclusionSuite.scala
sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/SetOperationSuite.scala

index 105623c..2ff6768 100644 (file)
@@ -135,7 +135,7 @@ abstract class Optimizer(sessionCatalog: SessionCatalog)
     Batch("Subquery", Once,
       OptimizeSubqueries) ::
     Batch("Replace Operators", fixedPoint,
-      RewriteExcepAll,
+      RewriteExceptAll,
       RewriteIntersectAll,
       ReplaceIntersectWithSemiJoin,
       ReplaceExceptWithFilter,
@@ -189,6 +189,8 @@ abstract class Optimizer(sessionCatalog: SessionCatalog)
       ReplaceIntersectWithSemiJoin.ruleName ::
       ReplaceExceptWithFilter.ruleName ::
       ReplaceExceptWithAntiJoin.ruleName ::
+      RewriteExceptAll.ruleName ::
+      RewriteIntersectAll.ruleName ::
       ReplaceDistinctWithAggregate.ruleName ::
       PullupCorrelatedPredicates.ruleName ::
       RewritePredicateSubquery.ruleName :: Nil
@@ -1462,7 +1464,7 @@ object ReplaceExceptWithAntiJoin extends Rule[LogicalPlan] {
  * }}}
  */
 
-object RewriteExcepAll extends Rule[LogicalPlan] {
+object RewriteExceptAll extends Rule[LogicalPlan] {
   def apply(plan: LogicalPlan): LogicalPlan = plan transform {
     case Except(left, right, true) =>
       assert(left.output.size == right.output.size)
index 30c80d2..eee8dc3 100644 (file)
@@ -80,12 +80,14 @@ class OptimizerRuleExclusionSuite extends PlanTest {
         "DummyRuleName"))
   }
 
-  test("Try to exclude a non-excludable rule") {
+  test("Try to exclude some non-excludable rules") {
     verifyExcludedRules(
       new SimpleTestOptimizer(),
       Seq(
         ReplaceIntersectWithSemiJoin.ruleName,
-        PullupCorrelatedPredicates.ruleName))
+        PullupCorrelatedPredicates.ruleName,
+        RewriteExceptAll.ruleName,
+        RewriteIntersectAll.ruleName))
   }
 
   test("Custom optimizer") {
index cb744be..da3923f 100644 (file)
@@ -148,7 +148,7 @@ class SetOperationSuite extends PlanTest {
 
   test("EXCEPT ALL rewrite") {
     val input = Except(testRelation, testRelation2, isAll = true)
-    val rewrittenPlan = RewriteExcepAll(input)
+    val rewrittenPlan = RewriteExceptAll(input)
 
     val planFragment = testRelation.select(Literal(1L).as("vcol"), 'a, 'b, 'c)
       .union(testRelation2.select(Literal(-1L).as("vcol"), 'd, 'e, 'f))