From 08dbd4cf00c34777d8154f6976666a8e851bf7db Mon Sep 17 00:00:00 2001
From: Radhakrishnan Pachyappan <gingeekrishna@gmail.com>
Date: Sat, 27 Jun 2026 13:16:27 +0530
Subject: [PATCH] test(ppl): remove invalid Spark SQL assertions from
 correlated-aggregate subquery tests (#5470)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The verifyPPLToSparkSQL assertions in testCorrelatedScalarSubqueryInWhere and
testCorrelatedScalarSubqueryInSelect pin SQL that Spark (4.1) refuses to execute:
SALGRADE has no SAL or EMPNO column, so those references bind to the outer EMP
table as correlated outer references. The SQL serializer emits AVG(`EMP`.`SAL`)
and MIN(`EMP`.`EMPNO`) inside subquery aggregate functions. Spark rejects this with
UNSUPPORTED_SUBQUERY_EXPRESSION_CATEGORY.CORRELATED_REFERENCE because outer-column
references in aggregate functions are not supported outside WHERE/HAVING clauses.

Changes:
* testCorrelatedScalarSubqueryInWhere: remove verifyPPLToSparkSQL; keep verifyLogical
  (the logical plan correctly models the correlated semantics per SQL-92 scoping rules)
* testCorrelatedScalarSubqueryInSelect: same — remove verifyPPLToSparkSQL; keep verifyLogical
* Remove testCorrelatedScalarSubqueryInWhereMaxOut: exact duplicate of
  testCorrelatedScalarSubqueryInWhere (same PPL, same expected logical, same invalid SQL)

The disjunctive tests (testDisjunctiveCorrelatedScalarSubqueryInWhere*) are unaffected:
their COUNT() aggregate does not reference outer columns, so the generated Spark SQL is valid.

Fixes #5470

Signed-off-by: Radhakrishnan Pachyappan <gingeekrishna@gmail.com>
---
 .../calcite/CalcitePPLScalarSubqueryTest.java | 59 ++++---------------
 1 file changed, 11 insertions(+), 48 deletions(-)

diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLScalarSubqueryTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLScalarSubqueryTest.java
index e19d896283d..014c1a716b9 100644
--- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLScalarSubqueryTest.java
+++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLScalarSubqueryTest.java
@@ -135,15 +135,12 @@ public void testCorrelatedScalarSubqueryInWhere() {
             + "}))], variablesSet=[[$cor0]])\n"
             + "  LogicalTableScan(table=[[scott, EMP]])\n";
     verifyLogical(root, expectedLogical);
-
-    String expectedSparkSql =
-        ""
-            + "SELECT *\n"
-            + "FROM `scott`.`EMP`\n"
-            + "WHERE `SAL` > (((SELECT AVG(`EMP`.`SAL`) `AVG(SAL)`\n"
-            + "FROM `scott`.`SALGRADE`\n"
-            + "WHERE `EMP`.`SAL` = `HISAL`)))";
-    verifyPPLToSparkSQL(root, expectedSparkSql);
+    // verifyPPLToSparkSQL is intentionally omitted: SALGRADE has no SAL column, so both SAL
+    // references in the subquery bind to the outer EMP.SAL (correlated outer reference). The
+    // SQL serializer emits AVG(`EMP`.`SAL`) in the subquery aggregate — outer-column references
+    // in aggregate functions are rejected by Spark with
+    // UNSUPPORTED_SUBQUERY_EXPRESSION_CATEGORY.CORRELATED_REFERENCE. The logical plan above is
+    // the authoritative contract for this query; the Spark SQL string is not executable.
   }
 
   @Test
@@ -167,14 +164,11 @@ public void testCorrelatedScalarSubqueryInSelect() {
             + "})], SAL=[$5])\n"
             + "  LogicalTableScan(table=[[scott, EMP]])\n";
     verifyLogical(root, expectedLogical);
-
-    String expectedSparkSql =
-        ""
-            + "SELECT (((SELECT MIN(`EMP`.`EMPNO`) `min(EMPNO)`\n"
-            + "FROM `scott`.`SALGRADE`\n"
-            + "WHERE `EMP`.`SAL` = `HISAL`))) `min_empno`, `SAL`\n"
-            + "FROM `scott`.`EMP`";
-    verifyPPLToSparkSQL(root, expectedSparkSql);
+    // verifyPPLToSparkSQL is intentionally omitted: SALGRADE has no EMPNO or SAL column, so
+    // both references in the subquery bind to outer EMP columns (correlated outer references).
+    // The SQL serializer emits MIN(`EMP`.`EMPNO`) inside a subquery aggregate — outer-column
+    // references in aggregate functions are rejected by Spark with
+    // UNSUPPORTED_SUBQUERY_EXPRESSION_CATEGORY.CORRELATED_REFERENCE. See issue #5470.
   }
 
   @Test
@@ -340,35 +334,4 @@ public void testNestedScalarSubquery() {
             + "LIMIT 1)))";
     verifyPPLToSparkSQL(root, expectedSparkSql);
   }
-
-  @Test
-  public void testCorrelatedScalarSubqueryInWhereMaxOut() {
-    String ppl =
-        """
-        source=EMP
-        | where SAL > [
-            source=SALGRADE | where SAL = HISAL | stats AVG(SAL)
-          ]
-        """;
-    RelNode root = getRelNode(ppl);
-    String expectedLogical =
-        ""
-            + "LogicalFilter(condition=[>($5, $SCALAR_QUERY({\n"
-            + "LogicalAggregate(group=[{}], AVG(SAL)=[AVG($0)])\n"
-            + "  LogicalProject($f0=[$cor0.SAL])\n"
-            + "    LogicalFilter(condition=[=($cor0.SAL, $2)])\n"
-            + "      LogicalTableScan(table=[[scott, SALGRADE]])\n"
-            + "}))], variablesSet=[[$cor0]])\n"
-            + "  LogicalTableScan(table=[[scott, EMP]])\n";
-    verifyLogical(root, expectedLogical);
-
-    String expectedSparkSql =
-        ""
-            + "SELECT *\n"
-            + "FROM `scott`.`EMP`\n"
-            + "WHERE `SAL` > (((SELECT AVG(`EMP`.`SAL`) `AVG(SAL)`\n"
-            + "FROM `scott`.`SALGRADE`\n"
-            + "WHERE `EMP`.`SAL` = `HISAL`)))";
-    verifyPPLToSparkSQL(root, expectedSparkSql);
-  }
 }