From 24730dc15143c55484cf77c25c7eda9c7d86a491 Mon Sep 17 00:00:00 2001 From: Louis Chu Date: Mon, 29 Jun 2026 21:31:29 -0700 Subject: [PATCH 01/14] [Feature] Add PPL `rest` command (Calcite system row source) Add a leading `rest ` command that exposes a curated, read-only, fixed-schema set of in-cluster management endpoints as a PPL table, modeled as a system row source bridged through visitRelation (the same seam as describe and the system-index family), so it runs on the Calcite engine without the unsupported table-function path. - Grammar/AST: REST/TIMEOUT tokens, restCommand rule, RestRelation, AstBuilder visit, query anonymizer. - Execution: RestSourceTable -> CalciteLogicalRestScan / CalciteEnumerableRestScan; RestEndpointRegistry (read-only allow-list + fixed schema + accepted args); RestEnumerator/RestRequest dispatch via OpenSearchClient (NodeClient in-cluster, RestClient standalone). - 9 endpoints: cluster health/state/settings, cat indices/nodes/cluster_manager/ plugins/shards, resolve/index. - Output shaping: numeric type normalization, id-to-name resolution, role-name expansion, structural flattening, graceful null degrade. - Args: count caps emitted rows; timeout reserved but rejected with 400; get-args applied server-side with per-arg value validation (local on health, health on cat/indices, expand_wildcards on resolve/index). Undeclared arg or out-of-domain value is rejected with a 400. level and include_defaults are deferred to a later release; flat_settings is dropped as redundant. - Error handling: blank endpoint, negative count, disallowed arg, and uncoercible value all surface clean 400s rather than 500s. Tests: CalcitePPLRestIT 25/25, RestEndpointRegistryTest 16, RestSourceTableTest 10. Signed-off-by: Louis Chu --- .../opensearch/sql/ast/tree/RestRelation.java | 25 ++ .../sql/utils/SystemIndexUtils.java | 106 +++++ docs/user/ppl/cmd/rest.md | 62 +++ docs/user/ppl/index.md | 1 + .../sql/calcite/CalciteNoPushdownIT.java | 1 + .../sql/calcite/remote/CalciteExplainIT.java | 9 + .../sql/calcite/remote/CalcitePPLRestIT.java | 213 +++++++++ .../sql/ppl/NewAddedCommandsIT.java | 13 + .../src/main/antlr4/OpenSearchPPLLexer.g4 | 2 + .../src/main/antlr4/OpenSearchPPLParser.g4 | 11 + .../opensearch/client/OpenSearchClient.java | 101 +++++ .../client/OpenSearchNodeClient.java | 252 +++++++++++ .../client/OpenSearchRestClient.java | 266 ++++++++++++ .../planner/rules/EnumerableRestScanRule.java | 46 ++ .../planner/rules/OpenSearchIndexRules.java | 2 + .../storage/OpenSearchStorageEngine.java | 7 +- .../storage/rest/AbstractCalciteRestScan.java | 41 ++ .../rest/CalciteEnumerableRestScan.java | 89 ++++ .../storage/rest/CalciteLogicalRestScan.java | 48 +++ .../storage/rest/RestEndpointRegistry.java | 407 ++++++++++++++++++ .../storage/rest/RestEnumerator.java | 86 ++++ .../opensearch/storage/rest/RestRequest.java | 47 ++ .../storage/rest/RestSourceTable.java | 80 ++++ .../rest/RestEndpointRegistryTest.java | 213 +++++++++ .../storage/rest/RestSourceTableTest.java | 135 ++++++ ppl/src/main/antlr/OpenSearchPPLLexer.g4 | 2 + ppl/src/main/antlr/OpenSearchPPLParser.g4 | 12 + .../opensearch/sql/ppl/parser/AstBuilder.java | 33 ++ .../sql/ppl/utils/PPLQueryDataAnonymizer.java | 19 + .../sql/ppl/calcite/CalcitePPLRestTest.java | 66 +++ .../sql/ppl/parser/AstBuilderTest.java | 28 ++ .../ppl/utils/PPLQueryDataAnonymizerTest.java | 12 + 32 files changed, 2434 insertions(+), 1 deletion(-) create mode 100644 core/src/main/java/org/opensearch/sql/ast/tree/RestRelation.java create mode 100644 docs/user/ppl/cmd/rest.md create mode 100644 integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLRestIT.java create mode 100644 opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/EnumerableRestScanRule.java create mode 100644 opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/AbstractCalciteRestScan.java create mode 100644 opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/CalciteEnumerableRestScan.java create mode 100644 opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/CalciteLogicalRestScan.java create mode 100644 opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistry.java create mode 100644 opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestEnumerator.java create mode 100644 opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestRequest.java create mode 100644 opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestSourceTable.java create mode 100644 opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistryTest.java create mode 100644 opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestSourceTableTest.java create mode 100644 ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLRestTest.java diff --git a/core/src/main/java/org/opensearch/sql/ast/tree/RestRelation.java b/core/src/main/java/org/opensearch/sql/ast/tree/RestRelation.java new file mode 100644 index 00000000000..d0a0e34d3c9 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/ast/tree/RestRelation.java @@ -0,0 +1,25 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.ast.tree; + +import java.util.Collections; +import lombok.EqualsAndHashCode; +import lombok.ToString; +import org.opensearch.sql.ast.expression.UnresolvedExpression; + +/** + * Extend Relation to mark a {@code rest} leading command. The single table name is a reserved, + * encoded token (produced by {@link org.opensearch.sql.utils.SystemIndexUtils#restTable}) that + * carries the validated REST endpoint spec; it resolves through the storage engine to a REST source + * table on the Calcite path, exactly as {@link DescribeRelation} resolves to a system index. + */ +@ToString +@EqualsAndHashCode(callSuper = false) +public class RestRelation extends Relation { + public RestRelation(UnresolvedExpression tableName) { + super(Collections.singletonList(tableName)); + } +} diff --git a/core/src/main/java/org/opensearch/sql/utils/SystemIndexUtils.java b/core/src/main/java/org/opensearch/sql/utils/SystemIndexUtils.java index c9fa35d7068..200a9f19681 100644 --- a/core/src/main/java/org/opensearch/sql/utils/SystemIndexUtils.java +++ b/core/src/main/java/org/opensearch/sql/utils/SystemIndexUtils.java @@ -5,6 +5,9 @@ package org.opensearch.sql.utils; +import java.nio.charset.StandardCharsets; +import java.util.LinkedHashMap; +import java.util.Map; import lombok.Getter; import lombok.RequiredArgsConstructor; import lombok.experimental.UtilityClass; @@ -34,6 +37,109 @@ public static Boolean isSystemIndex(String indexName) { return indexName.endsWith(SYS_TABLES_SUFFIX); } + /** + * Reserved suffix marking a {@code rest} source table. Distinct from {@link #SYS_TABLES_SUFFIX} + * so {@link #isSystemIndex} and {@link #isRestSource} never overlap. The whole reserved name is a + * single Calcite identifier (REST + lowercase hex + this suffix), so it survives name resolution + * the same way the uppercase system-mapping names do. + */ + private static final String REST_SOURCE_SUFFIX = "__REST_SOURCE"; + + private static final String REST_SOURCE_PREFIX = "REST"; + + /** True if the resolved table name is a {@code rest} source token. */ + public static boolean isRestSource(String indexName) { + return indexName.endsWith(REST_SOURCE_SUFFIX); + } + + /** + * Encode a validated {@link RestSpec} into a single reserved table name. Mirrors {@link + * #mappingTable}: structured metadata travels inside a reserved name rather than a side channel. + * The endpoint/args have already been allow-list-validated before this is called. + */ + public static String restTable(RestSpec spec) { + StringBuilder sb = new StringBuilder(); + sb.append("endpoint=").append(spec.getEndpoint()); + if (spec.getCount() != null) { + sb.append('\n').append("count=").append(spec.getCount()); + } + if (spec.getTimeout() != null) { + sb.append('\n').append("timeout=").append(spec.getTimeout()); + } + if (spec.getArgs() != null) { + for (Map.Entry e : spec.getArgs().entrySet()) { + sb.append('\n').append("arg.").append(e.getKey()).append('=').append(e.getValue()); + } + } + return REST_SOURCE_PREFIX + toHex(sb.toString()) + REST_SOURCE_SUFFIX; + } + + /** Decode a reserved {@code rest} table name back into its {@link RestSpec}. */ + public static RestSpec decodeRestSpec(String indexName) { + String body = + indexName.substring( + REST_SOURCE_PREFIX.length(), indexName.length() - REST_SOURCE_SUFFIX.length()); + String decoded = fromHex(body); + String endpoint = null; + Integer count = null; + String timeout = null; + LinkedHashMap args = new LinkedHashMap<>(); + for (String line : decoded.split("\n")) { + if (line.isEmpty()) { + continue; + } + int eq = line.indexOf('='); + if (eq < 0) { + continue; + } + String k = line.substring(0, eq); + String v = line.substring(eq + 1); + if (k.equals("endpoint")) { + endpoint = v; + } else if (k.equals("count")) { + count = Integer.parseInt(v); + } else if (k.equals("timeout")) { + timeout = v; + } else if (k.startsWith("arg.")) { + args.put(k.substring("arg.".length()), v); + } + } + return new RestSpec(endpoint, args, count, timeout); + } + + private static String toHex(String s) { + StringBuilder h = new StringBuilder(); + for (byte b : s.getBytes(StandardCharsets.UTF_8)) { + h.append(Character.forDigit((b >> 4) & 0xF, 16)).append(Character.forDigit(b & 0xF, 16)); + } + return h.toString(); + } + + private static String fromHex(String h) { + byte[] bytes = new byte[h.length() / 2]; + for (int i = 0; i < bytes.length; i++) { + bytes[i] = + (byte) + ((Character.digit(h.charAt(2 * i), 16) << 4) + + Character.digit(h.charAt(2 * i + 1), 16)); + } + return new String(bytes, StandardCharsets.UTF_8); + } + + /** + * The validated spec for a {@code rest} command: an allow-listed read-only endpoint plus optional + * count/timeout/query args. Lives in core so the parser (encode) and the storage engine (decode) + * share it without a cross-module dependency. + */ + @Getter + @RequiredArgsConstructor + public static class RestSpec { + private final String endpoint; + private final Map args; + private final Integer count; + private final String timeout; + } + /** * Compose system mapping table. * diff --git a/docs/user/ppl/cmd/rest.md b/docs/user/ppl/cmd/rest.md new file mode 100644 index 00000000000..11f35e2ee86 --- /dev/null +++ b/docs/user/ppl/cmd/rest.md @@ -0,0 +1,62 @@ +# rest + +The `rest` command is a leading command that reads an allow-listed, read-only in-cluster management endpoint (cluster/cat/nodes) and emits the response as PPL rows. Its rows come from the endpoint dispatch, not from an index, so `rest` appears at the start of a query. + +> **Note**: The `rest` command is supported only on the Calcite query engine (`plugins.calcite.enabled=true`). Each endpoint has a fixed output schema, and the dispatch runs under the caller's security context, so a user who cannot call an endpoint directly cannot call it through `rest`. The command is read-only; mutating and non-allow-listed endpoints are rejected. + +## Syntax + +The `rest` command has the following syntax: + +```syntax +rest [count=] [timeout=] [= ...] +``` + +## Parameters + +The `rest` command supports the following parameters. + +| Parameter | Required/Optional | Description | +| --- | --- | --- | +| `` | Required | An allow-listed, read-only endpoint path (see the allow-list below), for example `/_cluster/health`. | +| `count=` | Optional | Caps the number of emitted rows. | +| `timeout=` | Optional | Request timeout passed to the transport action, for example `30s`. | +| `=` | Optional | Endpoint query arguments, validated per endpoint (for example `level=indices` for `/_cluster/health`). | + +## Allow-list + +`rest` resolves only an explicit, curated set of read-only endpoints. Anything outside the list, including any mutating endpoint, is rejected with a clear error. + +| Endpoint | Output columns | +| --- | --- | +| `/_cluster/health` | `cluster_name` (string), `status` (string), `number_of_nodes` (integer), `number_of_data_nodes` (integer), `active_primary_shards` (integer), `active_shards` (integer), `relocating_shards` (integer), `initializing_shards` (integer), `unassigned_shards` (integer), `timed_out` (boolean) | +| `/_cat/indices` | `index` (string), `health` (string), `pri` (integer), `rep` (integer), `active_shards` (integer) | + +## Example 1: Reading cluster health + +The following query reads cluster health and projects two columns: + +```ppl +| rest /_cluster/health | fields status, number_of_nodes +``` + +The query returns the following results: + +```text +fetched rows / total rows = 1/1 ++--------+-----------------+ +| status | number_of_nodes | +|--------+-----------------| +| green | 1 | ++--------+-----------------+ +``` + +## Example 2: Listing indexes from cat indices + +The following query lists indexes and filters and sorts them on the Calcite plan: + +```ppl +| rest /_cat/indices | where health = "green" | sort index | fields index, health, pri +``` + +The downstream `where`, `sort`, and `fields` compose over the `rest` row source exactly like any index scan. diff --git a/docs/user/ppl/index.md b/docs/user/ppl/index.md index 37947113800..939684c0ecc 100644 --- a/docs/user/ppl/index.md +++ b/docs/user/ppl/index.md @@ -74,6 +74,7 @@ source=accounts | [lookup command](cmd/lookup.md) | 3.0 | experimental (since 3.0) | Add or replace data from a lookup index. | | [multisearch command](cmd/multisearch.md) | 3.4 | experimental (since 3.4) | Execute multiple search queries and combine their results. | | [union command](cmd/union.md) | 3.7 | experimental (since 3.7) | Combine results from multiple datasets using UNION ALL semantics. | +| [rest command](cmd/rest.md) | 3.8 | experimental (since 3.8) | Read an allow-listed, read-only in-cluster management endpoint (cluster/cat/nodes) as rows. Calcite engine only. | | [ml command](cmd/ml.md) | 2.5 | stable (since 2.5) | Apply machine learning algorithms to analyze data. | | [kmeans command](cmd/kmeans.md) | 1.3 | stable (since 1.3) | Apply the kmeans algorithm on the search result returned by a PPL command. | | [ad command](cmd/ad.md) | 1.3 | deprecated (since 2.5) | Apply Random Cut Forest algorithm on the search result returned by a PPL command. | diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/CalciteNoPushdownIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/CalciteNoPushdownIT.java index d46649d56d9..da434750a96 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/CalciteNoPushdownIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/CalciteNoPushdownIT.java @@ -57,6 +57,7 @@ CalciteObjectFieldOperateIT.class, CalciteOperatorIT.class, CalciteParseCommandIT.class, + CalcitePPLRestIT.class, CalcitePPLAggregationIT.class, CalcitePPLAppendcolIT.class, CalcitePPLAppendCommandIT.class, diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java index 95c47b9b0b7..784e324bfbd 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java @@ -64,6 +64,15 @@ public void init() throws Exception { loadIndex(Index.GRAPH_EMPLOYEES); } + // Only for Calcite: the rest row source explains as a CalciteEnumerableRestScan. + @Test + public void explainRestCommand() throws IOException { + String result = explainQueryToString("| rest \"/_cluster/health\" | fields status"); + Assert.assertTrue( + "Expected a rest scan node in the explain output, got: " + result, + result.contains("RestScan") || result.contains("rest")); + } + @Override @Ignore("test only in v2") public void testExplainModeUnsupportedInV2() throws IOException {} diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLRestIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLRestIT.java new file mode 100644 index 00000000000..c2ea6069996 --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLRestIT.java @@ -0,0 +1,213 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.calcite.remote; + +import static org.opensearch.sql.util.MatcherUtils.rows; +import static org.opensearch.sql.util.MatcherUtils.schema; +import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; +import static org.opensearch.sql.util.MatcherUtils.verifySchema; + +import java.io.IOException; +import org.json.JSONObject; +import org.junit.jupiter.api.Test; +import org.opensearch.client.Request; +import org.opensearch.client.ResponseException; +import org.opensearch.sql.ppl.PPLIntegTestCase; + +/** + * Integration tests for the {@code rest} leading command on the Calcite path. Uses {@code + * /_cluster/health} as the deterministic, single-row endpoint on a single-node test cluster. Also verifies that a non-allow-listed / mutating endpoint is refused. + */ +public class CalcitePPLRestIT extends PPLIntegTestCase { + + @Override + public void init() throws Exception { + super.init(); + enableCalcite(); + } + + @Test + public void testRestClusterHealthSchema() throws IOException { + JSONObject result = + executeQuery("| rest '/_cluster/health' | fields status, number_of_nodes"); + verifySchema(result, schema("status", "string"), schema("number_of_nodes", "int")); + } + + @Test + public void testRestClusterHealthDataRows() throws IOException { + // Single-node test cluster: exactly one node, status is green or yellow. + JSONObject result = executeQuery("| rest '/_cluster/health' | fields number_of_nodes"); + verifyDataRows(result, rows(1)); + } + + @Test + public void testRestRejectsNonAllowListedEndpoint() throws IOException { + assertRestBadRequest("| rest '/_cluster/reroute'", "allow-list"); + } + + @Test + public void testRestRejectsEmptyEndpoint() throws IOException { + assertRestBadRequest("| rest ''", "non-empty path"); + } + + @Test + public void testRestRejectsDisallowedArg() throws IOException { + assertRestBadRequest("| rest '/_cat/nodes' h='name'", "does not accept arg"); + } + + @Test + public void testRestRejectsNegativeCount() throws IOException { + assertRestBadRequest("| rest '/_cat/nodes' count=-1", "non-negative"); + } + + @Test + public void testRestRejectsTimeoutArg() throws IOException { + assertRestBadRequest("| rest '/_cluster/health' timeout='5s'", "timeout"); + } + + /** + * Assert a {@code rest} query is refused as a client error: HTTP 400 (not a 500 system error) + * with the given substring in the response body. The negative-case check for allow-list and secret-filter enforcement. + */ + private void assertRestBadRequest(String query, String expectedSubstring) { + ResponseException e = + org.junit.Assert.assertThrows(ResponseException.class, () -> executeQuery(query)); + org.junit.Assert.assertEquals( + 400, e.getResponse().getStatusLine().getStatusCode()); + org.junit.Assert.assertTrue( + "expected [" + expectedSubstring + "] in response body: " + e.getMessage(), + e.getMessage().contains(expectedSubstring)); + } + + @Test + public void testRestCatIndicesSchema() throws IOException { + // Schema is fixed by the registry, independent of how many indices exist. + JSONObject result = executeQuery("| rest '/_cat/indices' | fields index, health"); + verifySchema(result, schema("index", "string"), schema("health", "string")); + } + + @Test + public void testRestCatIndicesReturnsCreatedIndex() throws IOException { + // Create a known index, then confirm rest surfaces it and downstream where/fields compose. + client().performRequest(new Request("PUT", "/rest_cat_test")); + JSONObject result = + executeQuery("| rest '/_cat/indices' | where index = 'rest_cat_test' | fields index"); + verifyDataRows(result, rows("rest_cat_test")); + } + + @Test + public void testRestCatNodesSchema() throws IOException { + JSONObject result = executeQuery("| rest '/_cat/nodes' | fields name, cpu"); + verifySchema(result, schema("name", "string"), schema("cpu", "int")); + } + + @Test + public void testRestCatNodesSingleNode() throws IOException { + JSONObject result = executeQuery("| rest '/_cat/nodes' | stats count() as cnt"); + verifyDataRows(result, rows(1)); + } + + @Test + public void testRestCatClusterManagerSchema() throws IOException { + JSONObject result = executeQuery("| rest '/_cat/cluster_manager' | fields node, id"); + verifySchema(result, schema("node", "string"), schema("id", "string")); + } + + @Test + public void testRestCatClusterManagerSingleRow() throws IOException { + JSONObject result = executeQuery("| rest '/_cat/cluster_manager' | stats count() as cnt"); + verifyDataRows(result, rows(1)); + } + + @Test + public void testRestCatPluginsSchema() throws IOException { + JSONObject result = executeQuery("| rest '/_cat/plugins' | fields component, version"); + verifySchema(result, schema("component", "string"), schema("version", "string")); + } + + @Test + public void testRestCatShardsSchema() throws IOException { + JSONObject result = executeQuery("| rest '/_cat/shards' | fields index, shard, state"); + verifySchema( + result, schema("index", "string"), schema("shard", "int"), schema("state", "string")); + } + + @Test + public void testRestClusterStateSchema() throws IOException { + // Assert the string columns; version is the LONG epoch column. + JSONObject result = + executeQuery("| rest '/_cluster/state' | fields cluster_name, cluster_manager_node"); + verifySchema( + result, schema("cluster_name", "string"), schema("cluster_manager_node", "string")); + } + + @Test + public void testRestClusterStateSingleRow() throws IOException { + JSONObject result = executeQuery("| rest '/_cluster/state' | stats count() as cnt"); + verifyDataRows(result, rows(1)); + } + + @Test + public void testRestClusterSettingsSchema() throws IOException { + // Schema is registry-fixed regardless of how many settings are configured. + JSONObject result = + executeQuery("| rest '/_cluster/settings' | fields setting, value, tier"); + verifySchema( + result, schema("setting", "string"), schema("value", "string"), schema("tier", "string")); + } + + @Test + public void testRestResolveIndexSchema() throws IOException { + JSONObject result = executeQuery("| rest '/_resolve/index' | fields name, type"); + verifySchema(result, schema("name", "string"), schema("type", "string")); + } + + @Test + public void testRestResolveIndexSurfacesCreatedIndex() throws IOException { + client().performRequest(new Request("PUT", "/rest_resolve_test")); + JSONObject result = + executeQuery( + "| rest '/_resolve/index' | where name = 'rest_resolve_test' | fields name, type"); + verifyDataRows(result, rows("rest_resolve_test", "index")); + } + + // ---- get-arg server-side filtering (health, expand_wildcards, local) ---- + + @Test + public void testRestClusterHealthLocalArg() throws IOException { + // local=true reads health from the local node; on a single-node cluster the row is unchanged. + JSONObject result = + executeQuery("| rest '/_cluster/health' local='true' | fields number_of_nodes"); + verifyDataRows(result, rows(1)); + } + + @Test + public void testRestCatIndicesHealthFilterReturnsNoRed() throws IOException { + // health filters rows server-side; a healthy cluster has no red indices, so count is 0. + JSONObject result = + executeQuery("| rest '/_cat/indices' health='red' | stats count() as cnt"); + verifyDataRows(result, rows(0)); + } + + @Test + public void testRestResolveIndexExpandWildcardsArg() throws IOException { + // expand_wildcards is applied to the resolve request; schema stays fixed and the call succeeds. + JSONObject result = + executeQuery("| rest '/_resolve/index' expand_wildcards='open' | fields name, type"); + verifySchema(result, schema("name", "string"), schema("type", "string")); + } + + @Test + public void testRestRejectsDroppedLevelArg() throws IOException { + // level was dropped from the allow-list (no-op against the fixed health schema). + assertRestBadRequest("| rest '/_cluster/health' level='indices'", "does not accept arg"); + } + + @Test + public void testRestRejectsBadArgValue() throws IOException { + assertRestBadRequest("| rest '/_cat/indices' health='purple'", "unsupported value"); + } +} diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/NewAddedCommandsIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/NewAddedCommandsIT.java index 837865a3585..8d5912ab199 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/NewAddedCommandsIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/NewAddedCommandsIT.java @@ -32,6 +32,19 @@ public void init() throws Exception { loadIndex(Index.GRAPH_EMPLOYEES); } + @Test + public void testRest() throws IOException { + JSONObject result; + try { + result = executeQuery("| rest \"/_cluster/health\" | fields status, number_of_nodes"); + } catch (ResponseException e) { + result = new JSONObject(TestUtils.getResponseBody(e.getResponse())); + } + if (isCalciteEnabled()) { + assertFalse(result.getJSONArray("datarows").isEmpty()); + } + } + @Test public void testJoin() throws IOException { JSONObject result; diff --git a/language-grammar/src/main/antlr4/OpenSearchPPLLexer.g4 b/language-grammar/src/main/antlr4/OpenSearchPPLLexer.g4 index 2248374d8d9..8ba747614da 100644 --- a/language-grammar/src/main/antlr4/OpenSearchPPLLexer.g4 +++ b/language-grammar/src/main/antlr4/OpenSearchPPLLexer.g4 @@ -13,6 +13,8 @@ options { caseInsensitive = true; } SEARCH: 'SEARCH'; DESCRIBE: 'DESCRIBE'; SHOW: 'SHOW'; +REST: 'REST'; +TIMEOUT: 'TIMEOUT'; FROM: 'FROM'; WHERE: 'WHERE'; FIELDS: 'FIELDS'; diff --git a/language-grammar/src/main/antlr4/OpenSearchPPLParser.g4 b/language-grammar/src/main/antlr4/OpenSearchPPLParser.g4 index 451824d6992..1d13e8af046 100644 --- a/language-grammar/src/main/antlr4/OpenSearchPPLParser.g4 +++ b/language-grammar/src/main/antlr4/OpenSearchPPLParser.g4 @@ -62,6 +62,7 @@ commands commandName : SEARCH | DESCRIBE + | REST | SHOW | AD | ML @@ -113,6 +114,16 @@ describeCommand : DESCRIBE tableSourceClause ; + +restCommand + : REST stringLiteral (restArgument)* + ; + +restArgument + : COUNT EQUAL integerLiteral + | TIMEOUT EQUAL stringLiteral + | ident EQUAL literalValue + ; explainCommand : EXPLAIN explainMode ; diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchClient.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchClient.java index 68350c5a0fd..3b9c3619521 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchClient.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchClient.java @@ -114,4 +114,105 @@ public interface OpenSearchClient { * @param deletePitRequest Delete Point In Time request */ void deletePit(DeletePitRequest deletePitRequest); + + /** + * Read-only cluster health snapshot for the {@code rest} command (backs {@code + * /_cluster/health}). Returns a single flattened row of health fields. Runs under the caller's + * security thread-context; performs no privilege escalation and mutates nothing. + * + * @param params endpoint query args (already allow-list-validated) + * @return a single map of health field name to value + */ + default Map clusterHealth(Map params) { + throw new UnsupportedOperationException("clusterHealth is not supported by this client"); + } + + /** + * Read-only cat-indices listing for the {@code rest} command (backs {@code /_cat/indices}). One + * map per index. Runs under the caller's security thread-context; read-only. + * + * @param params endpoint query args (already allow-list-validated) + * @return one map of column name to value per index + */ + default List> catIndices(Map params) { + throw new UnsupportedOperationException("catIndices is not supported by this client"); + } + + /** + * Read-only cat-nodes listing for the {@code rest} command (backs {@code /_cat/nodes}). One map + * per node with resource state. Runs under the caller's security thread-context; read-only. + * + * @param params endpoint query args (already allow-list-validated) + * @return one map of column name to value per node + */ + default List> catNodes(Map params) { + throw new UnsupportedOperationException("catNodes is not supported by this client"); + } + + /** + * Read-only cat-cluster_manager listing for the {@code rest} command (backs {@code + * /_cat/cluster_manager}). Single map identifying the elected cluster manager. Read-only. + * + * @param params endpoint query args (already allow-list-validated) + * @return one map describing the cluster manager node + */ + default List> catClusterManager(Map params) { + throw new UnsupportedOperationException("catClusterManager is not supported by this client"); + } + + /** + * Read-only cat-plugins listing for the {@code rest} command (backs {@code /_cat/plugins}). One + * map per installed plugin per node. Read-only. + * + * @param params endpoint query args (already allow-list-validated) + * @return one map of column name to value per plugin + */ + default List> catPlugins(Map params) { + throw new UnsupportedOperationException("catPlugins is not supported by this client"); + } + + /** + * Read-only cat-shards listing for the {@code rest} command (backs {@code /_cat/shards}). One map + * per shard. Read-only. + * + * @param params endpoint query args (already allow-list-validated) + * @return one map of column name to value per shard + */ + default List> catShards(Map params) { + throw new UnsupportedOperationException("catShards is not supported by this client"); + } + + /** + * Read-only cluster-state epoch projection for the {@code rest} command (backs {@code + * /_cluster/state}). Single flattened row (cluster_name, state_uuid, version, + * cluster_manager_node). Read-only. + * + * @param params endpoint query args (already allow-list-validated) + * @return a single map of cluster-state field name to value + */ + default Map clusterState(Map params) { + throw new UnsupportedOperationException("clusterState is not supported by this client"); + } + + /** + * Read-only cluster-settings listing for the {@code rest} command (backs {@code + * /_cluster/settings}). One map per configured setting (setting, value, tier). Read-only. + * + * @param params endpoint query args (already allow-list-validated) + * @return one map of column name to value per setting + */ + default List> clusterSettings(Map params) { + throw new UnsupportedOperationException("clusterSettings is not supported by this client"); + } + + /** + * Read-only resolve-index listing for the {@code rest} command (backs {@code /_resolve/index}). + * One map per resolved index, alias, or data stream (name, type). Read-only. + * + * @param params endpoint query args (already allow-list-validated) + * @return one map of column name to value per resolved name + */ + default List> resolveIndex(Map params) { + throw new UnsupportedOperationException("resolveIndex is not supported by this client"); + } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClient.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClient.java index b491f38ef80..9ce9ff79ea8 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClient.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClient.java @@ -18,6 +18,8 @@ import java.util.stream.Collectors; import java.util.stream.Stream; import org.opensearch.OpenSearchSecurityException; +import org.opensearch.action.admin.cluster.health.ClusterHealthRequest; +import org.opensearch.action.admin.cluster.health.ClusterHealthResponse; import org.opensearch.action.admin.indices.create.CreateIndexRequest; import org.opensearch.action.admin.indices.exists.indices.IndicesExistsRequest; import org.opensearch.action.admin.indices.exists.indices.IndicesExistsResponse; @@ -25,6 +27,7 @@ import org.opensearch.action.admin.indices.mapping.get.GetMappingsResponse; import org.opensearch.action.admin.indices.settings.get.GetSettingsResponse; import org.opensearch.action.search.*; +import org.opensearch.cluster.health.ClusterIndexHealth; import org.opensearch.cluster.metadata.AliasMetadata; import org.opensearch.common.action.ActionFuture; import org.opensearch.common.settings.Settings; @@ -285,4 +288,253 @@ public void deletePit(DeletePitRequest deletePitRequest) { "Error occurred while deleting PIT for internal plugin operation", e); } } + + @Override + public Map clusterHealth(Map params) { + ClusterHealthRequest request = new ClusterHealthRequest(); + if (params != null && Boolean.parseBoolean(params.get("local"))) { + request.local(true); + } + ClusterHealthResponse response = client.admin().cluster().health(request).actionGet(); + return flattenHealth(response); + } + + @Override + public List> catIndices(Map params) { + ClusterHealthResponse response = + client.admin().cluster().health(new ClusterHealthRequest()).actionGet(); + List> rows = new java.util.ArrayList<>(); + for (Map.Entry entry : response.getIndices().entrySet()) { + ClusterIndexHealth health = entry.getValue(); + Map row = new java.util.LinkedHashMap<>(); + row.put("index", entry.getKey()); + row.put("health", health.getStatus().name().toLowerCase(java.util.Locale.ROOT)); + row.put("pri", health.getNumberOfShards()); + row.put("rep", health.getNumberOfReplicas()); + row.put("active_shards", health.getActiveShards()); + rows.add(row); + } + String healthFilter = params == null ? null : params.get("health"); + if (healthFilter != null) { + rows.removeIf(r -> !healthFilter.equalsIgnoreCase(String.valueOf(r.get("health")))); + } + return rows; + } + + @Override + public List> catNodes(Map params) { + org.opensearch.action.admin.cluster.node.stats.NodesStatsRequest statsRequest = + new org.opensearch.action.admin.cluster.node.stats.NodesStatsRequest(); + statsRequest.all(); + org.opensearch.action.admin.cluster.node.stats.NodesStatsResponse response = + client.admin().cluster().nodesStats(statsRequest).actionGet(); + List> rows = new java.util.ArrayList<>(); + for (org.opensearch.action.admin.cluster.node.stats.NodeStats ns : response.getNodes()) { + org.opensearch.cluster.node.DiscoveryNode node = ns.getNode(); + Map row = new java.util.LinkedHashMap<>(); + row.put("name", node.getName()); + row.put("ip", node.getHostAddress()); + row.put( + "node_role", + node.getRoles().stream() + .map(org.opensearch.cluster.node.DiscoveryNodeRole::roleName) + .sorted() + .collect(java.util.stream.Collectors.joining(","))); + row.put( + "heap_percent", + ns.getJvm() == null || ns.getJvm().getMem() == null + ? null + : (int) ns.getJvm().getMem().getHeapUsedPercent()); + row.put( + "ram_percent", + ns.getOs() == null || ns.getOs().getMem() == null + ? null + : (int) ns.getOs().getMem().getUsedPercent()); + row.put( + "cpu", + ns.getProcess() == null || ns.getProcess().getCpu() == null + ? null + : (int) ns.getProcess().getCpu().getPercent()); + rows.add(row); + } + return rows; + } + + @Override + public List> catClusterManager(Map params) { + org.opensearch.action.admin.cluster.state.ClusterStateResponse response = + client + .admin() + .cluster() + .state(new org.opensearch.action.admin.cluster.state.ClusterStateRequest()) + .actionGet(); + org.opensearch.cluster.node.DiscoveryNode cm = + response.getState().nodes().getClusterManagerNode(); + List> rows = new java.util.ArrayList<>(); + if (cm != null) { + Map row = new java.util.LinkedHashMap<>(); + row.put("id", cm.getId()); + row.put("host", cm.getHostName()); + row.put("ip", cm.getHostAddress()); + row.put("node", cm.getName()); + rows.add(row); + } + return rows; + } + + @Override + public List> catPlugins(Map params) { + org.opensearch.action.admin.cluster.node.info.NodesInfoRequest infoRequest = + new org.opensearch.action.admin.cluster.node.info.NodesInfoRequest(); + infoRequest.all(); + org.opensearch.action.admin.cluster.node.info.NodesInfoResponse response = + client.admin().cluster().nodesInfo(infoRequest).actionGet(); + List> rows = new java.util.ArrayList<>(); + for (org.opensearch.action.admin.cluster.node.info.NodeInfo info : response.getNodes()) { + org.opensearch.action.admin.cluster.node.info.PluginsAndModules plugins = + info.getInfo(org.opensearch.action.admin.cluster.node.info.PluginsAndModules.class); + if (plugins == null) { + continue; + } + for (org.opensearch.plugins.PluginInfo pi : plugins.getPluginInfos()) { + Map row = new java.util.LinkedHashMap<>(); + row.put("name", info.getNode().getName()); + row.put("component", pi.getName()); + row.put("version", pi.getVersion()); + rows.add(row); + } + } + return rows; + } + + @Override + public List> catShards(Map params) { + org.opensearch.action.admin.cluster.state.ClusterStateResponse response = + client + .admin() + .cluster() + .state(new org.opensearch.action.admin.cluster.state.ClusterStateRequest()) + .actionGet(); + org.opensearch.cluster.node.DiscoveryNodes nodes = response.getState().nodes(); + List> rows = new java.util.ArrayList<>(); + for (org.opensearch.cluster.routing.ShardRouting sr : + response.getState().getRoutingTable().allShards()) { + Map row = new java.util.LinkedHashMap<>(); + row.put("index", sr.getIndexName()); + row.put("shard", sr.id()); + row.put("prirep", sr.primary() ? "p" : "r"); + row.put("state", sr.state().name()); + org.opensearch.cluster.node.DiscoveryNode n = + sr.currentNodeId() == null ? null : nodes.get(sr.currentNodeId()); + row.put("node", n == null ? null : n.getName()); + rows.add(row); + } + return rows; + } + + @Override + public Map clusterState(Map params) { + org.opensearch.action.admin.cluster.state.ClusterStateResponse response = + client + .admin() + .cluster() + .state(new org.opensearch.action.admin.cluster.state.ClusterStateRequest()) + .actionGet(); + Map row = new java.util.LinkedHashMap<>(); + row.put("cluster_name", response.getClusterName().value()); + row.put("state_uuid", response.getState().stateUUID()); + row.put("version", response.getState().version()); + org.opensearch.cluster.node.DiscoveryNode cm = + response.getState().nodes().getClusterManagerNode(); + row.put("cluster_manager_node", cm == null ? null : cm.getName()); + return row; + } + + @Override + public List> clusterSettings(Map params) { + org.opensearch.action.admin.cluster.state.ClusterStateResponse response = + client + .admin() + .cluster() + .state(new org.opensearch.action.admin.cluster.state.ClusterStateRequest()) + .actionGet(); + List> rows = new java.util.ArrayList<>(); + collectSettings(response.getState().metadata().persistentSettings(), "persistent", rows); + collectSettings(response.getState().metadata().transientSettings(), "transient", rows); + return rows; + } + + private void collectSettings( + org.opensearch.common.settings.Settings settings, + String tier, + List> rows) { + if (settings == null) { + return; + } + for (String key : settings.keySet()) { + Map row = new java.util.LinkedHashMap<>(); + row.put("setting", key); + row.put("value", settings.get(key)); + row.put("tier", tier); + rows.add(row); + } + } + + @Override + public List> resolveIndex(Map params) { + String expandWildcards = params == null ? null : params.get("expand_wildcards"); + org.opensearch.action.admin.indices.resolve.ResolveIndexAction.Request request = + expandWildcards == null + ? new org.opensearch.action.admin.indices.resolve.ResolveIndexAction.Request( + new String[] {"*"}) + : new org.opensearch.action.admin.indices.resolve.ResolveIndexAction.Request( + new String[] {"*"}, + org.opensearch.action.support.IndicesOptions.fromParameters( + expandWildcards, + null, + null, + null, + org.opensearch.action.admin.indices.resolve.ResolveIndexAction.Request + .DEFAULT_INDICES_OPTIONS)); + org.opensearch.action.admin.indices.resolve.ResolveIndexAction.Response response = + client + .execute(org.opensearch.action.admin.indices.resolve.ResolveIndexAction.INSTANCE, request) + .actionGet(); + List> rows = new java.util.ArrayList<>(); + for (org.opensearch.action.admin.indices.resolve.ResolveIndexAction.ResolvedIndex idx : + response.getIndices()) { + rows.add(resolveRow(idx.getName(), "index")); + } + for (org.opensearch.action.admin.indices.resolve.ResolveIndexAction.ResolvedAlias alias : + response.getAliases()) { + rows.add(resolveRow(alias.getName(), "alias")); + } + for (org.opensearch.action.admin.indices.resolve.ResolveIndexAction.ResolvedDataStream ds : + response.getDataStreams()) { + rows.add(resolveRow(ds.getName(), "data_stream")); + } + return rows; + } + + private Map resolveRow(String name, String type) { + Map row = new java.util.LinkedHashMap<>(); + row.put("name", name); + row.put("type", type); + return row; + } + + private Map flattenHealth(ClusterHealthResponse response) { + Map row = new java.util.LinkedHashMap<>(); + row.put("cluster_name", response.getClusterName()); + row.put("status", response.getStatus().name().toLowerCase(java.util.Locale.ROOT)); + row.put("number_of_nodes", response.getNumberOfNodes()); + row.put("number_of_data_nodes", response.getNumberOfDataNodes()); + row.put("active_primary_shards", response.getActivePrimaryShards()); + row.put("active_shards", response.getActiveShards()); + row.put("relocating_shards", response.getRelocatingShards()); + row.put("initializing_shards", response.getInitializingShards()); + row.put("unassigned_shards", response.getUnassignedShards()); + row.put("timed_out", response.isTimedOut()); + return row; + } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchRestClient.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchRestClient.java index f369c0003b8..c9290408229 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchRestClient.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchRestClient.java @@ -8,15 +8,19 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import java.io.IOException; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.HashMap; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Optional; import java.util.stream.Collectors; import java.util.stream.Stream; import lombok.RequiredArgsConstructor; +import org.opensearch.action.admin.cluster.health.ClusterHealthRequest; +import org.opensearch.action.admin.cluster.health.ClusterHealthResponse; import org.opensearch.action.admin.cluster.settings.ClusterGetSettingsRequest; import org.opensearch.action.admin.indices.settings.get.GetSettingsRequest; import org.opensearch.action.admin.indices.settings.get.GetSettingsResponse; @@ -28,6 +32,7 @@ import org.opensearch.client.indices.GetIndexResponse; import org.opensearch.client.indices.GetMappingsRequest; import org.opensearch.client.indices.GetMappingsResponse; +import org.opensearch.cluster.health.ClusterIndexHealth; import org.opensearch.cluster.metadata.AliasMetadata; import org.opensearch.common.settings.Settings; import org.opensearch.index.IndexNotFoundException; @@ -272,4 +277,265 @@ public void deletePit(DeletePitRequest deletePitRequest) { "Error occurred while deleting PIT for internal plugin operation", e); } } + + @Override + public Map clusterHealth(Map params) { + try { + ClusterHealthRequest request = new ClusterHealthRequest(); + if (params != null && Boolean.parseBoolean(params.get("local"))) { + request.local(true); + } + ClusterHealthResponse response = + client.cluster().health(request, RequestOptions.DEFAULT); + return flattenHealth(response); + } catch (IOException e) { + throw new IllegalStateException("Failed to get cluster health", e); + } + } + + @Override + public List> catIndices(Map params) { + try { + ClusterHealthResponse response = + client.cluster().health(new ClusterHealthRequest(), RequestOptions.DEFAULT); + List> rows = new ArrayList<>(); + for (Map.Entry entry : response.getIndices().entrySet()) { + ClusterIndexHealth health = entry.getValue(); + Map row = new HashMap<>(); + row.put("index", entry.getKey()); + row.put("health", health.getStatus().name().toLowerCase(Locale.ROOT)); + row.put("pri", health.getNumberOfShards()); + row.put("rep", health.getNumberOfReplicas()); + row.put("active_shards", health.getActiveShards()); + rows.add(row); + } + String healthFilter = params == null ? null : params.get("health"); + if (healthFilter != null) { + rows.removeIf(r -> !healthFilter.equalsIgnoreCase(String.valueOf(r.get("health")))); + } + return rows; + } catch (IOException e) { + throw new IllegalStateException("Failed to get cat indices", e); + } + } + + @Override + public List> catNodes(Map params) { + List> raw = + catJson("/_cat/nodes", "name,ip,node.role,heap.percent,ram.percent,cpu"); + List> rows = new ArrayList<>(); + for (Map r : raw) { + Map row = new HashMap<>(); + row.put("name", r.get("name")); + row.put("ip", r.get("ip")); + row.put("node_role", r.get("node.role")); + row.put("heap_percent", asInt(r.get("heap.percent"))); + row.put("ram_percent", asInt(r.get("ram.percent"))); + row.put("cpu", asInt(r.get("cpu"))); + rows.add(row); + } + return rows; + } + + @Override + public List> catClusterManager(Map params) { + List> raw = catJson("/_cat/cluster_manager", "id,host,ip,node"); + List> rows = new ArrayList<>(); + for (Map r : raw) { + Map row = new HashMap<>(); + row.put("id", r.get("id")); + row.put("host", r.get("host")); + row.put("ip", r.get("ip")); + row.put("node", r.get("node")); + rows.add(row); + } + return rows; + } + + @Override + public List> catPlugins(Map params) { + List> raw = catJson("/_cat/plugins", "name,component,version"); + List> rows = new ArrayList<>(); + for (Map r : raw) { + Map row = new HashMap<>(); + row.put("name", r.get("name")); + row.put("component", r.get("component")); + row.put("version", r.get("version")); + rows.add(row); + } + return rows; + } + + @Override + public List> catShards(Map params) { + List> raw = catJson("/_cat/shards", "index,shard,prirep,state,node"); + List> rows = new ArrayList<>(); + for (Map r : raw) { + Map row = new HashMap<>(); + row.put("index", r.get("index")); + row.put("shard", asInt(r.get("shard"))); + row.put("prirep", r.get("prirep")); + row.put("state", r.get("state")); + row.put("node", r.get("node")); + rows.add(row); + } + return rows; + } + + /** Standalone-mode helper: GET a _cat endpoint as JSON via the low-level client. */ + @SuppressWarnings("unchecked") + private List> catJson(String path, String columns) { + try { + org.opensearch.client.Request request = new org.opensearch.client.Request("GET", path); + request.addParameter("format", "json"); + request.addParameter("h", columns); + org.opensearch.client.Response response = client.getLowLevelClient().performRequest(request); + try (org.opensearch.core.xcontent.XContentParser parser = + org.opensearch.common.xcontent.json.JsonXContent.jsonXContent.createParser( + org.opensearch.core.xcontent.NamedXContentRegistry.EMPTY, + org.opensearch.common.xcontent.LoggingDeprecationHandler.INSTANCE, + response.getEntity().getContent())) { + List list = parser.list(); + List> rows = new ArrayList<>(); + for (Object o : list) { + rows.add((Map) o); + } + return rows; + } + } catch (IOException e) { + throw new IllegalStateException("Failed GET " + path, e); + } + } + + private static Integer asInt(Object value) { + if (value == null) { + return null; + } + try { + return (int) Double.parseDouble(value.toString().trim()); + } catch (NumberFormatException e) { + return null; + } + } + + @Override + @SuppressWarnings("unchecked") + public Map clusterState(Map params) { + Map state = + getJsonMap( + "/_cluster/state/master_node,version,metadata,nodes", + Map.of("filter_path", "cluster_name,state_uuid,version,cluster_manager_node,nodes")); + Map row = new HashMap<>(); + row.put("cluster_name", state.get("cluster_name")); + row.put("state_uuid", state.get("state_uuid")); + row.put("version", asLong(state.get("version"))); + Object cmId = state.get("cluster_manager_node"); + String cmName = null; + Object nodes = state.get("nodes"); + if (cmId != null && nodes instanceof Map) { + Object n = ((Map) nodes).get(cmId.toString()); + if (n instanceof Map) { + Object name = ((Map) n).get("name"); + cmName = name == null ? null : name.toString(); + } + } + row.put("cluster_manager_node", cmName); + return row; + } + + @Override + @SuppressWarnings("unchecked") + public List> clusterSettings(Map params) { + Map body = getJsonMap("/_cluster/settings", Map.of("flat_settings", "true")); + List> rows = new ArrayList<>(); + for (String tier : new String[] {"persistent", "transient"}) { + Object section = body.get(tier); + if (section instanceof Map) { + for (Map.Entry e : ((Map) section).entrySet()) { + Map row = new HashMap<>(); + row.put("setting", e.getKey()); + row.put("value", e.getValue() == null ? null : e.getValue().toString()); + row.put("tier", tier); + rows.add(row); + } + } + } + return rows; + } + + /** Standalone-mode helper: GET a JSON-object endpoint via the low-level client. */ + @SuppressWarnings("unchecked") + private Map getJsonMap(String path, Map params) { + try { + org.opensearch.client.Request request = new org.opensearch.client.Request("GET", path); + if (params != null) { + params.forEach(request::addParameter); + } + org.opensearch.client.Response response = client.getLowLevelClient().performRequest(request); + try (org.opensearch.core.xcontent.XContentParser parser = + org.opensearch.common.xcontent.json.JsonXContent.jsonXContent.createParser( + org.opensearch.core.xcontent.NamedXContentRegistry.EMPTY, + org.opensearch.common.xcontent.LoggingDeprecationHandler.INSTANCE, + response.getEntity().getContent())) { + return parser.map(); + } + } catch (IOException e) { + throw new IllegalStateException("Failed GET " + path, e); + } + } + + private static Long asLong(Object value) { + if (value == null) { + return null; + } + try { + return (long) Double.parseDouble(value.toString().trim()); + } catch (NumberFormatException e) { + return null; + } + } + + @Override + @SuppressWarnings("unchecked") + public List> resolveIndex(Map params) { + String expandWildcards = params == null ? null : params.get("expand_wildcards"); + Map body = + getJsonMap( + "/_resolve/index/*", + expandWildcards == null ? Map.of() : Map.of("expand_wildcards", expandWildcards)); + List> rows = new ArrayList<>(); + addResolved(body.get("indices"), "index", rows); + addResolved(body.get("aliases"), "alias", rows); + addResolved(body.get("data_streams"), "data_stream", rows); + return rows; + } + + @SuppressWarnings("unchecked") + private void addResolved(Object section, String type, List> rows) { + if (section instanceof List) { + for (Object o : (List) section) { + if (o instanceof Map) { + Map row = new HashMap<>(); + row.put("name", ((Map) o).get("name")); + row.put("type", type); + rows.add(row); + } + } + } + } + + private Map flattenHealth(ClusterHealthResponse response) { + Map row = new HashMap<>(); + row.put("cluster_name", response.getClusterName()); + row.put("status", response.getStatus().name().toLowerCase(Locale.ROOT)); + row.put("number_of_nodes", response.getNumberOfNodes()); + row.put("number_of_data_nodes", response.getNumberOfDataNodes()); + row.put("active_primary_shards", response.getActivePrimaryShards()); + row.put("active_shards", response.getActiveShards()); + row.put("relocating_shards", response.getRelocatingShards()); + row.put("initializing_shards", response.getInitializingShards()); + row.put("unassigned_shards", response.getUnassignedShards()); + row.put("timed_out", response.isTimedOut()); + return row; + } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/EnumerableRestScanRule.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/EnumerableRestScanRule.java new file mode 100644 index 00000000000..282ba52b1ee --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/EnumerableRestScanRule.java @@ -0,0 +1,46 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.planner.rules; + +import org.apache.calcite.adapter.enumerable.EnumerableConvention; +import org.apache.calcite.plan.Convention; +import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.convert.ConverterRule; +import org.opensearch.sql.opensearch.storage.rest.CalciteEnumerableRestScan; +import org.opensearch.sql.opensearch.storage.rest.CalciteLogicalRestScan; + +/** Rule to convert a {@link CalciteLogicalRestScan} to a {@link CalciteEnumerableRestScan}. */ +public class EnumerableRestScanRule extends ConverterRule { + /** Default configuration. */ + public static final Config DEFAULT_CONFIG = + Config.INSTANCE + .as(Config.class) + .withConversion( + CalciteLogicalRestScan.class, + s -> s.getRestTable() != null, + Convention.NONE, + EnumerableConvention.INSTANCE, + "EnumerableRestScanRule") + .withRuleFactory(EnumerableRestScanRule::new); + + protected EnumerableRestScanRule(Config config) { + super(config); + } + + @Override + public boolean matches(RelOptRuleCall call) { + CalciteLogicalRestScan scan = call.rel(0); + return scan.getVariablesSet().isEmpty(); + } + + @Override + public RelNode convert(RelNode rel) { + final CalciteLogicalRestScan scan = (CalciteLogicalRestScan) rel; + return new CalciteEnumerableRestScan( + scan.getCluster(), scan.getHints(), scan.getTable(), scan.getRestTable(), scan.getSchema()); + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/OpenSearchIndexRules.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/OpenSearchIndexRules.java index 3c8508cc455..22a508b88cd 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/OpenSearchIndexRules.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/OpenSearchIndexRules.java @@ -14,6 +14,7 @@ public class OpenSearchIndexRules { private static final RelOptRule INDEX_SCAN_RULE = EnumerableIndexScanRule.DEFAULT_CONFIG.toRule(); private static final RelOptRule SYSTEM_INDEX_SCAN_RULE = EnumerableSystemIndexScanRule.DEFAULT_CONFIG.toRule(); + private static final RelOptRule REST_SCAN_RULE = EnumerableRestScanRule.DEFAULT_CONFIG.toRule(); private static final RelOptRule NESTED_AGGREGATE_RULE = EnumerableNestedAggregateRule.DEFAULT_CONFIG.toRule(); private static final RelOptRule GRAPH_LOOKUP_RULE = @@ -27,6 +28,7 @@ public class OpenSearchIndexRules { ImmutableList.of( INDEX_SCAN_RULE, SYSTEM_INDEX_SCAN_RULE, + REST_SCAN_RULE, NESTED_AGGREGATE_RULE, GRAPH_LOOKUP_RULE, RELEVANCE_FUNCTION_RULE); diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngine.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngine.java index 1b7de315fb6..0ec3c33c664 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngine.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngine.java @@ -5,6 +5,8 @@ package org.opensearch.sql.opensearch.storage; +import static org.opensearch.sql.utils.SystemIndexUtils.decodeRestSpec; +import static org.opensearch.sql.utils.SystemIndexUtils.isRestSource; import static org.opensearch.sql.utils.SystemIndexUtils.isSystemIndex; import java.util.Collection; @@ -15,6 +17,7 @@ import org.opensearch.sql.common.setting.Settings; import org.opensearch.sql.expression.function.FunctionResolver; import org.opensearch.sql.opensearch.client.OpenSearchClient; +import org.opensearch.sql.opensearch.storage.rest.RestSourceTable; import org.opensearch.sql.opensearch.storage.system.OpenSearchSystemIndex; import org.opensearch.sql.storage.StorageEngine; import org.opensearch.sql.storage.Table; @@ -35,7 +38,9 @@ public Collection getFunctions() { @Override public Table getTable(DataSourceSchemaName dataSourceSchemaName, String name) { - if (isSystemIndex(name)) { + if (isRestSource(name)) { + return new RestSourceTable(client, settings, decodeRestSpec(name)); + } else if (isSystemIndex(name)) { return new OpenSearchSystemIndex(client, settings, name); } else { return new OpenSearchIndex(client, settings, name); diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/AbstractCalciteRestScan.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/AbstractCalciteRestScan.java new file mode 100644 index 00000000000..fe07d9c584e --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/AbstractCalciteRestScan.java @@ -0,0 +1,41 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.rest; + +import static java.util.Objects.requireNonNull; + +import java.util.List; +import lombok.Getter; +import org.apache.calcite.plan.RelOptCluster; +import org.apache.calcite.plan.RelOptTable; +import org.apache.calcite.plan.RelTraitSet; +import org.apache.calcite.rel.core.TableScan; +import org.apache.calcite.rel.hint.RelHint; +import org.apache.calcite.rel.type.RelDataType; + +/** An abstract relational operator representing a scan of a {@link RestSourceTable}. */ +@Getter +public abstract class AbstractCalciteRestScan extends TableScan { + protected final RestSourceTable restTable; + protected final RelDataType schema; + + protected AbstractCalciteRestScan( + RelOptCluster cluster, + RelTraitSet traitSet, + List hints, + RelOptTable table, + RestSourceTable restTable, + RelDataType schema) { + super(cluster, traitSet, hints, table); + this.restTable = requireNonNull(restTable, "rest source table"); + this.schema = schema; + } + + @Override + public RelDataType deriveRowType() { + return this.schema; + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/CalciteEnumerableRestScan.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/CalciteEnumerableRestScan.java new file mode 100644 index 00000000000..a55d29e785a --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/CalciteEnumerableRestScan.java @@ -0,0 +1,89 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.rest; + +import java.util.List; +import org.apache.calcite.adapter.enumerable.EnumerableConvention; +import org.apache.calcite.adapter.enumerable.EnumerableRel; +import org.apache.calcite.adapter.enumerable.EnumerableRelImplementor; +import org.apache.calcite.adapter.enumerable.PhysType; +import org.apache.calcite.adapter.enumerable.PhysTypeImpl; +import org.apache.calcite.linq4j.AbstractEnumerable; +import org.apache.calcite.linq4j.Enumerable; +import org.apache.calcite.linq4j.Enumerator; +import org.apache.calcite.linq4j.tree.Blocks; +import org.apache.calcite.linq4j.tree.Expression; +import org.apache.calcite.linq4j.tree.Expressions; +import org.apache.calcite.plan.DeriveMode; +import org.apache.calcite.plan.RelOptCluster; +import org.apache.calcite.plan.RelOptTable; +import org.apache.calcite.plan.RelTraitSet; +import org.apache.calcite.rel.hint.RelHint; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.util.Pair; +import org.checkerframework.checker.nullness.qual.Nullable; +import org.opensearch.sql.calcite.plan.Scannable; + +/** The physical relational operator representing a scan of a {@link RestSourceTable}. */ +public class CalciteEnumerableRestScan extends AbstractCalciteRestScan + implements EnumerableRel, Scannable { + public CalciteEnumerableRestScan( + RelOptCluster cluster, + List hints, + RelOptTable table, + RestSourceTable restTable, + RelDataType schema) { + super( + cluster, + cluster.traitSetOf(EnumerableConvention.INSTANCE), + hints, + table, + restTable, + schema); + } + + @Override + public @Nullable Pair> passThroughTraits(RelTraitSet required) { + return EnumerableRel.super.passThroughTraits(required); + } + + @Override + public @Nullable Pair> deriveTraits( + RelTraitSet childTraits, int childId) { + return EnumerableRel.super.deriveTraits(childTraits, childId); + } + + @Override + public DeriveMode getDeriveMode() { + return EnumerableRel.super.getDeriveMode(); + } + + @Override + public Result implement(EnumerableRelImplementor implementor, Prefer pref) { + PhysType physType = + PhysTypeImpl.of(implementor.getTypeFactory(), getRowType(), pref.preferArray()); + + Expression scanOperator = implementor.stash(this, CalciteEnumerableRestScan.class); + return implementor.result(physType, Blocks.toBlock(Expressions.call(scanOperator, "scan"))); + } + + @Override + public Enumerable<@Nullable Object> scan() { + return new AbstractEnumerable<>() { + @Override + public Enumerator enumerator() { + return new RestEnumerator( + getFieldPath(), + restTable.createRestRequest(), + restTable.createOpenSearchResourceMonitor()); + } + }; + } + + private List getFieldPath() { + return getRowType().getFieldNames().stream().toList(); + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/CalciteLogicalRestScan.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/CalciteLogicalRestScan.java new file mode 100644 index 00000000000..4cb33b48646 --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/CalciteLogicalRestScan.java @@ -0,0 +1,48 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.rest; + +import com.google.common.collect.ImmutableList; +import java.util.List; +import org.apache.calcite.plan.Convention; +import org.apache.calcite.plan.RelOptCluster; +import org.apache.calcite.plan.RelOptPlanner; +import org.apache.calcite.plan.RelOptTable; +import org.apache.calcite.plan.RelTraitSet; +import org.apache.calcite.rel.hint.RelHint; +import org.apache.calcite.rel.type.RelDataType; +import org.opensearch.sql.opensearch.planner.rules.EnumerableRestScanRule; + +/** The logical relational operator representing a scan of a {@link RestSourceTable}. */ +public class CalciteLogicalRestScan extends AbstractCalciteRestScan { + + public CalciteLogicalRestScan( + RelOptCluster cluster, RelOptTable table, RestSourceTable restTable) { + this( + cluster, + cluster.traitSetOf(Convention.NONE), + ImmutableList.of(), + table, + restTable, + table.getRowType()); + } + + protected CalciteLogicalRestScan( + RelOptCluster cluster, + RelTraitSet traitSet, + List hints, + RelOptTable table, + RestSourceTable restTable, + RelDataType schema) { + super(cluster, traitSet, hints, table, restTable, schema); + } + + @Override + public void register(RelOptPlanner planner) { + super.register(planner); + planner.addRule(EnumerableRestScanRule.DEFAULT_CONFIG.toRule()); + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistry.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistry.java new file mode 100644 index 00000000000..9b79e8e57d4 --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistry.java @@ -0,0 +1,407 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.rest; + +import static org.opensearch.sql.data.model.ExprValueUtils.booleanValue; +import static org.opensearch.sql.data.model.ExprValueUtils.doubleValue; +import static org.opensearch.sql.data.model.ExprValueUtils.integerValue; +import static org.opensearch.sql.data.model.ExprValueUtils.longValue; +import static org.opensearch.sql.data.model.ExprValueUtils.stringValue; +import static org.opensearch.sql.data.type.ExprCoreType.BOOLEAN; +import static org.opensearch.sql.data.type.ExprCoreType.DOUBLE; +import static org.opensearch.sql.data.type.ExprCoreType.INTEGER; +import static org.opensearch.sql.data.type.ExprCoreType.LONG; +import static org.opensearch.sql.data.type.ExprCoreType.STRING; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import lombok.Getter; +import org.opensearch.sql.data.model.ExprNullValue; +import org.opensearch.sql.data.model.ExprTupleValue; +import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.data.type.ExprType; +import org.opensearch.sql.opensearch.client.OpenSearchClient; +import org.opensearch.sql.utils.SystemIndexUtils.RestSpec; + +/** + * The read-only endpoint allow-list expressed as data: each allow-listed, read-only endpoint maps to its transport + * action (a read-only call on {@link OpenSearchClient}), a fixed output schema (so the Calcite plan + * can fix its row type at plan time), the query args it accepts, and a secret-field filter. + * + *

This is the single place the read-only allow-list and secret filtering are enforced. + * Endpoints outside the registry, including every mutating endpoint, are rejected by {@link + * #resolve} with a clear exception. Adding an endpoint is a reviewed change here, never arbitrary + * pass-through. + */ +public final class RestEndpointRegistry { + + private RestEndpointRegistry() {} + + /** Produces the raw rows for an endpoint via a read-only client call. */ + @FunctionalInterface + public interface RowFetcher { + List> fetch(OpenSearchClient client, RestSpec spec); + } + + /** A single allow-listed endpoint description. */ + @Getter + public static final class Endpoint { + private final String path; + private final LinkedHashMap schema; + private final Set allowedArgs; + private final Set secretFields; + private final RowFetcher fetcher; + + Endpoint( + String path, + LinkedHashMap schema, + Set allowedArgs, + Set secretFields, + RowFetcher fetcher) { + this.path = path; + this.schema = schema; + this.allowedArgs = allowedArgs; + this.secretFields = secretFields; + this.fetcher = fetcher; + } + + /** Dispatch the read-only call and shape the response into fixed-schema rows. */ + public List toRows(OpenSearchClient client, RestSpec spec) { + List out = new ArrayList<>(); + for (Map raw : fetcher.fetch(client, spec)) { + LinkedHashMap tuple = new LinkedHashMap<>(); + for (Map.Entry col : schema.entrySet()) { + if (secretFields.contains(col.getKey())) { + // Never surface a secret-bearing field, even if the action returns it. + continue; + } + tuple.put(col.getKey(), coerce(col.getKey(), col.getValue(), raw.get(col.getKey()))); + } + out.add(new ExprTupleValue(tuple)); + } + return out; + } + } + + private static final Map REGISTRY = buildRegistry(); + + private static Map buildRegistry() { + Map m = new LinkedHashMap<>(); + + // /_cluster/health — single-row cluster health snapshot (read-only monitor action). + LinkedHashMap healthSchema = new LinkedHashMap<>(); + healthSchema.put("cluster_name", STRING); + healthSchema.put("status", STRING); + healthSchema.put("number_of_nodes", INTEGER); + healthSchema.put("number_of_data_nodes", INTEGER); + healthSchema.put("active_primary_shards", INTEGER); + healthSchema.put("active_shards", INTEGER); + healthSchema.put("relocating_shards", INTEGER); + healthSchema.put("initializing_shards", INTEGER); + healthSchema.put("unassigned_shards", INTEGER); + healthSchema.put("timed_out", BOOLEAN); + m.put( + "/_cluster/health", + new Endpoint( + "/_cluster/health", + healthSchema, + Set.of("local"), + Set.of(), + (client, spec) -> List.of(client.clusterHealth(spec.getArgs())))); + + // /_cat/indices — one row per index (read-only monitor action). + LinkedHashMap catSchema = new LinkedHashMap<>(); + catSchema.put("index", STRING); + catSchema.put("health", STRING); + catSchema.put("pri", INTEGER); + catSchema.put("rep", INTEGER); + catSchema.put("active_shards", INTEGER); + m.put( + "/_cat/indices", + new Endpoint( + "/_cat/indices", + catSchema, + Set.of("health"), + Set.of(), + (client, spec) -> client.catIndices(spec.getArgs()))); + + // /_cat/nodes — one row per node with resource state (read-only monitor action). + LinkedHashMap nodesSchema = new LinkedHashMap<>(); + nodesSchema.put("name", STRING); + nodesSchema.put("ip", STRING); + nodesSchema.put("node_role", STRING); + nodesSchema.put("heap_percent", INTEGER); + nodesSchema.put("ram_percent", INTEGER); + nodesSchema.put("cpu", INTEGER); + m.put( + "/_cat/nodes", + new Endpoint( + "/_cat/nodes", + nodesSchema, + Set.of(), + Set.of(), + (client, spec) -> client.catNodes(spec.getArgs()))); + + // /_cat/cluster_manager — single row identifying the elected cluster manager node. + LinkedHashMap clusterManagerSchema = new LinkedHashMap<>(); + clusterManagerSchema.put("id", STRING); + clusterManagerSchema.put("host", STRING); + clusterManagerSchema.put("ip", STRING); + clusterManagerSchema.put("node", STRING); + m.put( + "/_cat/cluster_manager", + new Endpoint( + "/_cat/cluster_manager", + clusterManagerSchema, + Set.of(), + Set.of(), + (client, spec) -> client.catClusterManager(spec.getArgs()))); + + // /_cat/plugins — one row per installed plugin per node (read-only monitor action). + LinkedHashMap pluginsSchema = new LinkedHashMap<>(); + pluginsSchema.put("name", STRING); + pluginsSchema.put("component", STRING); + pluginsSchema.put("version", STRING); + m.put( + "/_cat/plugins", + new Endpoint( + "/_cat/plugins", + pluginsSchema, + Set.of(), + Set.of(), + (client, spec) -> client.catPlugins(spec.getArgs()))); + + // /_cat/shards — one row per shard (read-only monitor action). + LinkedHashMap shardsSchema = new LinkedHashMap<>(); + shardsSchema.put("index", STRING); + shardsSchema.put("shard", INTEGER); + shardsSchema.put("prirep", STRING); + shardsSchema.put("state", STRING); + shardsSchema.put("node", STRING); + m.put( + "/_cat/shards", + new Endpoint( + "/_cat/shards", + shardsSchema, + Set.of(), + Set.of(), + (client, spec) -> client.catShards(spec.getArgs()))); + + // /_cluster/state — single-row cluster-state epoch (version, uuid, manager node). + LinkedHashMap stateSchema = new LinkedHashMap<>(); + stateSchema.put("cluster_name", STRING); + stateSchema.put("state_uuid", STRING); + stateSchema.put("version", LONG); + stateSchema.put("cluster_manager_node", STRING); + m.put( + "/_cluster/state", + new Endpoint( + "/_cluster/state", + stateSchema, + Set.of(), + Set.of(), + (client, spec) -> List.of(client.clusterState(spec.getArgs())))); + + // /_cluster/settings — one row per configured setting (persistent/transient tier). + LinkedHashMap settingsSchema = new LinkedHashMap<>(); + settingsSchema.put("setting", STRING); + settingsSchema.put("value", STRING); + settingsSchema.put("tier", STRING); + m.put( + "/_cluster/settings", + new Endpoint( + "/_cluster/settings", + settingsSchema, + Set.of(), + Set.of(), + (client, spec) -> client.clusterSettings(spec.getArgs()))); + + // /_resolve/index — one row per resolved index/alias/data_stream name. + LinkedHashMap resolveSchema = new LinkedHashMap<>(); + resolveSchema.put("name", STRING); + resolveSchema.put("type", STRING); + m.put( + "/_resolve/index", + new Endpoint( + "/_resolve/index", + resolveSchema, + Set.of("expand_wildcards"), + Set.of(), + (client, spec) -> client.resolveIndex(spec.getArgs()))); + + return m; + } + + /** + * Resolve an allow-listed endpoint. Anything outside the registry (unknown path, mutating verb, + * {@code /services/*}, plugin admin endpoints) is refused here. + */ + public static Endpoint resolve(String path) { + if (path == null || path.isBlank()) { + throw new IllegalArgumentException( + "rest endpoint must be a non-empty path. Supported read-only endpoints: " + + REGISTRY.keySet()); + } + Endpoint endpoint = REGISTRY.get(path); + if (endpoint == null) { + throw new IllegalArgumentException( + "rest endpoint [" + + path + + "] is not allow-listed. Only read-only in-cluster endpoints are supported: " + + REGISTRY.keySet()); + } + return endpoint; + } + + /** Validate that every supplied query arg is accepted by the endpoint. */ + public static void validate(RestSpec spec) { + Endpoint endpoint = resolve(spec.getEndpoint()); + if (spec.getCount() != null && spec.getCount() < 0) { + throw new IllegalArgumentException( + "rest endpoint [" + + spec.getEndpoint() + + "] count must be a non-negative integer, got [" + + spec.getCount() + + "]"); + } + if (spec.getTimeout() != null) { + // The timeout token is reserved in the grammar for forward compatibility, but a single + // uniform timeout cannot map cleanly across the endpoints (wait-for-status vs + // cluster-manager vs client socket timeouts differ per action). Reject it with a clear + // client error rather than silently ignoring it. + throw new IllegalArgumentException( + "rest endpoint [" + + spec.getEndpoint() + + "] does not support the timeout argument yet"); + } + if (spec.getArgs() != null) { + for (String arg : spec.getArgs().keySet()) { + if (!endpoint.getAllowedArgs().contains(arg)) { + throw new IllegalArgumentException( + "rest endpoint [" + + spec.getEndpoint() + + "] does not accept arg [" + + arg + + "]. Allowed args: " + + endpoint.getAllowedArgs()); + } + validateArgValue(spec.getEndpoint(), arg, spec.getArgs().get(arg)); + } + } + } + + // Allowed value domains for the get-args that are applied server-side. Keys are validated against + // the per-endpoint allow-list above; values are validated here so a user-supplied value is never + // passed unchecked into an admin transport request. + private static final Map> ARG_VALUE_DOMAINS = + Map.of( + "local", Set.of("true", "false"), + "health", Set.of("green", "yellow", "red")); + + private static final Set EXPAND_WILDCARDS_VALUES = + Set.of("open", "closed", "hidden", "none", "all"); + + /** Reject any get-arg value outside its allow-listed domain with a clear client error. */ + private static void validateArgValue(String endpoint, String arg, String value) { + Set domain = ARG_VALUE_DOMAINS.get(arg); + if (domain != null) { + if (value == null || !domain.contains(value.toLowerCase(java.util.Locale.ROOT))) { + throw new IllegalArgumentException( + "rest endpoint [" + + endpoint + + "] arg [" + + arg + + "] has an unsupported value [" + + value + + "]. Allowed values: " + + domain); + } + } else if ("expand_wildcards".equals(arg)) { + if (value == null || value.isBlank()) { + throw new IllegalArgumentException( + "rest endpoint [" + + endpoint + + "] arg [expand_wildcards] has an unsupported value [" + + value + + "]. Allowed values: " + + EXPAND_WILDCARDS_VALUES); + } + for (String token : value.toLowerCase(java.util.Locale.ROOT).split(",")) { + if (!EXPAND_WILDCARDS_VALUES.contains(token.trim())) { + throw new IllegalArgumentException( + "rest endpoint [" + + endpoint + + "] arg [expand_wildcards] has an unsupported value [" + + value + + "]. Allowed values: " + + EXPAND_WILDCARDS_VALUES); + } + } + } + } + + private static ExprValue coerce(String column, ExprType type, Object value) { + if (value == null) { + return ExprNullValue.of(); + } + try { + if (type == INTEGER) { + return integerValue(toNumber(value).intValue()); + } + if (type == LONG) { + return longValue(toNumber(value).longValue()); + } + if (type == DOUBLE) { + return doubleValue(toNumber(value).doubleValue()); + } + if (type == BOOLEAN) { + return booleanValue(toBoolean(value)); + } + } catch (RuntimeException e) { + // Surface a clear client error (HTTP 400) instead of a raw ClassCastException / + // NumberFormatException (HTTP 500) when an endpoint returns an unexpected value shape. + throw new IllegalArgumentException( + "rest endpoint value for column [" + + column + + "] could not be coerced to " + + type + + ": [" + + value + + "]"); + } + return stringValue(String.valueOf(value)); + } + + /** Coerce a transport/JSON value to a Number, parsing numeric strings (e.g. the cat JSON API). */ + private static Number toNumber(Object value) { + if (value instanceof Number n) { + return n; + } + String s = String.valueOf(value).trim(); + if (s.indexOf('.') >= 0 || s.indexOf('e') >= 0 || s.indexOf('E') >= 0) { + return Double.parseDouble(s); + } + return Long.parseLong(s); + } + + /** Coerce a transport/JSON value to a boolean, accepting Boolean or the strings true/false. */ + private static boolean toBoolean(Object value) { + if (value instanceof Boolean b) { + return b; + } + String s = String.valueOf(value).trim(); + if (s.equalsIgnoreCase("true")) { + return true; + } + if (s.equalsIgnoreCase("false")) { + return false; + } + throw new IllegalArgumentException("not a boolean: " + value); + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestEnumerator.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestEnumerator.java new file mode 100644 index 00000000000..ad678acb4c4 --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestEnumerator.java @@ -0,0 +1,86 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.rest; + +import java.util.Iterator; +import java.util.List; +import lombok.EqualsAndHashCode; +import lombok.ToString; +import org.apache.calcite.linq4j.Enumerator; +import org.opensearch.sql.data.model.ExprNullValue; +import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.exception.NonFallbackCalciteException; +import org.opensearch.sql.monitor.ResourceMonitor; + +/** + * A simple resource-monitored iteration over the rows produced by a {@code rest} endpoint dispatch. + * One-for-one parallel of {@code OpenSearchSystemIndexEnumerator}. + */ +public class RestEnumerator implements Enumerator { + /** How many moveNext() calls to perform a resource check once. */ + private static final long NUMBER_OF_NEXT_CALL_TO_CHECK = 1000; + + private final List fields; + + @EqualsAndHashCode.Include @ToString.Include private final RestRequest request; + + private Iterator iterator; + + private ExprValue current; + + /** Number of rows returned. */ + private Integer queryCount; + + /** ResourceMonitor. */ + private final ResourceMonitor monitor; + + public RestEnumerator(List fields, RestRequest request, ResourceMonitor monitor) { + this.fields = fields; + this.request = request; + this.monitor = monitor; + this.queryCount = 0; + this.current = null; + if (!this.monitor.isHealthy()) { + throw new NonFallbackCalciteException("insufficient resources to run the query, quit."); + } + this.iterator = request.search().iterator(); + } + + @Override + public Object current() { + return fields.stream() + .map(k -> current.tupleValue().getOrDefault(k, ExprNullValue.of()).valueForCalcite()) + .toArray(); + } + + @Override + public boolean moveNext() { + boolean shouldCheck = (queryCount % NUMBER_OF_NEXT_CALL_TO_CHECK == 0); + if (shouldCheck && !this.monitor.isHealthy()) { + throw new NonFallbackCalciteException("insufficient resources to load next row, quit."); + } + if (iterator.hasNext()) { + current = iterator.next(); + queryCount++; + return true; + } else { + return false; + } + } + + @Override + public void reset() { + iterator = request.search().iterator(); + queryCount = 0; + current = null; + } + + @Override + public void close() { + iterator = null; + current = null; + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestRequest.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestRequest.java new file mode 100644 index 00000000000..d6ee4dff1ac --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestRequest.java @@ -0,0 +1,47 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.rest; + +import java.util.List; +import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.opensearch.client.OpenSearchClient; +import org.opensearch.sql.opensearch.request.system.OpenSearchSystemRequest; +import org.opensearch.sql.utils.SystemIndexUtils.RestSpec; + +/** + * Dispatches an allow-listed, read-only management endpoint through the transport node client under + * the caller's security thread-context and returns the response shaped to the endpoint's fixed + * schema. The {@code rest} analogue of {@code OpenSearchCatIndicesRequest}; it implements {@link + * OpenSearchSystemRequest} so the enumerator pattern (resource-monitored iteration) is identical to + * the system-index scan family. + */ +public class RestRequest implements OpenSearchSystemRequest { + + private final OpenSearchClient client; + private final RestEndpointRegistry.Endpoint endpoint; + private final RestSpec spec; + + public RestRequest( + OpenSearchClient client, RestEndpointRegistry.Endpoint endpoint, RestSpec spec) { + this.client = client; + this.endpoint = endpoint; + this.spec = spec; + } + + @Override + public List search() { + List rows = endpoint.toRows(client, spec); + if (spec.getCount() != null && spec.getCount() >= 0 && rows.size() > spec.getCount()) { + return rows.subList(0, spec.getCount()); + } + return rows; + } + + @Override + public String toString() { + return "RestRequest{endpoint=" + endpoint.getPath() + "}"; + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestSourceTable.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestSourceTable.java new file mode 100644 index 00000000000..7269dce4955 --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestSourceTable.java @@ -0,0 +1,80 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.rest; + +import java.util.Map; +import lombok.Getter; +import org.apache.calcite.plan.RelOptCluster; +import org.apache.calcite.plan.RelOptTable; +import org.apache.calcite.rel.RelNode; +import org.opensearch.sql.calcite.plan.AbstractOpenSearchTable; +import org.opensearch.sql.common.setting.Settings; +import org.opensearch.sql.data.type.ExprType; +import org.opensearch.sql.opensearch.client.OpenSearchClient; +import org.opensearch.sql.opensearch.monitor.OpenSearchMemoryHealthy; +import org.opensearch.sql.opensearch.monitor.OpenSearchResourceMonitor; +import org.opensearch.sql.planner.logical.LogicalPlan; +import org.opensearch.sql.planner.physical.PhysicalPlan; +import org.opensearch.sql.utils.SystemIndexUtils.RestSpec; + +/** + * The {@code rest} command's row source: a fixed-schema, server-side dispatch table modeled on + * {@link org.opensearch.sql.opensearch.storage.system.OpenSearchSystemIndex}. It resolves the + * validated endpoint spec against {@link RestEndpointRegistry} (which enforces the read-only allow-list and + * secret filtering), exposes the endpoint's fixed schema, and produces a {@link + * CalciteLogicalRestScan} on the Calcite path. + */ +@Getter +public class RestSourceTable extends AbstractOpenSearchTable { + + private final OpenSearchClient client; + private final Settings settings; + private final RestSpec spec; + private final RestEndpointRegistry.Endpoint endpoint; + + public RestSourceTable(OpenSearchClient client, Settings settings, RestSpec spec) { + this.client = client; + this.settings = settings; + this.spec = spec; + // Allow-list and secret filtering enforced here: unknown/mutating endpoints and disallowed args are rejected. + this.endpoint = RestEndpointRegistry.resolve(spec.getEndpoint()); + RestEndpointRegistry.validate(spec); + } + + @Override + public boolean exists() { + return true; + } + + @Override + public void create(Map schema) { + throw new UnsupportedOperationException("rest endpoint is predefined and cannot be created"); + } + + @Override + public Map getFieldTypes() { + return endpoint.getSchema(); + } + + @Override + public RelNode toRel(RelOptTable.ToRelContext context, RelOptTable relOptTable) { + final RelOptCluster cluster = context.getCluster(); + return new CalciteLogicalRestScan(cluster, relOptTable, this); + } + + @Override + public PhysicalPlan implement(LogicalPlan plan) { + throw new UnsupportedOperationException("rest command is supported only on the Calcite engine"); + } + + public RestRequest createRestRequest() { + return new RestRequest(client, endpoint, spec); + } + + public OpenSearchResourceMonitor createOpenSearchResourceMonitor() { + return new OpenSearchResourceMonitor(settings, new OpenSearchMemoryHealthy(settings)); + } +} diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistryTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistryTest.java new file mode 100644 index 00000000000..4284aa6d25d --- /dev/null +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistryTest.java @@ -0,0 +1,213 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.rest; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.when; +import static org.opensearch.sql.data.type.ExprCoreType.INTEGER; +import static org.opensearch.sql.data.type.ExprCoreType.STRING; + +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.opensearch.client.OpenSearchClient; +import org.opensearch.sql.utils.SystemIndexUtils.RestSpec; + +@ExtendWith(MockitoExtension.class) +class RestEndpointRegistryTest { + + @Mock private OpenSearchClient client; + + @Test + void resolveAllowListedEndpoint() { + RestEndpointRegistry.Endpoint endpoint = RestEndpointRegistry.resolve("/_cluster/health"); + assertEquals("/_cluster/health", endpoint.getPath()); + assertEquals(STRING, endpoint.getSchema().get("status")); + assertEquals(INTEGER, endpoint.getSchema().get("number_of_nodes")); + } + + @Test + void resolveRejectsNonAllowListedEndpoint() { + // A mutating endpoint is simply absent from the registry and is refused here. + assertThrows( + IllegalArgumentException.class, () -> RestEndpointRegistry.resolve("/_cluster/reroute")); + assertThrows( + IllegalArgumentException.class, + () -> RestEndpointRegistry.resolve("/services/server/info")); + } + + @Test + void validateRejectsUnknownArg() { + RestSpec spec = new RestSpec("/_cluster/health", Map.of("not_allowed", "x"), null, null); + assertThrows(IllegalArgumentException.class, () -> RestEndpointRegistry.validate(spec)); + } + + @Test + void validateAcceptsAllowedArg() { + RestSpec spec = new RestSpec("/_cluster/health", Map.of("local", "true"), null, null); + RestEndpointRegistry.validate(spec); // no throw + } + + @Test + void validateRejectsDroppedLevelArg() { + // level was dropped (no-op against the fixed cluster-level health schema); now unknown. + RestSpec spec = new RestSpec("/_cluster/health", Map.of("level", "indices"), null, null); + IllegalArgumentException ex = + assertThrows(IllegalArgumentException.class, () -> RestEndpointRegistry.validate(spec)); + assertTrue(ex.getMessage().contains("does not accept arg")); + } + + @Test + void validateRejectsDroppedFlatSettingsArg() { + // flat_settings was dropped (redundant: settings are already flattened to dotted keys). + RestSpec spec = new RestSpec("/_cluster/settings", Map.of("flat_settings", "true"), null, null); + assertThrows(IllegalArgumentException.class, () -> RestEndpointRegistry.validate(spec)); + } + + @Test + void validateAcceptsValidArgValues() { + RestEndpointRegistry.validate( + new RestSpec("/_cat/indices", Map.of("health", "green"), null, null)); + RestEndpointRegistry.validate( + new RestSpec("/_resolve/index", Map.of("expand_wildcards", "open"), null, null)); + RestEndpointRegistry.validate( + new RestSpec("/_resolve/index", Map.of("expand_wildcards", "open,closed"), null, null)); + } + + @Test + void validateRejectsBadArgValue() { + IllegalArgumentException health = + assertThrows( + IllegalArgumentException.class, + () -> + RestEndpointRegistry.validate( + new RestSpec("/_cat/indices", Map.of("health", "purple"), null, null))); + assertTrue(health.getMessage().contains("unsupported value")); + + IllegalArgumentException local = + assertThrows( + IllegalArgumentException.class, + () -> + RestEndpointRegistry.validate( + new RestSpec("/_cluster/health", Map.of("local", "maybe"), null, null))); + assertTrue(local.getMessage().contains("unsupported value")); + + assertThrows( + IllegalArgumentException.class, + () -> + RestEndpointRegistry.validate( + new RestSpec("/_resolve/index", Map.of("expand_wildcards", "sideways"), null, null))); + } + + @Test + void resolveRejectsBlankEndpoint() { + IllegalArgumentException emptyEx = + assertThrows(IllegalArgumentException.class, () -> RestEndpointRegistry.resolve("")); + assertTrue(emptyEx.getMessage().contains("non-empty path")); + assertThrows(IllegalArgumentException.class, () -> RestEndpointRegistry.resolve(" ")); + assertThrows(IllegalArgumentException.class, () -> RestEndpointRegistry.resolve(null)); + } + + @Test + void validateRejectsNegativeCount() { + RestSpec spec = new RestSpec("/_cat/indices", Map.of(), -1, null); + IllegalArgumentException ex = + assertThrows(IllegalArgumentException.class, () -> RestEndpointRegistry.validate(spec)); + assertTrue(ex.getMessage().contains("non-negative")); + } + + @Test + void validateAcceptsZeroCount() { + RestSpec spec = new RestSpec("/_cat/indices", Map.of(), 0, null); + RestEndpointRegistry.validate(spec); // no throw: 0 is a valid limit + } + + @Test + void validateRejectsTimeoutArg() { + RestSpec spec = new RestSpec("/_cluster/health", Map.of(), null, "5s"); + IllegalArgumentException ex = + assertThrows(IllegalArgumentException.class, () -> RestEndpointRegistry.validate(spec)); + assertTrue(ex.getMessage().contains("timeout")); + } + + @Test + void coerceParsesNumericStringValues() { + // The cat JSON API returns numeric columns as strings; coerce must parse them. + Map health = new LinkedHashMap<>(); + health.put("status", "green"); + health.put("number_of_nodes", "3"); + when(client.clusterHealth(any())).thenReturn(health); + + RestEndpointRegistry.Endpoint endpoint = RestEndpointRegistry.resolve("/_cluster/health"); + List rows = + endpoint.toRows(client, new RestSpec("/_cluster/health", Map.of(), null, null)); + + assertEquals(3, rows.get(0).tupleValue().get("number_of_nodes").integerValue()); + } + + @Test + void coerceThrowsClearErrorOnUncoercibleValue() { + // A non-numeric value for an INTEGER column must surface a clear client error (HTTP 400), + // not a raw ClassCastException / NumberFormatException (HTTP 500). + Map health = new LinkedHashMap<>(); + health.put("status", "green"); + health.put("number_of_nodes", "not-a-number"); + when(client.clusterHealth(any())).thenReturn(health); + + RestEndpointRegistry.Endpoint endpoint = RestEndpointRegistry.resolve("/_cluster/health"); + IllegalArgumentException ex = + assertThrows( + IllegalArgumentException.class, + () -> endpoint.toRows(client, new RestSpec("/_cluster/health", Map.of(), null, null))); + assertTrue(ex.getMessage().contains("number_of_nodes")); + assertTrue(ex.getMessage().contains("not-a-number")); + } + + @Test + void clusterHealthRowsAreShapedToFixedSchema() { + Map health = new LinkedHashMap<>(); + health.put("cluster_name", "test-cluster"); + health.put("status", "green"); + health.put("number_of_nodes", 1); + when(client.clusterHealth(any())).thenReturn(health); + + RestEndpointRegistry.Endpoint endpoint = RestEndpointRegistry.resolve("/_cluster/health"); + List rows = + endpoint.toRows(client, new RestSpec("/_cluster/health", Map.of(), null, null)); + + assertEquals(1, rows.size()); + assertEquals("green", rows.get(0).tupleValue().get("status").stringValue()); + assertEquals(1, rows.get(0).tupleValue().get("number_of_nodes").integerValue()); + // a declared column the action did not return becomes null, never absent. + assertTrue(rows.get(0).tupleValue().get("relocating_shards").isNull()); + } + + @Test + void catIndicesRowsAreShapedToFixedSchema() { + Map idx = new LinkedHashMap<>(); + idx.put("index", "books"); + idx.put("health", "yellow"); + idx.put("pri", 1); + idx.put("rep", 1); + when(client.catIndices(any())).thenReturn(List.of(idx)); + + RestEndpointRegistry.Endpoint endpoint = RestEndpointRegistry.resolve("/_cat/indices"); + List rows = + endpoint.toRows(client, new RestSpec("/_cat/indices", Map.of(), null, null)); + + assertEquals(1, rows.size()); + assertEquals("books", rows.get(0).tupleValue().get("index").stringValue()); + assertEquals("yellow", rows.get(0).tupleValue().get("health").stringValue()); + } +} diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestSourceTableTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestSourceTableTest.java new file mode 100644 index 00000000000..a50bf27f042 --- /dev/null +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestSourceTableTest.java @@ -0,0 +1,135 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.rest; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.hasEntry; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.when; +import static org.opensearch.sql.data.type.ExprCoreType.INTEGER; +import static org.opensearch.sql.data.type.ExprCoreType.STRING; + +import com.google.common.collect.ImmutableMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.sql.common.setting.Settings; +import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.opensearch.client.OpenSearchClient; +import org.opensearch.sql.storage.Table; +import org.opensearch.sql.utils.SystemIndexUtils.RestSpec; + +@ExtendWith(MockitoExtension.class) +class RestSourceTableTest { + + @Mock private OpenSearchClient client; + + @Mock private Settings settings; + + private RestSpec healthSpec() { + return new RestSpec("/_cluster/health", Map.of(), null, null); + } + + @Test + void getFieldTypesReturnsFixedEndpointSchema() { + RestSourceTable table = new RestSourceTable(client, settings, healthSpec()); + Map fieldTypes = table.getFieldTypes(); + assertThat(fieldTypes, hasEntry("status", STRING)); + assertThat(fieldTypes, hasEntry("number_of_nodes", INTEGER)); + } + + @Test + void existsIsTrue() { + Table table = new RestSourceTable(client, settings, healthSpec()); + assertTrue(table.exists()); + } + + @Test + void createIsUnsupported() { + Table table = new RestSourceTable(client, settings, healthSpec()); + assertThrows(UnsupportedOperationException.class, () -> table.create(ImmutableMap.of())); + } + + @Test + void implementIsUnsupportedOnV2() { + RestSourceTable table = new RestSourceTable(client, settings, healthSpec()); + assertThrows(UnsupportedOperationException.class, () -> table.implement(null)); + } + + @Test + void constructorRejectsNonAllowListedEndpoint() { + assertThrows( + IllegalArgumentException.class, + () -> + new RestSourceTable( + client, settings, new RestSpec("/_cluster/reroute", Map.of(), null, null))); + } + + @Test + void constructorRejectsDisallowedArg() { + assertThrows( + IllegalArgumentException.class, + () -> + new RestSourceTable( + client, + settings, + new RestSpec("/_cluster/health", Map.of("bad", "x"), null, null))); + } + + @Test + void constructorRejectsNegativeCount() { + assertThrows( + IllegalArgumentException.class, + () -> + new RestSourceTable( + client, settings, new RestSpec("/_cat/indices", Map.of(), -1, null))); + } + + @Test + void constructorRejectsTimeoutArg() { + assertThrows( + IllegalArgumentException.class, + () -> + new RestSourceTable( + client, settings, new RestSpec("/_cluster/health", Map.of(), null, "5s"))); + } + + @Test + void restRequestShapesResponseRows() { + Map health = new LinkedHashMap<>(); + health.put("status", "green"); + health.put("number_of_nodes", 1); + when(client.clusterHealth(any())).thenReturn(health); + + RestSourceTable table = new RestSourceTable(client, settings, healthSpec()); + List rows = table.createRestRequest().search(); + assertEquals(1, rows.size()); + assertEquals("green", rows.get(0).tupleValue().get("status").stringValue()); + assertEquals(1, rows.get(0).tupleValue().get("number_of_nodes").integerValue()); + } + + @Test + void countTruncatesRows() { + Map idx1 = new LinkedHashMap<>(); + idx1.put("index", "a"); + Map idx2 = new LinkedHashMap<>(); + idx2.put("index", "b"); + when(client.catIndices(any())).thenReturn(List.of(idx1, idx2)); + + RestSourceTable table = + new RestSourceTable(client, settings, new RestSpec("/_cat/indices", Map.of(), 1, null)); + List rows = table.createRestRequest().search(); + assertEquals(1, rows.size()); + assertEquals("a", rows.get(0).tupleValue().get("index").stringValue()); + } +} diff --git a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 index 4bc69a8f295..197886342e7 100644 --- a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 @@ -13,6 +13,8 @@ options { caseInsensitive = true; } SEARCH: 'SEARCH'; DESCRIBE: 'DESCRIBE'; SHOW: 'SHOW'; +REST: 'REST'; +TIMEOUT: 'TIMEOUT'; EXPLAIN: 'EXPLAIN'; FROM: 'FROM'; WHERE: 'WHERE'; diff --git a/ppl/src/main/antlr/OpenSearchPPLParser.g4 b/ppl/src/main/antlr/OpenSearchPPLParser.g4 index 98f85b08282..0d4469893c2 100644 --- a/ppl/src/main/antlr/OpenSearchPPLParser.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLParser.g4 @@ -45,6 +45,7 @@ subSearch // commands pplCommands : describeCommand + | restCommand | showDataSourcesCommand | searchCommand | multisearchCommand @@ -103,6 +104,7 @@ commands commandName : SEARCH | DESCRIBE + | REST | SHOW | WHERE | FIELDS @@ -206,6 +208,16 @@ describeCommand : DESCRIBE tableSourceClause ; + +restCommand + : REST stringLiteral (restArgument)* + ; + +restArgument + : COUNT EQUAL integerLiteral + | TIMEOUT EQUAL stringLiteral + | ident EQUAL literalValue + ; showDataSourcesCommand : SHOW DATASOURCES ; diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java index 3741137f5a9..546450b8191 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java @@ -109,6 +109,7 @@ import org.opensearch.sql.ast.tree.Rename; import org.opensearch.sql.ast.tree.Replace; import org.opensearch.sql.ast.tree.ReplacePair; +import org.opensearch.sql.ast.tree.RestRelation; import org.opensearch.sql.ast.tree.Reverse; import org.opensearch.sql.ast.tree.Rex; import org.opensearch.sql.ast.tree.SPath; @@ -140,6 +141,7 @@ import org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParserBaseVisitor; import org.opensearch.sql.ppl.utils.ArgumentFactory; import org.opensearch.sql.ppl.utils.UnresolvedPlanHelper; +import org.opensearch.sql.utils.SystemIndexUtils; /** Class of building the AST. Refines the visit path and build the AST nodes */ public class AstBuilder extends OpenSearchPPLParserBaseVisitor { @@ -251,6 +253,37 @@ public UnresolvedPlan visitShowDataSourcesCommand( return new DescribeRelation(qualifiedName(DATASOURCES_TABLE_NAME)); } + /** + * Rest command.
+ * Leading command that reads an allow-listed, read-only in-cluster management endpoint + * (cluster/cat/nodes) as rows. The validated endpoint spec is encoded into a single reserved + * table name via {@link org.opensearch.sql.utils.SystemIndexUtils#restTable}; that name resolves + * through the storage engine to a REST source table on the Calcite path, mirroring how DESCRIBE + * resolves to a system index. Allow-list/authorization enforcement happens at source-table + * construction in the storage engine (it owns the transport actions and per-endpoint schemas). + */ + @Override + public UnresolvedPlan visitRestCommand(OpenSearchPPLParser.RestCommandContext ctx) { + String endpoint = StringUtils.unquoteText(ctx.stringLiteral().getText()); + LinkedHashMap args = new LinkedHashMap<>(); + Integer count = null; + String timeout = null; + for (OpenSearchPPLParser.RestArgumentContext arg : ctx.restArgument()) { + if (arg.COUNT() != null) { + count = Integer.parseInt(arg.integerLiteral().getText()); + } else if (arg.TIMEOUT() != null) { + timeout = StringUtils.unquoteText(arg.stringLiteral().getText()); + } else { + args.put( + StringUtils.unquoteIdentifier(arg.ident().getText()), + StringUtils.unquoteText(arg.literalValue().getText())); + } + } + String token = + SystemIndexUtils.restTable(new SystemIndexUtils.RestSpec(endpoint, args, count, timeout)); + return new RestRelation(new QualifiedName(token)); + } + /** Where command. */ @Override public UnresolvedPlan visitWhereCommand(WhereCommandContext ctx) { diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java b/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java index 4b75d444467..fbd447c4e0d 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java @@ -96,6 +96,7 @@ import org.opensearch.sql.ast.tree.Relation; import org.opensearch.sql.ast.tree.Rename; import org.opensearch.sql.ast.tree.Replace; +import org.opensearch.sql.ast.tree.RestRelation; import org.opensearch.sql.ast.tree.Reverse; import org.opensearch.sql.ast.tree.Rex; import org.opensearch.sql.ast.tree.SPath; @@ -122,6 +123,7 @@ import org.opensearch.sql.planner.logical.LogicalRemove; import org.opensearch.sql.planner.logical.LogicalRename; import org.opensearch.sql.planner.logical.LogicalSort; +import org.opensearch.sql.utils.SystemIndexUtils; /** Utility class to mask sensitive information in incoming PPL queries. */ public class PPLQueryDataAnonymizer extends AbstractNodeVisitor { @@ -165,6 +167,23 @@ public String visitExplain(Explain node, String context) { @Override public String visitRelation(Relation node, String context) { + if (node instanceof RestRelation) { + SystemIndexUtils.RestSpec spec = + SystemIndexUtils.decodeRestSpec(node.getTableQualifiedName().toString()); + StringBuilder sb = new StringBuilder("rest ").append(spec.getEndpoint()); + if (spec.getCount() != null) { + sb.append(" count=").append(MASK_LITERAL); + } + if (spec.getTimeout() != null) { + sb.append(" timeout=").append(MASK_LITERAL); + } + if (spec.getArgs() != null) { + for (String key : spec.getArgs().keySet()) { + sb.append(' ').append(key).append('=').append(MASK_LITERAL); + } + } + return sb.toString(); + } if (node instanceof DescribeRelation) { return StringUtils.format("describe %s", MASK_TABLE); } diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLRestTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLRestTest.java new file mode 100644 index 00000000000..790c071f6ae --- /dev/null +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLRestTest.java @@ -0,0 +1,66 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.ppl.calcite; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; + +import org.junit.Test; +import org.opensearch.sql.ast.Node; +import org.opensearch.sql.ast.tree.Project; +import org.opensearch.sql.ast.tree.RestRelation; +import org.opensearch.sql.common.setting.Settings; +import org.opensearch.sql.ppl.antlr.PPLSyntaxParser; +import org.opensearch.sql.ppl.parser.AstBuilder; +import org.opensearch.sql.utils.SystemIndexUtils; + +/** + * Calcite-path coverage for the {@code rest} leading command at the parse / AST tier. + * + *

The {@code rest} row source resolves through {@code OpenSearchStorageEngine.getTable} -> + * {@code RestSourceTable} -> {@code CalciteLogicalRestScan}, which lives in the {@code opensearch} + * module. This ppl-module Calcite harness binds a Calcite SCOTT schema rather than the OpenSearch + * storage engine, so the optimized {@code CalciteEnumerableRestScan} logical-plan assertion is exercised in {@code RestSourceTableTest} (logical scan + fixed row type, + * unit) and {@code CalcitePPLRestIT} (schema + datarows on a live single-node cluster). This test + * pins the Calcite-facing contract that the ppl module owns: the grammar/AST rewrite of {@code + * rest} into a {@code RestRelation} carrying the validated, reserved-name-encoded endpoint spec + * that rides {@code visitRelation} exactly like {@code DESCRIBE}. + */ +public class CalcitePPLRestTest { + + private final PPLSyntaxParser parser = new PPLSyntaxParser(); + private final Settings settings = mock(Settings.class); + + private Node parse(String ppl) { + return new AstBuilder(ppl, settings).visit(parser.parse(ppl)); + } + + @Test + public void restHealthProjectsDeclaredColumns() { + Project project = + (Project) parse("| rest \"/_cluster/health\" | fields status, number_of_nodes"); + RestRelation rest = (RestRelation) project.getChild().get(0); + SystemIndexUtils.RestSpec spec = + SystemIndexUtils.decodeRestSpec(rest.getTableQualifiedName().toString()); + assertEquals("/_cluster/health", spec.getEndpoint()); + // downstream fields compose on top of the rest row source. + assertEquals(2, project.getProjectList().size()); + } + + @Test + public void restReservedNameRoundTrips() { + RestRelation rest = + (RestRelation) parse("| rest \"/_cat/indices\" count=10 timeout=\"5s\" health=\"green\""); + String reserved = rest.getTableQualifiedName().toString(); + assertTrue(SystemIndexUtils.isRestSource(reserved)); + SystemIndexUtils.RestSpec spec = SystemIndexUtils.decodeRestSpec(reserved); + assertEquals("/_cat/indices", spec.getEndpoint()); + assertEquals(Integer.valueOf(10), spec.getCount()); + assertEquals("5s", spec.getTimeout()); + assertEquals("green", spec.getArgs().get("health")); + } +} diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java index 9d70487c741..d57ca8a69bb 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java @@ -1939,4 +1939,32 @@ public void testJoinNoPrefixComparisonStaysCondition() { public void testJoinPrefixWithoutCriteriaKeywordIsSyntaxError() { assertThrows(SyntaxCheckException.class, () -> plan("source=t1 | inner join a t2")); } + + // rest command tests + + @Test + public void testRestCommand() { + org.opensearch.sql.ast.tree.Project project = + (org.opensearch.sql.ast.tree.Project) + plan("| rest \"/_cluster/health\" | fields status, number_of_nodes"); + org.opensearch.sql.ast.tree.RestRelation rest = + (org.opensearch.sql.ast.tree.RestRelation) project.getChild().get(0); + SystemIndexUtils.RestSpec spec = + SystemIndexUtils.decodeRestSpec(rest.getTableQualifiedName().toString()); + assertEquals("/_cluster/health", spec.getEndpoint()); + assertTrue(spec.getArgs().isEmpty()); + } + + @Test + public void testRestCommandWithArgs() { + org.opensearch.sql.ast.tree.RestRelation rest = + (org.opensearch.sql.ast.tree.RestRelation) + plan("| rest \"/_cluster/health\" count=5 timeout=\"30s\" level=\"indices\""); + SystemIndexUtils.RestSpec spec = + SystemIndexUtils.decodeRestSpec(rest.getTableQualifiedName().toString()); + assertEquals("/_cluster/health", spec.getEndpoint()); + assertEquals(Integer.valueOf(5), spec.getCount()); + assertEquals("30s", spec.getTimeout()); + assertEquals("indices", spec.getArgs().get("level")); + } } diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java index 6756cdc198a..21c333166f1 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java @@ -48,6 +48,18 @@ public void testPrometheusPPLCommand() { assertEquals("source=table", anonymize("source=prometheus.http_requests_process")); } + @Test + public void testRestCommand() { + assertEquals("rest /_cluster/health", anonymize("| rest \"/_cluster/health\"")); + } + + @Test + public void testRestCommandMasksArgValues() { + assertEquals( + "rest /_cluster/health count=*** timeout=*** level=***", + anonymize("| rest \"/_cluster/health\" count=5 timeout=\"30s\" level=\"indices\"")); + } + @Test public void testWhereCommand() { assertEquals("source=table | where identifier = ***", anonymize("search source=t | where a=1")); From a7a8a2037e0977ee8c793739779f541a8337a125 Mon Sep 17 00:00:00 2001 From: Louis Chu Date: Tue, 30 Jun 2026 09:46:31 -0700 Subject: [PATCH 02/14] Align rest '/_cluster/settings' redaction with native endpoint; CI fixes; comment cleanup - /_cluster/settings: run the persistent and transient tiers through the node SettingsFilter (published via RestSettingsFilterHolder from SQLPlugin#getRestHandlers) so Property.Filtered and plugin-registered pattern settings are redacted exactly as the native GET /_cluster/settings endpoint. Remove the dead secretFields column-filter, which was the wrong shape for the (setting, value, tier) rows. - Parser: add TIMEOUT to searchableKeyWord so a bare 'timeout' term still matches searchLiteral. - coerce(): narrow the catch to IllegalArgumentException | ClassCastException; add empty-string guards in toNumber/toBoolean. - spotlessApply formatting; drop outdated and redundant comments. Tests: RestEndpointRegistryTest, RestSourceTableTest, OpenSearchNodeClientClusterSettingsFilterTest green. Signed-off-by: Louis Chu --- .../sql/calcite/remote/CalcitePPLRestIT.java | 17 ++- .../client/OpenSearchNodeClient.java | 21 +++- .../client/OpenSearchRestClient.java | 3 +- .../storage/rest/RestEndpointRegistry.java | 47 ++++---- .../rest/RestSettingsFilterHolder.java | 40 +++++++ .../storage/rest/RestSourceTable.java | 9 +- ...chNodeClientClusterSettingsFilterTest.java | 102 ++++++++++++++++++ .../rest/RestEndpointRegistryTest.java | 3 +- .../org/opensearch/sql/plugin/SQLPlugin.java | 4 + ppl/src/main/antlr/OpenSearchPPLParser.g4 | 2 + .../sql/ppl/calcite/CalcitePPLRestTest.java | 11 +- 11 files changed, 205 insertions(+), 54 deletions(-) create mode 100644 opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestSettingsFilterHolder.java create mode 100644 opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClientClusterSettingsFilterTest.java diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLRestIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLRestIT.java index c2ea6069996..70ee4ef0de6 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLRestIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLRestIT.java @@ -19,7 +19,8 @@ /** * Integration tests for the {@code rest} leading command on the Calcite path. Uses {@code - * /_cluster/health} as the deterministic, single-row endpoint on a single-node test cluster. Also verifies that a non-allow-listed / mutating endpoint is refused. + * /_cluster/health} as the deterministic, single-row endpoint on a single-node test cluster. Also + * verifies that a non-allow-listed / mutating endpoint is refused. */ public class CalcitePPLRestIT extends PPLIntegTestCase { @@ -31,8 +32,7 @@ public void init() throws Exception { @Test public void testRestClusterHealthSchema() throws IOException { - JSONObject result = - executeQuery("| rest '/_cluster/health' | fields status, number_of_nodes"); + JSONObject result = executeQuery("| rest '/_cluster/health' | fields status, number_of_nodes"); verifySchema(result, schema("status", "string"), schema("number_of_nodes", "int")); } @@ -70,13 +70,12 @@ public void testRestRejectsTimeoutArg() throws IOException { /** * Assert a {@code rest} query is refused as a client error: HTTP 400 (not a 500 system error) - * with the given substring in the response body. The negative-case check for allow-list and secret-filter enforcement. + * with the given substring in the response body. Covers allow-list and bad-argument rejection. */ private void assertRestBadRequest(String query, String expectedSubstring) { ResponseException e = org.junit.Assert.assertThrows(ResponseException.class, () -> executeQuery(query)); - org.junit.Assert.assertEquals( - 400, e.getResponse().getStatusLine().getStatusCode()); + org.junit.Assert.assertEquals(400, e.getResponse().getStatusLine().getStatusCode()); org.junit.Assert.assertTrue( "expected [" + expectedSubstring + "] in response body: " + e.getMessage(), e.getMessage().contains(expectedSubstring)); @@ -153,8 +152,7 @@ public void testRestClusterStateSingleRow() throws IOException { @Test public void testRestClusterSettingsSchema() throws IOException { // Schema is registry-fixed regardless of how many settings are configured. - JSONObject result = - executeQuery("| rest '/_cluster/settings' | fields setting, value, tier"); + JSONObject result = executeQuery("| rest '/_cluster/settings' | fields setting, value, tier"); verifySchema( result, schema("setting", "string"), schema("value", "string"), schema("tier", "string")); } @@ -187,8 +185,7 @@ public void testRestClusterHealthLocalArg() throws IOException { @Test public void testRestCatIndicesHealthFilterReturnsNoRed() throws IOException { // health filters rows server-side; a healthy cluster has no red indices, so count is 0. - JSONObject result = - executeQuery("| rest '/_cat/indices' health='red' | stats count() as cnt"); + JSONObject result = executeQuery("| rest '/_cat/indices' health='red' | stats count() as cnt"); verifyDataRows(result, rows(0)); } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClient.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClient.java index 9ce9ff79ea8..bc5ac2128ea 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClient.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClient.java @@ -459,8 +459,22 @@ public List> clusterSettings(Map params) { .state(new org.opensearch.action.admin.cluster.state.ClusterStateRequest()) .actionGet(); List> rows = new java.util.ArrayList<>(); - collectSettings(response.getState().metadata().persistentSettings(), "persistent", rows); - collectSettings(response.getState().metadata().transientSettings(), "transient", rows); + org.opensearch.common.settings.Settings persistent = + response.getState().metadata().persistentSettings(); + org.opensearch.common.settings.Settings transientSettings = + response.getState().metadata().transientSettings(); + // Mirror the native GET /_cluster/settings redaction: run both tiers through the node's + // SettingsFilter so Property.Filtered (and plugin-registered pattern) settings are not + // surfaced raw. The transport path carries no SettingsFilter of its own; the node instance is + // published into RestSettingsFilterHolder from SQLPlugin#getRestHandlers at startup. + org.opensearch.common.settings.SettingsFilter filter = + org.opensearch.sql.opensearch.storage.rest.RestSettingsFilterHolder.get(); + if (filter != null) { + persistent = filter.filter(persistent); + transientSettings = filter.filter(transientSettings); + } + collectSettings(persistent, "persistent", rows); + collectSettings(transientSettings, "transient", rows); return rows; } @@ -498,7 +512,8 @@ public List> resolveIndex(Map params) { .DEFAULT_INDICES_OPTIONS)); org.opensearch.action.admin.indices.resolve.ResolveIndexAction.Response response = client - .execute(org.opensearch.action.admin.indices.resolve.ResolveIndexAction.INSTANCE, request) + .execute( + org.opensearch.action.admin.indices.resolve.ResolveIndexAction.INSTANCE, request) .actionGet(); List> rows = new java.util.ArrayList<>(); for (org.opensearch.action.admin.indices.resolve.ResolveIndexAction.ResolvedIndex idx : diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchRestClient.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchRestClient.java index c9290408229..5afe3a927f1 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchRestClient.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchRestClient.java @@ -285,8 +285,7 @@ public Map clusterHealth(Map params) { if (params != null && Boolean.parseBoolean(params.get("local"))) { request.local(true); } - ClusterHealthResponse response = - client.cluster().health(request, RequestOptions.DEFAULT); + ClusterHealthResponse response = client.cluster().health(request, RequestOptions.DEFAULT); return flattenHealth(response); } catch (IOException e) { throw new IllegalStateException("Failed to get cluster health", e); diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistry.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistry.java index 9b79e8e57d4..feb6a6fafe3 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistry.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistry.java @@ -30,14 +30,13 @@ import org.opensearch.sql.utils.SystemIndexUtils.RestSpec; /** - * The read-only endpoint allow-list expressed as data: each allow-listed, read-only endpoint maps to its transport - * action (a read-only call on {@link OpenSearchClient}), a fixed output schema (so the Calcite plan - * can fix its row type at plan time), the query args it accepts, and a secret-field filter. + * The read-only endpoint allow-list expressed as data: each allow-listed, read-only endpoint maps + * to its transport action (a read-only call on {@link OpenSearchClient}), a fixed output schema (so + * the Calcite plan can fix its row type at plan time), and the query args it accepts. * - *

This is the single place the read-only allow-list and secret filtering are enforced. - * Endpoints outside the registry, including every mutating endpoint, are rejected by {@link - * #resolve} with a clear exception. Adding an endpoint is a reviewed change here, never arbitrary - * pass-through. + *

This is the single place the read-only allow-list is enforced. Endpoints outside the registry, + * including every mutating endpoint, are rejected by {@link #resolve} with a clear exception. + * Adding an endpoint is a reviewed change here, never arbitrary pass-through. */ public final class RestEndpointRegistry { @@ -55,19 +54,16 @@ public static final class Endpoint { private final String path; private final LinkedHashMap schema; private final Set allowedArgs; - private final Set secretFields; private final RowFetcher fetcher; Endpoint( String path, LinkedHashMap schema, Set allowedArgs, - Set secretFields, RowFetcher fetcher) { this.path = path; this.schema = schema; this.allowedArgs = allowedArgs; - this.secretFields = secretFields; this.fetcher = fetcher; } @@ -77,10 +73,6 @@ public List toRows(OpenSearchClient client, RestSpec spec) { for (Map raw : fetcher.fetch(client, spec)) { LinkedHashMap tuple = new LinkedHashMap<>(); for (Map.Entry col : schema.entrySet()) { - if (secretFields.contains(col.getKey())) { - // Never surface a secret-bearing field, even if the action returns it. - continue; - } tuple.put(col.getKey(), coerce(col.getKey(), col.getValue(), raw.get(col.getKey()))); } out.add(new ExprTupleValue(tuple)); @@ -112,7 +104,6 @@ private static Map buildRegistry() { "/_cluster/health", healthSchema, Set.of("local"), - Set.of(), (client, spec) -> List.of(client.clusterHealth(spec.getArgs())))); // /_cat/indices — one row per index (read-only monitor action). @@ -128,7 +119,6 @@ private static Map buildRegistry() { "/_cat/indices", catSchema, Set.of("health"), - Set.of(), (client, spec) -> client.catIndices(spec.getArgs()))); // /_cat/nodes — one row per node with resource state (read-only monitor action). @@ -145,7 +135,6 @@ private static Map buildRegistry() { "/_cat/nodes", nodesSchema, Set.of(), - Set.of(), (client, spec) -> client.catNodes(spec.getArgs()))); // /_cat/cluster_manager — single row identifying the elected cluster manager node. @@ -160,7 +149,6 @@ private static Map buildRegistry() { "/_cat/cluster_manager", clusterManagerSchema, Set.of(), - Set.of(), (client, spec) -> client.catClusterManager(spec.getArgs()))); // /_cat/plugins — one row per installed plugin per node (read-only monitor action). @@ -174,7 +162,6 @@ private static Map buildRegistry() { "/_cat/plugins", pluginsSchema, Set.of(), - Set.of(), (client, spec) -> client.catPlugins(spec.getArgs()))); // /_cat/shards — one row per shard (read-only monitor action). @@ -190,7 +177,6 @@ private static Map buildRegistry() { "/_cat/shards", shardsSchema, Set.of(), - Set.of(), (client, spec) -> client.catShards(spec.getArgs()))); // /_cluster/state — single-row cluster-state epoch (version, uuid, manager node). @@ -205,7 +191,6 @@ private static Map buildRegistry() { "/_cluster/state", stateSchema, Set.of(), - Set.of(), (client, spec) -> List.of(client.clusterState(spec.getArgs())))); // /_cluster/settings — one row per configured setting (persistent/transient tier). @@ -219,7 +204,6 @@ private static Map buildRegistry() { "/_cluster/settings", settingsSchema, Set.of(), - Set.of(), (client, spec) -> client.clusterSettings(spec.getArgs()))); // /_resolve/index — one row per resolved index/alias/data_stream name. @@ -232,7 +216,6 @@ private static Map buildRegistry() { "/_resolve/index", resolveSchema, Set.of("expand_wildcards"), - Set.of(), (client, spec) -> client.resolveIndex(spec.getArgs()))); return m; @@ -276,9 +259,7 @@ public static void validate(RestSpec spec) { // cluster-manager vs client socket timeouts differ per action). Reject it with a clear // client error rather than silently ignoring it. throw new IllegalArgumentException( - "rest endpoint [" - + spec.getEndpoint() - + "] does not support the timeout argument yet"); + "rest endpoint [" + spec.getEndpoint() + "] does not support the timeout argument yet"); } if (spec.getArgs() != null) { for (String arg : spec.getArgs().keySet()) { @@ -363,9 +344,11 @@ private static ExprValue coerce(String column, ExprType type, Object value) { if (type == BOOLEAN) { return booleanValue(toBoolean(value)); } - } catch (RuntimeException e) { - // Surface a clear client error (HTTP 400) instead of a raw ClassCastException / - // NumberFormatException (HTTP 500) when an endpoint returns an unexpected value shape. + } catch (IllegalArgumentException | ClassCastException e) { + // Surface a clear client error (HTTP 400) instead of a raw HTTP 500 when an endpoint + // returns an unexpected value shape. NumberFormatException extends IllegalArgumentException, + // so toNumber parse failures and toBoolean's "not a boolean" are both caught here; genuinely + // unexpected faults (NPE, etc.) are left to propagate. throw new IllegalArgumentException( "rest endpoint value for column [" + column @@ -384,6 +367,9 @@ private static Number toNumber(Object value) { return n; } String s = String.valueOf(value).trim(); + if (s.isEmpty()) { + throw new NumberFormatException("empty string"); + } if (s.indexOf('.') >= 0 || s.indexOf('e') >= 0 || s.indexOf('E') >= 0) { return Double.parseDouble(s); } @@ -396,6 +382,9 @@ private static boolean toBoolean(Object value) { return b; } String s = String.valueOf(value).trim(); + if (s.isEmpty()) { + throw new IllegalArgumentException("empty string is not a boolean"); + } if (s.equalsIgnoreCase("true")) { return true; } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestSettingsFilterHolder.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestSettingsFilterHolder.java new file mode 100644 index 00000000000..d425c361e6a --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestSettingsFilterHolder.java @@ -0,0 +1,40 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.rest; + +import org.opensearch.common.settings.SettingsFilter; + +/** + * Bridge for sharing the node-level {@link SettingsFilter} with the in-cluster {@code rest ' + * /_cluster/settings'} fetcher. + * + *

The native {@code GET /_cluster/settings} REST endpoint redacts settings registered with + * {@code Property.Filtered} (or matched by a plugin-registered filter pattern) by running the + * response through {@link SettingsFilter}. The PPL {@code rest} command's in-cluster path reads + * {@code persistentSettings()}/{@code transientSettings()} straight from cluster state via the + * transport layer, where no {@link SettingsFilter} is applied. To keep the command's redaction + * behavior identical to the native endpoint, the node's {@link SettingsFilter} is published here. + * + *

Why a static holder: the {@link SettingsFilter} instance is only handed to the plugin in + * {@code SQLPlugin#getRestHandlers}, which runs outside any Guice-managed lifecycle, while {@link + * OpenSearchNodeClient} is built through the Node injector. Persisting the filter here once {@code + * getRestHandlers} fires lets the fetcher read the same instance without going back through the + * injector. This mirrors the existing {@code AnalyticsExecutorHolder} pattern. + */ +public final class RestSettingsFilterHolder { + + private static volatile SettingsFilter settingsFilter; + + private RestSettingsFilterHolder() {} + + public static void set(SettingsFilter instance) { + settingsFilter = instance; + } + + public static SettingsFilter get() { + return settingsFilter; + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestSourceTable.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestSourceTable.java index 7269dce4955..a900a1d319e 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestSourceTable.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestSourceTable.java @@ -23,9 +23,9 @@ /** * The {@code rest} command's row source: a fixed-schema, server-side dispatch table modeled on * {@link org.opensearch.sql.opensearch.storage.system.OpenSearchSystemIndex}. It resolves the - * validated endpoint spec against {@link RestEndpointRegistry} (which enforces the read-only allow-list and - * secret filtering), exposes the endpoint's fixed schema, and produces a {@link - * CalciteLogicalRestScan} on the Calcite path. + * validated endpoint spec against {@link RestEndpointRegistry} (which enforces the read-only + * allow-list), exposes the endpoint's fixed schema, and produces a {@link CalciteLogicalRestScan} + * on the Calcite path. */ @Getter public class RestSourceTable extends AbstractOpenSearchTable { @@ -39,7 +39,8 @@ public RestSourceTable(OpenSearchClient client, Settings settings, RestSpec spec this.client = client; this.settings = settings; this.spec = spec; - // Allow-list and secret filtering enforced here: unknown/mutating endpoints and disallowed args are rejected. + // Allow-list enforced here: unknown/mutating endpoints and disallowed args + // are rejected. this.endpoint = RestEndpointRegistry.resolve(spec.getEndpoint()); RestEndpointRegistry.validate(spec); } diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClientClusterSettingsFilterTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClientClusterSettingsFilterTest.java new file mode 100644 index 00000000000..47ea968cb8f --- /dev/null +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClientClusterSettingsFilterTest.java @@ -0,0 +1,102 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.client; + +import static java.util.stream.Collectors.toSet; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Answers.RETURNS_DEEP_STUBS; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.util.List; +import java.util.Map; +import java.util.Set; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import org.opensearch.action.admin.cluster.state.ClusterStateRequest; +import org.opensearch.action.admin.cluster.state.ClusterStateResponse; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.settings.SettingsFilter; +import org.opensearch.sql.opensearch.storage.rest.RestSettingsFilterHolder; +import org.opensearch.transport.client.node.NodeClient; + +/** + * Verifies the in-cluster {@code rest '/_cluster/settings'} fetcher redacts filtered settings using + * the node {@link SettingsFilter}, matching the native {@code GET /_cluster/settings} endpoint. + */ +class OpenSearchNodeClientClusterSettingsFilterTest { + + @AfterEach + void clearHolder() { + RestSettingsFilterHolder.set(null); + } + + private OpenSearchNodeClient clientReturning(Settings persistent, Settings transientSettings) { + NodeClient nodeClient = mock(NodeClient.class, RETURNS_DEEP_STUBS); + ClusterStateResponse stateResp = mock(ClusterStateResponse.class, RETURNS_DEEP_STUBS); + when(nodeClient.admin().cluster().state(any(ClusterStateRequest.class)).actionGet()) + .thenReturn(stateResp); + when(stateResp.getState().metadata().persistentSettings()).thenReturn(persistent); + when(stateResp.getState().metadata().transientSettings()).thenReturn(transientSettings); + return new OpenSearchNodeClient(nodeClient); + } + + @Test + void clusterSettingsRedactsFilteredKeyWhenFilterPublished() { + Settings persistent = + Settings.builder() + .put("cluster.routing.allocation.enable", "all") + .put("plugins.secret.token", "supersecret") + .build(); + OpenSearchNodeClient client = clientReturning(persistent, Settings.EMPTY); + + // Publish a filter that redacts the secret key, exactly as the native endpoint would. + RestSettingsFilterHolder.set(new SettingsFilter(List.of("plugins.secret.token"))); + + List> rows = client.clusterSettings(Map.of()); + Set keys = rows.stream().map(r -> (String) r.get("setting")).collect(toSet()); + + assertTrue(keys.contains("cluster.routing.allocation.enable"), "non-secret setting kept"); + assertFalse(keys.contains("plugins.secret.token"), "filtered setting must be redacted"); + } + + @Test + void clusterSettingsRedactsByGlobPattern() { + Settings persistent = + Settings.builder() + .put("cluster.routing.allocation.enable", "all") + .put("s3.client.default.secret_key", "AKIAEXAMPLE") + .build(); + OpenSearchNodeClient client = clientReturning(persistent, Settings.EMPTY); + + RestSettingsFilterHolder.set(new SettingsFilter(List.of("s3.client.*.secret_key"))); + + Set keys = + client.clusterSettings(Map.of()).stream() + .map(r -> (String) r.get("setting")) + .collect(toSet()); + + assertTrue(keys.contains("cluster.routing.allocation.enable")); + assertFalse(keys.contains("s3.client.default.secret_key"), "glob-matched secret redacted"); + } + + @Test + void clusterSettingsReturnsRawWhenNoFilterPublished() { + Settings persistent = Settings.builder().put("plugins.secret.token", "supersecret").build(); + OpenSearchNodeClient client = clientReturning(persistent, Settings.EMPTY); + + // No filter published: confirms the SettingsFilter is the redaction mechanism (not some other + // code path). At runtime getRestHandlers always publishes the filter before any query. + Set keys = + client.clusterSettings(Map.of()).stream() + .map(r -> (String) r.get("setting")) + .collect(toSet()); + + assertTrue(keys.contains("plugins.secret.token")); + } +} diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistryTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistryTest.java index 4284aa6d25d..3254ee12f09 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistryTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistryTest.java @@ -107,7 +107,8 @@ void validateRejectsBadArgValue() { IllegalArgumentException.class, () -> RestEndpointRegistry.validate( - new RestSpec("/_resolve/index", Map.of("expand_wildcards", "sideways"), null, null))); + new RestSpec( + "/_resolve/index", Map.of("expand_wildcards", "sideways"), null, null))); } @Test diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java b/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java index ab8b923e3ae..48ad5c0b0a8 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java +++ b/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java @@ -197,6 +197,10 @@ public List getRestHandlers( Metrics.getInstance().registerDefaultMetrics(); + // Publish the node SettingsFilter so the in-cluster `rest '/_cluster/settings'` fetcher can + // redact filtered settings exactly as the native GET /_cluster/settings endpoint does. + org.opensearch.sql.opensearch.storage.rest.RestSettingsFilterHolder.set(settingsFilter); + return Arrays.asList( new RestPPLQueryAction(), new RestPPLGrammarAction(), diff --git a/ppl/src/main/antlr/OpenSearchPPLParser.g4 b/ppl/src/main/antlr/OpenSearchPPLParser.g4 index 0d4469893c2..8dca6a54ba0 100644 --- a/ppl/src/main/antlr/OpenSearchPPLParser.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLParser.g4 @@ -1785,4 +1785,6 @@ searchableKeyWord | MAX_DEPTH | DEPTH_FIELD | EDGE + // rest command token, also usable as a free-text search term / identifier + | TIMEOUT ; diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLRestTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLRestTest.java index 790c071f6ae..f26a2e617e4 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLRestTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLRestTest.java @@ -24,11 +24,12 @@ *

The {@code rest} row source resolves through {@code OpenSearchStorageEngine.getTable} -> * {@code RestSourceTable} -> {@code CalciteLogicalRestScan}, which lives in the {@code opensearch} * module. This ppl-module Calcite harness binds a Calcite SCOTT schema rather than the OpenSearch - * storage engine, so the optimized {@code CalciteEnumerableRestScan} logical-plan assertion is exercised in {@code RestSourceTableTest} (logical scan + fixed row type, - * unit) and {@code CalcitePPLRestIT} (schema + datarows on a live single-node cluster). This test - * pins the Calcite-facing contract that the ppl module owns: the grammar/AST rewrite of {@code - * rest} into a {@code RestRelation} carrying the validated, reserved-name-encoded endpoint spec - * that rides {@code visitRelation} exactly like {@code DESCRIBE}. + * storage engine, so the optimized {@code CalciteEnumerableRestScan} logical-plan assertion is + * exercised in {@code RestSourceTableTest} (logical scan + fixed row type, unit) and {@code + * CalcitePPLRestIT} (schema + datarows on a live single-node cluster). This test pins the + * Calcite-facing contract that the ppl module owns: the grammar/AST rewrite of {@code rest} into a + * {@code RestRelation} carrying the validated, reserved-name-encoded endpoint spec that rides + * {@code visitRelation} exactly like {@code DESCRIBE}. */ public class CalcitePPLRestTest { From eeb4e5bacbe1b789b7c0ea2f8cd39b5641555e94 Mon Sep 17 00:00:00 2001 From: Louis Chu Date: Tue, 30 Jun 2026 10:18:53 -0700 Subject: [PATCH 03/14] Address rest command bot-review findings; register rest doctest - clusterSettings: fail closed (throw IllegalStateException) when the node SettingsFilter is unavailable, instead of returning unredacted settings - collectSettings: handle list-type settings via getAsList fallback - decodeRestSpec: reject a blank/missing endpoint with a clear error - docs: correct rest.md allow-list table (9 endpoints + accepted args), quote endpoint literals, fix timeout/get-arg descriptions, add security note - register docs/user/ppl/cmd/rest.md in docs/category.json (deterministic single-node examples: number_of_nodes=1, cluster_manager count=1) Signed-off-by: Louis Chu --- .../sql/utils/SystemIndexUtils.java | 3 + docs/category.json | 1 + docs/user/ppl/cmd/rest.md | 58 +++++++++++++------ .../client/OpenSearchNodeClient.java | 21 +++++-- ...chNodeClientClusterSettingsFilterTest.java | 15 ++--- 5 files changed, 66 insertions(+), 32 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/utils/SystemIndexUtils.java b/core/src/main/java/org/opensearch/sql/utils/SystemIndexUtils.java index 200a9f19681..25ea68005d8 100644 --- a/core/src/main/java/org/opensearch/sql/utils/SystemIndexUtils.java +++ b/core/src/main/java/org/opensearch/sql/utils/SystemIndexUtils.java @@ -104,6 +104,9 @@ public static RestSpec decodeRestSpec(String indexName) { args.put(k.substring("arg.".length()), v); } } + if (endpoint == null) { + throw new IllegalArgumentException("rest source token is missing the endpoint: " + indexName); + } return new RestSpec(endpoint, args, count, timeout); } diff --git a/docs/category.json b/docs/category.json index ac1dda966d2..0defdaaa6d3 100644 --- a/docs/category.json +++ b/docs/category.json @@ -34,6 +34,7 @@ "user/ppl/cmd/rename.md", "user/ppl/cmd/multisearch.md", "user/ppl/cmd/replace.md", + "user/ppl/cmd/rest.md", "user/ppl/cmd/rex.md", "user/ppl/cmd/search.md", "user/ppl/cmd/showdatasources.md", diff --git a/docs/user/ppl/cmd/rest.md b/docs/user/ppl/cmd/rest.md index 11f35e2ee86..b97f247445f 100644 --- a/docs/user/ppl/cmd/rest.md +++ b/docs/user/ppl/cmd/rest.md @@ -2,7 +2,7 @@ The `rest` command is a leading command that reads an allow-listed, read-only in-cluster management endpoint (cluster/cat/nodes) and emits the response as PPL rows. Its rows come from the endpoint dispatch, not from an index, so `rest` appears at the start of a query. -> **Note**: The `rest` command is supported only on the Calcite query engine (`plugins.calcite.enabled=true`). Each endpoint has a fixed output schema, and the dispatch runs under the caller's security context, so a user who cannot call an endpoint directly cannot call it through `rest`. The command is read-only; mutating and non-allow-listed endpoints are rejected. +> **Note**: The `rest` command is supported only on the Calcite query engine (`plugins.calcite.enabled=true`). Each endpoint has a fixed output schema, and the dispatch runs under the caller's security context, so a user who cannot call an endpoint directly cannot call it through `rest`. The command is read-only; mutating and non-allow-listed endpoints are rejected. Each endpoint requires the same cluster-monitor privilege as calling it natively, so `rest` grants no extra access. Some allow-listed endpoints surface operational metadata (for example `/_cat/nodes` exposes node addresses and resource utilization, `/_cat/plugins` the installed plugin inventory, and `/_cluster/state` cluster-state identifiers); this is a deliberate, read-only, monitor-privileged trade-off. `/_cluster/settings` is redacted with the node's setting filter so `Property.Filtered` keys are not surfaced. ## Syntax @@ -20,43 +20,63 @@ The `rest` command supports the following parameters. | --- | --- | --- | | `` | Required | An allow-listed, read-only endpoint path (see the allow-list below), for example `/_cluster/health`. | | `count=` | Optional | Caps the number of emitted rows. | -| `timeout=` | Optional | Request timeout passed to the transport action, for example `30s`. | -| `=` | Optional | Endpoint query arguments, validated per endpoint (for example `level=indices` for `/_cluster/health`). | +| `timeout=` | Optional | Reserved for forward compatibility. It is currently rejected with a clear error, because a single uniform timeout does not map cleanly across the different endpoints. | +| `=` | Optional | Endpoint query arguments, validated per endpoint by both key and value (for example `local=true` for `/_cluster/health`, `health=green` for `/_cat/indices`, `expand_wildcards=open` for `/_resolve/index`). | ## Allow-list `rest` resolves only an explicit, curated set of read-only endpoints. Anything outside the list, including any mutating endpoint, is rejected with a clear error. -| Endpoint | Output columns | -| --- | --- | -| `/_cluster/health` | `cluster_name` (string), `status` (string), `number_of_nodes` (integer), `number_of_data_nodes` (integer), `active_primary_shards` (integer), `active_shards` (integer), `relocating_shards` (integer), `initializing_shards` (integer), `unassigned_shards` (integer), `timed_out` (boolean) | -| `/_cat/indices` | `index` (string), `health` (string), `pri` (integer), `rep` (integer), `active_shards` (integer) | +| Endpoint | Output columns | Accepted args | +| --- | --- | --- | +| `/_cluster/health` | `cluster_name` (string), `status` (string), `number_of_nodes` (integer), `number_of_data_nodes` (integer), `active_primary_shards` (integer), `active_shards` (integer), `relocating_shards` (integer), `initializing_shards` (integer), `unassigned_shards` (integer), `timed_out` (boolean) | `local` | +| `/_cluster/state` | `cluster_name` (string), `state_uuid` (string), `version` (long), `cluster_manager_node` (string) | (none) | +| `/_cluster/settings` | `setting` (string), `value` (string), `tier` (string) | (none) | +| `/_cat/indices` | `index` (string), `health` (string), `pri` (integer), `rep` (integer), `active_shards` (integer) | `health` | +| `/_cat/nodes` | `name` (string), `ip` (string), `node_role` (string), `heap_percent` (integer), `ram_percent` (integer), `cpu` (integer) | (none) | +| `/_cat/cluster_manager` | `id` (string), `host` (string), `ip` (string), `node` (string) | (none) | +| `/_cat/plugins` | `name` (string), `component` (string), `version` (string) | (none) | +| `/_cat/shards` | `index` (string), `shard` (integer), `prirep` (string), `state` (string), `node` (string) | (none) | +| `/_resolve/index` | `name` (string), `type` (string) | `expand_wildcards` | -## Example 1: Reading cluster health +## Example 1: Counting the nodes in the cluster -The following query reads cluster health and projects two columns: +The following query reads cluster health and projects a column that is deterministic on a single-node cluster: ```ppl -| rest /_cluster/health | fields status, number_of_nodes +| rest '/_cluster/health' | fields number_of_nodes ``` The query returns the following results: ```text fetched rows / total rows = 1/1 -+--------+-----------------+ -| status | number_of_nodes | -|--------+-----------------| -| green | 1 | -+--------+-----------------+ ++-----------------+ +| number_of_nodes | +|-----------------| +| 1 | ++-----------------+ ``` -## Example 2: Listing indexes from cat indices +`/_cluster/health` also exposes `status`, `active_shards`, and the other columns listed in the allow-list, which you can project and filter the same way. + +## Example 2: Composing downstream commands over a cat endpoint -The following query lists indexes and filters and sorts them on the Calcite plan: +The `rest` row source composes with downstream `where`, `sort`, `stats`, and `fields` exactly like an index scan. The following query reads `/_cat/cluster_manager` and counts the rows: ```ppl -| rest /_cat/indices | where health = "green" | sort index | fields index, health, pri +| rest '/_cat/cluster_manager' | stats count() as managers +``` + +The query returns the following results: + +```text +fetched rows / total rows = 1/1 ++----------+ +| managers | +|----------| +| 1 | ++----------+ ``` -The downstream `where`, `sort`, and `fields` compose over the `rest` row source exactly like any index scan. +For example, `| rest '/_cat/indices' | where health = 'green' | sort index | fields index, health, pri` lists green indexes; the projected columns come from the endpoint's fixed schema. diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClient.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClient.java index bc5ac2128ea..8efeda5f3cf 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClient.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClient.java @@ -469,10 +469,17 @@ public List> clusterSettings(Map params) { // published into RestSettingsFilterHolder from SQLPlugin#getRestHandlers at startup. org.opensearch.common.settings.SettingsFilter filter = org.opensearch.sql.opensearch.storage.rest.RestSettingsFilterHolder.get(); - if (filter != null) { - persistent = filter.filter(persistent); - transientSettings = filter.filter(transientSettings); + if (filter == null) { + // Fail closed: without the redaction filter (which strips Property.Filtered and + // plugin-registered secret-bearing keys) the command must refuse rather than surface raw + // settings. In-cluster the filter is published at startup (SQLPlugin#getRestHandlers) before + // any query runs, so this guards only against an unexpected uninitialized state. + throw new IllegalStateException( + "cluster settings redaction filter is not initialized; refusing to return unredacted" + + " settings"); } + persistent = filter.filter(persistent); + transientSettings = filter.filter(transientSettings); collectSettings(persistent, "persistent", rows); collectSettings(transientSettings, "transient", rows); return rows; @@ -488,7 +495,13 @@ private void collectSettings( for (String key : settings.keySet()) { Map row = new java.util.LinkedHashMap<>(); row.put("setting", key); - row.put("value", settings.get(key)); + String value = settings.get(key); + if (value == null) { + // List-valued settings return null from get(); fall back to the joined list form. + java.util.List list = settings.getAsList(key); + value = list.isEmpty() ? null : String.join(",", list); + } + row.put("value", value); row.put("tier", tier); rows.add(row); } diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClientClusterSettingsFilterTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClientClusterSettingsFilterTest.java index 47ea968cb8f..33be6cc8482 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClientClusterSettingsFilterTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClientClusterSettingsFilterTest.java @@ -7,6 +7,7 @@ import static java.util.stream.Collectors.toSet; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Answers.RETURNS_DEEP_STUBS; import static org.mockito.ArgumentMatchers.any; @@ -86,17 +87,13 @@ void clusterSettingsRedactsByGlobPattern() { } @Test - void clusterSettingsReturnsRawWhenNoFilterPublished() { + void clusterSettingsFailsClosedWhenNoFilterPublished() { Settings persistent = Settings.builder().put("plugins.secret.token", "supersecret").build(); OpenSearchNodeClient client = clientReturning(persistent, Settings.EMPTY); - // No filter published: confirms the SettingsFilter is the redaction mechanism (not some other - // code path). At runtime getRestHandlers always publishes the filter before any query. - Set keys = - client.clusterSettings(Map.of()).stream() - .map(r -> (String) r.get("setting")) - .collect(toSet()); - - assertTrue(keys.contains("plugins.secret.token")); + // Fail closed: without a published SettingsFilter the command must refuse rather than leak raw + // (potentially secret-bearing) settings. At runtime getRestHandlers always publishes the filter + // before any query, so this path is unreachable in cluster. + assertThrows(IllegalStateException.class, () -> client.clusterSettings(Map.of())); } } From 8b0a147fd4158cc3fd10768ba97e6b261e969cdc Mon Sep 17 00:00:00 2001 From: Louis Chu Date: Tue, 30 Jun 2026 10:50:41 -0700 Subject: [PATCH 04/14] Harden decodeRestSpec: reject non-rest-source tokens with a clear error decodeRestSpec is only ever called behind an isRestSource gate today, but as a public decoder it must not assume its precondition. Without the guard a malformed token would surface an opaque StringIndexOutOfBoundsException from substring; now it throws a clear IllegalArgumentException instead. Addresses the PR #5599 Code Suggestions finding (importance 8). Signed-off-by: Louis Chu --- .../java/org/opensearch/sql/utils/SystemIndexUtils.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/core/src/main/java/org/opensearch/sql/utils/SystemIndexUtils.java b/core/src/main/java/org/opensearch/sql/utils/SystemIndexUtils.java index 25ea68005d8..4df2f34c78c 100644 --- a/core/src/main/java/org/opensearch/sql/utils/SystemIndexUtils.java +++ b/core/src/main/java/org/opensearch/sql/utils/SystemIndexUtils.java @@ -76,6 +76,12 @@ public static String restTable(RestSpec spec) { /** Decode a reserved {@code rest} table name back into its {@link RestSpec}. */ public static RestSpec decodeRestSpec(String indexName) { + // Validate the token shape before slicing it: callers gate on isRestSource today, but a + // public decoder must not assume its precondition, otherwise a malformed token would throw an + // opaque StringIndexOutOfBoundsException from substring rather than a clear input error. + if (!isRestSource(indexName)) { + throw new IllegalArgumentException("not a valid rest source token: " + indexName); + } String body = indexName.substring( REST_SOURCE_PREFIX.length(), indexName.length() - REST_SOURCE_SUFFIX.length()); From 343e93a0d4cc54c3d426ef6897a7f299d9401bf7 Mon Sep 17 00:00:00 2001 From: Louis Chu Date: Tue, 30 Jun 2026 19:21:06 -0700 Subject: [PATCH 05/14] Fix rest explain IT (JSON payload) and harden cluster-settings/state fetch - CalciteExplainIT.explainRestCommand: single-quote the endpoint literal; the explain harness inlines the query into a JSON body without escaping, so a double-quoted literal produced an invalid payload and a 400 (integration CI failure). - OpenSearchNodeClient.clusterSettings: resolve the SettingsFilter and fail closed BEFORE fetching cluster state, so settings are never read into memory when the redaction filter is unavailable. - OpenSearchRestClient.clusterState: narrow filter_path to nodes.*.name so node IPs/attributes are not over-fetched; manager-name resolution is preserved. Signed-off-by: Louis Chu --- .../sql/calcite/remote/CalciteExplainIT.java | 2 +- .../sql/ppl/NewAddedCommandsIT.java | 2 +- .../opensearch/sql/sql/VectorSearchIT.java | 1 + .../client/OpenSearchNodeClient.java | 31 +++++++------------ .../client/OpenSearchRestClient.java | 5 ++- 5 files changed, 19 insertions(+), 22 deletions(-) diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java index 784e324bfbd..f8623c70d6b 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java @@ -67,7 +67,7 @@ public void init() throws Exception { // Only for Calcite: the rest row source explains as a CalciteEnumerableRestScan. @Test public void explainRestCommand() throws IOException { - String result = explainQueryToString("| rest \"/_cluster/health\" | fields status"); + String result = explainQueryToString("| rest '/_cluster/health' | fields status"); Assert.assertTrue( "Expected a rest scan node in the explain output, got: " + result, result.contains("RestScan") || result.contains("rest")); diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/NewAddedCommandsIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/NewAddedCommandsIT.java index 8d5912ab199..2ccda31eea7 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/NewAddedCommandsIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/NewAddedCommandsIT.java @@ -36,7 +36,7 @@ public void init() throws Exception { public void testRest() throws IOException { JSONObject result; try { - result = executeQuery("| rest \"/_cluster/health\" | fields status, number_of_nodes"); + result = executeQuery("| rest '/_cluster/health' | fields status, number_of_nodes"); } catch (ResponseException e) { result = new JSONObject(TestUtils.getResponseBody(e.getResponse())); } diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/VectorSearchIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/VectorSearchIT.java index 8ae3167b40b..e08735ebda7 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/VectorSearchIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/VectorSearchIT.java @@ -25,6 +25,7 @@ public class VectorSearchIT extends SQLIntegTestCase { @Override protected void init() throws Exception { + super.init(); loadIndex(Index.ACCOUNT); } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClient.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClient.java index 8efeda5f3cf..080a8627894 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClient.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClient.java @@ -452,6 +452,16 @@ public Map clusterState(Map params) { @Override public List> clusterSettings(Map params) { + // The transport path has no SettingsFilter of its own; it is published from + // SQLPlugin#getRestHandlers at startup. Fail closed before fetching so we never read settings + // into memory when we cannot redact them, matching native GET /_cluster/settings. + org.opensearch.common.settings.SettingsFilter filter = + org.opensearch.sql.opensearch.storage.rest.RestSettingsFilterHolder.get(); + if (filter == null) { + throw new IllegalStateException( + "cluster settings redaction filter is not initialized; refusing to return unredacted" + + " settings"); + } org.opensearch.action.admin.cluster.state.ClusterStateResponse response = client .admin() @@ -460,26 +470,9 @@ public List> clusterSettings(Map params) { .actionGet(); List> rows = new java.util.ArrayList<>(); org.opensearch.common.settings.Settings persistent = - response.getState().metadata().persistentSettings(); + filter.filter(response.getState().metadata().persistentSettings()); org.opensearch.common.settings.Settings transientSettings = - response.getState().metadata().transientSettings(); - // Mirror the native GET /_cluster/settings redaction: run both tiers through the node's - // SettingsFilter so Property.Filtered (and plugin-registered pattern) settings are not - // surfaced raw. The transport path carries no SettingsFilter of its own; the node instance is - // published into RestSettingsFilterHolder from SQLPlugin#getRestHandlers at startup. - org.opensearch.common.settings.SettingsFilter filter = - org.opensearch.sql.opensearch.storage.rest.RestSettingsFilterHolder.get(); - if (filter == null) { - // Fail closed: without the redaction filter (which strips Property.Filtered and - // plugin-registered secret-bearing keys) the command must refuse rather than surface raw - // settings. In-cluster the filter is published at startup (SQLPlugin#getRestHandlers) before - // any query runs, so this guards only against an unexpected uninitialized state. - throw new IllegalStateException( - "cluster settings redaction filter is not initialized; refusing to return unredacted" - + " settings"); - } - persistent = filter.filter(persistent); - transientSettings = filter.filter(transientSettings); + filter.filter(response.getState().metadata().transientSettings()); collectSettings(persistent, "persistent", rows); collectSettings(transientSettings, "transient", rows); return rows; diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchRestClient.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchRestClient.java index 5afe3a927f1..e98c5bf95f4 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchRestClient.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchRestClient.java @@ -423,7 +423,10 @@ public Map clusterState(Map params) { Map state = getJsonMap( "/_cluster/state/master_node,version,metadata,nodes", - Map.of("filter_path", "cluster_name,state_uuid,version,cluster_manager_node,nodes")); + // nodes.*.name resolves the manager id to a name without over-fetching node IPs. + Map.of( + "filter_path", + "cluster_name,state_uuid,version,cluster_manager_node,nodes.*.name")); Map row = new HashMap<>(); row.put("cluster_name", state.get("cluster_name")); row.put("state_uuid", state.get("state_uuid")); From 42f77a540a14c48790865a095a2283af025492df Mon Sep 17 00:00:00 2001 From: Louis Chu Date: Thu, 2 Jul 2026 07:03:19 -0700 Subject: [PATCH 06/14] Rest command: analytics-engine coexistence IT + hardened source-token decode - AnalyticsEngineCompatIT: assert | rest '/_cluster/health' behaves identically with the analytics engine enabled (rest is never routed to DataFusion). - SystemIndexUtils.fromHex: reject an odd-length hex body so a crafted source name that passes the isRestSource suffix check fails clearly rather than silently dropping the trailing half-byte. Signed-off-by: Louis Chu --- .../sql/utils/SystemIndexUtils.java | 3 ++ .../sql/plugin/AnalyticsEngineCompatIT.java | 31 +++++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/core/src/main/java/org/opensearch/sql/utils/SystemIndexUtils.java b/core/src/main/java/org/opensearch/sql/utils/SystemIndexUtils.java index 4df2f34c78c..7589cf522f6 100644 --- a/core/src/main/java/org/opensearch/sql/utils/SystemIndexUtils.java +++ b/core/src/main/java/org/opensearch/sql/utils/SystemIndexUtils.java @@ -125,6 +125,9 @@ private static String toHex(String s) { } private static String fromHex(String h) { + if (h.length() % 2 != 0) { + throw new IllegalArgumentException("not a valid rest source token: odd-length hex body"); + } byte[] bytes = new byte[h.length() / 2]; for (int i = 0; i < bytes.length; i++) { bytes[i] = diff --git a/integ-test/src/test/java/org/opensearch/sql/plugin/AnalyticsEngineCompatIT.java b/integ-test/src/test/java/org/opensearch/sql/plugin/AnalyticsEngineCompatIT.java index f6ec903c395..b5c08cc8ad0 100644 --- a/integ-test/src/test/java/org/opensearch/sql/plugin/AnalyticsEngineCompatIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/plugin/AnalyticsEngineCompatIT.java @@ -5,9 +5,13 @@ package org.opensearch.sql.plugin; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; import static org.junit.Assume.assumeTrue; import java.io.IOException; +import org.json.JSONArray; +import org.json.JSONObject; import org.junit.Before; import org.junit.Test; import org.opensearch.client.Request; @@ -56,4 +60,31 @@ public void testClusterStarted() { // If the cluster booted with analytics-engine present, all plugins loaded without classloader // errors. The assumption above guarantees we only assert this where it is meaningful. } + + /** + * The {@code rest} row source is a Calcite Enumerable/Scannable scan with no backing index, so it + * is never routed to the analytics (DataFusion) engine. This pins that {@code rest} returns its + * fixed schema and correct data unchanged when the analytics-engine plugin is present. + */ + @Test + public void testRestCommandUnaffectedByAnalyticsEngine() throws IOException { + Request request = new Request("POST", "/_plugins/_ppl"); + request.setJsonEntity( + "{\"query\": \"| rest '/_cluster/health' | fields status, number_of_nodes\"}"); + Response response = client().performRequest(request); + assertEquals(200, response.getStatusLine().getStatusCode()); + + JSONObject result = new JSONObject(TestUtils.getResponseBody(response, true)); + + JSONArray schema = result.getJSONArray("schema"); + assertEquals(2, schema.length()); + assertEquals("status", schema.getJSONObject(0).getString("name")); + assertEquals("string", schema.getJSONObject(0).getString("type")); + assertEquals("number_of_nodes", schema.getJSONObject(1).getString("name")); + assertEquals("int", schema.getJSONObject(1).getString("type")); + + JSONArray datarows = result.getJSONArray("datarows"); + assertEquals(1, datarows.length()); + assertTrue(datarows.getJSONArray(0).getInt(1) >= 1); + } } From 2db942ffd388e293be12d23c7d21742c3269f305 Mon Sep 17 00:00:00 2001 From: Louis Chu Date: Tue, 7 Jul 2026 19:00:46 +0800 Subject: [PATCH 07/14] [Bugfix] Keep rest command on Calcite path under cluster-composite On a cluster started with cluster.pluggable.dataformat=composite, RestUnifiedQueryAction.isAnalyticsIndex() routed every non-system-catalog PPL query to the analytics engine. The rest command's reserved in-cluster source (REST...__REST_SOURCE) has no backing index and only resolves on the Calcite path, so it was routed to DataFusion and failed with "Table 'REST...__REST_SOURCE' not found". Fix: exclude isRestSource(name) alongside isSystemCatalog(name) so a rest source falls back to the default (Calcite) pipeline and is never routed to the analytics engine. - RestUnifiedQueryActionTest: unit repro under cluster-composite. - integ-test analyticsEngineCompat testcluster set composite-default so the existing rest coexistence IT exercises this routing exclusion. Signed-off-by: Louis Chu --- integ-test/build.gradle | 2 ++ .../sql/plugin/rest/RestUnifiedQueryAction.java | 9 +++++---- .../sql/plugin/rest/RestUnifiedQueryActionTest.java | 6 ++++++ 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/integ-test/build.gradle b/integ-test/build.gradle index 2e565aab8f2..d04c4484cc1 100644 --- a/integ-test/build.gradle +++ b/integ-test/build.gradle @@ -419,6 +419,8 @@ testClusters { plugin(getArrowFlightRpcPlugin()) plugin(getAnalyticsEnginePlugin()) plugin ":opensearch-sql-plugin" + // Composite-default cluster: PPL queries route to the analytics engine unless excluded. + setting 'cluster.pluggable.dataformat', 'composite' } } diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestUnifiedQueryAction.java b/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestUnifiedQueryAction.java index 62f3dd0c346..943bae38dc6 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestUnifiedQueryAction.java +++ b/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestUnifiedQueryAction.java @@ -99,13 +99,14 @@ public boolean isAnalyticsIndex(String query, QueryType queryType) { .equals( IndicesService.CLUSTER_PLUGGABLE_DATAFORMAT_VALUE_SETTING.get( clusterService.getSettings()))) { - // Analytics engine can't serve system catalog; SHOW/DESCRIBE fall back to default pipeline + // Analytics engine serves neither the system catalog nor the rest command's reserved + // in-cluster source; both fall back to the default (Calcite) pipeline. try (UnifiedQueryContext context = buildParsingContext(queryType)) { - boolean systemCatalog = + boolean defaultPipeline = extractIndexName(query, queryType, context) - .map(RestUnifiedQueryAction::isSystemCatalog) + .map(name -> isSystemCatalog(name) || SystemIndexUtils.isRestSource(name)) .orElse(false); - return !systemCatalog; + return !defaultPipeline; } catch (Exception e) { // Check legacy-syntax SHOW/DESCRIBE; otherwise let AE handle and surface the error. return !isLegacySystemCatalogQuery(query); diff --git a/plugin/src/test/java/org/opensearch/sql/plugin/rest/RestUnifiedQueryActionTest.java b/plugin/src/test/java/org/opensearch/sql/plugin/rest/RestUnifiedQueryActionTest.java index 111597bb587..0cf87f0604e 100644 --- a/plugin/src/test/java/org/opensearch/sql/plugin/rest/RestUnifiedQueryActionTest.java +++ b/plugin/src/test/java/org/opensearch/sql/plugin/rest/RestUnifiedQueryActionTest.java @@ -178,6 +178,12 @@ public void describeStatementNotRoutedToAnalyticsEngineUnderClusterComposite() { assertFalse(action.isAnalyticsIndex("DESCRIBE TABLES LIKE 'parquet_logs'", QueryType.SQL)); } + @Test + public void restCommandNotRoutedToAnalyticsEngineUnderClusterComposite() { + enableClusterComposite(); + assertFalse(action.isAnalyticsIndex("| rest '/_cluster/health'", QueryType.PPL)); + } + @Test public void dataQueryStillRoutesToAnalyticsUnderClusterComposite() { enableClusterComposite(); From 08191587ed45190a49f56c9807f8128b80919e65 Mon Sep 17 00:00:00 2001 From: Louis Chu Date: Wed, 8 Jul 2026 14:28:07 +0800 Subject: [PATCH 08/14] [Refactor] Unify system-index and rest scans behind one catalog table Collapse the two near-duplicate Calcite scan hierarchies -- SHOW/DESCRIBE system tables and the rest command -- into one generic OpenSearchCatalogTable whose per-endpoint behavior is supplied by a pluggable CatalogSource. - OpenSearchSystemIndex + RestSourceTable -> one OpenSearchCatalogTable backed by SystemIndexCatalogSource / RestCatalogSource. - Two Abstract/Logical/Enumerable scans + two enumerators + two converter rules -> AbstractCalciteCatalogScan / CalciteLogicalCatalogScan / CalciteEnumerableCatalogScan (+ rest-only CalciteScannableCatalogScan) / OpenSearchCatalogEnumerator / EnumerableCatalogScanRule. - Concerns stay per-source: system tables keep the real V2 implement() path; rest is Calcite-only (implement() throws) and opts into Scannable for the collect short-circuit. No behavior change: dispatch, schemas, V2 support, and the Scannable marker are preserved; this is pure de-duplication of the Calcite scan plumbing. Verified: opensearch compileJava/compileTestJava green; ppl and integ-test test-compile green; affected unit tests pass. Signed-off-by: Louis Chu --- .../sql/calcite/remote/CalciteExplainIT.java | 4 +- .../rules/EnumerableCatalogScanRule.java | 63 +++++++++++ .../planner/rules/EnumerableRestScanRule.java | 46 -------- .../rules/EnumerableSystemIndexScanRule.java | 50 --------- .../planner/rules/OpenSearchIndexRules.java | 8 +- .../storage/OpenSearchStorageEngine.java | 10 +- .../storage/rest/AbstractCalciteRestScan.java | 41 ------- .../storage/rest/CalciteLogicalRestScan.java | 48 -------- .../storage/rest/RestCatalogSource.java | 57 ++++++++++ .../storage/rest/RestEnumerator.java | 86 -------------- .../storage/rest/RestSourceTable.java | 81 -------------- ...n.java => AbstractCalciteCatalogScan.java} | 12 +- .../CalciteEnumerableCatalogScan.java} | 24 ++-- .../CalciteEnumerableSystemIndexScan.java | 82 -------------- ...an.java => CalciteLogicalCatalogScan.java} | 20 ++-- .../system/CalciteScannableCatalogScan.java | 29 +++++ .../storage/system/CatalogSource.java | 39 +++++++ ....java => OpenSearchCatalogEnumerator.java} | 9 +- .../system/OpenSearchCatalogTable.java | 67 +++++++++++ .../storage/system/OpenSearchSystemIndex.java | 105 ------------------ .../system/SystemIndexCatalogSource.java | 70 ++++++++++++ .../storage/OpenSearchStorageEngineTest.java | 5 +- ...leTest.java => RestCatalogSourceTest.java} | 60 ++++------ ...t.java => OpenSearchCatalogTableTest.java} | 25 +++-- .../sql/ppl/calcite/CalcitePPLRestTest.java | 10 +- 25 files changed, 416 insertions(+), 635 deletions(-) create mode 100644 opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/EnumerableCatalogScanRule.java delete mode 100644 opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/EnumerableRestScanRule.java delete mode 100644 opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/EnumerableSystemIndexScanRule.java delete mode 100644 opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/AbstractCalciteRestScan.java delete mode 100644 opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/CalciteLogicalRestScan.java create mode 100644 opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestCatalogSource.java delete mode 100644 opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestEnumerator.java delete mode 100644 opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestSourceTable.java rename opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/{AbstractCalciteSystemIndexScan.java => AbstractCalciteCatalogScan.java} (68%) rename opensearch/src/main/java/org/opensearch/sql/opensearch/storage/{rest/CalciteEnumerableRestScan.java => system/CalciteEnumerableCatalogScan.java} (81%) delete mode 100644 opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteEnumerableSystemIndexScan.java rename opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/{CalciteLogicalSystemIndexScan.java => CalciteLogicalCatalogScan.java} (59%) create mode 100644 opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteScannableCatalogScan.java create mode 100644 opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CatalogSource.java rename opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/{OpenSearchSystemIndexEnumerator.java => OpenSearchCatalogEnumerator.java} (90%) create mode 100644 opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/OpenSearchCatalogTable.java delete mode 100644 opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/OpenSearchSystemIndex.java create mode 100644 opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/SystemIndexCatalogSource.java rename opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/{RestSourceTableTest.java => RestCatalogSourceTest.java} (60%) rename opensearch/src/test/java/org/opensearch/sql/opensearch/storage/system/{OpenSearchSystemIndexTest.java => OpenSearchCatalogTableTest.java} (76%) diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java index f8623c70d6b..fd8515aecfd 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java @@ -64,13 +64,13 @@ public void init() throws Exception { loadIndex(Index.GRAPH_EMPLOYEES); } - // Only for Calcite: the rest row source explains as a CalciteEnumerableRestScan. + // Only for Calcite: the rest row source explains as a CalciteScannableCatalogScan. @Test public void explainRestCommand() throws IOException { String result = explainQueryToString("| rest '/_cluster/health' | fields status"); Assert.assertTrue( "Expected a rest scan node in the explain output, got: " + result, - result.contains("RestScan") || result.contains("rest")); + result.contains("CatalogScan")); } @Override diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/EnumerableCatalogScanRule.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/EnumerableCatalogScanRule.java new file mode 100644 index 00000000000..9905ca564ab --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/EnumerableCatalogScanRule.java @@ -0,0 +1,63 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.planner.rules; + +import org.apache.calcite.adapter.enumerable.EnumerableConvention; +import org.apache.calcite.plan.Convention; +import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.convert.ConverterRule; +import org.opensearch.sql.opensearch.storage.system.CalciteEnumerableCatalogScan; +import org.opensearch.sql.opensearch.storage.system.CalciteLogicalCatalogScan; +import org.opensearch.sql.opensearch.storage.system.CalciteScannableCatalogScan; + +/** + * Rule to convert a {@link CalciteLogicalCatalogScan} into an enumerable scan: a {@link + * CalciteScannableCatalogScan} when the source opts into {@code Scannable}, otherwise a plain + * {@link CalciteEnumerableCatalogScan}. + */ +public class EnumerableCatalogScanRule extends ConverterRule { + /** Default configuration. */ + public static final Config DEFAULT_CONFIG = + Config.INSTANCE + .as(Config.class) + .withConversion( + CalciteLogicalCatalogScan.class, + s -> s.getCatalogTable() != null, + Convention.NONE, + EnumerableConvention.INSTANCE, + "EnumerableCatalogScanRule") + .withRuleFactory(EnumerableCatalogScanRule::new); + + protected EnumerableCatalogScanRule(Config config) { + super(config); + } + + @Override + public boolean matches(RelOptRuleCall call) { + CalciteLogicalCatalogScan scan = call.rel(0); + return scan.getVariablesSet().isEmpty(); + } + + @Override + public RelNode convert(RelNode rel) { + final CalciteLogicalCatalogScan scan = (CalciteLogicalCatalogScan) rel; + if (scan.getCatalogTable().getSource().isScannable()) { + return new CalciteScannableCatalogScan( + scan.getCluster(), + scan.getHints(), + scan.getTable(), + scan.getCatalogTable(), + scan.getSchema()); + } + return new CalciteEnumerableCatalogScan( + scan.getCluster(), + scan.getHints(), + scan.getTable(), + scan.getCatalogTable(), + scan.getSchema()); + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/EnumerableRestScanRule.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/EnumerableRestScanRule.java deleted file mode 100644 index 282ba52b1ee..00000000000 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/EnumerableRestScanRule.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.opensearch.planner.rules; - -import org.apache.calcite.adapter.enumerable.EnumerableConvention; -import org.apache.calcite.plan.Convention; -import org.apache.calcite.plan.RelOptRuleCall; -import org.apache.calcite.rel.RelNode; -import org.apache.calcite.rel.convert.ConverterRule; -import org.opensearch.sql.opensearch.storage.rest.CalciteEnumerableRestScan; -import org.opensearch.sql.opensearch.storage.rest.CalciteLogicalRestScan; - -/** Rule to convert a {@link CalciteLogicalRestScan} to a {@link CalciteEnumerableRestScan}. */ -public class EnumerableRestScanRule extends ConverterRule { - /** Default configuration. */ - public static final Config DEFAULT_CONFIG = - Config.INSTANCE - .as(Config.class) - .withConversion( - CalciteLogicalRestScan.class, - s -> s.getRestTable() != null, - Convention.NONE, - EnumerableConvention.INSTANCE, - "EnumerableRestScanRule") - .withRuleFactory(EnumerableRestScanRule::new); - - protected EnumerableRestScanRule(Config config) { - super(config); - } - - @Override - public boolean matches(RelOptRuleCall call) { - CalciteLogicalRestScan scan = call.rel(0); - return scan.getVariablesSet().isEmpty(); - } - - @Override - public RelNode convert(RelNode rel) { - final CalciteLogicalRestScan scan = (CalciteLogicalRestScan) rel; - return new CalciteEnumerableRestScan( - scan.getCluster(), scan.getHints(), scan.getTable(), scan.getRestTable(), scan.getSchema()); - } -} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/EnumerableSystemIndexScanRule.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/EnumerableSystemIndexScanRule.java deleted file mode 100644 index 616d1178873..00000000000 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/EnumerableSystemIndexScanRule.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.opensearch.planner.rules; - -import org.apache.calcite.adapter.enumerable.EnumerableConvention; -import org.apache.calcite.plan.Convention; -import org.apache.calcite.plan.RelOptRuleCall; -import org.apache.calcite.rel.RelNode; -import org.apache.calcite.rel.convert.ConverterRule; -import org.opensearch.sql.opensearch.storage.system.CalciteEnumerableSystemIndexScan; -import org.opensearch.sql.opensearch.storage.system.CalciteLogicalSystemIndexScan; - -/** - * Rule to convert a {@link CalciteLogicalSystemIndexScan} to a {@link - * CalciteEnumerableSystemIndexScan}. - */ -public class EnumerableSystemIndexScanRule extends ConverterRule { - /** Default configuration. */ - public static final Config DEFAULT_CONFIG = - Config.INSTANCE - .as(Config.class) - .withConversion( - CalciteLogicalSystemIndexScan.class, - s -> s.getSysIndex() != null, - Convention.NONE, - EnumerableConvention.INSTANCE, - "EnumerableSystemIndexScanRule") - .withRuleFactory(EnumerableSystemIndexScanRule::new); - - /** Creates an EnumerableProjectRule. */ - protected EnumerableSystemIndexScanRule(Config config) { - super(config); - } - - @Override - public boolean matches(RelOptRuleCall call) { - CalciteLogicalSystemIndexScan scan = call.rel(0); - return scan.getVariablesSet().isEmpty(); - } - - @Override - public RelNode convert(RelNode rel) { - final CalciteLogicalSystemIndexScan scan = (CalciteLogicalSystemIndexScan) rel; - return new CalciteEnumerableSystemIndexScan( - scan.getCluster(), scan.getHints(), scan.getTable(), scan.getSysIndex(), scan.getSchema()); - } -} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/OpenSearchIndexRules.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/OpenSearchIndexRules.java index 22a508b88cd..c200ffa8909 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/OpenSearchIndexRules.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/OpenSearchIndexRules.java @@ -12,9 +12,8 @@ public class OpenSearchIndexRules { private static final RelOptRule INDEX_SCAN_RULE = EnumerableIndexScanRule.DEFAULT_CONFIG.toRule(); - private static final RelOptRule SYSTEM_INDEX_SCAN_RULE = - EnumerableSystemIndexScanRule.DEFAULT_CONFIG.toRule(); - private static final RelOptRule REST_SCAN_RULE = EnumerableRestScanRule.DEFAULT_CONFIG.toRule(); + private static final RelOptRule CATALOG_SCAN_RULE = + EnumerableCatalogScanRule.DEFAULT_CONFIG.toRule(); private static final RelOptRule NESTED_AGGREGATE_RULE = EnumerableNestedAggregateRule.DEFAULT_CONFIG.toRule(); private static final RelOptRule GRAPH_LOOKUP_RULE = @@ -27,8 +26,7 @@ public class OpenSearchIndexRules { public static final List OPEN_SEARCH_NON_PUSHDOWN_RULES = ImmutableList.of( INDEX_SCAN_RULE, - SYSTEM_INDEX_SCAN_RULE, - REST_SCAN_RULE, + CATALOG_SCAN_RULE, NESTED_AGGREGATE_RULE, GRAPH_LOOKUP_RULE, RELEVANCE_FUNCTION_RULE); diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngine.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngine.java index 0ec3c33c664..64bc83211b1 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngine.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngine.java @@ -17,8 +17,9 @@ import org.opensearch.sql.common.setting.Settings; import org.opensearch.sql.expression.function.FunctionResolver; import org.opensearch.sql.opensearch.client.OpenSearchClient; -import org.opensearch.sql.opensearch.storage.rest.RestSourceTable; -import org.opensearch.sql.opensearch.storage.system.OpenSearchSystemIndex; +import org.opensearch.sql.opensearch.storage.rest.RestCatalogSource; +import org.opensearch.sql.opensearch.storage.system.OpenSearchCatalogTable; +import org.opensearch.sql.opensearch.storage.system.SystemIndexCatalogSource; import org.opensearch.sql.storage.StorageEngine; import org.opensearch.sql.storage.Table; @@ -39,9 +40,10 @@ public Collection getFunctions() { @Override public Table getTable(DataSourceSchemaName dataSourceSchemaName, String name) { if (isRestSource(name)) { - return new RestSourceTable(client, settings, decodeRestSpec(name)); + return new OpenSearchCatalogTable( + new RestCatalogSource(client, decodeRestSpec(name)), settings); } else if (isSystemIndex(name)) { - return new OpenSearchSystemIndex(client, settings, name); + return new OpenSearchCatalogTable(new SystemIndexCatalogSource(client, name), settings); } else { return new OpenSearchIndex(client, settings, name); } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/AbstractCalciteRestScan.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/AbstractCalciteRestScan.java deleted file mode 100644 index fe07d9c584e..00000000000 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/AbstractCalciteRestScan.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.opensearch.storage.rest; - -import static java.util.Objects.requireNonNull; - -import java.util.List; -import lombok.Getter; -import org.apache.calcite.plan.RelOptCluster; -import org.apache.calcite.plan.RelOptTable; -import org.apache.calcite.plan.RelTraitSet; -import org.apache.calcite.rel.core.TableScan; -import org.apache.calcite.rel.hint.RelHint; -import org.apache.calcite.rel.type.RelDataType; - -/** An abstract relational operator representing a scan of a {@link RestSourceTable}. */ -@Getter -public abstract class AbstractCalciteRestScan extends TableScan { - protected final RestSourceTable restTable; - protected final RelDataType schema; - - protected AbstractCalciteRestScan( - RelOptCluster cluster, - RelTraitSet traitSet, - List hints, - RelOptTable table, - RestSourceTable restTable, - RelDataType schema) { - super(cluster, traitSet, hints, table); - this.restTable = requireNonNull(restTable, "rest source table"); - this.schema = schema; - } - - @Override - public RelDataType deriveRowType() { - return this.schema; - } -} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/CalciteLogicalRestScan.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/CalciteLogicalRestScan.java deleted file mode 100644 index 4cb33b48646..00000000000 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/CalciteLogicalRestScan.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.opensearch.storage.rest; - -import com.google.common.collect.ImmutableList; -import java.util.List; -import org.apache.calcite.plan.Convention; -import org.apache.calcite.plan.RelOptCluster; -import org.apache.calcite.plan.RelOptPlanner; -import org.apache.calcite.plan.RelOptTable; -import org.apache.calcite.plan.RelTraitSet; -import org.apache.calcite.rel.hint.RelHint; -import org.apache.calcite.rel.type.RelDataType; -import org.opensearch.sql.opensearch.planner.rules.EnumerableRestScanRule; - -/** The logical relational operator representing a scan of a {@link RestSourceTable}. */ -public class CalciteLogicalRestScan extends AbstractCalciteRestScan { - - public CalciteLogicalRestScan( - RelOptCluster cluster, RelOptTable table, RestSourceTable restTable) { - this( - cluster, - cluster.traitSetOf(Convention.NONE), - ImmutableList.of(), - table, - restTable, - table.getRowType()); - } - - protected CalciteLogicalRestScan( - RelOptCluster cluster, - RelTraitSet traitSet, - List hints, - RelOptTable table, - RestSourceTable restTable, - RelDataType schema) { - super(cluster, traitSet, hints, table, restTable, schema); - } - - @Override - public void register(RelOptPlanner planner) { - super.register(planner); - planner.addRule(EnumerableRestScanRule.DEFAULT_CONFIG.toRule()); - } -} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestCatalogSource.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestCatalogSource.java new file mode 100644 index 00000000000..807f83df4bf --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestCatalogSource.java @@ -0,0 +1,57 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.rest; + +import java.util.Map; +import lombok.Getter; +import org.opensearch.sql.data.type.ExprType; +import org.opensearch.sql.opensearch.client.OpenSearchClient; +import org.opensearch.sql.opensearch.request.system.OpenSearchSystemRequest; +import org.opensearch.sql.opensearch.storage.system.CatalogSource; +import org.opensearch.sql.planner.logical.LogicalPlan; +import org.opensearch.sql.planner.physical.PhysicalPlan; +import org.opensearch.sql.utils.SystemIndexUtils.RestSpec; + +/** + * {@link CatalogSource} for the {@code rest} command: an allow-listed, read-only management + * endpoint resolved against {@link RestEndpointRegistry}, exposing the fixed endpoint schema. + * Calcite only (no V2 path) and {@code Scannable} for the {@code collect} short-circuit. + */ +@Getter +public class RestCatalogSource implements CatalogSource { + + private final OpenSearchClient client; + private final RestSpec spec; + private final RestEndpointRegistry.Endpoint endpoint; + + public RestCatalogSource(OpenSearchClient client, RestSpec spec) { + this.client = client; + this.spec = spec; + // Allow-list enforced here: unknown or mutating endpoints and disallowed args are rejected. + this.endpoint = RestEndpointRegistry.resolve(spec.getEndpoint()); + RestEndpointRegistry.validate(spec); + } + + @Override + public Map getFieldTypes() { + return endpoint.getSchema(); + } + + @Override + public OpenSearchSystemRequest createRequest() { + return new RestRequest(client, endpoint, spec); + } + + @Override + public boolean isScannable() { + return true; + } + + @Override + public PhysicalPlan implementV2(LogicalPlan plan) { + throw new UnsupportedOperationException("rest command is supported only on the Calcite engine"); + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestEnumerator.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestEnumerator.java deleted file mode 100644 index ad678acb4c4..00000000000 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestEnumerator.java +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.opensearch.storage.rest; - -import java.util.Iterator; -import java.util.List; -import lombok.EqualsAndHashCode; -import lombok.ToString; -import org.apache.calcite.linq4j.Enumerator; -import org.opensearch.sql.data.model.ExprNullValue; -import org.opensearch.sql.data.model.ExprValue; -import org.opensearch.sql.exception.NonFallbackCalciteException; -import org.opensearch.sql.monitor.ResourceMonitor; - -/** - * A simple resource-monitored iteration over the rows produced by a {@code rest} endpoint dispatch. - * One-for-one parallel of {@code OpenSearchSystemIndexEnumerator}. - */ -public class RestEnumerator implements Enumerator { - /** How many moveNext() calls to perform a resource check once. */ - private static final long NUMBER_OF_NEXT_CALL_TO_CHECK = 1000; - - private final List fields; - - @EqualsAndHashCode.Include @ToString.Include private final RestRequest request; - - private Iterator iterator; - - private ExprValue current; - - /** Number of rows returned. */ - private Integer queryCount; - - /** ResourceMonitor. */ - private final ResourceMonitor monitor; - - public RestEnumerator(List fields, RestRequest request, ResourceMonitor monitor) { - this.fields = fields; - this.request = request; - this.monitor = monitor; - this.queryCount = 0; - this.current = null; - if (!this.monitor.isHealthy()) { - throw new NonFallbackCalciteException("insufficient resources to run the query, quit."); - } - this.iterator = request.search().iterator(); - } - - @Override - public Object current() { - return fields.stream() - .map(k -> current.tupleValue().getOrDefault(k, ExprNullValue.of()).valueForCalcite()) - .toArray(); - } - - @Override - public boolean moveNext() { - boolean shouldCheck = (queryCount % NUMBER_OF_NEXT_CALL_TO_CHECK == 0); - if (shouldCheck && !this.monitor.isHealthy()) { - throw new NonFallbackCalciteException("insufficient resources to load next row, quit."); - } - if (iterator.hasNext()) { - current = iterator.next(); - queryCount++; - return true; - } else { - return false; - } - } - - @Override - public void reset() { - iterator = request.search().iterator(); - queryCount = 0; - current = null; - } - - @Override - public void close() { - iterator = null; - current = null; - } -} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestSourceTable.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestSourceTable.java deleted file mode 100644 index a900a1d319e..00000000000 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestSourceTable.java +++ /dev/null @@ -1,81 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.opensearch.storage.rest; - -import java.util.Map; -import lombok.Getter; -import org.apache.calcite.plan.RelOptCluster; -import org.apache.calcite.plan.RelOptTable; -import org.apache.calcite.rel.RelNode; -import org.opensearch.sql.calcite.plan.AbstractOpenSearchTable; -import org.opensearch.sql.common.setting.Settings; -import org.opensearch.sql.data.type.ExprType; -import org.opensearch.sql.opensearch.client.OpenSearchClient; -import org.opensearch.sql.opensearch.monitor.OpenSearchMemoryHealthy; -import org.opensearch.sql.opensearch.monitor.OpenSearchResourceMonitor; -import org.opensearch.sql.planner.logical.LogicalPlan; -import org.opensearch.sql.planner.physical.PhysicalPlan; -import org.opensearch.sql.utils.SystemIndexUtils.RestSpec; - -/** - * The {@code rest} command's row source: a fixed-schema, server-side dispatch table modeled on - * {@link org.opensearch.sql.opensearch.storage.system.OpenSearchSystemIndex}. It resolves the - * validated endpoint spec against {@link RestEndpointRegistry} (which enforces the read-only - * allow-list), exposes the endpoint's fixed schema, and produces a {@link CalciteLogicalRestScan} - * on the Calcite path. - */ -@Getter -public class RestSourceTable extends AbstractOpenSearchTable { - - private final OpenSearchClient client; - private final Settings settings; - private final RestSpec spec; - private final RestEndpointRegistry.Endpoint endpoint; - - public RestSourceTable(OpenSearchClient client, Settings settings, RestSpec spec) { - this.client = client; - this.settings = settings; - this.spec = spec; - // Allow-list enforced here: unknown/mutating endpoints and disallowed args - // are rejected. - this.endpoint = RestEndpointRegistry.resolve(spec.getEndpoint()); - RestEndpointRegistry.validate(spec); - } - - @Override - public boolean exists() { - return true; - } - - @Override - public void create(Map schema) { - throw new UnsupportedOperationException("rest endpoint is predefined and cannot be created"); - } - - @Override - public Map getFieldTypes() { - return endpoint.getSchema(); - } - - @Override - public RelNode toRel(RelOptTable.ToRelContext context, RelOptTable relOptTable) { - final RelOptCluster cluster = context.getCluster(); - return new CalciteLogicalRestScan(cluster, relOptTable, this); - } - - @Override - public PhysicalPlan implement(LogicalPlan plan) { - throw new UnsupportedOperationException("rest command is supported only on the Calcite engine"); - } - - public RestRequest createRestRequest() { - return new RestRequest(client, endpoint, spec); - } - - public OpenSearchResourceMonitor createOpenSearchResourceMonitor() { - return new OpenSearchResourceMonitor(settings, new OpenSearchMemoryHealthy(settings)); - } -} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/AbstractCalciteSystemIndexScan.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/AbstractCalciteCatalogScan.java similarity index 68% rename from opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/AbstractCalciteSystemIndexScan.java rename to opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/AbstractCalciteCatalogScan.java index fa543ab266b..19d585ea684 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/AbstractCalciteSystemIndexScan.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/AbstractCalciteCatalogScan.java @@ -16,21 +16,21 @@ import org.apache.calcite.rel.hint.RelHint; import org.apache.calcite.rel.type.RelDataType; -/** An abstract relational operator representing a scan of an OpenSearchSystemIndex type. */ +/** An abstract relational operator representing a scan of an {@link OpenSearchCatalogTable}. */ @Getter -public abstract class AbstractCalciteSystemIndexScan extends TableScan { - public final OpenSearchSystemIndex sysIndex; +public abstract class AbstractCalciteCatalogScan extends TableScan { + public final OpenSearchCatalogTable catalogTable; protected final RelDataType schema; - protected AbstractCalciteSystemIndexScan( + protected AbstractCalciteCatalogScan( RelOptCluster cluster, RelTraitSet traitSet, List hints, RelOptTable table, - OpenSearchSystemIndex sysIndex, + OpenSearchCatalogTable catalogTable, RelDataType schema) { super(cluster, traitSet, hints, table); - this.sysIndex = requireNonNull(sysIndex, "OpenSearch system index"); + this.catalogTable = requireNonNull(catalogTable, "OpenSearch catalog table"); this.schema = schema; } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/CalciteEnumerableRestScan.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteEnumerableCatalogScan.java similarity index 81% rename from opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/CalciteEnumerableRestScan.java rename to opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteEnumerableCatalogScan.java index a55d29e785a..58f86874c73 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/CalciteEnumerableRestScan.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteEnumerableCatalogScan.java @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.sql.opensearch.storage.rest; +package org.opensearch.sql.opensearch.storage.system; import java.util.List; import org.apache.calcite.adapter.enumerable.EnumerableConvention; @@ -25,23 +25,22 @@ import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.util.Pair; import org.checkerframework.checker.nullness.qual.Nullable; -import org.opensearch.sql.calcite.plan.Scannable; -/** The physical relational operator representing a scan of a {@link RestSourceTable}. */ -public class CalciteEnumerableRestScan extends AbstractCalciteRestScan - implements EnumerableRel, Scannable { - public CalciteEnumerableRestScan( +/** The physical relational operator representing a scan of an {@link OpenSearchCatalogTable}. */ +public class CalciteEnumerableCatalogScan extends AbstractCalciteCatalogScan + implements EnumerableRel { + public CalciteEnumerableCatalogScan( RelOptCluster cluster, List hints, RelOptTable table, - RestSourceTable restTable, + OpenSearchCatalogTable catalogTable, RelDataType schema) { super( cluster, cluster.traitSetOf(EnumerableConvention.INSTANCE), hints, table, - restTable, + catalogTable, schema); } @@ -66,19 +65,18 @@ public Result implement(EnumerableRelImplementor implementor, Prefer pref) { PhysType physType = PhysTypeImpl.of(implementor.getTypeFactory(), getRowType(), pref.preferArray()); - Expression scanOperator = implementor.stash(this, CalciteEnumerableRestScan.class); + Expression scanOperator = implementor.stash(this, CalciteEnumerableCatalogScan.class); return implementor.result(physType, Blocks.toBlock(Expressions.call(scanOperator, "scan"))); } - @Override public Enumerable<@Nullable Object> scan() { return new AbstractEnumerable<>() { @Override public Enumerator enumerator() { - return new RestEnumerator( + return new OpenSearchCatalogEnumerator( getFieldPath(), - restTable.createRestRequest(), - restTable.createOpenSearchResourceMonitor()); + catalogTable.getSource().createRequest(), + catalogTable.createOpenSearchResourceMonitor()); } }; } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteEnumerableSystemIndexScan.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteEnumerableSystemIndexScan.java deleted file mode 100644 index b0c92dce8f9..00000000000 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteEnumerableSystemIndexScan.java +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.opensearch.storage.system; - -import java.util.List; -import org.apache.calcite.adapter.enumerable.EnumerableConvention; -import org.apache.calcite.adapter.enumerable.EnumerableRel; -import org.apache.calcite.adapter.enumerable.EnumerableRelImplementor; -import org.apache.calcite.adapter.enumerable.PhysType; -import org.apache.calcite.adapter.enumerable.PhysTypeImpl; -import org.apache.calcite.linq4j.AbstractEnumerable; -import org.apache.calcite.linq4j.Enumerable; -import org.apache.calcite.linq4j.Enumerator; -import org.apache.calcite.linq4j.tree.Blocks; -import org.apache.calcite.linq4j.tree.Expression; -import org.apache.calcite.linq4j.tree.Expressions; -import org.apache.calcite.plan.DeriveMode; -import org.apache.calcite.plan.RelOptCluster; -import org.apache.calcite.plan.RelOptTable; -import org.apache.calcite.plan.RelTraitSet; -import org.apache.calcite.rel.hint.RelHint; -import org.apache.calcite.rel.type.RelDataType; -import org.apache.calcite.util.Pair; -import org.checkerframework.checker.nullness.qual.Nullable; - -/** The physical relational operator representing a scan of an OpenSearchSystemIndex type. */ -public class CalciteEnumerableSystemIndexScan extends AbstractCalciteSystemIndexScan - implements EnumerableRel { - public CalciteEnumerableSystemIndexScan( - RelOptCluster cluster, - List hints, - RelOptTable table, - OpenSearchSystemIndex sysIndex, - RelDataType schema) { - super( - cluster, cluster.traitSetOf(EnumerableConvention.INSTANCE), hints, table, sysIndex, schema); - } - - @Override - public @Nullable Pair> passThroughTraits(RelTraitSet required) { - return EnumerableRel.super.passThroughTraits(required); - } - - @Override - public @Nullable Pair> deriveTraits( - RelTraitSet childTraits, int childId) { - return EnumerableRel.super.deriveTraits(childTraits, childId); - } - - @Override - public DeriveMode getDeriveMode() { - return EnumerableRel.super.getDeriveMode(); - } - - @Override - public Result implement(EnumerableRelImplementor implementor, Prefer pref) { - PhysType physType = - PhysTypeImpl.of(implementor.getTypeFactory(), getRowType(), pref.preferArray()); - - Expression scanOperator = implementor.stash(this, CalciteEnumerableSystemIndexScan.class); - return implementor.result(physType, Blocks.toBlock(Expressions.call(scanOperator, "scan"))); - } - - public Enumerable<@Nullable Object> scan() { - return new AbstractEnumerable<>() { - @Override - public Enumerator enumerator() { - return new OpenSearchSystemIndexEnumerator( - getFieldPath(), - sysIndex.getSystemIndexBundle().getRight(), - sysIndex.createOpenSearchResourceMonitor()); - } - }; - } - - private List getFieldPath() { - return getRowType().getFieldNames().stream().toList(); - } -} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteLogicalSystemIndexScan.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteLogicalCatalogScan.java similarity index 59% rename from opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteLogicalSystemIndexScan.java rename to opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteLogicalCatalogScan.java index 012ceec8c13..4278539c2b0 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteLogicalSystemIndexScan.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteLogicalCatalogScan.java @@ -14,35 +14,35 @@ import org.apache.calcite.plan.RelTraitSet; import org.apache.calcite.rel.hint.RelHint; import org.apache.calcite.rel.type.RelDataType; -import org.opensearch.sql.opensearch.planner.rules.EnumerableSystemIndexScanRule; +import org.opensearch.sql.opensearch.planner.rules.EnumerableCatalogScanRule; -/** The logical relational operator representing a scan of an OpenSearchSystemIndex type. */ -public class CalciteLogicalSystemIndexScan extends AbstractCalciteSystemIndexScan { +/** The logical relational operator representing a scan of an {@link OpenSearchCatalogTable}. */ +public class CalciteLogicalCatalogScan extends AbstractCalciteCatalogScan { - public CalciteLogicalSystemIndexScan( - RelOptCluster cluster, RelOptTable table, OpenSearchSystemIndex sysIndex) { + public CalciteLogicalCatalogScan( + RelOptCluster cluster, RelOptTable table, OpenSearchCatalogTable catalogTable) { this( cluster, cluster.traitSetOf(Convention.NONE), ImmutableList.of(), table, - sysIndex, + catalogTable, table.getRowType()); } - protected CalciteLogicalSystemIndexScan( + protected CalciteLogicalCatalogScan( RelOptCluster cluster, RelTraitSet traitSet, List hints, RelOptTable table, - OpenSearchSystemIndex sysIndex, + OpenSearchCatalogTable catalogTable, RelDataType schema) { - super(cluster, traitSet, hints, table, sysIndex, schema); + super(cluster, traitSet, hints, table, catalogTable, schema); } @Override public void register(RelOptPlanner planner) { super.register(planner); - planner.addRule(EnumerableSystemIndexScanRule.DEFAULT_CONFIG.toRule()); + planner.addRule(EnumerableCatalogScanRule.DEFAULT_CONFIG.toRule()); } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteScannableCatalogScan.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteScannableCatalogScan.java new file mode 100644 index 00000000000..1d021cda354 --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteScannableCatalogScan.java @@ -0,0 +1,29 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.system; + +import java.util.List; +import org.apache.calcite.plan.RelOptCluster; +import org.apache.calcite.plan.RelOptTable; +import org.apache.calcite.rel.hint.RelHint; +import org.apache.calcite.rel.type.RelDataType; +import org.opensearch.sql.calcite.plan.Scannable; + +/** + * A {@link CalciteEnumerableCatalogScan} that additionally carries the {@link Scannable} marker, + * enabling the {@code collect} short-circuit. Produced when the {@link CatalogSource} opts in via + * {@link CatalogSource#isScannable()}. + */ +public class CalciteScannableCatalogScan extends CalciteEnumerableCatalogScan implements Scannable { + public CalciteScannableCatalogScan( + RelOptCluster cluster, + List hints, + RelOptTable table, + OpenSearchCatalogTable catalogTable, + RelDataType schema) { + super(cluster, hints, table, catalogTable, schema); + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CatalogSource.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CatalogSource.java new file mode 100644 index 00000000000..85dabe06b9a --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CatalogSource.java @@ -0,0 +1,39 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.system; + +import java.util.Map; +import org.opensearch.sql.data.type.ExprType; +import org.opensearch.sql.opensearch.request.system.OpenSearchSystemRequest; +import org.opensearch.sql.planner.logical.LogicalPlan; +import org.opensearch.sql.planner.physical.PhysicalPlan; + +/** + * Strategy supplying the schema and row source for an {@link OpenSearchCatalogTable} backed by a + * read-only OpenSearch admin API. Each endpoint family plugs in its own {@code CatalogSource} + * rather than defining a separate table and Calcite scan hierarchy. + */ +public interface CatalogSource { + + /** Fixed schema mapping each column name to its type. */ + Map getFieldTypes(); + + /** + * Builds the read-only request whose {@link OpenSearchSystemRequest#search()} yields the rows. + */ + OpenSearchSystemRequest createRequest(); + + /** + * Whether the enumerable scan should carry the {@link org.opensearch.sql.calcite.plan.Scannable} + * marker for the {@code collect} short-circuit. Defaults to {@code false}. + */ + default boolean isScannable() { + return false; + } + + /** The V2 physical plan path, or throws when the source is Calcite only. */ + PhysicalPlan implementV2(LogicalPlan plan); +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/OpenSearchSystemIndexEnumerator.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/OpenSearchCatalogEnumerator.java similarity index 90% rename from opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/OpenSearchSystemIndexEnumerator.java rename to opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/OpenSearchCatalogEnumerator.java index 1bb8f9d5293..dff9b47265a 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/OpenSearchSystemIndexEnumerator.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/OpenSearchCatalogEnumerator.java @@ -16,8 +16,11 @@ import org.opensearch.sql.monitor.ResourceMonitor; import org.opensearch.sql.opensearch.request.system.OpenSearchSystemRequest; -/** Supports a simple iteration over a collection for OpenSearch system index */ -public class OpenSearchSystemIndexEnumerator implements Enumerator { +/** + * Resource-monitored iteration over the rows produced by a read-only catalog {@link + * OpenSearchSystemRequest}. + */ +public class OpenSearchCatalogEnumerator implements Enumerator { /** How many moveNext() calls to perform resource check once. */ private static final long NUMBER_OF_NEXT_CALL_TO_CHECK = 1000; @@ -35,7 +38,7 @@ public class OpenSearchSystemIndexEnumerator implements Enumerator { /** ResourceMonitor. */ private final ResourceMonitor monitor; - public OpenSearchSystemIndexEnumerator( + public OpenSearchCatalogEnumerator( List fields, OpenSearchSystemRequest request, ResourceMonitor monitor) { this.fields = fields; this.request = request; diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/OpenSearchCatalogTable.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/OpenSearchCatalogTable.java new file mode 100644 index 00000000000..9bcb3a3ff0c --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/OpenSearchCatalogTable.java @@ -0,0 +1,67 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.system; + +import java.util.Map; +import lombok.Getter; +import org.apache.calcite.plan.RelOptCluster; +import org.apache.calcite.plan.RelOptTable; +import org.apache.calcite.rel.RelNode; +import org.opensearch.sql.calcite.plan.AbstractOpenSearchTable; +import org.opensearch.sql.common.setting.Settings; +import org.opensearch.sql.data.type.ExprType; +import org.opensearch.sql.opensearch.monitor.OpenSearchMemoryHealthy; +import org.opensearch.sql.opensearch.monitor.OpenSearchResourceMonitor; +import org.opensearch.sql.planner.logical.LogicalPlan; +import org.opensearch.sql.planner.physical.PhysicalPlan; + +/** + * A single generic Calcite and V2 table over a read-only OpenSearch catalog endpoint. Per-endpoint + * behavior (schema, row source, V2 support, and the {@code Scannable} marker) is supplied by a + * pluggable {@link CatalogSource}. + */ +@Getter +public class OpenSearchCatalogTable extends AbstractOpenSearchTable { + + private final CatalogSource source; + private final Settings settings; + + public OpenSearchCatalogTable(CatalogSource source, Settings settings) { + this.source = source; + this.settings = settings; + } + + @Override + public boolean exists() { + return true; + } + + @Override + public void create(Map schema) { + throw new UnsupportedOperationException( + "OpenSearch catalog table is predefined and cannot be created"); + } + + @Override + public Map getFieldTypes() { + return source.getFieldTypes(); + } + + @Override + public RelNode toRel(RelOptTable.ToRelContext context, RelOptTable relOptTable) { + final RelOptCluster cluster = context.getCluster(); + return new CalciteLogicalCatalogScan(cluster, relOptTable, this); + } + + @Override + public PhysicalPlan implement(LogicalPlan plan) { + return source.implementV2(plan); + } + + public OpenSearchResourceMonitor createOpenSearchResourceMonitor() { + return new OpenSearchResourceMonitor(settings, new OpenSearchMemoryHealthy(settings)); + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/OpenSearchSystemIndex.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/OpenSearchSystemIndex.java deleted file mode 100644 index 6bae4d21aba..00000000000 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/OpenSearchSystemIndex.java +++ /dev/null @@ -1,105 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.opensearch.storage.system; - -import static org.opensearch.sql.utils.SystemIndexUtils.systemTable; - -import com.google.common.annotations.VisibleForTesting; -import java.util.Map; -import lombok.Getter; -import lombok.RequiredArgsConstructor; -import org.apache.calcite.plan.RelOptCluster; -import org.apache.calcite.plan.RelOptTable; -import org.apache.calcite.rel.RelNode; -import org.apache.commons.lang3.tuple.Pair; -import org.opensearch.sql.calcite.plan.AbstractOpenSearchTable; -import org.opensearch.sql.common.setting.Settings; -import org.opensearch.sql.data.type.ExprType; -import org.opensearch.sql.opensearch.client.OpenSearchClient; -import org.opensearch.sql.opensearch.monitor.OpenSearchMemoryHealthy; -import org.opensearch.sql.opensearch.monitor.OpenSearchResourceMonitor; -import org.opensearch.sql.opensearch.request.system.OpenSearchCatIndicesRequest; -import org.opensearch.sql.opensearch.request.system.OpenSearchDescribeIndexRequest; -import org.opensearch.sql.opensearch.request.system.OpenSearchSystemRequest; -import org.opensearch.sql.planner.DefaultImplementor; -import org.opensearch.sql.planner.logical.LogicalPlan; -import org.opensearch.sql.planner.logical.LogicalRelation; -import org.opensearch.sql.planner.physical.PhysicalPlan; -import org.opensearch.sql.utils.SystemIndexUtils; - -/** OpenSearch System Index Table Implementation. */ -@Getter -public class OpenSearchSystemIndex extends AbstractOpenSearchTable { - /** System Index Name. */ - private final Pair systemIndexBundle; - - @Getter private final Settings settings; - - public OpenSearchSystemIndex(OpenSearchClient client, Settings settings, String indexName) { - this.systemIndexBundle = buildIndexBundle(client, indexName); - this.settings = settings; - } - - @Override - public boolean exists() { - return true; // TODO: implement for system index later - } - - @Override - public void create(Map schema) { - throw new UnsupportedOperationException( - "OpenSearch system index is predefined and cannot be created"); - } - - @Override - public Map getFieldTypes() { - return systemIndexBundle.getLeft().getMapping(); - } - - @Override - public RelNode toRel(RelOptTable.ToRelContext context, RelOptTable relOptTable) { - final RelOptCluster cluster = context.getCluster(); - return new CalciteLogicalSystemIndexScan(cluster, relOptTable, this); - } - - @Override - public PhysicalPlan implement(LogicalPlan plan) { - return plan.accept(new OpenSearchSystemIndexDefaultImplementor(), null); - } - - public OpenSearchResourceMonitor createOpenSearchResourceMonitor() { - return new OpenSearchResourceMonitor(getSettings(), new OpenSearchMemoryHealthy(settings)); - } - - @VisibleForTesting - @RequiredArgsConstructor - public class OpenSearchSystemIndexDefaultImplementor extends DefaultImplementor { - - @Override - public PhysicalPlan visitRelation(LogicalRelation node, Object context) { - return new OpenSearchSystemIndexScan(systemIndexBundle.getRight()); - } - } - - /** - * Constructor of ElasticsearchSystemIndexName. - * - * @param indexName index name; - */ - private Pair buildIndexBundle( - OpenSearchClient client, String indexName) { - SystemIndexUtils.SystemTable systemTable = systemTable(indexName); - if (systemTable.isSystemInfoTable()) { - return Pair.of( - OpenSearchSystemIndexSchema.SYS_TABLE_TABLES, new OpenSearchCatIndicesRequest(client)); - } else { - return Pair.of( - OpenSearchSystemIndexSchema.SYS_TABLE_MAPPINGS, - new OpenSearchDescribeIndexRequest( - client, systemTable.getTableName(), systemTable.getLangSpec())); - } - } -} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/SystemIndexCatalogSource.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/SystemIndexCatalogSource.java new file mode 100644 index 00000000000..3541e745e9d --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/SystemIndexCatalogSource.java @@ -0,0 +1,70 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.system; + +import static org.opensearch.sql.utils.SystemIndexUtils.systemTable; + +import java.util.Map; +import org.apache.commons.lang3.tuple.Pair; +import org.opensearch.sql.data.type.ExprType; +import org.opensearch.sql.opensearch.client.OpenSearchClient; +import org.opensearch.sql.opensearch.request.system.OpenSearchCatIndicesRequest; +import org.opensearch.sql.opensearch.request.system.OpenSearchDescribeIndexRequest; +import org.opensearch.sql.opensearch.request.system.OpenSearchSystemRequest; +import org.opensearch.sql.planner.DefaultImplementor; +import org.opensearch.sql.planner.logical.LogicalPlan; +import org.opensearch.sql.planner.logical.LogicalRelation; +import org.opensearch.sql.planner.physical.PhysicalPlan; +import org.opensearch.sql.utils.SystemIndexUtils; + +/** + * {@link CatalogSource} for the SHOW TABLES and DESCRIBE system tables, backed by index listing and + * index field mappings. + */ +public class SystemIndexCatalogSource implements CatalogSource { + + private final Pair bundle; + + public SystemIndexCatalogSource(OpenSearchClient client, String indexName) { + this.bundle = buildBundle(client, indexName); + } + + @Override + public Map getFieldTypes() { + return bundle.getLeft().getMapping(); + } + + @Override + public OpenSearchSystemRequest createRequest() { + return bundle.getRight(); + } + + @Override + public PhysicalPlan implementV2(LogicalPlan plan) { + return plan.accept( + new DefaultImplementor() { + @Override + public PhysicalPlan visitRelation(LogicalRelation node, Object context) { + return new OpenSearchSystemIndexScan(bundle.getRight()); + } + }, + null); + } + + private static Pair buildBundle( + OpenSearchClient client, String indexName) { + SystemIndexUtils.SystemTable systemTable = systemTable(indexName); + if (systemTable.isSystemInfoTable()) { + return Pair.of( + OpenSearchSystemIndexSchema.SYS_TABLE_TABLES, new OpenSearchCatIndicesRequest(client)); + } else { + return Pair.of( + OpenSearchSystemIndexSchema.SYS_TABLE_MAPPINGS, + new OpenSearchDescribeIndexRequest( + client, systemTable.getTableName(), systemTable.getLangSpec())); + } + } +} diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngineTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngineTest.java index fa04395e065..d065e9ac0b6 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngineTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngineTest.java @@ -20,7 +20,7 @@ import org.opensearch.sql.common.setting.Settings; import org.opensearch.sql.expression.function.FunctionResolver; import org.opensearch.sql.opensearch.client.OpenSearchClient; -import org.opensearch.sql.opensearch.storage.system.OpenSearchSystemIndex; +import org.opensearch.sql.opensearch.storage.system.OpenSearchCatalogTable; import org.opensearch.sql.storage.Table; @ExtendWith(MockitoExtension.class) @@ -52,6 +52,7 @@ public void getSystemTable() { OpenSearchStorageEngine engine = new OpenSearchStorageEngine(client, settings); Table table = engine.getTable(new DataSourceSchemaName(DEFAULT_DATASOURCE_NAME, "default"), TABLE_INFO); - assertAll(() -> assertNotNull(table), () -> assertTrue(table instanceof OpenSearchSystemIndex)); + assertAll( + () -> assertNotNull(table), () -> assertTrue(table instanceof OpenSearchCatalogTable)); } } diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestSourceTableTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestCatalogSourceTest.java similarity index 60% rename from opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestSourceTableTest.java rename to opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestCatalogSourceTest.java index a50bf27f042..8676373d2a6 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestSourceTableTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestCatalogSourceTest.java @@ -15,7 +15,6 @@ import static org.opensearch.sql.data.type.ExprCoreType.INTEGER; import static org.opensearch.sql.data.type.ExprCoreType.STRING; -import com.google.common.collect.ImmutableMap; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -23,47 +22,42 @@ import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; -import org.opensearch.sql.common.setting.Settings; import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.data.type.ExprType; import org.opensearch.sql.opensearch.client.OpenSearchClient; -import org.opensearch.sql.storage.Table; import org.opensearch.sql.utils.SystemIndexUtils.RestSpec; +/** + * Covers the {@code rest} {@link RestCatalogSource}: fixed endpoint schema, allow-list enforcement, + * response row shaping and truncation, the {@code Scannable} opt-in, and the Calcite only (no V2) + * path. + */ @ExtendWith(MockitoExtension.class) -class RestSourceTableTest { +class RestCatalogSourceTest { @Mock private OpenSearchClient client; - @Mock private Settings settings; - private RestSpec healthSpec() { return new RestSpec("/_cluster/health", Map.of(), null, null); } @Test void getFieldTypesReturnsFixedEndpointSchema() { - RestSourceTable table = new RestSourceTable(client, settings, healthSpec()); - Map fieldTypes = table.getFieldTypes(); + RestCatalogSource source = new RestCatalogSource(client, healthSpec()); + Map fieldTypes = source.getFieldTypes(); assertThat(fieldTypes, hasEntry("status", STRING)); assertThat(fieldTypes, hasEntry("number_of_nodes", INTEGER)); } @Test - void existsIsTrue() { - Table table = new RestSourceTable(client, settings, healthSpec()); - assertTrue(table.exists()); - } - - @Test - void createIsUnsupported() { - Table table = new RestSourceTable(client, settings, healthSpec()); - assertThrows(UnsupportedOperationException.class, () -> table.create(ImmutableMap.of())); + void isScannable() { + assertTrue(new RestCatalogSource(client, healthSpec()).isScannable()); } @Test - void implementIsUnsupportedOnV2() { - RestSourceTable table = new RestSourceTable(client, settings, healthSpec()); - assertThrows(UnsupportedOperationException.class, () -> table.implement(null)); + void implementV2IsUnsupported() { + RestCatalogSource source = new RestCatalogSource(client, healthSpec()); + assertThrows(UnsupportedOperationException.class, () -> source.implementV2(null)); } @Test @@ -71,8 +65,7 @@ void constructorRejectsNonAllowListedEndpoint() { assertThrows( IllegalArgumentException.class, () -> - new RestSourceTable( - client, settings, new RestSpec("/_cluster/reroute", Map.of(), null, null))); + new RestCatalogSource(client, new RestSpec("/_cluster/reroute", Map.of(), null, null))); } @Test @@ -80,19 +73,15 @@ void constructorRejectsDisallowedArg() { assertThrows( IllegalArgumentException.class, () -> - new RestSourceTable( - client, - settings, - new RestSpec("/_cluster/health", Map.of("bad", "x"), null, null))); + new RestCatalogSource( + client, new RestSpec("/_cluster/health", Map.of("bad", "x"), null, null))); } @Test void constructorRejectsNegativeCount() { assertThrows( IllegalArgumentException.class, - () -> - new RestSourceTable( - client, settings, new RestSpec("/_cat/indices", Map.of(), -1, null))); + () -> new RestCatalogSource(client, new RestSpec("/_cat/indices", Map.of(), -1, null))); } @Test @@ -100,8 +89,7 @@ void constructorRejectsTimeoutArg() { assertThrows( IllegalArgumentException.class, () -> - new RestSourceTable( - client, settings, new RestSpec("/_cluster/health", Map.of(), null, "5s"))); + new RestCatalogSource(client, new RestSpec("/_cluster/health", Map.of(), null, "5s"))); } @Test @@ -111,8 +99,8 @@ void restRequestShapesResponseRows() { health.put("number_of_nodes", 1); when(client.clusterHealth(any())).thenReturn(health); - RestSourceTable table = new RestSourceTable(client, settings, healthSpec()); - List rows = table.createRestRequest().search(); + RestCatalogSource source = new RestCatalogSource(client, healthSpec()); + List rows = source.createRequest().search(); assertEquals(1, rows.size()); assertEquals("green", rows.get(0).tupleValue().get("status").stringValue()); assertEquals(1, rows.get(0).tupleValue().get("number_of_nodes").integerValue()); @@ -126,9 +114,9 @@ void countTruncatesRows() { idx2.put("index", "b"); when(client.catIndices(any())).thenReturn(List.of(idx1, idx2)); - RestSourceTable table = - new RestSourceTable(client, settings, new RestSpec("/_cat/indices", Map.of(), 1, null)); - List rows = table.createRestRequest().search(); + RestCatalogSource source = + new RestCatalogSource(client, new RestSpec("/_cat/indices", Map.of(), 1, null)); + List rows = source.createRequest().search(); assertEquals(1, rows.size()); assertEquals("a", rows.get(0).tupleValue().get("index").stringValue()); } diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/system/OpenSearchSystemIndexTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/system/OpenSearchCatalogTableTest.java similarity index 76% rename from opensearch/src/test/java/org/opensearch/sql/opensearch/storage/system/OpenSearchSystemIndexTest.java rename to opensearch/src/test/java/org/opensearch/sql/opensearch/storage/system/OpenSearchCatalogTableTest.java index 0b0aa1ec521..df81225afbe 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/system/OpenSearchSystemIndexTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/system/OpenSearchCatalogTableTest.java @@ -32,8 +32,12 @@ import org.opensearch.sql.planner.physical.ProjectOperator; import org.opensearch.sql.storage.Table; +/** + * Covers the generic {@link OpenSearchCatalogTable} through a {@link SystemIndexCatalogSource}: + * schema delegation, the predefined-table contract, and the V2 physical path. + */ @ExtendWith(MockitoExtension.class) -class OpenSearchSystemIndexTest { +class OpenSearchCatalogTableTest { @Mock private OpenSearchClient client; @@ -41,36 +45,37 @@ class OpenSearchSystemIndexTest { @Mock private Settings settings; + private OpenSearchCatalogTable systemTable(String name) { + return new OpenSearchCatalogTable(new SystemIndexCatalogSource(client, name), settings); + } + @Test void testGetFieldTypesOfMetaTable() { - OpenSearchSystemIndex systemIndex = new OpenSearchSystemIndex(client, settings, TABLE_INFO); - final Map fieldTypes = systemIndex.getFieldTypes(); + final Map fieldTypes = systemTable(TABLE_INFO).getFieldTypes(); assertThat(fieldTypes, anyOf(hasEntry("TABLE_CAT", STRING))); } @Test void testGetFieldTypesOfMappingTable() { - OpenSearchSystemIndex systemIndex = - new OpenSearchSystemIndex(client, settings, mappingTable("test_index")); - final Map fieldTypes = systemIndex.getFieldTypes(); + final Map fieldTypes = + systemTable(mappingTable("test_index")).getFieldTypes(); assertThat(fieldTypes, anyOf(hasEntry("COLUMN_NAME", STRING))); } @Test void testIsExist() { - Table systemIndex = new OpenSearchSystemIndex(client, settings, TABLE_INFO); - assertTrue(systemIndex.exists()); + assertTrue(systemTable(TABLE_INFO).exists()); } @Test void testCreateTable() { - Table systemIndex = new OpenSearchSystemIndex(client, settings, TABLE_INFO); + Table systemIndex = systemTable(TABLE_INFO); assertThrows(UnsupportedOperationException.class, () -> systemIndex.create(ImmutableMap.of())); } @Test void implement() { - OpenSearchSystemIndex systemIndex = new OpenSearchSystemIndex(client, settings, TABLE_INFO); + OpenSearchCatalogTable systemIndex = systemTable(TABLE_INFO); NamedExpression projectExpr = named("TABLE_NAME", ref("TABLE_NAME", STRING)); final PhysicalPlan plan = diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLRestTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLRestTest.java index f26a2e617e4..da9424c9c8f 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLRestTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLRestTest.java @@ -22,11 +22,11 @@ * Calcite-path coverage for the {@code rest} leading command at the parse / AST tier. * *

The {@code rest} row source resolves through {@code OpenSearchStorageEngine.getTable} -> - * {@code RestSourceTable} -> {@code CalciteLogicalRestScan}, which lives in the {@code opensearch} - * module. This ppl-module Calcite harness binds a Calcite SCOTT schema rather than the OpenSearch - * storage engine, so the optimized {@code CalciteEnumerableRestScan} logical-plan assertion is - * exercised in {@code RestSourceTableTest} (logical scan + fixed row type, unit) and {@code - * CalcitePPLRestIT} (schema + datarows on a live single-node cluster). This test pins the + * {@code RestCatalogSource} -> {@code CalciteLogicalCatalogScan}, which lives in the {@code + * opensearch} module. This ppl-module Calcite harness binds a Calcite SCOTT schema rather than the + * OpenSearch storage engine, so the optimized {@code CalciteScannableCatalogScan} logical-plan + * assertion is exercised in {@code RestCatalogSourceTest} (source schema + request, unit) and + * {@code CalcitePPLRestIT} (schema + datarows on a live single-node cluster). This test pins the * Calcite-facing contract that the ppl module owns: the grammar/AST rewrite of {@code rest} into a * {@code RestRelation} carrying the validated, reserved-name-encoded endpoint spec that rides * {@code visitRelation} exactly like {@code DESCRIBE}. From 95f0618ce17654b58ddd7514166bcba7d037d8e3 Mon Sep 17 00:00:00 2001 From: Louis Chu Date: Thu, 9 Jul 2026 04:06:50 +0800 Subject: [PATCH 09/14] [Feature] Add rest command response redaction and endpoint allow-list Two dynamic cluster settings gate the rest command per deployment: - plugins.ppl.rest.redaction.enabled (default false): mask network identifiers in _cat/* cells and availability-zone names in _cluster/settings values. - plugins.ppl.rest.allowed_endpoints (default all): restrict which endpoints are served; an empty list disables the rest command. Both default to open-source parity: all endpoints served, no masking. Signed-off-by: Louis Chu --- .../sql/common/setting/Settings.java | 2 + .../setting/OpenSearchSettings.java | 29 ++++++ .../storage/OpenSearchStorageEngine.java | 20 +++- .../storage/rest/RestCatalogSource.java | 8 +- .../storage/rest/RestEndpointRegistry.java | 34 ++++++- .../opensearch/storage/rest/RestRequest.java | 12 ++- .../storage/rest/RestResponseRedactor.java | 61 ++++++++++++ .../storage/OpenSearchStorageEngineTest.java | 66 +++++++++++++ .../rest/RestEndpointRegistryTest.java | 76 +++++++++++++++ .../rest/RestResponseRedactorTest.java | 94 +++++++++++++++++++ 10 files changed, 397 insertions(+), 5 deletions(-) create mode 100644 opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestResponseRedactor.java create mode 100644 opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestResponseRedactorTest.java diff --git a/common/src/main/java/org/opensearch/sql/common/setting/Settings.java b/common/src/main/java/org/opensearch/sql/common/setting/Settings.java index 96fe2e04eea..9419b179bb8 100644 --- a/common/src/main/java/org/opensearch/sql/common/setting/Settings.java +++ b/common/src/main/java/org/opensearch/sql/common/setting/Settings.java @@ -36,6 +36,8 @@ public enum Key { PPL_SYNTAX_LEGACY_PREFERRED("plugins.ppl.syntax.legacy.preferred"), PPL_SUBSEARCH_MAXOUT("plugins.ppl.subsearch.maxout"), PPL_JOIN_SUBSEARCH_MAXOUT("plugins.ppl.join.subsearch_maxout"), + PPL_REST_REDACTION_ENABLED("plugins.ppl.rest.redaction.enabled"), + PPL_REST_ALLOWED_ENDPOINTS("plugins.ppl.rest.allowed_endpoints"), /** Enable Calcite as execution engine */ CALCITE_ENGINE_ENABLED("plugins.calcite.enabled"), diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/setting/OpenSearchSettings.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/setting/OpenSearchSettings.java index bd8001f589d..902d9a55684 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/setting/OpenSearchSettings.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/setting/OpenSearchSettings.java @@ -70,6 +70,21 @@ public class OpenSearchSettings extends Settings { Setting.Property.NodeScope, Setting.Property.Dynamic); + public static final Setting PPL_REST_REDACTION_ENABLED_SETTING = + Setting.boolSetting( + Key.PPL_REST_REDACTION_ENABLED.getKeyValue(), + false, + Setting.Property.NodeScope, + Setting.Property.Dynamic); + + public static final Setting> PPL_REST_ALLOWED_ENDPOINTS_SETTING = + Setting.listSetting( + Key.PPL_REST_ALLOWED_ENDPOINTS.getKeyValue(), + List.of("*"), + Function.identity(), + Setting.Property.NodeScope, + Setting.Property.Dynamic); + public static final Setting PPL_QUERY_TIMEOUT_SETTING = Setting.positiveTimeSetting( Key.PPL_QUERY_TIMEOUT.getKeyValue(), @@ -371,6 +386,18 @@ public OpenSearchSettings(ClusterSettings clusterSettings) { Key.PPL_ENABLED, PPL_ENABLED_SETTING, new Updater(Key.PPL_ENABLED)); + register( + settingBuilder, + clusterSettings, + Key.PPL_REST_REDACTION_ENABLED, + PPL_REST_REDACTION_ENABLED_SETTING, + new Updater(Key.PPL_REST_REDACTION_ENABLED)); + register( + settingBuilder, + clusterSettings, + Key.PPL_REST_ALLOWED_ENDPOINTS, + PPL_REST_ALLOWED_ENDPOINTS_SETTING, + new Updater(Key.PPL_REST_ALLOWED_ENDPOINTS)); register( settingBuilder, clusterSettings, @@ -651,6 +678,8 @@ public static List> pluginSettings() { .add(SQL_SLOWLOG_SETTING) .add(SQL_CURSOR_KEEP_ALIVE_SETTING) .add(PPL_ENABLED_SETTING) + .add(PPL_REST_REDACTION_ENABLED_SETTING) + .add(PPL_REST_ALLOWED_ENDPOINTS_SETTING) .add(PPL_QUERY_TIMEOUT_SETTING) .add(PPL_SYNTAX_LEGACY_PREFERRED_SETTING) .add(CALCITE_ENGINE_ENABLED_SETTING) diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngine.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngine.java index 64bc83211b1..a2afd9a9992 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngine.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngine.java @@ -22,6 +22,7 @@ import org.opensearch.sql.opensearch.storage.system.SystemIndexCatalogSource; import org.opensearch.sql.storage.StorageEngine; import org.opensearch.sql.storage.Table; +import org.opensearch.sql.utils.SystemIndexUtils.RestSpec; /** OpenSearch storage engine implementation. */ @RequiredArgsConstructor @@ -40,12 +41,27 @@ public Collection getFunctions() { @Override public Table getTable(DataSourceSchemaName dataSourceSchemaName, String name) { if (isRestSource(name)) { - return new OpenSearchCatalogTable( - new RestCatalogSource(client, decodeRestSpec(name)), settings); + return restTable(name); } else if (isSystemIndex(name)) { return new OpenSearchCatalogTable(new SystemIndexCatalogSource(client, name), settings); } else { return new OpenSearchIndex(client, settings, name); } } + + private Table restTable(String name) { + RestSpec spec = decodeRestSpec(name); + List allowed = settings.getSettingValue(Settings.Key.PPL_REST_ALLOWED_ENDPOINTS); + if (allowed == null || !(allowed.contains("*") || allowed.contains(spec.getEndpoint()))) { + throw new IllegalArgumentException( + allowed == null || allowed.isEmpty() + ? "the rest command is disabled on this cluster" + : "rest endpoint [" + + spec.getEndpoint() + + "] is not enabled on this cluster. Enabled endpoints: " + + allowed); + } + boolean redact = settings.getSettingValue(Settings.Key.PPL_REST_REDACTION_ENABLED); + return new OpenSearchCatalogTable(new RestCatalogSource(client, spec, redact), settings); + } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestCatalogSource.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestCatalogSource.java index 807f83df4bf..96779726a7e 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestCatalogSource.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestCatalogSource.java @@ -26,10 +26,16 @@ public class RestCatalogSource implements CatalogSource { private final OpenSearchClient client; private final RestSpec spec; private final RestEndpointRegistry.Endpoint endpoint; + private final boolean redact; public RestCatalogSource(OpenSearchClient client, RestSpec spec) { + this(client, spec, false); + } + + public RestCatalogSource(OpenSearchClient client, RestSpec spec, boolean redact) { this.client = client; this.spec = spec; + this.redact = redact; // Allow-list enforced here: unknown or mutating endpoints and disallowed args are rejected. this.endpoint = RestEndpointRegistry.resolve(spec.getEndpoint()); RestEndpointRegistry.validate(spec); @@ -42,7 +48,7 @@ public Map getFieldTypes() { @Override public OpenSearchSystemRequest createRequest() { - return new RestRequest(client, endpoint, spec); + return new RestRequest(client, endpoint, spec, redact); } @Override diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistry.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistry.java index feb6a6fafe3..e64a91ab54a 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistry.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistry.java @@ -69,16 +69,48 @@ public static final class Endpoint { /** Dispatch the read-only call and shape the response into fixed-schema rows. */ public List toRows(OpenSearchClient client, RestSpec spec) { + return toRows(client, spec, false); + } + + /** + * Shape the response into fixed-schema rows, masking network identifiers when redaction is + * enabled. {@code /_cat/*} cells are fully masked and the {@code /_cluster/settings} value + * column is zone-masked. Off by default. + */ + public List toRows(OpenSearchClient client, RestSpec spec, boolean redact) { + boolean redactCat = redact && path.startsWith("/_cat"); + boolean redactSettingsValue = redact && "/_cluster/settings".equals(path); List out = new ArrayList<>(); for (Map raw : fetcher.fetch(client, spec)) { LinkedHashMap tuple = new LinkedHashMap<>(); for (Map.Entry col : schema.entrySet()) { - tuple.put(col.getKey(), coerce(col.getKey(), col.getValue(), raw.get(col.getKey()))); + ExprValue value = coerce(col.getKey(), col.getValue(), raw.get(col.getKey())); + tuple.put( + col.getKey(), + maskCell(col.getKey(), col.getValue(), value, redactCat, redactSettingsValue)); } out.add(new ExprTupleValue(tuple)); } return out; } + + private static ExprValue maskCell( + String column, + ExprType type, + ExprValue value, + boolean redactCat, + boolean redactSettingsValue) { + if (type != STRING || value.isNull()) { + return value; + } + if (redactCat) { + return stringValue(RestResponseRedactor.redact(value.stringValue())); + } + if (redactSettingsValue && "value".equals(column)) { + return stringValue(RestResponseRedactor.maskAvailabilityZone(value.stringValue())); + } + return value; + } } private static final Map REGISTRY = buildRegistry(); diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestRequest.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestRequest.java index d6ee4dff1ac..868dabdd64e 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestRequest.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestRequest.java @@ -23,17 +23,27 @@ public class RestRequest implements OpenSearchSystemRequest { private final OpenSearchClient client; private final RestEndpointRegistry.Endpoint endpoint; private final RestSpec spec; + private final boolean redact; public RestRequest( OpenSearchClient client, RestEndpointRegistry.Endpoint endpoint, RestSpec spec) { + this(client, endpoint, spec, false); + } + + public RestRequest( + OpenSearchClient client, + RestEndpointRegistry.Endpoint endpoint, + RestSpec spec, + boolean redact) { this.client = client; this.endpoint = endpoint; this.spec = spec; + this.redact = redact; } @Override public List search() { - List rows = endpoint.toRows(client, spec); + List rows = endpoint.toRows(client, spec, redact); if (spec.getCount() != null && spec.getCount() >= 0 && rows.size() > spec.getCount()) { return rows.subList(0, spec.getCount()); } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestResponseRedactor.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestResponseRedactor.java new file mode 100644 index 00000000000..89ba8d47244 --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestResponseRedactor.java @@ -0,0 +1,61 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.rest; + +import java.util.List; +import java.util.regex.Pattern; + +/** + * Masks network identifiers in rest command cell values. Enabled per deployment via {@code + * plugins.ppl.rest.redaction.enabled}; off by default. + */ +public final class RestResponseRedactor { + + private RestResponseRedactor() {} + + private static final String OCTET = "(25[0-5]|2[0-4]\\d|[0-1]?\\d\\d?)"; + private static final Pattern IPV4 = + Pattern.compile("\\b" + OCTET + "\\." + OCTET + "\\." + OCTET + "\\." + OCTET + "\\b"); + private static final Pattern INET = Pattern.compile("inet\\[/[\\d.:]+\\]"); + private static final Pattern EC2_HOST = + Pattern.compile("\\bip-" + OCTET + "-" + OCTET + "-" + OCTET + "-" + OCTET + "\\b"); + private static final Pattern IPV6 = + Pattern.compile("([0-9a-f]{1,4}:){7}[0-9a-f]{1,4}", Pattern.CASE_INSENSITIVE); + private static final Pattern AZ_NAME = + Pattern.compile( + "(us(-(gov|iso[a-z]?))?|af|ap|ca|cn|eu|sa|me|il)-(central|(north|south)?(east|west)?)-\\d[a-z]", + Pattern.CASE_INSENSITIVE); + + private record Mask(Pattern pattern, String replacement) {} + + private static final List MASKS = + List.of( + new Mask(IPV4, "x.x.x.x"), + new Mask(INET, "inet[/x.x.x.x:y]"), + new Mask(EC2_HOST, ""), + new Mask(IPV6, "x.x.x.x"), + new Mask(AZ_NAME, "xx-xxxxx-xx")); + + /** Mask IPv4, inet, EC2 host names, IPv6, and availability-zone names in the text. */ + public static String redact(String text) { + if (text == null || text.isEmpty()) { + return text; + } + String out = text; + for (Mask mask : MASKS) { + out = mask.pattern().matcher(out).replaceAll(mask.replacement()); + } + return out; + } + + /** Mask availability-zone names only. */ + public static String maskAvailabilityZone(String text) { + if (text == null || text.isEmpty()) { + return text; + } + return AZ_NAME.matcher(text).replaceAll("xx-xxxxx-xx"); + } +} diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngineTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngineTest.java index d065e9ac0b6..102ec4da8f7 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngineTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngineTest.java @@ -7,11 +7,15 @@ import static org.junit.jupiter.api.Assertions.assertAll; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.when; import static org.opensearch.sql.analysis.DataSourceSchemaIdentifierNameResolver.DEFAULT_DATASOURCE_NAME; import static org.opensearch.sql.utils.SystemIndexUtils.TABLE_INFO; import java.util.Collection; +import java.util.List; +import java.util.Map; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; @@ -22,6 +26,7 @@ import org.opensearch.sql.opensearch.client.OpenSearchClient; import org.opensearch.sql.opensearch.storage.system.OpenSearchCatalogTable; import org.opensearch.sql.storage.Table; +import org.opensearch.sql.utils.SystemIndexUtils; @ExtendWith(MockitoExtension.class) class OpenSearchStorageEngineTest { @@ -55,4 +60,65 @@ public void getSystemTable() { assertAll( () -> assertNotNull(table), () -> assertTrue(table instanceof OpenSearchCatalogTable)); } + + @Test + public void getRestTableAllowedByWildcard() { + when(settings.getSettingValue(Settings.Key.PPL_REST_ALLOWED_ENDPOINTS)) + .thenReturn(List.of("*")); + when(settings.getSettingValue(Settings.Key.PPL_REST_REDACTION_ENABLED)).thenReturn(false); + OpenSearchStorageEngine engine = new OpenSearchStorageEngine(client, settings); + String name = + SystemIndexUtils.restTable( + new SystemIndexUtils.RestSpec("/_cat/nodes", Map.of(), null, null)); + Table table = + engine.getTable(new DataSourceSchemaName(DEFAULT_DATASOURCE_NAME, "default"), name); + assertTrue(table instanceof OpenSearchCatalogTable); + } + + @Test + public void getRestTableAllowedBySubset() { + when(settings.getSettingValue(Settings.Key.PPL_REST_ALLOWED_ENDPOINTS)) + .thenReturn(List.of("/_cat/nodes")); + when(settings.getSettingValue(Settings.Key.PPL_REST_REDACTION_ENABLED)).thenReturn(false); + OpenSearchStorageEngine engine = new OpenSearchStorageEngine(client, settings); + String name = + SystemIndexUtils.restTable( + new SystemIndexUtils.RestSpec("/_cat/nodes", Map.of(), null, null)); + assertTrue( + engine.getTable(new DataSourceSchemaName(DEFAULT_DATASOURCE_NAME, "default"), name) + instanceof OpenSearchCatalogTable); + } + + @Test + public void getRestTableRejectedWhenEndpointNotInSubset() { + when(settings.getSettingValue(Settings.Key.PPL_REST_ALLOWED_ENDPOINTS)) + .thenReturn(List.of("/_cat/nodes")); + OpenSearchStorageEngine engine = new OpenSearchStorageEngine(client, settings); + String name = + SystemIndexUtils.restTable( + new SystemIndexUtils.RestSpec("/_cluster/settings", Map.of(), null, null)); + IllegalArgumentException e = + assertThrows( + IllegalArgumentException.class, + () -> + engine.getTable( + new DataSourceSchemaName(DEFAULT_DATASOURCE_NAME, "default"), name)); + assertTrue(e.getMessage().contains("is not enabled on this cluster")); + } + + @Test + public void getRestTableDisabledWhenListEmpty() { + when(settings.getSettingValue(Settings.Key.PPL_REST_ALLOWED_ENDPOINTS)).thenReturn(List.of()); + OpenSearchStorageEngine engine = new OpenSearchStorageEngine(client, settings); + String name = + SystemIndexUtils.restTable( + new SystemIndexUtils.RestSpec("/_cat/nodes", Map.of(), null, null)); + IllegalArgumentException e = + assertThrows( + IllegalArgumentException.class, + () -> + engine.getTable( + new DataSourceSchemaName(DEFAULT_DATASOURCE_NAME, "default"), name)); + assertTrue(e.getMessage().contains("disabled on this cluster")); + } } diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistryTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistryTest.java index 3254ee12f09..c5a978bb495 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistryTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistryTest.java @@ -59,6 +59,82 @@ void validateAcceptsAllowedArg() { RestEndpointRegistry.validate(spec); // no throw } + @Test + void catEndpointsRedactAddressesWhenRedactionEnabled() { + Map node = new LinkedHashMap<>(); + node.put("name", "ip-10-0-0-7"); + node.put("ip", "10.0.0.7"); + node.put("node_role", "dir"); + node.put("heap_percent", 44); + node.put("ram_percent", 95); + node.put("cpu", 2); + when(client.catNodes(any())).thenReturn(List.of(node)); + + RestEndpointRegistry.Endpoint endpoint = RestEndpointRegistry.resolve("/_cat/nodes"); + RestSpec spec = new RestSpec("/_cat/nodes", Map.of(), null, null); + + Map redacted = endpoint.toRows(client, spec, true).get(0).tupleValue(); + assertEquals("x.x.x.x", redacted.get("ip").stringValue()); + assertEquals("", redacted.get("name").stringValue()); + assertEquals(44, redacted.get("heap_percent").integerValue()); + + Map plain = endpoint.toRows(client, spec, false).get(0).tupleValue(); + assertEquals("10.0.0.7", plain.get("ip").stringValue()); + assertEquals("ip-10-0-0-7", plain.get("name").stringValue()); + } + + @Test + void catClusterManagerRedactsHostAndIp() { + Map row = new LinkedHashMap<>(); + row.put("id", "fWhl6_ZQTaSJD9cJ82Ln2w"); + row.put("host", "10.0.0.7"); + row.put("ip", "10.0.0.7"); + row.put("node", "71d03b567bb755839a73d437b2b066d4"); + when(client.catClusterManager(any())).thenReturn(List.of(row)); + + RestEndpointRegistry.Endpoint endpoint = RestEndpointRegistry.resolve("/_cat/cluster_manager"); + RestSpec spec = new RestSpec("/_cat/cluster_manager", Map.of(), null, null); + + Map redacted = endpoint.toRows(client, spec, true).get(0).tupleValue(); + assertEquals("x.x.x.x", redacted.get("host").stringValue()); + assertEquals("x.x.x.x", redacted.get("ip").stringValue()); + assertEquals("fWhl6_ZQTaSJD9cJ82Ln2w", redacted.get("id").stringValue()); + assertEquals("71d03b567bb755839a73d437b2b066d4", redacted.get("node").stringValue()); + } + + @Test + void nonCatEndpointNotRedactedEvenWhenEnabled() { + Map health = new LinkedHashMap<>(); + health.put("cluster_name", "10.0.0.7"); + health.put("status", "green"); + health.put("number_of_nodes", 3); + when(client.clusterHealth(any())).thenReturn(health); + + RestEndpointRegistry.Endpoint endpoint = RestEndpointRegistry.resolve("/_cluster/health"); + RestSpec spec = new RestSpec("/_cluster/health", Map.of(), null, null); + + Map row = endpoint.toRows(client, spec, true).get(0).tupleValue(); + assertEquals("10.0.0.7", row.get("cluster_name").stringValue()); + } + + @Test + void clusterSettingsMasksAvailabilityZoneInValue() { + Map setting = new LinkedHashMap<>(); + setting.put("setting", "cluster.routing.allocation.awareness.attributes"); + setting.put("value", "zone:us-east-1a"); + setting.put("tier", "persistent"); + when(client.clusterSettings(any())).thenReturn(List.of(setting)); + + RestEndpointRegistry.Endpoint endpoint = RestEndpointRegistry.resolve("/_cluster/settings"); + RestSpec spec = new RestSpec("/_cluster/settings", Map.of(), null, null); + + Map row = endpoint.toRows(client, spec, true).get(0).tupleValue(); + assertEquals("zone:xx-xxxxx-xx", row.get("value").stringValue()); + assertEquals( + "cluster.routing.allocation.awareness.attributes", row.get("setting").stringValue()); + assertEquals("persistent", row.get("tier").stringValue()); + } + @Test void validateRejectsDroppedLevelArg() { // level was dropped (no-op against the fixed cluster-level health schema); now unknown. diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestResponseRedactorTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestResponseRedactorTest.java new file mode 100644 index 00000000000..81d60661f50 --- /dev/null +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestResponseRedactorTest.java @@ -0,0 +1,94 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.rest; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.junit.jupiter.api.Test; + +class RestResponseRedactorTest { + + @Test + void masksIpv4() { + assertEquals("x.x.x.x", RestResponseRedactor.redact("0.0.0.0")); + assertEquals("x.x.x.x", RestResponseRedactor.redact("255.255.255.255")); + assertEquals("x.x.x.x", RestResponseRedactor.redact("192.168.1.1")); + assertEquals("x.x.x.x", RestResponseRedactor.redact("1.2.3.4")); + assertEquals("x.x.x.x:9200", RestResponseRedactor.redact("10.0.0.1:9200")); + assertEquals("a x.x.x.x b x.x.x.x c", RestResponseRedactor.redact("a 10.0.0.1 b 172.16.5.4 c")); + } + + @Test + void doesNotMaskInvalidOrPartialIpv4() { + assertEquals("256.1.1.1", RestResponseRedactor.redact("256.1.1.1")); + assertEquals("1.2.3", RestResponseRedactor.redact("1.2.3")); + assertEquals("44", RestResponseRedactor.redact("44")); + } + + @Test + void masksEc2HostName() { + assertEquals("", RestResponseRedactor.redact("ip-10-0-0-1")); + assertEquals("", RestResponseRedactor.redact("ip-172-31-255-9")); + assertEquals("node here", RestResponseRedactor.redact("node ip-10-1-2-3 here")); + assertEquals("ip-256-0-0-1", RestResponseRedactor.redact("ip-256-0-0-1")); + } + + @Test + void masksFullIpv6() { + assertEquals("x.x.x.x", RestResponseRedactor.redact("fe80:0:0:0:0:0:0:1")); + assertEquals("x.x.x.x", RestResponseRedactor.redact("2001:0db8:85a3:0000:0000:8a2e:0370:7334")); + assertEquals("x.x.x.x", RestResponseRedactor.redact("FE80:0:0:0:0:0:0:1")); + } + + @Test + void doesNotMaskCompressedIpv6() { + assertEquals("::1", RestResponseRedactor.redact("::1")); + assertEquals("2001:db8::1", RestResponseRedactor.redact("2001:db8::1")); + } + + @Test + void masksInetAddress() { + assertEquals("inet[/x.x.x.x:9200]", RestResponseRedactor.redact("inet[/10.0.0.7:9200]")); + } + + @Test + void masksAvailabilityZones() { + assertEquals("xx-xxxxx-xx", RestResponseRedactor.redact("us-east-1a")); + assertEquals("xx-xxxxx-xx", RestResponseRedactor.redact("ap-southeast-2b")); + assertEquals("xx-xxxxx-xx", RestResponseRedactor.redact("eu-west-1c")); + assertEquals("xx-xxxxx-xx", RestResponseRedactor.redact("us-gov-west-1a")); + assertEquals( + "a xx-xxxxx-xx b xx-xxxxx-xx", RestResponseRedactor.redact("a us-east-1a b us-west-2b")); + } + + @Test + void maskAvailabilityZoneMasksOnlyZones() { + assertEquals("xx-xxxxx-xx", RestResponseRedactor.maskAvailabilityZone("us-east-1a")); + assertEquals("10.0.0.7", RestResponseRedactor.maskAvailabilityZone("10.0.0.7")); + assertEquals("ip-10-0-0-1", RestResponseRedactor.maskAvailabilityZone("ip-10-0-0-1")); + } + + @Test + void leavesNonAddressesIntact() { + assertEquals( + "e4e136ea81e27370ff73cf753ba22d39", + RestResponseRedactor.redact("e4e136ea81e27370ff73cf753ba22d39")); + assertEquals( + "data,ingest,remote_cluster_client", + RestResponseRedactor.redact("data,ingest,remote_cluster_client")); + assertEquals( + "x.x.x.x 44 95 imr - e4e136ea", + RestResponseRedactor.redact("10.0.0.7 44 95 imr - e4e136ea")); + } + + @Test + void handlesNullAndEmpty() { + assertEquals(null, RestResponseRedactor.redact(null)); + assertEquals("", RestResponseRedactor.redact("")); + assertEquals(null, RestResponseRedactor.maskAvailabilityZone(null)); + assertEquals("", RestResponseRedactor.maskAvailabilityZone("")); + } +} From 06ff820dedb45843da9f30d4b00eb24a951c5540 Mon Sep 17 00:00:00 2001 From: Louis Chu Date: Thu, 9 Jul 2026 04:18:00 +0800 Subject: [PATCH 10/14] [Refactor] Remove unused REST/TIMEOUT rules from shared language grammar The shared language-grammar carried REST and TIMEOUT lexer tokens and the restCommand/restArgument parser rules with no consumer: async-query-core has no rest visitor, and the rest command grammar lives in the ppl module. Remove the dead rules. Signed-off-by: Louis Chu --- .../src/main/antlr4/OpenSearchPPLLexer.g4 | 2 -- .../src/main/antlr4/OpenSearchPPLParser.g4 | 11 ----------- 2 files changed, 13 deletions(-) diff --git a/language-grammar/src/main/antlr4/OpenSearchPPLLexer.g4 b/language-grammar/src/main/antlr4/OpenSearchPPLLexer.g4 index 8ba747614da..2248374d8d9 100644 --- a/language-grammar/src/main/antlr4/OpenSearchPPLLexer.g4 +++ b/language-grammar/src/main/antlr4/OpenSearchPPLLexer.g4 @@ -13,8 +13,6 @@ options { caseInsensitive = true; } SEARCH: 'SEARCH'; DESCRIBE: 'DESCRIBE'; SHOW: 'SHOW'; -REST: 'REST'; -TIMEOUT: 'TIMEOUT'; FROM: 'FROM'; WHERE: 'WHERE'; FIELDS: 'FIELDS'; diff --git a/language-grammar/src/main/antlr4/OpenSearchPPLParser.g4 b/language-grammar/src/main/antlr4/OpenSearchPPLParser.g4 index 1d13e8af046..451824d6992 100644 --- a/language-grammar/src/main/antlr4/OpenSearchPPLParser.g4 +++ b/language-grammar/src/main/antlr4/OpenSearchPPLParser.g4 @@ -62,7 +62,6 @@ commands commandName : SEARCH | DESCRIBE - | REST | SHOW | AD | ML @@ -114,16 +113,6 @@ describeCommand : DESCRIBE tableSourceClause ; - -restCommand - : REST stringLiteral (restArgument)* - ; - -restArgument - : COUNT EQUAL integerLiteral - | TIMEOUT EQUAL stringLiteral - | ident EQUAL literalValue - ; explainCommand : EXPLAIN explainMode ; From 7d9df84765881c034f6af5952ca0e170d4275aa9 Mon Sep 17 00:00:00 2001 From: Louis Chu Date: Thu, 9 Jul 2026 12:32:07 +0800 Subject: [PATCH 11/14] [Test] Add rest command security integration tests Verify the rest command is subject to the security plugin fine grained access control: a caller without cluster:monitor privilege is denied the cat and cluster endpoints, a caller holding the privilege can run them, and the resolve index endpoint is filtered to the caller authorized indices. Test indices are created idempotently, and denials are asserted by the security denial reason in the response body because a denied action on the Calcite only rest path surfaces as a wrapped error carrying that reason. Calcite fallback is disabled so the denial reason is not replaced by an unsupported command error. Signed-off-by: Louis Chu --- .../sql/security/RestCommandSecurityIT.java | 154 ++++++++++++++++++ 1 file changed, 154 insertions(+) create mode 100644 integ-test/src/test/java/org/opensearch/sql/security/RestCommandSecurityIT.java diff --git a/integ-test/src/test/java/org/opensearch/sql/security/RestCommandSecurityIT.java b/integ-test/src/test/java/org/opensearch/sql/security/RestCommandSecurityIT.java new file mode 100644 index 00000000000..042a3aefbf7 --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/security/RestCommandSecurityIT.java @@ -0,0 +1,154 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.security; + +import static org.opensearch.sql.util.MatcherUtils.columnName; +import static org.opensearch.sql.util.MatcherUtils.verifyColumn; + +import java.io.IOException; +import java.util.HashSet; +import java.util.Set; +import org.json.JSONArray; +import org.json.JSONObject; +import org.junit.Test; +import org.opensearch.client.Request; +import org.opensearch.client.ResponseException; +import org.opensearch.sql.legacy.TestUtils; + +/** + * Integration tests that verify the rest command is subject to the security plugin fine grained + * access control. The command dispatches standard transport actions under the caller identity, so + * the security ActionFilter authorizes each one by action name. A caller without the required + * cluster monitor privilege is denied the cat and cluster endpoints, a caller holding the privilege + * can run them, and the resolve index endpoint requires the resolve index privilege because the + * command resolves all indices. + */ +public class RestCommandSecurityIT extends SecurityTestBase { + + private static final String ALPHA_INDEX = "rest_sec_alpha"; + private static final String BETA_INDEX = "rest_sec_beta"; + + private static final String MONITOR_USER = "rest_monitor_user"; + private static final String MONITOR_ROLE = "rest_monitor_role"; + + private static final String NO_MONITOR_USER = "rest_no_monitor_user"; + private static final String NO_MONITOR_ROLE = "rest_no_monitor_role"; + + @Override + protected void init() throws Exception { + super.init(); + setupRolesUsersAndIndices(); + enableCalcite(); + // rest is Calcite only, so a V2 fallback would replace the security denial with an unsupported + // command error. Disable fallback so the denial reason surfaces to the caller. + disallowCalciteFallback(); + } + + private void setupRolesUsersAndIndices() throws IOException { + createIndexIfAbsent(ALPHA_INDEX); + createIndexIfAbsent(BETA_INDEX); + + createRoleWithPermissions( + MONITOR_ROLE, + "*", + new String[] { + "cluster:admin/opensearch/ppl", + "cluster:monitor/health", + "cluster:monitor/state", + "cluster:monitor/nodes/stats", + "cluster:monitor/nodes/info" + }, + new String[] {"indices:admin/resolve/index"}); + createUser(MONITOR_USER, MONITOR_ROLE); + + createRoleWithPermissions( + NO_MONITOR_ROLE, + ALPHA_INDEX, + new String[] {"cluster:admin/opensearch/ppl"}, + new String[] {"indices:data/read/search*"}); + createUser(NO_MONITOR_USER, NO_MONITOR_ROLE); + } + + @Test + public void monitorUserCanRunCatNodes() throws IOException { + JSONObject result = executeQueryAsUser("| rest '/_cat/nodes' | fields name", MONITOR_USER); + verifyColumn(result, columnName("name")); + } + + @Test + public void monitorUserCanResolveIndex() throws IOException { + JSONObject result = + executeQueryAsUser("| rest '/_resolve/index' | fields name, type", MONITOR_USER); + Set names = resolvedNames(result); + assertTrue("resolve should list authorized indices: " + names, names.contains(ALPHA_INDEX)); + assertTrue("resolve should list authorized indices: " + names, names.contains(BETA_INDEX)); + } + + @Test + public void userWithoutClusterMonitorCannotRunCatNodes() throws IOException { + assertDenied( + "| rest '/_cat/nodes' | fields name", NO_MONITOR_USER, "cluster:monitor/nodes/stats"); + } + + @Test + public void userWithoutClusterMonitorCannotRunClusterState() throws IOException { + assertDenied( + "| rest '/_cluster/state' | fields cluster_name", NO_MONITOR_USER, "cluster:monitor/state"); + } + + @Test + public void userWithoutResolvePrivilegeCannotResolveIndex() throws IOException { + assertDenied( + "| rest '/_resolve/index' | fields name, type", + NO_MONITOR_USER, + "indices:admin/resolve/index"); + } + + /** + * Asserts the query is rejected for a caller lacking the privilege. A denied transport action on + * the Calcite only rest path surfaces as a client or server error whose body carries the security + * denial reason, so this checks the denial signal rather than a fixed status code. + */ + private void assertDenied(String query, String user, String deniedAction) throws IOException { + try { + executeQueryAsUser(query, user); + fail("Expected a permission denial for user without privilege: " + user); + } catch (ResponseException e) { + int status = e.getResponse().getStatusLine().getStatusCode(); + String body = TestUtils.getResponseBody(e.getResponse(), false); + assertTrue("Expected an error status, got " + status, status >= 400); + assertTrue( + "Response should indicate a permission denial. Status " + status + ", body: " + body, + body.contains("no permissions") + || body.contains("Forbidden") + || body.contains("security_exception") + || body.contains(deniedAction)); + } + } + + private void createIndexIfAbsent(String name) throws IOException { + Request request = new Request("PUT", "/" + name); + request.setJsonEntity( + "{ \"settings\": { \"number_of_shards\": 1, \"number_of_replicas\": 0 } }"); + try { + client().performRequest(request); + } catch (ResponseException e) { + String body = TestUtils.getResponseBody(e.getResponse(), false); + if (!body.contains("resource_already_exists_exception")) { + throw e; + } + } + } + + private Set resolvedNames(JSONObject result) { + Set names = new HashSet<>(); + JSONArray datarows = result.getJSONArray("datarows"); + for (int i = 0; i < datarows.length(); i++) { + names.add(datarows.getJSONArray(i).getString(0)); + } + return names; + } +} From aef46635798e65c9a3c7962bfec4d38f3d844eb0 Mon Sep 17 00:00:00 2001 From: Louis Chu Date: Fri, 10 Jul 2026 09:50:13 +0800 Subject: [PATCH 12/14] [Bugfix] Make rest redaction and allow-list settings node-level plugins.ppl.rest.redaction.enabled and plugins.ppl.rest.allowed_endpoints were dynamic cluster settings, so they could be changed at runtime through _cluster/settings or the _plugins/_query/settings endpoint. On a managed deployment that let a caller disable redaction or widen the allow-list an operator had configured. Drop Setting.Property.Dynamic so both are node-level settings set in the node config; the engine rejects runtime updates on both paths. Register them without an update consumer and read the node-configured value. Signed-off-by: Louis Chu --- .../setting/OpenSearchSettings.java | 26 ++++++++----------- .../setting/OpenSearchSettingsTest.java | 12 +++++++++ 2 files changed, 23 insertions(+), 15 deletions(-) diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/setting/OpenSearchSettings.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/setting/OpenSearchSettings.java index 902d9a55684..13af16a297f 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/setting/OpenSearchSettings.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/setting/OpenSearchSettings.java @@ -72,18 +72,14 @@ public class OpenSearchSettings extends Settings { public static final Setting PPL_REST_REDACTION_ENABLED_SETTING = Setting.boolSetting( - Key.PPL_REST_REDACTION_ENABLED.getKeyValue(), - false, - Setting.Property.NodeScope, - Setting.Property.Dynamic); + Key.PPL_REST_REDACTION_ENABLED.getKeyValue(), false, Setting.Property.NodeScope); public static final Setting> PPL_REST_ALLOWED_ENDPOINTS_SETTING = Setting.listSetting( Key.PPL_REST_ALLOWED_ENDPOINTS.getKeyValue(), List.of("*"), Function.identity(), - Setting.Property.NodeScope, - Setting.Property.Dynamic); + Setting.Property.NodeScope); public static final Setting PPL_QUERY_TIMEOUT_SETTING = Setting.positiveTimeSetting( @@ -386,18 +382,16 @@ public OpenSearchSettings(ClusterSettings clusterSettings) { Key.PPL_ENABLED, PPL_ENABLED_SETTING, new Updater(Key.PPL_ENABLED)); - register( + registerNonDynamicSettings( settingBuilder, clusterSettings, Key.PPL_REST_REDACTION_ENABLED, - PPL_REST_REDACTION_ENABLED_SETTING, - new Updater(Key.PPL_REST_REDACTION_ENABLED)); - register( + PPL_REST_REDACTION_ENABLED_SETTING); + registerNonDynamicSettings( settingBuilder, clusterSettings, Key.PPL_REST_ALLOWED_ENDPOINTS, - PPL_REST_ALLOWED_ENDPOINTS_SETTING, - new Updater(Key.PPL_REST_ALLOWED_ENDPOINTS)); + PPL_REST_ALLOWED_ENDPOINTS_SETTING); register( settingBuilder, clusterSettings, @@ -652,7 +646,9 @@ private void registerNonDynamicSettings( Settings.Key key, Setting setting) { settingBuilder.put(key, setting); - latestSettings.put(key, clusterSettings.get(setting)); + if (clusterSettings.get(setting) != null) { + latestSettings.put(key, clusterSettings.get(setting)); + } } /** @@ -678,8 +674,6 @@ public static List> pluginSettings() { .add(SQL_SLOWLOG_SETTING) .add(SQL_CURSOR_KEEP_ALIVE_SETTING) .add(PPL_ENABLED_SETTING) - .add(PPL_REST_REDACTION_ENABLED_SETTING) - .add(PPL_REST_ALLOWED_ENDPOINTS_SETTING) .add(PPL_QUERY_TIMEOUT_SETTING) .add(PPL_SYNTAX_LEGACY_PREFERRED_SETTING) .add(CALCITE_ENGINE_ENABLED_SETTING) @@ -724,6 +718,8 @@ public static List> pluginNonDynamicSettings() { return new ImmutableList.Builder>() .add(DATASOURCE_MASTER_SECRET_KEY) .add(DATASOURCE_CONFIG) + .add(PPL_REST_REDACTION_ENABLED_SETTING) + .add(PPL_REST_ALLOWED_ENDPOINTS_SETTING) .build(); } diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/setting/OpenSearchSettingsTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/setting/OpenSearchSettingsTest.java index 5024d416086..0c570098924 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/setting/OpenSearchSettingsTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/setting/OpenSearchSettingsTest.java @@ -9,12 +9,15 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.AdditionalMatchers.not; import static org.mockito.AdditionalMatchers.or; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.when; import static org.opensearch.sql.opensearch.setting.OpenSearchSettings.ASYNC_QUERY_EXTERNAL_SCHEDULER_ENABLED_SETTING; import static org.opensearch.sql.opensearch.setting.OpenSearchSettings.ASYNC_QUERY_EXTERNAL_SCHEDULER_INTERVAL_SETTING; +import static org.opensearch.sql.opensearch.setting.OpenSearchSettings.PPL_REST_ALLOWED_ENDPOINTS_SETTING; +import static org.opensearch.sql.opensearch.setting.OpenSearchSettings.PPL_REST_REDACTION_ENABLED_SETTING; import static org.opensearch.sql.opensearch.setting.OpenSearchSettings.QUERY_MEMORY_LIMIT_SETTING; import static org.opensearch.sql.opensearch.setting.OpenSearchSettings.SPARK_EXECUTION_ENGINE_CONFIG; @@ -74,6 +77,15 @@ void pluginNonDynamicSettings() { assertFalse(settings.isEmpty()); } + @Test + void restSettingsAreNonDynamic() { + assertFalse(PPL_REST_REDACTION_ENABLED_SETTING.isDynamic()); + assertFalse(PPL_REST_ALLOWED_ENDPOINTS_SETTING.isDynamic()); + List> nonDynamic = OpenSearchSettings.pluginNonDynamicSettings(); + assertTrue(nonDynamic.contains(PPL_REST_REDACTION_ENABLED_SETTING)); + assertTrue(nonDynamic.contains(PPL_REST_ALLOWED_ENDPOINTS_SETTING)); + } + @Test void getSettings() { when(clusterSettings.get(ClusterName.CLUSTER_NAME_SETTING)).thenReturn(ClusterName.DEFAULT); From 8e4b5651b5f71bf84e48e17b7d439469a8d99993 Mon Sep 17 00:00:00 2001 From: Louis Chu Date: Wed, 15 Jul 2026 03:57:02 +0800 Subject: [PATCH 13/14] Enhance redaction logic Signed-off-by: Louis Chu --- .../opensearch/storage/rest/RestResponseRedactor.java | 7 +++++-- .../storage/rest/RestResponseRedactorTest.java | 10 +++++++--- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestResponseRedactor.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestResponseRedactor.java index 89ba8d47244..fd674dccde4 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestResponseRedactor.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestResponseRedactor.java @@ -23,10 +23,13 @@ private RestResponseRedactor() {} private static final Pattern EC2_HOST = Pattern.compile("\\bip-" + OCTET + "-" + OCTET + "-" + OCTET + "-" + OCTET + "\\b"); private static final Pattern IPV6 = - Pattern.compile("([0-9a-f]{1,4}:){7}[0-9a-f]{1,4}", Pattern.CASE_INSENSITIVE); + Pattern.compile( + "([0-9a-f]{1,4}:){7}[0-9a-f]{1,4}" + + "|([0-9a-f]{1,4}(:[0-9a-f]{1,4})*)?::([0-9a-f]{1,4}(:[0-9a-f]{1,4})*)?", + Pattern.CASE_INSENSITIVE); private static final Pattern AZ_NAME = Pattern.compile( - "(us(-(gov|iso[a-z]?))?|af|ap|ca|cn|eu|sa|me|il)-(central|(north|south)?(east|west)?)-\\d[a-z]", + "\\b[a-z]{2}(-(gov|iso[a-z]?))?-(central|(north|south)?(east|west)?)-\\d[a-z]\\b", Pattern.CASE_INSENSITIVE); private record Mask(Pattern pattern, String replacement) {} diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestResponseRedactorTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestResponseRedactorTest.java index 81d60661f50..3b038306da5 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestResponseRedactorTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestResponseRedactorTest.java @@ -44,9 +44,11 @@ void masksFullIpv6() { } @Test - void doesNotMaskCompressedIpv6() { - assertEquals("::1", RestResponseRedactor.redact("::1")); - assertEquals("2001:db8::1", RestResponseRedactor.redact("2001:db8::1")); + void masksCompressedIpv6() { + assertEquals("x.x.x.x", RestResponseRedactor.redact("::1")); + assertEquals("x.x.x.x", RestResponseRedactor.redact("fe80::1")); + assertEquals("x.x.x.x", RestResponseRedactor.redact("2001:db8::1")); + assertEquals("x.x.x.x", RestResponseRedactor.redact("2001:db8::8a2e:370:7334")); } @Test @@ -60,6 +62,8 @@ void masksAvailabilityZones() { assertEquals("xx-xxxxx-xx", RestResponseRedactor.redact("ap-southeast-2b")); assertEquals("xx-xxxxx-xx", RestResponseRedactor.redact("eu-west-1c")); assertEquals("xx-xxxxx-xx", RestResponseRedactor.redact("us-gov-west-1a")); + // Shape-based match covers regions not in any hard-coded list (e.g. mx-central-1). + assertEquals("xx-xxxxx-xx", RestResponseRedactor.redact("mx-central-1a")); assertEquals( "a xx-xxxxx-xx b xx-xxxxx-xx", RestResponseRedactor.redact("a us-east-1a b us-west-2b")); } From 03d21c3d0138afade9fd673ffca9bf9a2c6493dd Mon Sep 17 00:00:00 2001 From: Louis Chu Date: Wed, 15 Jul 2026 10:34:52 +0800 Subject: [PATCH 14/14] [Change] Disable all rest endpoints by default (empty allow-list) Flip plugins.ppl.rest.allowed_endpoints default from ["*"] to empty so open source ships with the rest command closed. Deployments opt specific endpoints in via the setting (AOS enables the ones it supports; AOSS leaves it empty and stays disabled). Enforcement already treats an empty or missing list as disabled. Opt the integration-test clusters into all endpoints so the rest ITs still exercise the enabled path. Signed-off-by: Louis Chu --- docs/user/ppl/cmd/rest.md | 12 ++++++++++++ doctest/build.gradle | 5 +++++ integ-test/build.gradle | 8 ++++++++ .../sql/opensearch/setting/OpenSearchSettings.java | 2 +- .../opensearch/storage/OpenSearchStorageEngine.java | 2 ++ 5 files changed, 28 insertions(+), 1 deletion(-) diff --git a/docs/user/ppl/cmd/rest.md b/docs/user/ppl/cmd/rest.md index b97f247445f..a9761aa87e8 100644 --- a/docs/user/ppl/cmd/rest.md +++ b/docs/user/ppl/cmd/rest.md @@ -4,6 +4,18 @@ The `rest` command is a leading command that reads an allow-listed, read-only in > **Note**: The `rest` command is supported only on the Calcite query engine (`plugins.calcite.enabled=true`). Each endpoint has a fixed output schema, and the dispatch runs under the caller's security context, so a user who cannot call an endpoint directly cannot call it through `rest`. The command is read-only; mutating and non-allow-listed endpoints are rejected. Each endpoint requires the same cluster-monitor privilege as calling it natively, so `rest` grants no extra access. Some allow-listed endpoints surface operational metadata (for example `/_cat/nodes` exposes node addresses and resource utilization, `/_cat/plugins` the installed plugin inventory, and `/_cluster/state` cluster-state identifiers); this is a deliberate, read-only, monitor-privileged trade-off. `/_cluster/settings` is redacted with the node's setting filter so `Property.Filtered` keys are not surfaced. +## Enabling the command + +The `rest` command is **disabled by default**: `plugins.ppl.rest.allowed_endpoints` defaults to an empty list, so every endpoint is rejected until a deployment explicitly opts in. Enable specific endpoints by setting the allow-list (a node-level setting, so it is applied at node startup and cannot be changed at runtime): + +```yaml +plugins.ppl.rest.allowed_endpoints: ["/_cluster/health", "/_cat/nodes"] +``` + +Use `["*"]` to allow every endpoint in the curated list below. An empty list (the default) disables the command entirely. + +The `rest` command also supports optional response redaction of network identifiers (IPv4/IPv6 addresses, `inet[...]` forms, EC2-style host names, and availability-zone names) in `/_cat/*` and `/_cluster/settings` cell values, controlled by `plugins.ppl.rest.redaction.enabled` (a node-level setting, default `false`). Managed deployments that must not expose host topology should set it to `true`. + ## Syntax The `rest` command has the following syntax: diff --git a/doctest/build.gradle b/doctest/build.gradle index cce64170f56..1ac658457dc 100644 --- a/doctest/build.gradle +++ b/doctest/build.gradle @@ -205,6 +205,11 @@ testClusters { plugin(getJobSchedulerPlugin()) plugin ':opensearch-sql-plugin' testDistribution = 'archive' + // The rest command is disabled by default (empty allow-list). Opt the doctest cluster into + // the registered endpoints so the rest.md examples run against an enabled command. A literal + // "*" cannot be used because a bare * is a YAML alias indicator in opensearch.yml. + setting 'plugins.ppl.rest.allowed_endpoints', + '/_cluster/health,/_cluster/state,/_cluster/settings,/_cat/indices,/_cat/nodes,/_cat/cluster_manager,/_cat/plugins,/_cat/shards,/_resolve/index' } } tasks.register("runRestTestCluster", RunTask) { diff --git a/integ-test/build.gradle b/integ-test/build.gradle index d04c4484cc1..a377d343c1b 100644 --- a/integ-test/build.gradle +++ b/integ-test/build.gradle @@ -387,6 +387,11 @@ testClusters { plugin(getGeoSpatialPlugin()) plugin ":opensearch-sql-plugin" setting "plugins.query.datasources.encryption.masterkey", "1234567812345678" + // The rest command is disabled by default (empty allow-list). Opt this cluster into the + // registered endpoints so the rest integration tests exercise the enabled path. A literal + // "*" cannot be used because a bare * is a YAML alias indicator in opensearch.yml. + setting 'plugins.ppl.rest.allowed_endpoints', + '/_cluster/health,/_cluster/state,/_cluster/settings,/_cat/indices,/_cat/nodes,/_cat/cluster_manager,/_cat/plugins,/_cat/shards,/_resolve/index' } yamlRestTest { testDistribution = 'archive' @@ -405,6 +410,9 @@ testClusters { testDistribution = 'archive' plugin(getJobSchedulerPlugin()) plugin ":opensearch-sql-plugin" + // Opt into the rest endpoints (disabled by default) so RestCommandSecurityIT runs. + setting 'plugins.ppl.rest.allowed_endpoints', + '/_cluster/health,/_cluster/state,/_cluster/settings,/_cat/indices,/_cat/nodes,/_cat/cluster_manager,/_cat/plugins,/_cat/shards,/_resolve/index' } remoteIntegTestWithSecurity { testDistribution = 'archive' diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/setting/OpenSearchSettings.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/setting/OpenSearchSettings.java index 13af16a297f..55212248b43 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/setting/OpenSearchSettings.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/setting/OpenSearchSettings.java @@ -77,7 +77,7 @@ public class OpenSearchSettings extends Settings { public static final Setting> PPL_REST_ALLOWED_ENDPOINTS_SETTING = Setting.listSetting( Key.PPL_REST_ALLOWED_ENDPOINTS.getKeyValue(), - List.of("*"), + List.of(), Function.identity(), Setting.Property.NodeScope); diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngine.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngine.java index a2afd9a9992..7b911471242 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngine.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngine.java @@ -18,6 +18,7 @@ import org.opensearch.sql.expression.function.FunctionResolver; import org.opensearch.sql.opensearch.client.OpenSearchClient; import org.opensearch.sql.opensearch.storage.rest.RestCatalogSource; +import org.opensearch.sql.opensearch.storage.rest.RestEndpointRegistry; import org.opensearch.sql.opensearch.storage.system.OpenSearchCatalogTable; import org.opensearch.sql.opensearch.storage.system.SystemIndexCatalogSource; import org.opensearch.sql.storage.StorageEngine; @@ -51,6 +52,7 @@ public Table getTable(DataSourceSchemaName dataSourceSchemaName, String name) { private Table restTable(String name) { RestSpec spec = decodeRestSpec(name); + RestEndpointRegistry.resolve(spec.getEndpoint()); List allowed = settings.getSettingValue(Settings.Key.PPL_REST_ALLOWED_ENDPOINTS); if (allowed == null || !(allowed.contains("*") || allowed.contains(spec.getEndpoint()))) { throw new IllegalArgumentException(