diff --git a/common/src/main/java/org/opensearch/sql/common/setting/Settings.java b/common/src/main/java/org/opensearch/sql/common/setting/Settings.java index 96fe2e04eea..9419b179bb8 100644 --- a/common/src/main/java/org/opensearch/sql/common/setting/Settings.java +++ b/common/src/main/java/org/opensearch/sql/common/setting/Settings.java @@ -36,6 +36,8 @@ public enum Key { PPL_SYNTAX_LEGACY_PREFERRED("plugins.ppl.syntax.legacy.preferred"), PPL_SUBSEARCH_MAXOUT("plugins.ppl.subsearch.maxout"), PPL_JOIN_SUBSEARCH_MAXOUT("plugins.ppl.join.subsearch_maxout"), + PPL_REST_REDACTION_ENABLED("plugins.ppl.rest.redaction.enabled"), + PPL_REST_ALLOWED_ENDPOINTS("plugins.ppl.rest.allowed_endpoints"), /** Enable Calcite as execution engine */ CALCITE_ENGINE_ENABLED("plugins.calcite.enabled"), diff --git a/core/src/main/java/org/opensearch/sql/ast/tree/RestRelation.java b/core/src/main/java/org/opensearch/sql/ast/tree/RestRelation.java new file mode 100644 index 00000000000..d0a0e34d3c9 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/ast/tree/RestRelation.java @@ -0,0 +1,25 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.ast.tree; + +import java.util.Collections; +import lombok.EqualsAndHashCode; +import lombok.ToString; +import org.opensearch.sql.ast.expression.UnresolvedExpression; + +/** + * Extend Relation to mark a {@code rest} leading command. The single table name is a reserved, + * encoded token (produced by {@link org.opensearch.sql.utils.SystemIndexUtils#restTable}) that + * carries the validated REST endpoint spec; it resolves through the storage engine to a REST source + * table on the Calcite path, exactly as {@link DescribeRelation} resolves to a system index. + */ +@ToString +@EqualsAndHashCode(callSuper = false) +public class RestRelation extends Relation { + public RestRelation(UnresolvedExpression tableName) { + super(Collections.singletonList(tableName)); + } +} diff --git a/core/src/main/java/org/opensearch/sql/utils/SystemIndexUtils.java b/core/src/main/java/org/opensearch/sql/utils/SystemIndexUtils.java index c9fa35d7068..7589cf522f6 100644 --- a/core/src/main/java/org/opensearch/sql/utils/SystemIndexUtils.java +++ b/core/src/main/java/org/opensearch/sql/utils/SystemIndexUtils.java @@ -5,6 +5,9 @@ package org.opensearch.sql.utils; +import java.nio.charset.StandardCharsets; +import java.util.LinkedHashMap; +import java.util.Map; import lombok.Getter; import lombok.RequiredArgsConstructor; import lombok.experimental.UtilityClass; @@ -34,6 +37,121 @@ public static Boolean isSystemIndex(String indexName) { return indexName.endsWith(SYS_TABLES_SUFFIX); } + /** + * Reserved suffix marking a {@code rest} source table. Distinct from {@link #SYS_TABLES_SUFFIX} + * so {@link #isSystemIndex} and {@link #isRestSource} never overlap. The whole reserved name is a + * single Calcite identifier (REST + lowercase hex + this suffix), so it survives name resolution + * the same way the uppercase system-mapping names do. + */ + private static final String REST_SOURCE_SUFFIX = "__REST_SOURCE"; + + private static final String REST_SOURCE_PREFIX = "REST"; + + /** True if the resolved table name is a {@code rest} source token. */ + public static boolean isRestSource(String indexName) { + return indexName.endsWith(REST_SOURCE_SUFFIX); + } + + /** + * Encode a validated {@link RestSpec} into a single reserved table name. Mirrors {@link + * #mappingTable}: structured metadata travels inside a reserved name rather than a side channel. + * The endpoint/args have already been allow-list-validated before this is called. + */ + public static String restTable(RestSpec spec) { + StringBuilder sb = new StringBuilder(); + sb.append("endpoint=").append(spec.getEndpoint()); + if (spec.getCount() != null) { + sb.append('\n').append("count=").append(spec.getCount()); + } + if (spec.getTimeout() != null) { + sb.append('\n').append("timeout=").append(spec.getTimeout()); + } + if (spec.getArgs() != null) { + for (Map.Entry e : spec.getArgs().entrySet()) { + sb.append('\n').append("arg.").append(e.getKey()).append('=').append(e.getValue()); + } + } + return REST_SOURCE_PREFIX + toHex(sb.toString()) + REST_SOURCE_SUFFIX; + } + + /** Decode a reserved {@code rest} table name back into its {@link RestSpec}. */ + public static RestSpec decodeRestSpec(String indexName) { + // Validate the token shape before slicing it: callers gate on isRestSource today, but a + // public decoder must not assume its precondition, otherwise a malformed token would throw an + // opaque StringIndexOutOfBoundsException from substring rather than a clear input error. + if (!isRestSource(indexName)) { + throw new IllegalArgumentException("not a valid rest source token: " + indexName); + } + String body = + indexName.substring( + REST_SOURCE_PREFIX.length(), indexName.length() - REST_SOURCE_SUFFIX.length()); + String decoded = fromHex(body); + String endpoint = null; + Integer count = null; + String timeout = null; + LinkedHashMap args = new LinkedHashMap<>(); + for (String line : decoded.split("\n")) { + if (line.isEmpty()) { + continue; + } + int eq = line.indexOf('='); + if (eq < 0) { + continue; + } + String k = line.substring(0, eq); + String v = line.substring(eq + 1); + if (k.equals("endpoint")) { + endpoint = v; + } else if (k.equals("count")) { + count = Integer.parseInt(v); + } else if (k.equals("timeout")) { + timeout = v; + } else if (k.startsWith("arg.")) { + args.put(k.substring("arg.".length()), v); + } + } + if (endpoint == null) { + throw new IllegalArgumentException("rest source token is missing the endpoint: " + indexName); + } + return new RestSpec(endpoint, args, count, timeout); + } + + private static String toHex(String s) { + StringBuilder h = new StringBuilder(); + for (byte b : s.getBytes(StandardCharsets.UTF_8)) { + h.append(Character.forDigit((b >> 4) & 0xF, 16)).append(Character.forDigit(b & 0xF, 16)); + } + return h.toString(); + } + + private static String fromHex(String h) { + if (h.length() % 2 != 0) { + throw new IllegalArgumentException("not a valid rest source token: odd-length hex body"); + } + byte[] bytes = new byte[h.length() / 2]; + for (int i = 0; i < bytes.length; i++) { + bytes[i] = + (byte) + ((Character.digit(h.charAt(2 * i), 16) << 4) + + Character.digit(h.charAt(2 * i + 1), 16)); + } + return new String(bytes, StandardCharsets.UTF_8); + } + + /** + * The validated spec for a {@code rest} command: an allow-listed read-only endpoint plus optional + * count/timeout/query args. Lives in core so the parser (encode) and the storage engine (decode) + * share it without a cross-module dependency. + */ + @Getter + @RequiredArgsConstructor + public static class RestSpec { + private final String endpoint; + private final Map args; + private final Integer count; + private final String timeout; + } + /** * Compose system mapping table. * diff --git a/docs/category.json b/docs/category.json index ac1dda966d2..0defdaaa6d3 100644 --- a/docs/category.json +++ b/docs/category.json @@ -34,6 +34,7 @@ "user/ppl/cmd/rename.md", "user/ppl/cmd/multisearch.md", "user/ppl/cmd/replace.md", + "user/ppl/cmd/rest.md", "user/ppl/cmd/rex.md", "user/ppl/cmd/search.md", "user/ppl/cmd/showdatasources.md", diff --git a/docs/user/ppl/cmd/rest.md b/docs/user/ppl/cmd/rest.md new file mode 100644 index 00000000000..a9761aa87e8 --- /dev/null +++ b/docs/user/ppl/cmd/rest.md @@ -0,0 +1,94 @@ +# rest + +The `rest` command is a leading command that reads an allow-listed, read-only in-cluster management endpoint (cluster/cat/nodes) and emits the response as PPL rows. Its rows come from the endpoint dispatch, not from an index, so `rest` appears at the start of a query. + +> **Note**: The `rest` command is supported only on the Calcite query engine (`plugins.calcite.enabled=true`). Each endpoint has a fixed output schema, and the dispatch runs under the caller's security context, so a user who cannot call an endpoint directly cannot call it through `rest`. The command is read-only; mutating and non-allow-listed endpoints are rejected. Each endpoint requires the same cluster-monitor privilege as calling it natively, so `rest` grants no extra access. Some allow-listed endpoints surface operational metadata (for example `/_cat/nodes` exposes node addresses and resource utilization, `/_cat/plugins` the installed plugin inventory, and `/_cluster/state` cluster-state identifiers); this is a deliberate, read-only, monitor-privileged trade-off. `/_cluster/settings` is redacted with the node's setting filter so `Property.Filtered` keys are not surfaced. + +## Enabling the command + +The `rest` command is **disabled by default**: `plugins.ppl.rest.allowed_endpoints` defaults to an empty list, so every endpoint is rejected until a deployment explicitly opts in. Enable specific endpoints by setting the allow-list (a node-level setting, so it is applied at node startup and cannot be changed at runtime): + +```yaml +plugins.ppl.rest.allowed_endpoints: ["/_cluster/health", "/_cat/nodes"] +``` + +Use `["*"]` to allow every endpoint in the curated list below. An empty list (the default) disables the command entirely. + +The `rest` command also supports optional response redaction of network identifiers (IPv4/IPv6 addresses, `inet[...]` forms, EC2-style host names, and availability-zone names) in `/_cat/*` and `/_cluster/settings` cell values, controlled by `plugins.ppl.rest.redaction.enabled` (a node-level setting, default `false`). Managed deployments that must not expose host topology should set it to `true`. + +## Syntax + +The `rest` command has the following syntax: + +```syntax +rest [count=] [timeout=] [= ...] +``` + +## Parameters + +The `rest` command supports the following parameters. + +| Parameter | Required/Optional | Description | +| --- | --- | --- | +| `` | Required | An allow-listed, read-only endpoint path (see the allow-list below), for example `/_cluster/health`. | +| `count=` | Optional | Caps the number of emitted rows. | +| `timeout=` | Optional | Reserved for forward compatibility. It is currently rejected with a clear error, because a single uniform timeout does not map cleanly across the different endpoints. | +| `=` | Optional | Endpoint query arguments, validated per endpoint by both key and value (for example `local=true` for `/_cluster/health`, `health=green` for `/_cat/indices`, `expand_wildcards=open` for `/_resolve/index`). | + +## Allow-list + +`rest` resolves only an explicit, curated set of read-only endpoints. Anything outside the list, including any mutating endpoint, is rejected with a clear error. + +| Endpoint | Output columns | Accepted args | +| --- | --- | --- | +| `/_cluster/health` | `cluster_name` (string), `status` (string), `number_of_nodes` (integer), `number_of_data_nodes` (integer), `active_primary_shards` (integer), `active_shards` (integer), `relocating_shards` (integer), `initializing_shards` (integer), `unassigned_shards` (integer), `timed_out` (boolean) | `local` | +| `/_cluster/state` | `cluster_name` (string), `state_uuid` (string), `version` (long), `cluster_manager_node` (string) | (none) | +| `/_cluster/settings` | `setting` (string), `value` (string), `tier` (string) | (none) | +| `/_cat/indices` | `index` (string), `health` (string), `pri` (integer), `rep` (integer), `active_shards` (integer) | `health` | +| `/_cat/nodes` | `name` (string), `ip` (string), `node_role` (string), `heap_percent` (integer), `ram_percent` (integer), `cpu` (integer) | (none) | +| `/_cat/cluster_manager` | `id` (string), `host` (string), `ip` (string), `node` (string) | (none) | +| `/_cat/plugins` | `name` (string), `component` (string), `version` (string) | (none) | +| `/_cat/shards` | `index` (string), `shard` (integer), `prirep` (string), `state` (string), `node` (string) | (none) | +| `/_resolve/index` | `name` (string), `type` (string) | `expand_wildcards` | + +## Example 1: Counting the nodes in the cluster + +The following query reads cluster health and projects a column that is deterministic on a single-node cluster: + +```ppl +| rest '/_cluster/health' | fields number_of_nodes +``` + +The query returns the following results: + +```text +fetched rows / total rows = 1/1 ++-----------------+ +| number_of_nodes | +|-----------------| +| 1 | ++-----------------+ +``` + +`/_cluster/health` also exposes `status`, `active_shards`, and the other columns listed in the allow-list, which you can project and filter the same way. + +## Example 2: Composing downstream commands over a cat endpoint + +The `rest` row source composes with downstream `where`, `sort`, `stats`, and `fields` exactly like an index scan. The following query reads `/_cat/cluster_manager` and counts the rows: + +```ppl +| rest '/_cat/cluster_manager' | stats count() as managers +``` + +The query returns the following results: + +```text +fetched rows / total rows = 1/1 ++----------+ +| managers | +|----------| +| 1 | ++----------+ +``` + +For example, `| rest '/_cat/indices' | where health = 'green' | sort index | fields index, health, pri` lists green indexes; the projected columns come from the endpoint's fixed schema. diff --git a/docs/user/ppl/index.md b/docs/user/ppl/index.md index 37947113800..939684c0ecc 100644 --- a/docs/user/ppl/index.md +++ b/docs/user/ppl/index.md @@ -74,6 +74,7 @@ source=accounts | [lookup command](cmd/lookup.md) | 3.0 | experimental (since 3.0) | Add or replace data from a lookup index. | | [multisearch command](cmd/multisearch.md) | 3.4 | experimental (since 3.4) | Execute multiple search queries and combine their results. | | [union command](cmd/union.md) | 3.7 | experimental (since 3.7) | Combine results from multiple datasets using UNION ALL semantics. | +| [rest command](cmd/rest.md) | 3.8 | experimental (since 3.8) | Read an allow-listed, read-only in-cluster management endpoint (cluster/cat/nodes) as rows. Calcite engine only. | | [ml command](cmd/ml.md) | 2.5 | stable (since 2.5) | Apply machine learning algorithms to analyze data. | | [kmeans command](cmd/kmeans.md) | 1.3 | stable (since 1.3) | Apply the kmeans algorithm on the search result returned by a PPL command. | | [ad command](cmd/ad.md) | 1.3 | deprecated (since 2.5) | Apply Random Cut Forest algorithm on the search result returned by a PPL command. | diff --git a/doctest/build.gradle b/doctest/build.gradle index cce64170f56..1ac658457dc 100644 --- a/doctest/build.gradle +++ b/doctest/build.gradle @@ -205,6 +205,11 @@ testClusters { plugin(getJobSchedulerPlugin()) plugin ':opensearch-sql-plugin' testDistribution = 'archive' + // The rest command is disabled by default (empty allow-list). Opt the doctest cluster into + // the registered endpoints so the rest.md examples run against an enabled command. A literal + // "*" cannot be used because a bare * is a YAML alias indicator in opensearch.yml. + setting 'plugins.ppl.rest.allowed_endpoints', + '/_cluster/health,/_cluster/state,/_cluster/settings,/_cat/indices,/_cat/nodes,/_cat/cluster_manager,/_cat/plugins,/_cat/shards,/_resolve/index' } } tasks.register("runRestTestCluster", RunTask) { diff --git a/integ-test/build.gradle b/integ-test/build.gradle index 2e565aab8f2..a377d343c1b 100644 --- a/integ-test/build.gradle +++ b/integ-test/build.gradle @@ -387,6 +387,11 @@ testClusters { plugin(getGeoSpatialPlugin()) plugin ":opensearch-sql-plugin" setting "plugins.query.datasources.encryption.masterkey", "1234567812345678" + // The rest command is disabled by default (empty allow-list). Opt this cluster into the + // registered endpoints so the rest integration tests exercise the enabled path. A literal + // "*" cannot be used because a bare * is a YAML alias indicator in opensearch.yml. + setting 'plugins.ppl.rest.allowed_endpoints', + '/_cluster/health,/_cluster/state,/_cluster/settings,/_cat/indices,/_cat/nodes,/_cat/cluster_manager,/_cat/plugins,/_cat/shards,/_resolve/index' } yamlRestTest { testDistribution = 'archive' @@ -405,6 +410,9 @@ testClusters { testDistribution = 'archive' plugin(getJobSchedulerPlugin()) plugin ":opensearch-sql-plugin" + // Opt into the rest endpoints (disabled by default) so RestCommandSecurityIT runs. + setting 'plugins.ppl.rest.allowed_endpoints', + '/_cluster/health,/_cluster/state,/_cluster/settings,/_cat/indices,/_cat/nodes,/_cat/cluster_manager,/_cat/plugins,/_cat/shards,/_resolve/index' } remoteIntegTestWithSecurity { testDistribution = 'archive' @@ -419,6 +427,8 @@ testClusters { plugin(getArrowFlightRpcPlugin()) plugin(getAnalyticsEnginePlugin()) plugin ":opensearch-sql-plugin" + // Composite-default cluster: PPL queries route to the analytics engine unless excluded. + setting 'cluster.pluggable.dataformat', 'composite' } } diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/CalciteNoPushdownIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/CalciteNoPushdownIT.java index d46649d56d9..da434750a96 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/CalciteNoPushdownIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/CalciteNoPushdownIT.java @@ -57,6 +57,7 @@ CalciteObjectFieldOperateIT.class, CalciteOperatorIT.class, CalciteParseCommandIT.class, + CalcitePPLRestIT.class, CalcitePPLAggregationIT.class, CalcitePPLAppendcolIT.class, CalcitePPLAppendCommandIT.class, diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java index 95c47b9b0b7..fd8515aecfd 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java @@ -64,6 +64,15 @@ public void init() throws Exception { loadIndex(Index.GRAPH_EMPLOYEES); } + // Only for Calcite: the rest row source explains as a CalciteScannableCatalogScan. + @Test + public void explainRestCommand() throws IOException { + String result = explainQueryToString("| rest '/_cluster/health' | fields status"); + Assert.assertTrue( + "Expected a rest scan node in the explain output, got: " + result, + result.contains("CatalogScan")); + } + @Override @Ignore("test only in v2") public void testExplainModeUnsupportedInV2() throws IOException {} diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLRestIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLRestIT.java new file mode 100644 index 00000000000..70ee4ef0de6 --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLRestIT.java @@ -0,0 +1,210 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.calcite.remote; + +import static org.opensearch.sql.util.MatcherUtils.rows; +import static org.opensearch.sql.util.MatcherUtils.schema; +import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; +import static org.opensearch.sql.util.MatcherUtils.verifySchema; + +import java.io.IOException; +import org.json.JSONObject; +import org.junit.jupiter.api.Test; +import org.opensearch.client.Request; +import org.opensearch.client.ResponseException; +import org.opensearch.sql.ppl.PPLIntegTestCase; + +/** + * Integration tests for the {@code rest} leading command on the Calcite path. Uses {@code + * /_cluster/health} as the deterministic, single-row endpoint on a single-node test cluster. Also + * verifies that a non-allow-listed / mutating endpoint is refused. + */ +public class CalcitePPLRestIT extends PPLIntegTestCase { + + @Override + public void init() throws Exception { + super.init(); + enableCalcite(); + } + + @Test + public void testRestClusterHealthSchema() throws IOException { + JSONObject result = executeQuery("| rest '/_cluster/health' | fields status, number_of_nodes"); + verifySchema(result, schema("status", "string"), schema("number_of_nodes", "int")); + } + + @Test + public void testRestClusterHealthDataRows() throws IOException { + // Single-node test cluster: exactly one node, status is green or yellow. + JSONObject result = executeQuery("| rest '/_cluster/health' | fields number_of_nodes"); + verifyDataRows(result, rows(1)); + } + + @Test + public void testRestRejectsNonAllowListedEndpoint() throws IOException { + assertRestBadRequest("| rest '/_cluster/reroute'", "allow-list"); + } + + @Test + public void testRestRejectsEmptyEndpoint() throws IOException { + assertRestBadRequest("| rest ''", "non-empty path"); + } + + @Test + public void testRestRejectsDisallowedArg() throws IOException { + assertRestBadRequest("| rest '/_cat/nodes' h='name'", "does not accept arg"); + } + + @Test + public void testRestRejectsNegativeCount() throws IOException { + assertRestBadRequest("| rest '/_cat/nodes' count=-1", "non-negative"); + } + + @Test + public void testRestRejectsTimeoutArg() throws IOException { + assertRestBadRequest("| rest '/_cluster/health' timeout='5s'", "timeout"); + } + + /** + * Assert a {@code rest} query is refused as a client error: HTTP 400 (not a 500 system error) + * with the given substring in the response body. Covers allow-list and bad-argument rejection. + */ + private void assertRestBadRequest(String query, String expectedSubstring) { + ResponseException e = + org.junit.Assert.assertThrows(ResponseException.class, () -> executeQuery(query)); + org.junit.Assert.assertEquals(400, e.getResponse().getStatusLine().getStatusCode()); + org.junit.Assert.assertTrue( + "expected [" + expectedSubstring + "] in response body: " + e.getMessage(), + e.getMessage().contains(expectedSubstring)); + } + + @Test + public void testRestCatIndicesSchema() throws IOException { + // Schema is fixed by the registry, independent of how many indices exist. + JSONObject result = executeQuery("| rest '/_cat/indices' | fields index, health"); + verifySchema(result, schema("index", "string"), schema("health", "string")); + } + + @Test + public void testRestCatIndicesReturnsCreatedIndex() throws IOException { + // Create a known index, then confirm rest surfaces it and downstream where/fields compose. + client().performRequest(new Request("PUT", "/rest_cat_test")); + JSONObject result = + executeQuery("| rest '/_cat/indices' | where index = 'rest_cat_test' | fields index"); + verifyDataRows(result, rows("rest_cat_test")); + } + + @Test + public void testRestCatNodesSchema() throws IOException { + JSONObject result = executeQuery("| rest '/_cat/nodes' | fields name, cpu"); + verifySchema(result, schema("name", "string"), schema("cpu", "int")); + } + + @Test + public void testRestCatNodesSingleNode() throws IOException { + JSONObject result = executeQuery("| rest '/_cat/nodes' | stats count() as cnt"); + verifyDataRows(result, rows(1)); + } + + @Test + public void testRestCatClusterManagerSchema() throws IOException { + JSONObject result = executeQuery("| rest '/_cat/cluster_manager' | fields node, id"); + verifySchema(result, schema("node", "string"), schema("id", "string")); + } + + @Test + public void testRestCatClusterManagerSingleRow() throws IOException { + JSONObject result = executeQuery("| rest '/_cat/cluster_manager' | stats count() as cnt"); + verifyDataRows(result, rows(1)); + } + + @Test + public void testRestCatPluginsSchema() throws IOException { + JSONObject result = executeQuery("| rest '/_cat/plugins' | fields component, version"); + verifySchema(result, schema("component", "string"), schema("version", "string")); + } + + @Test + public void testRestCatShardsSchema() throws IOException { + JSONObject result = executeQuery("| rest '/_cat/shards' | fields index, shard, state"); + verifySchema( + result, schema("index", "string"), schema("shard", "int"), schema("state", "string")); + } + + @Test + public void testRestClusterStateSchema() throws IOException { + // Assert the string columns; version is the LONG epoch column. + JSONObject result = + executeQuery("| rest '/_cluster/state' | fields cluster_name, cluster_manager_node"); + verifySchema( + result, schema("cluster_name", "string"), schema("cluster_manager_node", "string")); + } + + @Test + public void testRestClusterStateSingleRow() throws IOException { + JSONObject result = executeQuery("| rest '/_cluster/state' | stats count() as cnt"); + verifyDataRows(result, rows(1)); + } + + @Test + public void testRestClusterSettingsSchema() throws IOException { + // Schema is registry-fixed regardless of how many settings are configured. + JSONObject result = executeQuery("| rest '/_cluster/settings' | fields setting, value, tier"); + verifySchema( + result, schema("setting", "string"), schema("value", "string"), schema("tier", "string")); + } + + @Test + public void testRestResolveIndexSchema() throws IOException { + JSONObject result = executeQuery("| rest '/_resolve/index' | fields name, type"); + verifySchema(result, schema("name", "string"), schema("type", "string")); + } + + @Test + public void testRestResolveIndexSurfacesCreatedIndex() throws IOException { + client().performRequest(new Request("PUT", "/rest_resolve_test")); + JSONObject result = + executeQuery( + "| rest '/_resolve/index' | where name = 'rest_resolve_test' | fields name, type"); + verifyDataRows(result, rows("rest_resolve_test", "index")); + } + + // ---- get-arg server-side filtering (health, expand_wildcards, local) ---- + + @Test + public void testRestClusterHealthLocalArg() throws IOException { + // local=true reads health from the local node; on a single-node cluster the row is unchanged. + JSONObject result = + executeQuery("| rest '/_cluster/health' local='true' | fields number_of_nodes"); + verifyDataRows(result, rows(1)); + } + + @Test + public void testRestCatIndicesHealthFilterReturnsNoRed() throws IOException { + // health filters rows server-side; a healthy cluster has no red indices, so count is 0. + JSONObject result = executeQuery("| rest '/_cat/indices' health='red' | stats count() as cnt"); + verifyDataRows(result, rows(0)); + } + + @Test + public void testRestResolveIndexExpandWildcardsArg() throws IOException { + // expand_wildcards is applied to the resolve request; schema stays fixed and the call succeeds. + JSONObject result = + executeQuery("| rest '/_resolve/index' expand_wildcards='open' | fields name, type"); + verifySchema(result, schema("name", "string"), schema("type", "string")); + } + + @Test + public void testRestRejectsDroppedLevelArg() throws IOException { + // level was dropped from the allow-list (no-op against the fixed health schema). + assertRestBadRequest("| rest '/_cluster/health' level='indices'", "does not accept arg"); + } + + @Test + public void testRestRejectsBadArgValue() throws IOException { + assertRestBadRequest("| rest '/_cat/indices' health='purple'", "unsupported value"); + } +} diff --git a/integ-test/src/test/java/org/opensearch/sql/plugin/AnalyticsEngineCompatIT.java b/integ-test/src/test/java/org/opensearch/sql/plugin/AnalyticsEngineCompatIT.java index f6ec903c395..b5c08cc8ad0 100644 --- a/integ-test/src/test/java/org/opensearch/sql/plugin/AnalyticsEngineCompatIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/plugin/AnalyticsEngineCompatIT.java @@ -5,9 +5,13 @@ package org.opensearch.sql.plugin; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; import static org.junit.Assume.assumeTrue; import java.io.IOException; +import org.json.JSONArray; +import org.json.JSONObject; import org.junit.Before; import org.junit.Test; import org.opensearch.client.Request; @@ -56,4 +60,31 @@ public void testClusterStarted() { // If the cluster booted with analytics-engine present, all plugins loaded without classloader // errors. The assumption above guarantees we only assert this where it is meaningful. } + + /** + * The {@code rest} row source is a Calcite Enumerable/Scannable scan with no backing index, so it + * is never routed to the analytics (DataFusion) engine. This pins that {@code rest} returns its + * fixed schema and correct data unchanged when the analytics-engine plugin is present. + */ + @Test + public void testRestCommandUnaffectedByAnalyticsEngine() throws IOException { + Request request = new Request("POST", "/_plugins/_ppl"); + request.setJsonEntity( + "{\"query\": \"| rest '/_cluster/health' | fields status, number_of_nodes\"}"); + Response response = client().performRequest(request); + assertEquals(200, response.getStatusLine().getStatusCode()); + + JSONObject result = new JSONObject(TestUtils.getResponseBody(response, true)); + + JSONArray schema = result.getJSONArray("schema"); + assertEquals(2, schema.length()); + assertEquals("status", schema.getJSONObject(0).getString("name")); + assertEquals("string", schema.getJSONObject(0).getString("type")); + assertEquals("number_of_nodes", schema.getJSONObject(1).getString("name")); + assertEquals("int", schema.getJSONObject(1).getString("type")); + + JSONArray datarows = result.getJSONArray("datarows"); + assertEquals(1, datarows.length()); + assertTrue(datarows.getJSONArray(0).getInt(1) >= 1); + } } diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/NewAddedCommandsIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/NewAddedCommandsIT.java index 837865a3585..2ccda31eea7 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/NewAddedCommandsIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/NewAddedCommandsIT.java @@ -32,6 +32,19 @@ public void init() throws Exception { loadIndex(Index.GRAPH_EMPLOYEES); } + @Test + public void testRest() throws IOException { + JSONObject result; + try { + result = executeQuery("| rest '/_cluster/health' | fields status, number_of_nodes"); + } catch (ResponseException e) { + result = new JSONObject(TestUtils.getResponseBody(e.getResponse())); + } + if (isCalciteEnabled()) { + assertFalse(result.getJSONArray("datarows").isEmpty()); + } + } + @Test public void testJoin() throws IOException { JSONObject result; diff --git a/integ-test/src/test/java/org/opensearch/sql/security/RestCommandSecurityIT.java b/integ-test/src/test/java/org/opensearch/sql/security/RestCommandSecurityIT.java new file mode 100644 index 00000000000..042a3aefbf7 --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/security/RestCommandSecurityIT.java @@ -0,0 +1,154 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.security; + +import static org.opensearch.sql.util.MatcherUtils.columnName; +import static org.opensearch.sql.util.MatcherUtils.verifyColumn; + +import java.io.IOException; +import java.util.HashSet; +import java.util.Set; +import org.json.JSONArray; +import org.json.JSONObject; +import org.junit.Test; +import org.opensearch.client.Request; +import org.opensearch.client.ResponseException; +import org.opensearch.sql.legacy.TestUtils; + +/** + * Integration tests that verify the rest command is subject to the security plugin fine grained + * access control. The command dispatches standard transport actions under the caller identity, so + * the security ActionFilter authorizes each one by action name. A caller without the required + * cluster monitor privilege is denied the cat and cluster endpoints, a caller holding the privilege + * can run them, and the resolve index endpoint requires the resolve index privilege because the + * command resolves all indices. + */ +public class RestCommandSecurityIT extends SecurityTestBase { + + private static final String ALPHA_INDEX = "rest_sec_alpha"; + private static final String BETA_INDEX = "rest_sec_beta"; + + private static final String MONITOR_USER = "rest_monitor_user"; + private static final String MONITOR_ROLE = "rest_monitor_role"; + + private static final String NO_MONITOR_USER = "rest_no_monitor_user"; + private static final String NO_MONITOR_ROLE = "rest_no_monitor_role"; + + @Override + protected void init() throws Exception { + super.init(); + setupRolesUsersAndIndices(); + enableCalcite(); + // rest is Calcite only, so a V2 fallback would replace the security denial with an unsupported + // command error. Disable fallback so the denial reason surfaces to the caller. + disallowCalciteFallback(); + } + + private void setupRolesUsersAndIndices() throws IOException { + createIndexIfAbsent(ALPHA_INDEX); + createIndexIfAbsent(BETA_INDEX); + + createRoleWithPermissions( + MONITOR_ROLE, + "*", + new String[] { + "cluster:admin/opensearch/ppl", + "cluster:monitor/health", + "cluster:monitor/state", + "cluster:monitor/nodes/stats", + "cluster:monitor/nodes/info" + }, + new String[] {"indices:admin/resolve/index"}); + createUser(MONITOR_USER, MONITOR_ROLE); + + createRoleWithPermissions( + NO_MONITOR_ROLE, + ALPHA_INDEX, + new String[] {"cluster:admin/opensearch/ppl"}, + new String[] {"indices:data/read/search*"}); + createUser(NO_MONITOR_USER, NO_MONITOR_ROLE); + } + + @Test + public void monitorUserCanRunCatNodes() throws IOException { + JSONObject result = executeQueryAsUser("| rest '/_cat/nodes' | fields name", MONITOR_USER); + verifyColumn(result, columnName("name")); + } + + @Test + public void monitorUserCanResolveIndex() throws IOException { + JSONObject result = + executeQueryAsUser("| rest '/_resolve/index' | fields name, type", MONITOR_USER); + Set names = resolvedNames(result); + assertTrue("resolve should list authorized indices: " + names, names.contains(ALPHA_INDEX)); + assertTrue("resolve should list authorized indices: " + names, names.contains(BETA_INDEX)); + } + + @Test + public void userWithoutClusterMonitorCannotRunCatNodes() throws IOException { + assertDenied( + "| rest '/_cat/nodes' | fields name", NO_MONITOR_USER, "cluster:monitor/nodes/stats"); + } + + @Test + public void userWithoutClusterMonitorCannotRunClusterState() throws IOException { + assertDenied( + "| rest '/_cluster/state' | fields cluster_name", NO_MONITOR_USER, "cluster:monitor/state"); + } + + @Test + public void userWithoutResolvePrivilegeCannotResolveIndex() throws IOException { + assertDenied( + "| rest '/_resolve/index' | fields name, type", + NO_MONITOR_USER, + "indices:admin/resolve/index"); + } + + /** + * Asserts the query is rejected for a caller lacking the privilege. A denied transport action on + * the Calcite only rest path surfaces as a client or server error whose body carries the security + * denial reason, so this checks the denial signal rather than a fixed status code. + */ + private void assertDenied(String query, String user, String deniedAction) throws IOException { + try { + executeQueryAsUser(query, user); + fail("Expected a permission denial for user without privilege: " + user); + } catch (ResponseException e) { + int status = e.getResponse().getStatusLine().getStatusCode(); + String body = TestUtils.getResponseBody(e.getResponse(), false); + assertTrue("Expected an error status, got " + status, status >= 400); + assertTrue( + "Response should indicate a permission denial. Status " + status + ", body: " + body, + body.contains("no permissions") + || body.contains("Forbidden") + || body.contains("security_exception") + || body.contains(deniedAction)); + } + } + + private void createIndexIfAbsent(String name) throws IOException { + Request request = new Request("PUT", "/" + name); + request.setJsonEntity( + "{ \"settings\": { \"number_of_shards\": 1, \"number_of_replicas\": 0 } }"); + try { + client().performRequest(request); + } catch (ResponseException e) { + String body = TestUtils.getResponseBody(e.getResponse(), false); + if (!body.contains("resource_already_exists_exception")) { + throw e; + } + } + } + + private Set resolvedNames(JSONObject result) { + Set names = new HashSet<>(); + JSONArray datarows = result.getJSONArray("datarows"); + for (int i = 0; i < datarows.length(); i++) { + names.add(datarows.getJSONArray(i).getString(0)); + } + return names; + } +} diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/VectorSearchIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/VectorSearchIT.java index 8ae3167b40b..e08735ebda7 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/VectorSearchIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/VectorSearchIT.java @@ -25,6 +25,7 @@ public class VectorSearchIT extends SQLIntegTestCase { @Override protected void init() throws Exception { + super.init(); loadIndex(Index.ACCOUNT); } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchClient.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchClient.java index 68350c5a0fd..3b9c3619521 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchClient.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchClient.java @@ -114,4 +114,105 @@ public interface OpenSearchClient { * @param deletePitRequest Delete Point In Time request */ void deletePit(DeletePitRequest deletePitRequest); + + /** + * Read-only cluster health snapshot for the {@code rest} command (backs {@code + * /_cluster/health}). Returns a single flattened row of health fields. Runs under the caller's + * security thread-context; performs no privilege escalation and mutates nothing. + * + * @param params endpoint query args (already allow-list-validated) + * @return a single map of health field name to value + */ + default Map clusterHealth(Map params) { + throw new UnsupportedOperationException("clusterHealth is not supported by this client"); + } + + /** + * Read-only cat-indices listing for the {@code rest} command (backs {@code /_cat/indices}). One + * map per index. Runs under the caller's security thread-context; read-only. + * + * @param params endpoint query args (already allow-list-validated) + * @return one map of column name to value per index + */ + default List> catIndices(Map params) { + throw new UnsupportedOperationException("catIndices is not supported by this client"); + } + + /** + * Read-only cat-nodes listing for the {@code rest} command (backs {@code /_cat/nodes}). One map + * per node with resource state. Runs under the caller's security thread-context; read-only. + * + * @param params endpoint query args (already allow-list-validated) + * @return one map of column name to value per node + */ + default List> catNodes(Map params) { + throw new UnsupportedOperationException("catNodes is not supported by this client"); + } + + /** + * Read-only cat-cluster_manager listing for the {@code rest} command (backs {@code + * /_cat/cluster_manager}). Single map identifying the elected cluster manager. Read-only. + * + * @param params endpoint query args (already allow-list-validated) + * @return one map describing the cluster manager node + */ + default List> catClusterManager(Map params) { + throw new UnsupportedOperationException("catClusterManager is not supported by this client"); + } + + /** + * Read-only cat-plugins listing for the {@code rest} command (backs {@code /_cat/plugins}). One + * map per installed plugin per node. Read-only. + * + * @param params endpoint query args (already allow-list-validated) + * @return one map of column name to value per plugin + */ + default List> catPlugins(Map params) { + throw new UnsupportedOperationException("catPlugins is not supported by this client"); + } + + /** + * Read-only cat-shards listing for the {@code rest} command (backs {@code /_cat/shards}). One map + * per shard. Read-only. + * + * @param params endpoint query args (already allow-list-validated) + * @return one map of column name to value per shard + */ + default List> catShards(Map params) { + throw new UnsupportedOperationException("catShards is not supported by this client"); + } + + /** + * Read-only cluster-state epoch projection for the {@code rest} command (backs {@code + * /_cluster/state}). Single flattened row (cluster_name, state_uuid, version, + * cluster_manager_node). Read-only. + * + * @param params endpoint query args (already allow-list-validated) + * @return a single map of cluster-state field name to value + */ + default Map clusterState(Map params) { + throw new UnsupportedOperationException("clusterState is not supported by this client"); + } + + /** + * Read-only cluster-settings listing for the {@code rest} command (backs {@code + * /_cluster/settings}). One map per configured setting (setting, value, tier). Read-only. + * + * @param params endpoint query args (already allow-list-validated) + * @return one map of column name to value per setting + */ + default List> clusterSettings(Map params) { + throw new UnsupportedOperationException("clusterSettings is not supported by this client"); + } + + /** + * Read-only resolve-index listing for the {@code rest} command (backs {@code /_resolve/index}). + * One map per resolved index, alias, or data stream (name, type). Read-only. + * + * @param params endpoint query args (already allow-list-validated) + * @return one map of column name to value per resolved name + */ + default List> resolveIndex(Map params) { + throw new UnsupportedOperationException("resolveIndex is not supported by this client"); + } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClient.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClient.java index b491f38ef80..080a8627894 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClient.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClient.java @@ -18,6 +18,8 @@ import java.util.stream.Collectors; import java.util.stream.Stream; import org.opensearch.OpenSearchSecurityException; +import org.opensearch.action.admin.cluster.health.ClusterHealthRequest; +import org.opensearch.action.admin.cluster.health.ClusterHealthResponse; import org.opensearch.action.admin.indices.create.CreateIndexRequest; import org.opensearch.action.admin.indices.exists.indices.IndicesExistsRequest; import org.opensearch.action.admin.indices.exists.indices.IndicesExistsResponse; @@ -25,6 +27,7 @@ import org.opensearch.action.admin.indices.mapping.get.GetMappingsResponse; import org.opensearch.action.admin.indices.settings.get.GetSettingsResponse; import org.opensearch.action.search.*; +import org.opensearch.cluster.health.ClusterIndexHealth; import org.opensearch.cluster.metadata.AliasMetadata; import org.opensearch.common.action.ActionFuture; import org.opensearch.common.settings.Settings; @@ -285,4 +288,274 @@ public void deletePit(DeletePitRequest deletePitRequest) { "Error occurred while deleting PIT for internal plugin operation", e); } } + + @Override + public Map clusterHealth(Map params) { + ClusterHealthRequest request = new ClusterHealthRequest(); + if (params != null && Boolean.parseBoolean(params.get("local"))) { + request.local(true); + } + ClusterHealthResponse response = client.admin().cluster().health(request).actionGet(); + return flattenHealth(response); + } + + @Override + public List> catIndices(Map params) { + ClusterHealthResponse response = + client.admin().cluster().health(new ClusterHealthRequest()).actionGet(); + List> rows = new java.util.ArrayList<>(); + for (Map.Entry entry : response.getIndices().entrySet()) { + ClusterIndexHealth health = entry.getValue(); + Map row = new java.util.LinkedHashMap<>(); + row.put("index", entry.getKey()); + row.put("health", health.getStatus().name().toLowerCase(java.util.Locale.ROOT)); + row.put("pri", health.getNumberOfShards()); + row.put("rep", health.getNumberOfReplicas()); + row.put("active_shards", health.getActiveShards()); + rows.add(row); + } + String healthFilter = params == null ? null : params.get("health"); + if (healthFilter != null) { + rows.removeIf(r -> !healthFilter.equalsIgnoreCase(String.valueOf(r.get("health")))); + } + return rows; + } + + @Override + public List> catNodes(Map params) { + org.opensearch.action.admin.cluster.node.stats.NodesStatsRequest statsRequest = + new org.opensearch.action.admin.cluster.node.stats.NodesStatsRequest(); + statsRequest.all(); + org.opensearch.action.admin.cluster.node.stats.NodesStatsResponse response = + client.admin().cluster().nodesStats(statsRequest).actionGet(); + List> rows = new java.util.ArrayList<>(); + for (org.opensearch.action.admin.cluster.node.stats.NodeStats ns : response.getNodes()) { + org.opensearch.cluster.node.DiscoveryNode node = ns.getNode(); + Map row = new java.util.LinkedHashMap<>(); + row.put("name", node.getName()); + row.put("ip", node.getHostAddress()); + row.put( + "node_role", + node.getRoles().stream() + .map(org.opensearch.cluster.node.DiscoveryNodeRole::roleName) + .sorted() + .collect(java.util.stream.Collectors.joining(","))); + row.put( + "heap_percent", + ns.getJvm() == null || ns.getJvm().getMem() == null + ? null + : (int) ns.getJvm().getMem().getHeapUsedPercent()); + row.put( + "ram_percent", + ns.getOs() == null || ns.getOs().getMem() == null + ? null + : (int) ns.getOs().getMem().getUsedPercent()); + row.put( + "cpu", + ns.getProcess() == null || ns.getProcess().getCpu() == null + ? null + : (int) ns.getProcess().getCpu().getPercent()); + rows.add(row); + } + return rows; + } + + @Override + public List> catClusterManager(Map params) { + org.opensearch.action.admin.cluster.state.ClusterStateResponse response = + client + .admin() + .cluster() + .state(new org.opensearch.action.admin.cluster.state.ClusterStateRequest()) + .actionGet(); + org.opensearch.cluster.node.DiscoveryNode cm = + response.getState().nodes().getClusterManagerNode(); + List> rows = new java.util.ArrayList<>(); + if (cm != null) { + Map row = new java.util.LinkedHashMap<>(); + row.put("id", cm.getId()); + row.put("host", cm.getHostName()); + row.put("ip", cm.getHostAddress()); + row.put("node", cm.getName()); + rows.add(row); + } + return rows; + } + + @Override + public List> catPlugins(Map params) { + org.opensearch.action.admin.cluster.node.info.NodesInfoRequest infoRequest = + new org.opensearch.action.admin.cluster.node.info.NodesInfoRequest(); + infoRequest.all(); + org.opensearch.action.admin.cluster.node.info.NodesInfoResponse response = + client.admin().cluster().nodesInfo(infoRequest).actionGet(); + List> rows = new java.util.ArrayList<>(); + for (org.opensearch.action.admin.cluster.node.info.NodeInfo info : response.getNodes()) { + org.opensearch.action.admin.cluster.node.info.PluginsAndModules plugins = + info.getInfo(org.opensearch.action.admin.cluster.node.info.PluginsAndModules.class); + if (plugins == null) { + continue; + } + for (org.opensearch.plugins.PluginInfo pi : plugins.getPluginInfos()) { + Map row = new java.util.LinkedHashMap<>(); + row.put("name", info.getNode().getName()); + row.put("component", pi.getName()); + row.put("version", pi.getVersion()); + rows.add(row); + } + } + return rows; + } + + @Override + public List> catShards(Map params) { + org.opensearch.action.admin.cluster.state.ClusterStateResponse response = + client + .admin() + .cluster() + .state(new org.opensearch.action.admin.cluster.state.ClusterStateRequest()) + .actionGet(); + org.opensearch.cluster.node.DiscoveryNodes nodes = response.getState().nodes(); + List> rows = new java.util.ArrayList<>(); + for (org.opensearch.cluster.routing.ShardRouting sr : + response.getState().getRoutingTable().allShards()) { + Map row = new java.util.LinkedHashMap<>(); + row.put("index", sr.getIndexName()); + row.put("shard", sr.id()); + row.put("prirep", sr.primary() ? "p" : "r"); + row.put("state", sr.state().name()); + org.opensearch.cluster.node.DiscoveryNode n = + sr.currentNodeId() == null ? null : nodes.get(sr.currentNodeId()); + row.put("node", n == null ? null : n.getName()); + rows.add(row); + } + return rows; + } + + @Override + public Map clusterState(Map params) { + org.opensearch.action.admin.cluster.state.ClusterStateResponse response = + client + .admin() + .cluster() + .state(new org.opensearch.action.admin.cluster.state.ClusterStateRequest()) + .actionGet(); + Map row = new java.util.LinkedHashMap<>(); + row.put("cluster_name", response.getClusterName().value()); + row.put("state_uuid", response.getState().stateUUID()); + row.put("version", response.getState().version()); + org.opensearch.cluster.node.DiscoveryNode cm = + response.getState().nodes().getClusterManagerNode(); + row.put("cluster_manager_node", cm == null ? null : cm.getName()); + return row; + } + + @Override + public List> clusterSettings(Map params) { + // The transport path has no SettingsFilter of its own; it is published from + // SQLPlugin#getRestHandlers at startup. Fail closed before fetching so we never read settings + // into memory when we cannot redact them, matching native GET /_cluster/settings. + org.opensearch.common.settings.SettingsFilter filter = + org.opensearch.sql.opensearch.storage.rest.RestSettingsFilterHolder.get(); + if (filter == null) { + throw new IllegalStateException( + "cluster settings redaction filter is not initialized; refusing to return unredacted" + + " settings"); + } + org.opensearch.action.admin.cluster.state.ClusterStateResponse response = + client + .admin() + .cluster() + .state(new org.opensearch.action.admin.cluster.state.ClusterStateRequest()) + .actionGet(); + List> rows = new java.util.ArrayList<>(); + org.opensearch.common.settings.Settings persistent = + filter.filter(response.getState().metadata().persistentSettings()); + org.opensearch.common.settings.Settings transientSettings = + filter.filter(response.getState().metadata().transientSettings()); + collectSettings(persistent, "persistent", rows); + collectSettings(transientSettings, "transient", rows); + return rows; + } + + private void collectSettings( + org.opensearch.common.settings.Settings settings, + String tier, + List> rows) { + if (settings == null) { + return; + } + for (String key : settings.keySet()) { + Map row = new java.util.LinkedHashMap<>(); + row.put("setting", key); + String value = settings.get(key); + if (value == null) { + // List-valued settings return null from get(); fall back to the joined list form. + java.util.List list = settings.getAsList(key); + value = list.isEmpty() ? null : String.join(",", list); + } + row.put("value", value); + row.put("tier", tier); + rows.add(row); + } + } + + @Override + public List> resolveIndex(Map params) { + String expandWildcards = params == null ? null : params.get("expand_wildcards"); + org.opensearch.action.admin.indices.resolve.ResolveIndexAction.Request request = + expandWildcards == null + ? new org.opensearch.action.admin.indices.resolve.ResolveIndexAction.Request( + new String[] {"*"}) + : new org.opensearch.action.admin.indices.resolve.ResolveIndexAction.Request( + new String[] {"*"}, + org.opensearch.action.support.IndicesOptions.fromParameters( + expandWildcards, + null, + null, + null, + org.opensearch.action.admin.indices.resolve.ResolveIndexAction.Request + .DEFAULT_INDICES_OPTIONS)); + org.opensearch.action.admin.indices.resolve.ResolveIndexAction.Response response = + client + .execute( + org.opensearch.action.admin.indices.resolve.ResolveIndexAction.INSTANCE, request) + .actionGet(); + List> rows = new java.util.ArrayList<>(); + for (org.opensearch.action.admin.indices.resolve.ResolveIndexAction.ResolvedIndex idx : + response.getIndices()) { + rows.add(resolveRow(idx.getName(), "index")); + } + for (org.opensearch.action.admin.indices.resolve.ResolveIndexAction.ResolvedAlias alias : + response.getAliases()) { + rows.add(resolveRow(alias.getName(), "alias")); + } + for (org.opensearch.action.admin.indices.resolve.ResolveIndexAction.ResolvedDataStream ds : + response.getDataStreams()) { + rows.add(resolveRow(ds.getName(), "data_stream")); + } + return rows; + } + + private Map resolveRow(String name, String type) { + Map row = new java.util.LinkedHashMap<>(); + row.put("name", name); + row.put("type", type); + return row; + } + + private Map flattenHealth(ClusterHealthResponse response) { + Map row = new java.util.LinkedHashMap<>(); + row.put("cluster_name", response.getClusterName()); + row.put("status", response.getStatus().name().toLowerCase(java.util.Locale.ROOT)); + row.put("number_of_nodes", response.getNumberOfNodes()); + row.put("number_of_data_nodes", response.getNumberOfDataNodes()); + row.put("active_primary_shards", response.getActivePrimaryShards()); + row.put("active_shards", response.getActiveShards()); + row.put("relocating_shards", response.getRelocatingShards()); + row.put("initializing_shards", response.getInitializingShards()); + row.put("unassigned_shards", response.getUnassignedShards()); + row.put("timed_out", response.isTimedOut()); + return row; + } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchRestClient.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchRestClient.java index f369c0003b8..e98c5bf95f4 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchRestClient.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchRestClient.java @@ -8,15 +8,19 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import java.io.IOException; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.HashMap; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Optional; import java.util.stream.Collectors; import java.util.stream.Stream; import lombok.RequiredArgsConstructor; +import org.opensearch.action.admin.cluster.health.ClusterHealthRequest; +import org.opensearch.action.admin.cluster.health.ClusterHealthResponse; import org.opensearch.action.admin.cluster.settings.ClusterGetSettingsRequest; import org.opensearch.action.admin.indices.settings.get.GetSettingsRequest; import org.opensearch.action.admin.indices.settings.get.GetSettingsResponse; @@ -28,6 +32,7 @@ import org.opensearch.client.indices.GetIndexResponse; import org.opensearch.client.indices.GetMappingsRequest; import org.opensearch.client.indices.GetMappingsResponse; +import org.opensearch.cluster.health.ClusterIndexHealth; import org.opensearch.cluster.metadata.AliasMetadata; import org.opensearch.common.settings.Settings; import org.opensearch.index.IndexNotFoundException; @@ -272,4 +277,267 @@ public void deletePit(DeletePitRequest deletePitRequest) { "Error occurred while deleting PIT for internal plugin operation", e); } } + + @Override + public Map clusterHealth(Map params) { + try { + ClusterHealthRequest request = new ClusterHealthRequest(); + if (params != null && Boolean.parseBoolean(params.get("local"))) { + request.local(true); + } + ClusterHealthResponse response = client.cluster().health(request, RequestOptions.DEFAULT); + return flattenHealth(response); + } catch (IOException e) { + throw new IllegalStateException("Failed to get cluster health", e); + } + } + + @Override + public List> catIndices(Map params) { + try { + ClusterHealthResponse response = + client.cluster().health(new ClusterHealthRequest(), RequestOptions.DEFAULT); + List> rows = new ArrayList<>(); + for (Map.Entry entry : response.getIndices().entrySet()) { + ClusterIndexHealth health = entry.getValue(); + Map row = new HashMap<>(); + row.put("index", entry.getKey()); + row.put("health", health.getStatus().name().toLowerCase(Locale.ROOT)); + row.put("pri", health.getNumberOfShards()); + row.put("rep", health.getNumberOfReplicas()); + row.put("active_shards", health.getActiveShards()); + rows.add(row); + } + String healthFilter = params == null ? null : params.get("health"); + if (healthFilter != null) { + rows.removeIf(r -> !healthFilter.equalsIgnoreCase(String.valueOf(r.get("health")))); + } + return rows; + } catch (IOException e) { + throw new IllegalStateException("Failed to get cat indices", e); + } + } + + @Override + public List> catNodes(Map params) { + List> raw = + catJson("/_cat/nodes", "name,ip,node.role,heap.percent,ram.percent,cpu"); + List> rows = new ArrayList<>(); + for (Map r : raw) { + Map row = new HashMap<>(); + row.put("name", r.get("name")); + row.put("ip", r.get("ip")); + row.put("node_role", r.get("node.role")); + row.put("heap_percent", asInt(r.get("heap.percent"))); + row.put("ram_percent", asInt(r.get("ram.percent"))); + row.put("cpu", asInt(r.get("cpu"))); + rows.add(row); + } + return rows; + } + + @Override + public List> catClusterManager(Map params) { + List> raw = catJson("/_cat/cluster_manager", "id,host,ip,node"); + List> rows = new ArrayList<>(); + for (Map r : raw) { + Map row = new HashMap<>(); + row.put("id", r.get("id")); + row.put("host", r.get("host")); + row.put("ip", r.get("ip")); + row.put("node", r.get("node")); + rows.add(row); + } + return rows; + } + + @Override + public List> catPlugins(Map params) { + List> raw = catJson("/_cat/plugins", "name,component,version"); + List> rows = new ArrayList<>(); + for (Map r : raw) { + Map row = new HashMap<>(); + row.put("name", r.get("name")); + row.put("component", r.get("component")); + row.put("version", r.get("version")); + rows.add(row); + } + return rows; + } + + @Override + public List> catShards(Map params) { + List> raw = catJson("/_cat/shards", "index,shard,prirep,state,node"); + List> rows = new ArrayList<>(); + for (Map r : raw) { + Map row = new HashMap<>(); + row.put("index", r.get("index")); + row.put("shard", asInt(r.get("shard"))); + row.put("prirep", r.get("prirep")); + row.put("state", r.get("state")); + row.put("node", r.get("node")); + rows.add(row); + } + return rows; + } + + /** Standalone-mode helper: GET a _cat endpoint as JSON via the low-level client. */ + @SuppressWarnings("unchecked") + private List> catJson(String path, String columns) { + try { + org.opensearch.client.Request request = new org.opensearch.client.Request("GET", path); + request.addParameter("format", "json"); + request.addParameter("h", columns); + org.opensearch.client.Response response = client.getLowLevelClient().performRequest(request); + try (org.opensearch.core.xcontent.XContentParser parser = + org.opensearch.common.xcontent.json.JsonXContent.jsonXContent.createParser( + org.opensearch.core.xcontent.NamedXContentRegistry.EMPTY, + org.opensearch.common.xcontent.LoggingDeprecationHandler.INSTANCE, + response.getEntity().getContent())) { + List list = parser.list(); + List> rows = new ArrayList<>(); + for (Object o : list) { + rows.add((Map) o); + } + return rows; + } + } catch (IOException e) { + throw new IllegalStateException("Failed GET " + path, e); + } + } + + private static Integer asInt(Object value) { + if (value == null) { + return null; + } + try { + return (int) Double.parseDouble(value.toString().trim()); + } catch (NumberFormatException e) { + return null; + } + } + + @Override + @SuppressWarnings("unchecked") + public Map clusterState(Map params) { + Map state = + getJsonMap( + "/_cluster/state/master_node,version,metadata,nodes", + // nodes.*.name resolves the manager id to a name without over-fetching node IPs. + Map.of( + "filter_path", + "cluster_name,state_uuid,version,cluster_manager_node,nodes.*.name")); + Map row = new HashMap<>(); + row.put("cluster_name", state.get("cluster_name")); + row.put("state_uuid", state.get("state_uuid")); + row.put("version", asLong(state.get("version"))); + Object cmId = state.get("cluster_manager_node"); + String cmName = null; + Object nodes = state.get("nodes"); + if (cmId != null && nodes instanceof Map) { + Object n = ((Map) nodes).get(cmId.toString()); + if (n instanceof Map) { + Object name = ((Map) n).get("name"); + cmName = name == null ? null : name.toString(); + } + } + row.put("cluster_manager_node", cmName); + return row; + } + + @Override + @SuppressWarnings("unchecked") + public List> clusterSettings(Map params) { + Map body = getJsonMap("/_cluster/settings", Map.of("flat_settings", "true")); + List> rows = new ArrayList<>(); + for (String tier : new String[] {"persistent", "transient"}) { + Object section = body.get(tier); + if (section instanceof Map) { + for (Map.Entry e : ((Map) section).entrySet()) { + Map row = new HashMap<>(); + row.put("setting", e.getKey()); + row.put("value", e.getValue() == null ? null : e.getValue().toString()); + row.put("tier", tier); + rows.add(row); + } + } + } + return rows; + } + + /** Standalone-mode helper: GET a JSON-object endpoint via the low-level client. */ + @SuppressWarnings("unchecked") + private Map getJsonMap(String path, Map params) { + try { + org.opensearch.client.Request request = new org.opensearch.client.Request("GET", path); + if (params != null) { + params.forEach(request::addParameter); + } + org.opensearch.client.Response response = client.getLowLevelClient().performRequest(request); + try (org.opensearch.core.xcontent.XContentParser parser = + org.opensearch.common.xcontent.json.JsonXContent.jsonXContent.createParser( + org.opensearch.core.xcontent.NamedXContentRegistry.EMPTY, + org.opensearch.common.xcontent.LoggingDeprecationHandler.INSTANCE, + response.getEntity().getContent())) { + return parser.map(); + } + } catch (IOException e) { + throw new IllegalStateException("Failed GET " + path, e); + } + } + + private static Long asLong(Object value) { + if (value == null) { + return null; + } + try { + return (long) Double.parseDouble(value.toString().trim()); + } catch (NumberFormatException e) { + return null; + } + } + + @Override + @SuppressWarnings("unchecked") + public List> resolveIndex(Map params) { + String expandWildcards = params == null ? null : params.get("expand_wildcards"); + Map body = + getJsonMap( + "/_resolve/index/*", + expandWildcards == null ? Map.of() : Map.of("expand_wildcards", expandWildcards)); + List> rows = new ArrayList<>(); + addResolved(body.get("indices"), "index", rows); + addResolved(body.get("aliases"), "alias", rows); + addResolved(body.get("data_streams"), "data_stream", rows); + return rows; + } + + @SuppressWarnings("unchecked") + private void addResolved(Object section, String type, List> rows) { + if (section instanceof List) { + for (Object o : (List) section) { + if (o instanceof Map) { + Map row = new HashMap<>(); + row.put("name", ((Map) o).get("name")); + row.put("type", type); + rows.add(row); + } + } + } + } + + private Map flattenHealth(ClusterHealthResponse response) { + Map row = new HashMap<>(); + row.put("cluster_name", response.getClusterName()); + row.put("status", response.getStatus().name().toLowerCase(Locale.ROOT)); + row.put("number_of_nodes", response.getNumberOfNodes()); + row.put("number_of_data_nodes", response.getNumberOfDataNodes()); + row.put("active_primary_shards", response.getActivePrimaryShards()); + row.put("active_shards", response.getActiveShards()); + row.put("relocating_shards", response.getRelocatingShards()); + row.put("initializing_shards", response.getInitializingShards()); + row.put("unassigned_shards", response.getUnassignedShards()); + row.put("timed_out", response.isTimedOut()); + return row; + } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/EnumerableCatalogScanRule.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/EnumerableCatalogScanRule.java new file mode 100644 index 00000000000..9905ca564ab --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/EnumerableCatalogScanRule.java @@ -0,0 +1,63 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.planner.rules; + +import org.apache.calcite.adapter.enumerable.EnumerableConvention; +import org.apache.calcite.plan.Convention; +import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.convert.ConverterRule; +import org.opensearch.sql.opensearch.storage.system.CalciteEnumerableCatalogScan; +import org.opensearch.sql.opensearch.storage.system.CalciteLogicalCatalogScan; +import org.opensearch.sql.opensearch.storage.system.CalciteScannableCatalogScan; + +/** + * Rule to convert a {@link CalciteLogicalCatalogScan} into an enumerable scan: a {@link + * CalciteScannableCatalogScan} when the source opts into {@code Scannable}, otherwise a plain + * {@link CalciteEnumerableCatalogScan}. + */ +public class EnumerableCatalogScanRule extends ConverterRule { + /** Default configuration. */ + public static final Config DEFAULT_CONFIG = + Config.INSTANCE + .as(Config.class) + .withConversion( + CalciteLogicalCatalogScan.class, + s -> s.getCatalogTable() != null, + Convention.NONE, + EnumerableConvention.INSTANCE, + "EnumerableCatalogScanRule") + .withRuleFactory(EnumerableCatalogScanRule::new); + + protected EnumerableCatalogScanRule(Config config) { + super(config); + } + + @Override + public boolean matches(RelOptRuleCall call) { + CalciteLogicalCatalogScan scan = call.rel(0); + return scan.getVariablesSet().isEmpty(); + } + + @Override + public RelNode convert(RelNode rel) { + final CalciteLogicalCatalogScan scan = (CalciteLogicalCatalogScan) rel; + if (scan.getCatalogTable().getSource().isScannable()) { + return new CalciteScannableCatalogScan( + scan.getCluster(), + scan.getHints(), + scan.getTable(), + scan.getCatalogTable(), + scan.getSchema()); + } + return new CalciteEnumerableCatalogScan( + scan.getCluster(), + scan.getHints(), + scan.getTable(), + scan.getCatalogTable(), + scan.getSchema()); + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/EnumerableSystemIndexScanRule.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/EnumerableSystemIndexScanRule.java deleted file mode 100644 index 616d1178873..00000000000 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/EnumerableSystemIndexScanRule.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.opensearch.planner.rules; - -import org.apache.calcite.adapter.enumerable.EnumerableConvention; -import org.apache.calcite.plan.Convention; -import org.apache.calcite.plan.RelOptRuleCall; -import org.apache.calcite.rel.RelNode; -import org.apache.calcite.rel.convert.ConverterRule; -import org.opensearch.sql.opensearch.storage.system.CalciteEnumerableSystemIndexScan; -import org.opensearch.sql.opensearch.storage.system.CalciteLogicalSystemIndexScan; - -/** - * Rule to convert a {@link CalciteLogicalSystemIndexScan} to a {@link - * CalciteEnumerableSystemIndexScan}. - */ -public class EnumerableSystemIndexScanRule extends ConverterRule { - /** Default configuration. */ - public static final Config DEFAULT_CONFIG = - Config.INSTANCE - .as(Config.class) - .withConversion( - CalciteLogicalSystemIndexScan.class, - s -> s.getSysIndex() != null, - Convention.NONE, - EnumerableConvention.INSTANCE, - "EnumerableSystemIndexScanRule") - .withRuleFactory(EnumerableSystemIndexScanRule::new); - - /** Creates an EnumerableProjectRule. */ - protected EnumerableSystemIndexScanRule(Config config) { - super(config); - } - - @Override - public boolean matches(RelOptRuleCall call) { - CalciteLogicalSystemIndexScan scan = call.rel(0); - return scan.getVariablesSet().isEmpty(); - } - - @Override - public RelNode convert(RelNode rel) { - final CalciteLogicalSystemIndexScan scan = (CalciteLogicalSystemIndexScan) rel; - return new CalciteEnumerableSystemIndexScan( - scan.getCluster(), scan.getHints(), scan.getTable(), scan.getSysIndex(), scan.getSchema()); - } -} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/OpenSearchIndexRules.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/OpenSearchIndexRules.java index 3c8508cc455..c200ffa8909 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/OpenSearchIndexRules.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/OpenSearchIndexRules.java @@ -12,8 +12,8 @@ public class OpenSearchIndexRules { private static final RelOptRule INDEX_SCAN_RULE = EnumerableIndexScanRule.DEFAULT_CONFIG.toRule(); - private static final RelOptRule SYSTEM_INDEX_SCAN_RULE = - EnumerableSystemIndexScanRule.DEFAULT_CONFIG.toRule(); + private static final RelOptRule CATALOG_SCAN_RULE = + EnumerableCatalogScanRule.DEFAULT_CONFIG.toRule(); private static final RelOptRule NESTED_AGGREGATE_RULE = EnumerableNestedAggregateRule.DEFAULT_CONFIG.toRule(); private static final RelOptRule GRAPH_LOOKUP_RULE = @@ -26,7 +26,7 @@ public class OpenSearchIndexRules { public static final List OPEN_SEARCH_NON_PUSHDOWN_RULES = ImmutableList.of( INDEX_SCAN_RULE, - SYSTEM_INDEX_SCAN_RULE, + CATALOG_SCAN_RULE, NESTED_AGGREGATE_RULE, GRAPH_LOOKUP_RULE, RELEVANCE_FUNCTION_RULE); diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/setting/OpenSearchSettings.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/setting/OpenSearchSettings.java index bd8001f589d..55212248b43 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/setting/OpenSearchSettings.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/setting/OpenSearchSettings.java @@ -70,6 +70,17 @@ public class OpenSearchSettings extends Settings { Setting.Property.NodeScope, Setting.Property.Dynamic); + public static final Setting PPL_REST_REDACTION_ENABLED_SETTING = + Setting.boolSetting( + Key.PPL_REST_REDACTION_ENABLED.getKeyValue(), false, Setting.Property.NodeScope); + + public static final Setting> PPL_REST_ALLOWED_ENDPOINTS_SETTING = + Setting.listSetting( + Key.PPL_REST_ALLOWED_ENDPOINTS.getKeyValue(), + List.of(), + Function.identity(), + Setting.Property.NodeScope); + public static final Setting PPL_QUERY_TIMEOUT_SETTING = Setting.positiveTimeSetting( Key.PPL_QUERY_TIMEOUT.getKeyValue(), @@ -371,6 +382,16 @@ public OpenSearchSettings(ClusterSettings clusterSettings) { Key.PPL_ENABLED, PPL_ENABLED_SETTING, new Updater(Key.PPL_ENABLED)); + registerNonDynamicSettings( + settingBuilder, + clusterSettings, + Key.PPL_REST_REDACTION_ENABLED, + PPL_REST_REDACTION_ENABLED_SETTING); + registerNonDynamicSettings( + settingBuilder, + clusterSettings, + Key.PPL_REST_ALLOWED_ENDPOINTS, + PPL_REST_ALLOWED_ENDPOINTS_SETTING); register( settingBuilder, clusterSettings, @@ -625,7 +646,9 @@ private void registerNonDynamicSettings( Settings.Key key, Setting setting) { settingBuilder.put(key, setting); - latestSettings.put(key, clusterSettings.get(setting)); + if (clusterSettings.get(setting) != null) { + latestSettings.put(key, clusterSettings.get(setting)); + } } /** @@ -695,6 +718,8 @@ public static List> pluginNonDynamicSettings() { return new ImmutableList.Builder>() .add(DATASOURCE_MASTER_SECRET_KEY) .add(DATASOURCE_CONFIG) + .add(PPL_REST_REDACTION_ENABLED_SETTING) + .add(PPL_REST_ALLOWED_ENDPOINTS_SETTING) .build(); } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngine.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngine.java index 1b7de315fb6..7b911471242 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngine.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngine.java @@ -5,6 +5,8 @@ package org.opensearch.sql.opensearch.storage; +import static org.opensearch.sql.utils.SystemIndexUtils.decodeRestSpec; +import static org.opensearch.sql.utils.SystemIndexUtils.isRestSource; import static org.opensearch.sql.utils.SystemIndexUtils.isSystemIndex; import java.util.Collection; @@ -15,9 +17,13 @@ import org.opensearch.sql.common.setting.Settings; import org.opensearch.sql.expression.function.FunctionResolver; import org.opensearch.sql.opensearch.client.OpenSearchClient; -import org.opensearch.sql.opensearch.storage.system.OpenSearchSystemIndex; +import org.opensearch.sql.opensearch.storage.rest.RestCatalogSource; +import org.opensearch.sql.opensearch.storage.rest.RestEndpointRegistry; +import org.opensearch.sql.opensearch.storage.system.OpenSearchCatalogTable; +import org.opensearch.sql.opensearch.storage.system.SystemIndexCatalogSource; import org.opensearch.sql.storage.StorageEngine; import org.opensearch.sql.storage.Table; +import org.opensearch.sql.utils.SystemIndexUtils.RestSpec; /** OpenSearch storage engine implementation. */ @RequiredArgsConstructor @@ -35,10 +41,29 @@ public Collection getFunctions() { @Override public Table getTable(DataSourceSchemaName dataSourceSchemaName, String name) { - if (isSystemIndex(name)) { - return new OpenSearchSystemIndex(client, settings, name); + if (isRestSource(name)) { + return restTable(name); + } else if (isSystemIndex(name)) { + return new OpenSearchCatalogTable(new SystemIndexCatalogSource(client, name), settings); } else { return new OpenSearchIndex(client, settings, name); } } + + private Table restTable(String name) { + RestSpec spec = decodeRestSpec(name); + RestEndpointRegistry.resolve(spec.getEndpoint()); + List allowed = settings.getSettingValue(Settings.Key.PPL_REST_ALLOWED_ENDPOINTS); + if (allowed == null || !(allowed.contains("*") || allowed.contains(spec.getEndpoint()))) { + throw new IllegalArgumentException( + allowed == null || allowed.isEmpty() + ? "the rest command is disabled on this cluster" + : "rest endpoint [" + + spec.getEndpoint() + + "] is not enabled on this cluster. Enabled endpoints: " + + allowed); + } + boolean redact = settings.getSettingValue(Settings.Key.PPL_REST_REDACTION_ENABLED); + return new OpenSearchCatalogTable(new RestCatalogSource(client, spec, redact), settings); + } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestCatalogSource.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestCatalogSource.java new file mode 100644 index 00000000000..96779726a7e --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestCatalogSource.java @@ -0,0 +1,63 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.rest; + +import java.util.Map; +import lombok.Getter; +import org.opensearch.sql.data.type.ExprType; +import org.opensearch.sql.opensearch.client.OpenSearchClient; +import org.opensearch.sql.opensearch.request.system.OpenSearchSystemRequest; +import org.opensearch.sql.opensearch.storage.system.CatalogSource; +import org.opensearch.sql.planner.logical.LogicalPlan; +import org.opensearch.sql.planner.physical.PhysicalPlan; +import org.opensearch.sql.utils.SystemIndexUtils.RestSpec; + +/** + * {@link CatalogSource} for the {@code rest} command: an allow-listed, read-only management + * endpoint resolved against {@link RestEndpointRegistry}, exposing the fixed endpoint schema. + * Calcite only (no V2 path) and {@code Scannable} for the {@code collect} short-circuit. + */ +@Getter +public class RestCatalogSource implements CatalogSource { + + private final OpenSearchClient client; + private final RestSpec spec; + private final RestEndpointRegistry.Endpoint endpoint; + private final boolean redact; + + public RestCatalogSource(OpenSearchClient client, RestSpec spec) { + this(client, spec, false); + } + + public RestCatalogSource(OpenSearchClient client, RestSpec spec, boolean redact) { + this.client = client; + this.spec = spec; + this.redact = redact; + // Allow-list enforced here: unknown or mutating endpoints and disallowed args are rejected. + this.endpoint = RestEndpointRegistry.resolve(spec.getEndpoint()); + RestEndpointRegistry.validate(spec); + } + + @Override + public Map getFieldTypes() { + return endpoint.getSchema(); + } + + @Override + public OpenSearchSystemRequest createRequest() { + return new RestRequest(client, endpoint, spec, redact); + } + + @Override + public boolean isScannable() { + return true; + } + + @Override + public PhysicalPlan implementV2(LogicalPlan plan) { + throw new UnsupportedOperationException("rest command is supported only on the Calcite engine"); + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistry.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistry.java new file mode 100644 index 00000000000..e64a91ab54a --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistry.java @@ -0,0 +1,428 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.rest; + +import static org.opensearch.sql.data.model.ExprValueUtils.booleanValue; +import static org.opensearch.sql.data.model.ExprValueUtils.doubleValue; +import static org.opensearch.sql.data.model.ExprValueUtils.integerValue; +import static org.opensearch.sql.data.model.ExprValueUtils.longValue; +import static org.opensearch.sql.data.model.ExprValueUtils.stringValue; +import static org.opensearch.sql.data.type.ExprCoreType.BOOLEAN; +import static org.opensearch.sql.data.type.ExprCoreType.DOUBLE; +import static org.opensearch.sql.data.type.ExprCoreType.INTEGER; +import static org.opensearch.sql.data.type.ExprCoreType.LONG; +import static org.opensearch.sql.data.type.ExprCoreType.STRING; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import lombok.Getter; +import org.opensearch.sql.data.model.ExprNullValue; +import org.opensearch.sql.data.model.ExprTupleValue; +import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.data.type.ExprType; +import org.opensearch.sql.opensearch.client.OpenSearchClient; +import org.opensearch.sql.utils.SystemIndexUtils.RestSpec; + +/** + * The read-only endpoint allow-list expressed as data: each allow-listed, read-only endpoint maps + * to its transport action (a read-only call on {@link OpenSearchClient}), a fixed output schema (so + * the Calcite plan can fix its row type at plan time), and the query args it accepts. + * + *

This is the single place the read-only allow-list is enforced. Endpoints outside the registry, + * including every mutating endpoint, are rejected by {@link #resolve} with a clear exception. + * Adding an endpoint is a reviewed change here, never arbitrary pass-through. + */ +public final class RestEndpointRegistry { + + private RestEndpointRegistry() {} + + /** Produces the raw rows for an endpoint via a read-only client call. */ + @FunctionalInterface + public interface RowFetcher { + List> fetch(OpenSearchClient client, RestSpec spec); + } + + /** A single allow-listed endpoint description. */ + @Getter + public static final class Endpoint { + private final String path; + private final LinkedHashMap schema; + private final Set allowedArgs; + private final RowFetcher fetcher; + + Endpoint( + String path, + LinkedHashMap schema, + Set allowedArgs, + RowFetcher fetcher) { + this.path = path; + this.schema = schema; + this.allowedArgs = allowedArgs; + this.fetcher = fetcher; + } + + /** Dispatch the read-only call and shape the response into fixed-schema rows. */ + public List toRows(OpenSearchClient client, RestSpec spec) { + return toRows(client, spec, false); + } + + /** + * Shape the response into fixed-schema rows, masking network identifiers when redaction is + * enabled. {@code /_cat/*} cells are fully masked and the {@code /_cluster/settings} value + * column is zone-masked. Off by default. + */ + public List toRows(OpenSearchClient client, RestSpec spec, boolean redact) { + boolean redactCat = redact && path.startsWith("/_cat"); + boolean redactSettingsValue = redact && "/_cluster/settings".equals(path); + List out = new ArrayList<>(); + for (Map raw : fetcher.fetch(client, spec)) { + LinkedHashMap tuple = new LinkedHashMap<>(); + for (Map.Entry col : schema.entrySet()) { + ExprValue value = coerce(col.getKey(), col.getValue(), raw.get(col.getKey())); + tuple.put( + col.getKey(), + maskCell(col.getKey(), col.getValue(), value, redactCat, redactSettingsValue)); + } + out.add(new ExprTupleValue(tuple)); + } + return out; + } + + private static ExprValue maskCell( + String column, + ExprType type, + ExprValue value, + boolean redactCat, + boolean redactSettingsValue) { + if (type != STRING || value.isNull()) { + return value; + } + if (redactCat) { + return stringValue(RestResponseRedactor.redact(value.stringValue())); + } + if (redactSettingsValue && "value".equals(column)) { + return stringValue(RestResponseRedactor.maskAvailabilityZone(value.stringValue())); + } + return value; + } + } + + private static final Map REGISTRY = buildRegistry(); + + private static Map buildRegistry() { + Map m = new LinkedHashMap<>(); + + // /_cluster/health — single-row cluster health snapshot (read-only monitor action). + LinkedHashMap healthSchema = new LinkedHashMap<>(); + healthSchema.put("cluster_name", STRING); + healthSchema.put("status", STRING); + healthSchema.put("number_of_nodes", INTEGER); + healthSchema.put("number_of_data_nodes", INTEGER); + healthSchema.put("active_primary_shards", INTEGER); + healthSchema.put("active_shards", INTEGER); + healthSchema.put("relocating_shards", INTEGER); + healthSchema.put("initializing_shards", INTEGER); + healthSchema.put("unassigned_shards", INTEGER); + healthSchema.put("timed_out", BOOLEAN); + m.put( + "/_cluster/health", + new Endpoint( + "/_cluster/health", + healthSchema, + Set.of("local"), + (client, spec) -> List.of(client.clusterHealth(spec.getArgs())))); + + // /_cat/indices — one row per index (read-only monitor action). + LinkedHashMap catSchema = new LinkedHashMap<>(); + catSchema.put("index", STRING); + catSchema.put("health", STRING); + catSchema.put("pri", INTEGER); + catSchema.put("rep", INTEGER); + catSchema.put("active_shards", INTEGER); + m.put( + "/_cat/indices", + new Endpoint( + "/_cat/indices", + catSchema, + Set.of("health"), + (client, spec) -> client.catIndices(spec.getArgs()))); + + // /_cat/nodes — one row per node with resource state (read-only monitor action). + LinkedHashMap nodesSchema = new LinkedHashMap<>(); + nodesSchema.put("name", STRING); + nodesSchema.put("ip", STRING); + nodesSchema.put("node_role", STRING); + nodesSchema.put("heap_percent", INTEGER); + nodesSchema.put("ram_percent", INTEGER); + nodesSchema.put("cpu", INTEGER); + m.put( + "/_cat/nodes", + new Endpoint( + "/_cat/nodes", + nodesSchema, + Set.of(), + (client, spec) -> client.catNodes(spec.getArgs()))); + + // /_cat/cluster_manager — single row identifying the elected cluster manager node. + LinkedHashMap clusterManagerSchema = new LinkedHashMap<>(); + clusterManagerSchema.put("id", STRING); + clusterManagerSchema.put("host", STRING); + clusterManagerSchema.put("ip", STRING); + clusterManagerSchema.put("node", STRING); + m.put( + "/_cat/cluster_manager", + new Endpoint( + "/_cat/cluster_manager", + clusterManagerSchema, + Set.of(), + (client, spec) -> client.catClusterManager(spec.getArgs()))); + + // /_cat/plugins — one row per installed plugin per node (read-only monitor action). + LinkedHashMap pluginsSchema = new LinkedHashMap<>(); + pluginsSchema.put("name", STRING); + pluginsSchema.put("component", STRING); + pluginsSchema.put("version", STRING); + m.put( + "/_cat/plugins", + new Endpoint( + "/_cat/plugins", + pluginsSchema, + Set.of(), + (client, spec) -> client.catPlugins(spec.getArgs()))); + + // /_cat/shards — one row per shard (read-only monitor action). + LinkedHashMap shardsSchema = new LinkedHashMap<>(); + shardsSchema.put("index", STRING); + shardsSchema.put("shard", INTEGER); + shardsSchema.put("prirep", STRING); + shardsSchema.put("state", STRING); + shardsSchema.put("node", STRING); + m.put( + "/_cat/shards", + new Endpoint( + "/_cat/shards", + shardsSchema, + Set.of(), + (client, spec) -> client.catShards(spec.getArgs()))); + + // /_cluster/state — single-row cluster-state epoch (version, uuid, manager node). + LinkedHashMap stateSchema = new LinkedHashMap<>(); + stateSchema.put("cluster_name", STRING); + stateSchema.put("state_uuid", STRING); + stateSchema.put("version", LONG); + stateSchema.put("cluster_manager_node", STRING); + m.put( + "/_cluster/state", + new Endpoint( + "/_cluster/state", + stateSchema, + Set.of(), + (client, spec) -> List.of(client.clusterState(spec.getArgs())))); + + // /_cluster/settings — one row per configured setting (persistent/transient tier). + LinkedHashMap settingsSchema = new LinkedHashMap<>(); + settingsSchema.put("setting", STRING); + settingsSchema.put("value", STRING); + settingsSchema.put("tier", STRING); + m.put( + "/_cluster/settings", + new Endpoint( + "/_cluster/settings", + settingsSchema, + Set.of(), + (client, spec) -> client.clusterSettings(spec.getArgs()))); + + // /_resolve/index — one row per resolved index/alias/data_stream name. + LinkedHashMap resolveSchema = new LinkedHashMap<>(); + resolveSchema.put("name", STRING); + resolveSchema.put("type", STRING); + m.put( + "/_resolve/index", + new Endpoint( + "/_resolve/index", + resolveSchema, + Set.of("expand_wildcards"), + (client, spec) -> client.resolveIndex(spec.getArgs()))); + + return m; + } + + /** + * Resolve an allow-listed endpoint. Anything outside the registry (unknown path, mutating verb, + * {@code /services/*}, plugin admin endpoints) is refused here. + */ + public static Endpoint resolve(String path) { + if (path == null || path.isBlank()) { + throw new IllegalArgumentException( + "rest endpoint must be a non-empty path. Supported read-only endpoints: " + + REGISTRY.keySet()); + } + Endpoint endpoint = REGISTRY.get(path); + if (endpoint == null) { + throw new IllegalArgumentException( + "rest endpoint [" + + path + + "] is not allow-listed. Only read-only in-cluster endpoints are supported: " + + REGISTRY.keySet()); + } + return endpoint; + } + + /** Validate that every supplied query arg is accepted by the endpoint. */ + public static void validate(RestSpec spec) { + Endpoint endpoint = resolve(spec.getEndpoint()); + if (spec.getCount() != null && spec.getCount() < 0) { + throw new IllegalArgumentException( + "rest endpoint [" + + spec.getEndpoint() + + "] count must be a non-negative integer, got [" + + spec.getCount() + + "]"); + } + if (spec.getTimeout() != null) { + // The timeout token is reserved in the grammar for forward compatibility, but a single + // uniform timeout cannot map cleanly across the endpoints (wait-for-status vs + // cluster-manager vs client socket timeouts differ per action). Reject it with a clear + // client error rather than silently ignoring it. + throw new IllegalArgumentException( + "rest endpoint [" + spec.getEndpoint() + "] does not support the timeout argument yet"); + } + if (spec.getArgs() != null) { + for (String arg : spec.getArgs().keySet()) { + if (!endpoint.getAllowedArgs().contains(arg)) { + throw new IllegalArgumentException( + "rest endpoint [" + + spec.getEndpoint() + + "] does not accept arg [" + + arg + + "]. Allowed args: " + + endpoint.getAllowedArgs()); + } + validateArgValue(spec.getEndpoint(), arg, spec.getArgs().get(arg)); + } + } + } + + // Allowed value domains for the get-args that are applied server-side. Keys are validated against + // the per-endpoint allow-list above; values are validated here so a user-supplied value is never + // passed unchecked into an admin transport request. + private static final Map> ARG_VALUE_DOMAINS = + Map.of( + "local", Set.of("true", "false"), + "health", Set.of("green", "yellow", "red")); + + private static final Set EXPAND_WILDCARDS_VALUES = + Set.of("open", "closed", "hidden", "none", "all"); + + /** Reject any get-arg value outside its allow-listed domain with a clear client error. */ + private static void validateArgValue(String endpoint, String arg, String value) { + Set domain = ARG_VALUE_DOMAINS.get(arg); + if (domain != null) { + if (value == null || !domain.contains(value.toLowerCase(java.util.Locale.ROOT))) { + throw new IllegalArgumentException( + "rest endpoint [" + + endpoint + + "] arg [" + + arg + + "] has an unsupported value [" + + value + + "]. Allowed values: " + + domain); + } + } else if ("expand_wildcards".equals(arg)) { + if (value == null || value.isBlank()) { + throw new IllegalArgumentException( + "rest endpoint [" + + endpoint + + "] arg [expand_wildcards] has an unsupported value [" + + value + + "]. Allowed values: " + + EXPAND_WILDCARDS_VALUES); + } + for (String token : value.toLowerCase(java.util.Locale.ROOT).split(",")) { + if (!EXPAND_WILDCARDS_VALUES.contains(token.trim())) { + throw new IllegalArgumentException( + "rest endpoint [" + + endpoint + + "] arg [expand_wildcards] has an unsupported value [" + + value + + "]. Allowed values: " + + EXPAND_WILDCARDS_VALUES); + } + } + } + } + + private static ExprValue coerce(String column, ExprType type, Object value) { + if (value == null) { + return ExprNullValue.of(); + } + try { + if (type == INTEGER) { + return integerValue(toNumber(value).intValue()); + } + if (type == LONG) { + return longValue(toNumber(value).longValue()); + } + if (type == DOUBLE) { + return doubleValue(toNumber(value).doubleValue()); + } + if (type == BOOLEAN) { + return booleanValue(toBoolean(value)); + } + } catch (IllegalArgumentException | ClassCastException e) { + // Surface a clear client error (HTTP 400) instead of a raw HTTP 500 when an endpoint + // returns an unexpected value shape. NumberFormatException extends IllegalArgumentException, + // so toNumber parse failures and toBoolean's "not a boolean" are both caught here; genuinely + // unexpected faults (NPE, etc.) are left to propagate. + throw new IllegalArgumentException( + "rest endpoint value for column [" + + column + + "] could not be coerced to " + + type + + ": [" + + value + + "]"); + } + return stringValue(String.valueOf(value)); + } + + /** Coerce a transport/JSON value to a Number, parsing numeric strings (e.g. the cat JSON API). */ + private static Number toNumber(Object value) { + if (value instanceof Number n) { + return n; + } + String s = String.valueOf(value).trim(); + if (s.isEmpty()) { + throw new NumberFormatException("empty string"); + } + if (s.indexOf('.') >= 0 || s.indexOf('e') >= 0 || s.indexOf('E') >= 0) { + return Double.parseDouble(s); + } + return Long.parseLong(s); + } + + /** Coerce a transport/JSON value to a boolean, accepting Boolean or the strings true/false. */ + private static boolean toBoolean(Object value) { + if (value instanceof Boolean b) { + return b; + } + String s = String.valueOf(value).trim(); + if (s.isEmpty()) { + throw new IllegalArgumentException("empty string is not a boolean"); + } + if (s.equalsIgnoreCase("true")) { + return true; + } + if (s.equalsIgnoreCase("false")) { + return false; + } + throw new IllegalArgumentException("not a boolean: " + value); + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestRequest.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestRequest.java new file mode 100644 index 00000000000..868dabdd64e --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestRequest.java @@ -0,0 +1,57 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.rest; + +import java.util.List; +import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.opensearch.client.OpenSearchClient; +import org.opensearch.sql.opensearch.request.system.OpenSearchSystemRequest; +import org.opensearch.sql.utils.SystemIndexUtils.RestSpec; + +/** + * Dispatches an allow-listed, read-only management endpoint through the transport node client under + * the caller's security thread-context and returns the response shaped to the endpoint's fixed + * schema. The {@code rest} analogue of {@code OpenSearchCatIndicesRequest}; it implements {@link + * OpenSearchSystemRequest} so the enumerator pattern (resource-monitored iteration) is identical to + * the system-index scan family. + */ +public class RestRequest implements OpenSearchSystemRequest { + + private final OpenSearchClient client; + private final RestEndpointRegistry.Endpoint endpoint; + private final RestSpec spec; + private final boolean redact; + + public RestRequest( + OpenSearchClient client, RestEndpointRegistry.Endpoint endpoint, RestSpec spec) { + this(client, endpoint, spec, false); + } + + public RestRequest( + OpenSearchClient client, + RestEndpointRegistry.Endpoint endpoint, + RestSpec spec, + boolean redact) { + this.client = client; + this.endpoint = endpoint; + this.spec = spec; + this.redact = redact; + } + + @Override + public List search() { + List rows = endpoint.toRows(client, spec, redact); + if (spec.getCount() != null && spec.getCount() >= 0 && rows.size() > spec.getCount()) { + return rows.subList(0, spec.getCount()); + } + return rows; + } + + @Override + public String toString() { + return "RestRequest{endpoint=" + endpoint.getPath() + "}"; + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestResponseRedactor.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestResponseRedactor.java new file mode 100644 index 00000000000..fd674dccde4 --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestResponseRedactor.java @@ -0,0 +1,64 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.rest; + +import java.util.List; +import java.util.regex.Pattern; + +/** + * Masks network identifiers in rest command cell values. Enabled per deployment via {@code + * plugins.ppl.rest.redaction.enabled}; off by default. + */ +public final class RestResponseRedactor { + + private RestResponseRedactor() {} + + private static final String OCTET = "(25[0-5]|2[0-4]\\d|[0-1]?\\d\\d?)"; + private static final Pattern IPV4 = + Pattern.compile("\\b" + OCTET + "\\." + OCTET + "\\." + OCTET + "\\." + OCTET + "\\b"); + private static final Pattern INET = Pattern.compile("inet\\[/[\\d.:]+\\]"); + private static final Pattern EC2_HOST = + Pattern.compile("\\bip-" + OCTET + "-" + OCTET + "-" + OCTET + "-" + OCTET + "\\b"); + private static final Pattern IPV6 = + Pattern.compile( + "([0-9a-f]{1,4}:){7}[0-9a-f]{1,4}" + + "|([0-9a-f]{1,4}(:[0-9a-f]{1,4})*)?::([0-9a-f]{1,4}(:[0-9a-f]{1,4})*)?", + Pattern.CASE_INSENSITIVE); + private static final Pattern AZ_NAME = + Pattern.compile( + "\\b[a-z]{2}(-(gov|iso[a-z]?))?-(central|(north|south)?(east|west)?)-\\d[a-z]\\b", + Pattern.CASE_INSENSITIVE); + + private record Mask(Pattern pattern, String replacement) {} + + private static final List MASKS = + List.of( + new Mask(IPV4, "x.x.x.x"), + new Mask(INET, "inet[/x.x.x.x:y]"), + new Mask(EC2_HOST, ""), + new Mask(IPV6, "x.x.x.x"), + new Mask(AZ_NAME, "xx-xxxxx-xx")); + + /** Mask IPv4, inet, EC2 host names, IPv6, and availability-zone names in the text. */ + public static String redact(String text) { + if (text == null || text.isEmpty()) { + return text; + } + String out = text; + for (Mask mask : MASKS) { + out = mask.pattern().matcher(out).replaceAll(mask.replacement()); + } + return out; + } + + /** Mask availability-zone names only. */ + public static String maskAvailabilityZone(String text) { + if (text == null || text.isEmpty()) { + return text; + } + return AZ_NAME.matcher(text).replaceAll("xx-xxxxx-xx"); + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestSettingsFilterHolder.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestSettingsFilterHolder.java new file mode 100644 index 00000000000..d425c361e6a --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestSettingsFilterHolder.java @@ -0,0 +1,40 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.rest; + +import org.opensearch.common.settings.SettingsFilter; + +/** + * Bridge for sharing the node-level {@link SettingsFilter} with the in-cluster {@code rest ' + * /_cluster/settings'} fetcher. + * + *

The native {@code GET /_cluster/settings} REST endpoint redacts settings registered with + * {@code Property.Filtered} (or matched by a plugin-registered filter pattern) by running the + * response through {@link SettingsFilter}. The PPL {@code rest} command's in-cluster path reads + * {@code persistentSettings()}/{@code transientSettings()} straight from cluster state via the + * transport layer, where no {@link SettingsFilter} is applied. To keep the command's redaction + * behavior identical to the native endpoint, the node's {@link SettingsFilter} is published here. + * + *

Why a static holder: the {@link SettingsFilter} instance is only handed to the plugin in + * {@code SQLPlugin#getRestHandlers}, which runs outside any Guice-managed lifecycle, while {@link + * OpenSearchNodeClient} is built through the Node injector. Persisting the filter here once {@code + * getRestHandlers} fires lets the fetcher read the same instance without going back through the + * injector. This mirrors the existing {@code AnalyticsExecutorHolder} pattern. + */ +public final class RestSettingsFilterHolder { + + private static volatile SettingsFilter settingsFilter; + + private RestSettingsFilterHolder() {} + + public static void set(SettingsFilter instance) { + settingsFilter = instance; + } + + public static SettingsFilter get() { + return settingsFilter; + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/AbstractCalciteSystemIndexScan.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/AbstractCalciteCatalogScan.java similarity index 68% rename from opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/AbstractCalciteSystemIndexScan.java rename to opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/AbstractCalciteCatalogScan.java index fa543ab266b..19d585ea684 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/AbstractCalciteSystemIndexScan.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/AbstractCalciteCatalogScan.java @@ -16,21 +16,21 @@ import org.apache.calcite.rel.hint.RelHint; import org.apache.calcite.rel.type.RelDataType; -/** An abstract relational operator representing a scan of an OpenSearchSystemIndex type. */ +/** An abstract relational operator representing a scan of an {@link OpenSearchCatalogTable}. */ @Getter -public abstract class AbstractCalciteSystemIndexScan extends TableScan { - public final OpenSearchSystemIndex sysIndex; +public abstract class AbstractCalciteCatalogScan extends TableScan { + public final OpenSearchCatalogTable catalogTable; protected final RelDataType schema; - protected AbstractCalciteSystemIndexScan( + protected AbstractCalciteCatalogScan( RelOptCluster cluster, RelTraitSet traitSet, List hints, RelOptTable table, - OpenSearchSystemIndex sysIndex, + OpenSearchCatalogTable catalogTable, RelDataType schema) { super(cluster, traitSet, hints, table); - this.sysIndex = requireNonNull(sysIndex, "OpenSearch system index"); + this.catalogTable = requireNonNull(catalogTable, "OpenSearch catalog table"); this.schema = schema; } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteEnumerableSystemIndexScan.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteEnumerableCatalogScan.java similarity index 80% rename from opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteEnumerableSystemIndexScan.java rename to opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteEnumerableCatalogScan.java index b0c92dce8f9..58f86874c73 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteEnumerableSystemIndexScan.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteEnumerableCatalogScan.java @@ -26,17 +26,22 @@ import org.apache.calcite.util.Pair; import org.checkerframework.checker.nullness.qual.Nullable; -/** The physical relational operator representing a scan of an OpenSearchSystemIndex type. */ -public class CalciteEnumerableSystemIndexScan extends AbstractCalciteSystemIndexScan +/** The physical relational operator representing a scan of an {@link OpenSearchCatalogTable}. */ +public class CalciteEnumerableCatalogScan extends AbstractCalciteCatalogScan implements EnumerableRel { - public CalciteEnumerableSystemIndexScan( + public CalciteEnumerableCatalogScan( RelOptCluster cluster, List hints, RelOptTable table, - OpenSearchSystemIndex sysIndex, + OpenSearchCatalogTable catalogTable, RelDataType schema) { super( - cluster, cluster.traitSetOf(EnumerableConvention.INSTANCE), hints, table, sysIndex, schema); + cluster, + cluster.traitSetOf(EnumerableConvention.INSTANCE), + hints, + table, + catalogTable, + schema); } @Override @@ -60,7 +65,7 @@ public Result implement(EnumerableRelImplementor implementor, Prefer pref) { PhysType physType = PhysTypeImpl.of(implementor.getTypeFactory(), getRowType(), pref.preferArray()); - Expression scanOperator = implementor.stash(this, CalciteEnumerableSystemIndexScan.class); + Expression scanOperator = implementor.stash(this, CalciteEnumerableCatalogScan.class); return implementor.result(physType, Blocks.toBlock(Expressions.call(scanOperator, "scan"))); } @@ -68,10 +73,10 @@ public Result implement(EnumerableRelImplementor implementor, Prefer pref) { return new AbstractEnumerable<>() { @Override public Enumerator enumerator() { - return new OpenSearchSystemIndexEnumerator( + return new OpenSearchCatalogEnumerator( getFieldPath(), - sysIndex.getSystemIndexBundle().getRight(), - sysIndex.createOpenSearchResourceMonitor()); + catalogTable.getSource().createRequest(), + catalogTable.createOpenSearchResourceMonitor()); } }; } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteLogicalSystemIndexScan.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteLogicalCatalogScan.java similarity index 59% rename from opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteLogicalSystemIndexScan.java rename to opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteLogicalCatalogScan.java index 012ceec8c13..4278539c2b0 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteLogicalSystemIndexScan.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteLogicalCatalogScan.java @@ -14,35 +14,35 @@ import org.apache.calcite.plan.RelTraitSet; import org.apache.calcite.rel.hint.RelHint; import org.apache.calcite.rel.type.RelDataType; -import org.opensearch.sql.opensearch.planner.rules.EnumerableSystemIndexScanRule; +import org.opensearch.sql.opensearch.planner.rules.EnumerableCatalogScanRule; -/** The logical relational operator representing a scan of an OpenSearchSystemIndex type. */ -public class CalciteLogicalSystemIndexScan extends AbstractCalciteSystemIndexScan { +/** The logical relational operator representing a scan of an {@link OpenSearchCatalogTable}. */ +public class CalciteLogicalCatalogScan extends AbstractCalciteCatalogScan { - public CalciteLogicalSystemIndexScan( - RelOptCluster cluster, RelOptTable table, OpenSearchSystemIndex sysIndex) { + public CalciteLogicalCatalogScan( + RelOptCluster cluster, RelOptTable table, OpenSearchCatalogTable catalogTable) { this( cluster, cluster.traitSetOf(Convention.NONE), ImmutableList.of(), table, - sysIndex, + catalogTable, table.getRowType()); } - protected CalciteLogicalSystemIndexScan( + protected CalciteLogicalCatalogScan( RelOptCluster cluster, RelTraitSet traitSet, List hints, RelOptTable table, - OpenSearchSystemIndex sysIndex, + OpenSearchCatalogTable catalogTable, RelDataType schema) { - super(cluster, traitSet, hints, table, sysIndex, schema); + super(cluster, traitSet, hints, table, catalogTable, schema); } @Override public void register(RelOptPlanner planner) { super.register(planner); - planner.addRule(EnumerableSystemIndexScanRule.DEFAULT_CONFIG.toRule()); + planner.addRule(EnumerableCatalogScanRule.DEFAULT_CONFIG.toRule()); } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteScannableCatalogScan.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteScannableCatalogScan.java new file mode 100644 index 00000000000..1d021cda354 --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteScannableCatalogScan.java @@ -0,0 +1,29 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.system; + +import java.util.List; +import org.apache.calcite.plan.RelOptCluster; +import org.apache.calcite.plan.RelOptTable; +import org.apache.calcite.rel.hint.RelHint; +import org.apache.calcite.rel.type.RelDataType; +import org.opensearch.sql.calcite.plan.Scannable; + +/** + * A {@link CalciteEnumerableCatalogScan} that additionally carries the {@link Scannable} marker, + * enabling the {@code collect} short-circuit. Produced when the {@link CatalogSource} opts in via + * {@link CatalogSource#isScannable()}. + */ +public class CalciteScannableCatalogScan extends CalciteEnumerableCatalogScan implements Scannable { + public CalciteScannableCatalogScan( + RelOptCluster cluster, + List hints, + RelOptTable table, + OpenSearchCatalogTable catalogTable, + RelDataType schema) { + super(cluster, hints, table, catalogTable, schema); + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CatalogSource.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CatalogSource.java new file mode 100644 index 00000000000..85dabe06b9a --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CatalogSource.java @@ -0,0 +1,39 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.system; + +import java.util.Map; +import org.opensearch.sql.data.type.ExprType; +import org.opensearch.sql.opensearch.request.system.OpenSearchSystemRequest; +import org.opensearch.sql.planner.logical.LogicalPlan; +import org.opensearch.sql.planner.physical.PhysicalPlan; + +/** + * Strategy supplying the schema and row source for an {@link OpenSearchCatalogTable} backed by a + * read-only OpenSearch admin API. Each endpoint family plugs in its own {@code CatalogSource} + * rather than defining a separate table and Calcite scan hierarchy. + */ +public interface CatalogSource { + + /** Fixed schema mapping each column name to its type. */ + Map getFieldTypes(); + + /** + * Builds the read-only request whose {@link OpenSearchSystemRequest#search()} yields the rows. + */ + OpenSearchSystemRequest createRequest(); + + /** + * Whether the enumerable scan should carry the {@link org.opensearch.sql.calcite.plan.Scannable} + * marker for the {@code collect} short-circuit. Defaults to {@code false}. + */ + default boolean isScannable() { + return false; + } + + /** The V2 physical plan path, or throws when the source is Calcite only. */ + PhysicalPlan implementV2(LogicalPlan plan); +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/OpenSearchSystemIndexEnumerator.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/OpenSearchCatalogEnumerator.java similarity index 90% rename from opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/OpenSearchSystemIndexEnumerator.java rename to opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/OpenSearchCatalogEnumerator.java index 1bb8f9d5293..dff9b47265a 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/OpenSearchSystemIndexEnumerator.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/OpenSearchCatalogEnumerator.java @@ -16,8 +16,11 @@ import org.opensearch.sql.monitor.ResourceMonitor; import org.opensearch.sql.opensearch.request.system.OpenSearchSystemRequest; -/** Supports a simple iteration over a collection for OpenSearch system index */ -public class OpenSearchSystemIndexEnumerator implements Enumerator { +/** + * Resource-monitored iteration over the rows produced by a read-only catalog {@link + * OpenSearchSystemRequest}. + */ +public class OpenSearchCatalogEnumerator implements Enumerator { /** How many moveNext() calls to perform resource check once. */ private static final long NUMBER_OF_NEXT_CALL_TO_CHECK = 1000; @@ -35,7 +38,7 @@ public class OpenSearchSystemIndexEnumerator implements Enumerator { /** ResourceMonitor. */ private final ResourceMonitor monitor; - public OpenSearchSystemIndexEnumerator( + public OpenSearchCatalogEnumerator( List fields, OpenSearchSystemRequest request, ResourceMonitor monitor) { this.fields = fields; this.request = request; diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/OpenSearchCatalogTable.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/OpenSearchCatalogTable.java new file mode 100644 index 00000000000..9bcb3a3ff0c --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/OpenSearchCatalogTable.java @@ -0,0 +1,67 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.system; + +import java.util.Map; +import lombok.Getter; +import org.apache.calcite.plan.RelOptCluster; +import org.apache.calcite.plan.RelOptTable; +import org.apache.calcite.rel.RelNode; +import org.opensearch.sql.calcite.plan.AbstractOpenSearchTable; +import org.opensearch.sql.common.setting.Settings; +import org.opensearch.sql.data.type.ExprType; +import org.opensearch.sql.opensearch.monitor.OpenSearchMemoryHealthy; +import org.opensearch.sql.opensearch.monitor.OpenSearchResourceMonitor; +import org.opensearch.sql.planner.logical.LogicalPlan; +import org.opensearch.sql.planner.physical.PhysicalPlan; + +/** + * A single generic Calcite and V2 table over a read-only OpenSearch catalog endpoint. Per-endpoint + * behavior (schema, row source, V2 support, and the {@code Scannable} marker) is supplied by a + * pluggable {@link CatalogSource}. + */ +@Getter +public class OpenSearchCatalogTable extends AbstractOpenSearchTable { + + private final CatalogSource source; + private final Settings settings; + + public OpenSearchCatalogTable(CatalogSource source, Settings settings) { + this.source = source; + this.settings = settings; + } + + @Override + public boolean exists() { + return true; + } + + @Override + public void create(Map schema) { + throw new UnsupportedOperationException( + "OpenSearch catalog table is predefined and cannot be created"); + } + + @Override + public Map getFieldTypes() { + return source.getFieldTypes(); + } + + @Override + public RelNode toRel(RelOptTable.ToRelContext context, RelOptTable relOptTable) { + final RelOptCluster cluster = context.getCluster(); + return new CalciteLogicalCatalogScan(cluster, relOptTable, this); + } + + @Override + public PhysicalPlan implement(LogicalPlan plan) { + return source.implementV2(plan); + } + + public OpenSearchResourceMonitor createOpenSearchResourceMonitor() { + return new OpenSearchResourceMonitor(settings, new OpenSearchMemoryHealthy(settings)); + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/OpenSearchSystemIndex.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/OpenSearchSystemIndex.java deleted file mode 100644 index 6bae4d21aba..00000000000 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/OpenSearchSystemIndex.java +++ /dev/null @@ -1,105 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.opensearch.storage.system; - -import static org.opensearch.sql.utils.SystemIndexUtils.systemTable; - -import com.google.common.annotations.VisibleForTesting; -import java.util.Map; -import lombok.Getter; -import lombok.RequiredArgsConstructor; -import org.apache.calcite.plan.RelOptCluster; -import org.apache.calcite.plan.RelOptTable; -import org.apache.calcite.rel.RelNode; -import org.apache.commons.lang3.tuple.Pair; -import org.opensearch.sql.calcite.plan.AbstractOpenSearchTable; -import org.opensearch.sql.common.setting.Settings; -import org.opensearch.sql.data.type.ExprType; -import org.opensearch.sql.opensearch.client.OpenSearchClient; -import org.opensearch.sql.opensearch.monitor.OpenSearchMemoryHealthy; -import org.opensearch.sql.opensearch.monitor.OpenSearchResourceMonitor; -import org.opensearch.sql.opensearch.request.system.OpenSearchCatIndicesRequest; -import org.opensearch.sql.opensearch.request.system.OpenSearchDescribeIndexRequest; -import org.opensearch.sql.opensearch.request.system.OpenSearchSystemRequest; -import org.opensearch.sql.planner.DefaultImplementor; -import org.opensearch.sql.planner.logical.LogicalPlan; -import org.opensearch.sql.planner.logical.LogicalRelation; -import org.opensearch.sql.planner.physical.PhysicalPlan; -import org.opensearch.sql.utils.SystemIndexUtils; - -/** OpenSearch System Index Table Implementation. */ -@Getter -public class OpenSearchSystemIndex extends AbstractOpenSearchTable { - /** System Index Name. */ - private final Pair systemIndexBundle; - - @Getter private final Settings settings; - - public OpenSearchSystemIndex(OpenSearchClient client, Settings settings, String indexName) { - this.systemIndexBundle = buildIndexBundle(client, indexName); - this.settings = settings; - } - - @Override - public boolean exists() { - return true; // TODO: implement for system index later - } - - @Override - public void create(Map schema) { - throw new UnsupportedOperationException( - "OpenSearch system index is predefined and cannot be created"); - } - - @Override - public Map getFieldTypes() { - return systemIndexBundle.getLeft().getMapping(); - } - - @Override - public RelNode toRel(RelOptTable.ToRelContext context, RelOptTable relOptTable) { - final RelOptCluster cluster = context.getCluster(); - return new CalciteLogicalSystemIndexScan(cluster, relOptTable, this); - } - - @Override - public PhysicalPlan implement(LogicalPlan plan) { - return plan.accept(new OpenSearchSystemIndexDefaultImplementor(), null); - } - - public OpenSearchResourceMonitor createOpenSearchResourceMonitor() { - return new OpenSearchResourceMonitor(getSettings(), new OpenSearchMemoryHealthy(settings)); - } - - @VisibleForTesting - @RequiredArgsConstructor - public class OpenSearchSystemIndexDefaultImplementor extends DefaultImplementor { - - @Override - public PhysicalPlan visitRelation(LogicalRelation node, Object context) { - return new OpenSearchSystemIndexScan(systemIndexBundle.getRight()); - } - } - - /** - * Constructor of ElasticsearchSystemIndexName. - * - * @param indexName index name; - */ - private Pair buildIndexBundle( - OpenSearchClient client, String indexName) { - SystemIndexUtils.SystemTable systemTable = systemTable(indexName); - if (systemTable.isSystemInfoTable()) { - return Pair.of( - OpenSearchSystemIndexSchema.SYS_TABLE_TABLES, new OpenSearchCatIndicesRequest(client)); - } else { - return Pair.of( - OpenSearchSystemIndexSchema.SYS_TABLE_MAPPINGS, - new OpenSearchDescribeIndexRequest( - client, systemTable.getTableName(), systemTable.getLangSpec())); - } - } -} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/SystemIndexCatalogSource.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/SystemIndexCatalogSource.java new file mode 100644 index 00000000000..3541e745e9d --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/SystemIndexCatalogSource.java @@ -0,0 +1,70 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.system; + +import static org.opensearch.sql.utils.SystemIndexUtils.systemTable; + +import java.util.Map; +import org.apache.commons.lang3.tuple.Pair; +import org.opensearch.sql.data.type.ExprType; +import org.opensearch.sql.opensearch.client.OpenSearchClient; +import org.opensearch.sql.opensearch.request.system.OpenSearchCatIndicesRequest; +import org.opensearch.sql.opensearch.request.system.OpenSearchDescribeIndexRequest; +import org.opensearch.sql.opensearch.request.system.OpenSearchSystemRequest; +import org.opensearch.sql.planner.DefaultImplementor; +import org.opensearch.sql.planner.logical.LogicalPlan; +import org.opensearch.sql.planner.logical.LogicalRelation; +import org.opensearch.sql.planner.physical.PhysicalPlan; +import org.opensearch.sql.utils.SystemIndexUtils; + +/** + * {@link CatalogSource} for the SHOW TABLES and DESCRIBE system tables, backed by index listing and + * index field mappings. + */ +public class SystemIndexCatalogSource implements CatalogSource { + + private final Pair bundle; + + public SystemIndexCatalogSource(OpenSearchClient client, String indexName) { + this.bundle = buildBundle(client, indexName); + } + + @Override + public Map getFieldTypes() { + return bundle.getLeft().getMapping(); + } + + @Override + public OpenSearchSystemRequest createRequest() { + return bundle.getRight(); + } + + @Override + public PhysicalPlan implementV2(LogicalPlan plan) { + return plan.accept( + new DefaultImplementor() { + @Override + public PhysicalPlan visitRelation(LogicalRelation node, Object context) { + return new OpenSearchSystemIndexScan(bundle.getRight()); + } + }, + null); + } + + private static Pair buildBundle( + OpenSearchClient client, String indexName) { + SystemIndexUtils.SystemTable systemTable = systemTable(indexName); + if (systemTable.isSystemInfoTable()) { + return Pair.of( + OpenSearchSystemIndexSchema.SYS_TABLE_TABLES, new OpenSearchCatIndicesRequest(client)); + } else { + return Pair.of( + OpenSearchSystemIndexSchema.SYS_TABLE_MAPPINGS, + new OpenSearchDescribeIndexRequest( + client, systemTable.getTableName(), systemTable.getLangSpec())); + } + } +} diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClientClusterSettingsFilterTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClientClusterSettingsFilterTest.java new file mode 100644 index 00000000000..33be6cc8482 --- /dev/null +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClientClusterSettingsFilterTest.java @@ -0,0 +1,99 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.client; + +import static java.util.stream.Collectors.toSet; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Answers.RETURNS_DEEP_STUBS; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.util.List; +import java.util.Map; +import java.util.Set; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import org.opensearch.action.admin.cluster.state.ClusterStateRequest; +import org.opensearch.action.admin.cluster.state.ClusterStateResponse; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.settings.SettingsFilter; +import org.opensearch.sql.opensearch.storage.rest.RestSettingsFilterHolder; +import org.opensearch.transport.client.node.NodeClient; + +/** + * Verifies the in-cluster {@code rest '/_cluster/settings'} fetcher redacts filtered settings using + * the node {@link SettingsFilter}, matching the native {@code GET /_cluster/settings} endpoint. + */ +class OpenSearchNodeClientClusterSettingsFilterTest { + + @AfterEach + void clearHolder() { + RestSettingsFilterHolder.set(null); + } + + private OpenSearchNodeClient clientReturning(Settings persistent, Settings transientSettings) { + NodeClient nodeClient = mock(NodeClient.class, RETURNS_DEEP_STUBS); + ClusterStateResponse stateResp = mock(ClusterStateResponse.class, RETURNS_DEEP_STUBS); + when(nodeClient.admin().cluster().state(any(ClusterStateRequest.class)).actionGet()) + .thenReturn(stateResp); + when(stateResp.getState().metadata().persistentSettings()).thenReturn(persistent); + when(stateResp.getState().metadata().transientSettings()).thenReturn(transientSettings); + return new OpenSearchNodeClient(nodeClient); + } + + @Test + void clusterSettingsRedactsFilteredKeyWhenFilterPublished() { + Settings persistent = + Settings.builder() + .put("cluster.routing.allocation.enable", "all") + .put("plugins.secret.token", "supersecret") + .build(); + OpenSearchNodeClient client = clientReturning(persistent, Settings.EMPTY); + + // Publish a filter that redacts the secret key, exactly as the native endpoint would. + RestSettingsFilterHolder.set(new SettingsFilter(List.of("plugins.secret.token"))); + + List> rows = client.clusterSettings(Map.of()); + Set keys = rows.stream().map(r -> (String) r.get("setting")).collect(toSet()); + + assertTrue(keys.contains("cluster.routing.allocation.enable"), "non-secret setting kept"); + assertFalse(keys.contains("plugins.secret.token"), "filtered setting must be redacted"); + } + + @Test + void clusterSettingsRedactsByGlobPattern() { + Settings persistent = + Settings.builder() + .put("cluster.routing.allocation.enable", "all") + .put("s3.client.default.secret_key", "AKIAEXAMPLE") + .build(); + OpenSearchNodeClient client = clientReturning(persistent, Settings.EMPTY); + + RestSettingsFilterHolder.set(new SettingsFilter(List.of("s3.client.*.secret_key"))); + + Set keys = + client.clusterSettings(Map.of()).stream() + .map(r -> (String) r.get("setting")) + .collect(toSet()); + + assertTrue(keys.contains("cluster.routing.allocation.enable")); + assertFalse(keys.contains("s3.client.default.secret_key"), "glob-matched secret redacted"); + } + + @Test + void clusterSettingsFailsClosedWhenNoFilterPublished() { + Settings persistent = Settings.builder().put("plugins.secret.token", "supersecret").build(); + OpenSearchNodeClient client = clientReturning(persistent, Settings.EMPTY); + + // Fail closed: without a published SettingsFilter the command must refuse rather than leak raw + // (potentially secret-bearing) settings. At runtime getRestHandlers always publishes the filter + // before any query, so this path is unreachable in cluster. + assertThrows(IllegalStateException.class, () -> client.clusterSettings(Map.of())); + } +} diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/setting/OpenSearchSettingsTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/setting/OpenSearchSettingsTest.java index 5024d416086..0c570098924 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/setting/OpenSearchSettingsTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/setting/OpenSearchSettingsTest.java @@ -9,12 +9,15 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.AdditionalMatchers.not; import static org.mockito.AdditionalMatchers.or; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.when; import static org.opensearch.sql.opensearch.setting.OpenSearchSettings.ASYNC_QUERY_EXTERNAL_SCHEDULER_ENABLED_SETTING; import static org.opensearch.sql.opensearch.setting.OpenSearchSettings.ASYNC_QUERY_EXTERNAL_SCHEDULER_INTERVAL_SETTING; +import static org.opensearch.sql.opensearch.setting.OpenSearchSettings.PPL_REST_ALLOWED_ENDPOINTS_SETTING; +import static org.opensearch.sql.opensearch.setting.OpenSearchSettings.PPL_REST_REDACTION_ENABLED_SETTING; import static org.opensearch.sql.opensearch.setting.OpenSearchSettings.QUERY_MEMORY_LIMIT_SETTING; import static org.opensearch.sql.opensearch.setting.OpenSearchSettings.SPARK_EXECUTION_ENGINE_CONFIG; @@ -74,6 +77,15 @@ void pluginNonDynamicSettings() { assertFalse(settings.isEmpty()); } + @Test + void restSettingsAreNonDynamic() { + assertFalse(PPL_REST_REDACTION_ENABLED_SETTING.isDynamic()); + assertFalse(PPL_REST_ALLOWED_ENDPOINTS_SETTING.isDynamic()); + List> nonDynamic = OpenSearchSettings.pluginNonDynamicSettings(); + assertTrue(nonDynamic.contains(PPL_REST_REDACTION_ENABLED_SETTING)); + assertTrue(nonDynamic.contains(PPL_REST_ALLOWED_ENDPOINTS_SETTING)); + } + @Test void getSettings() { when(clusterSettings.get(ClusterName.CLUSTER_NAME_SETTING)).thenReturn(ClusterName.DEFAULT); diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngineTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngineTest.java index fa04395e065..102ec4da8f7 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngineTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngineTest.java @@ -7,11 +7,15 @@ import static org.junit.jupiter.api.Assertions.assertAll; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.when; import static org.opensearch.sql.analysis.DataSourceSchemaIdentifierNameResolver.DEFAULT_DATASOURCE_NAME; import static org.opensearch.sql.utils.SystemIndexUtils.TABLE_INFO; import java.util.Collection; +import java.util.List; +import java.util.Map; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; @@ -20,8 +24,9 @@ import org.opensearch.sql.common.setting.Settings; import org.opensearch.sql.expression.function.FunctionResolver; import org.opensearch.sql.opensearch.client.OpenSearchClient; -import org.opensearch.sql.opensearch.storage.system.OpenSearchSystemIndex; +import org.opensearch.sql.opensearch.storage.system.OpenSearchCatalogTable; import org.opensearch.sql.storage.Table; +import org.opensearch.sql.utils.SystemIndexUtils; @ExtendWith(MockitoExtension.class) class OpenSearchStorageEngineTest { @@ -52,6 +57,68 @@ public void getSystemTable() { OpenSearchStorageEngine engine = new OpenSearchStorageEngine(client, settings); Table table = engine.getTable(new DataSourceSchemaName(DEFAULT_DATASOURCE_NAME, "default"), TABLE_INFO); - assertAll(() -> assertNotNull(table), () -> assertTrue(table instanceof OpenSearchSystemIndex)); + assertAll( + () -> assertNotNull(table), () -> assertTrue(table instanceof OpenSearchCatalogTable)); + } + + @Test + public void getRestTableAllowedByWildcard() { + when(settings.getSettingValue(Settings.Key.PPL_REST_ALLOWED_ENDPOINTS)) + .thenReturn(List.of("*")); + when(settings.getSettingValue(Settings.Key.PPL_REST_REDACTION_ENABLED)).thenReturn(false); + OpenSearchStorageEngine engine = new OpenSearchStorageEngine(client, settings); + String name = + SystemIndexUtils.restTable( + new SystemIndexUtils.RestSpec("/_cat/nodes", Map.of(), null, null)); + Table table = + engine.getTable(new DataSourceSchemaName(DEFAULT_DATASOURCE_NAME, "default"), name); + assertTrue(table instanceof OpenSearchCatalogTable); + } + + @Test + public void getRestTableAllowedBySubset() { + when(settings.getSettingValue(Settings.Key.PPL_REST_ALLOWED_ENDPOINTS)) + .thenReturn(List.of("/_cat/nodes")); + when(settings.getSettingValue(Settings.Key.PPL_REST_REDACTION_ENABLED)).thenReturn(false); + OpenSearchStorageEngine engine = new OpenSearchStorageEngine(client, settings); + String name = + SystemIndexUtils.restTable( + new SystemIndexUtils.RestSpec("/_cat/nodes", Map.of(), null, null)); + assertTrue( + engine.getTable(new DataSourceSchemaName(DEFAULT_DATASOURCE_NAME, "default"), name) + instanceof OpenSearchCatalogTable); + } + + @Test + public void getRestTableRejectedWhenEndpointNotInSubset() { + when(settings.getSettingValue(Settings.Key.PPL_REST_ALLOWED_ENDPOINTS)) + .thenReturn(List.of("/_cat/nodes")); + OpenSearchStorageEngine engine = new OpenSearchStorageEngine(client, settings); + String name = + SystemIndexUtils.restTable( + new SystemIndexUtils.RestSpec("/_cluster/settings", Map.of(), null, null)); + IllegalArgumentException e = + assertThrows( + IllegalArgumentException.class, + () -> + engine.getTable( + new DataSourceSchemaName(DEFAULT_DATASOURCE_NAME, "default"), name)); + assertTrue(e.getMessage().contains("is not enabled on this cluster")); + } + + @Test + public void getRestTableDisabledWhenListEmpty() { + when(settings.getSettingValue(Settings.Key.PPL_REST_ALLOWED_ENDPOINTS)).thenReturn(List.of()); + OpenSearchStorageEngine engine = new OpenSearchStorageEngine(client, settings); + String name = + SystemIndexUtils.restTable( + new SystemIndexUtils.RestSpec("/_cat/nodes", Map.of(), null, null)); + IllegalArgumentException e = + assertThrows( + IllegalArgumentException.class, + () -> + engine.getTable( + new DataSourceSchemaName(DEFAULT_DATASOURCE_NAME, "default"), name)); + assertTrue(e.getMessage().contains("disabled on this cluster")); } } diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestCatalogSourceTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestCatalogSourceTest.java new file mode 100644 index 00000000000..8676373d2a6 --- /dev/null +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestCatalogSourceTest.java @@ -0,0 +1,123 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.rest; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.hasEntry; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.when; +import static org.opensearch.sql.data.type.ExprCoreType.INTEGER; +import static org.opensearch.sql.data.type.ExprCoreType.STRING; + +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.data.type.ExprType; +import org.opensearch.sql.opensearch.client.OpenSearchClient; +import org.opensearch.sql.utils.SystemIndexUtils.RestSpec; + +/** + * Covers the {@code rest} {@link RestCatalogSource}: fixed endpoint schema, allow-list enforcement, + * response row shaping and truncation, the {@code Scannable} opt-in, and the Calcite only (no V2) + * path. + */ +@ExtendWith(MockitoExtension.class) +class RestCatalogSourceTest { + + @Mock private OpenSearchClient client; + + private RestSpec healthSpec() { + return new RestSpec("/_cluster/health", Map.of(), null, null); + } + + @Test + void getFieldTypesReturnsFixedEndpointSchema() { + RestCatalogSource source = new RestCatalogSource(client, healthSpec()); + Map fieldTypes = source.getFieldTypes(); + assertThat(fieldTypes, hasEntry("status", STRING)); + assertThat(fieldTypes, hasEntry("number_of_nodes", INTEGER)); + } + + @Test + void isScannable() { + assertTrue(new RestCatalogSource(client, healthSpec()).isScannable()); + } + + @Test + void implementV2IsUnsupported() { + RestCatalogSource source = new RestCatalogSource(client, healthSpec()); + assertThrows(UnsupportedOperationException.class, () -> source.implementV2(null)); + } + + @Test + void constructorRejectsNonAllowListedEndpoint() { + assertThrows( + IllegalArgumentException.class, + () -> + new RestCatalogSource(client, new RestSpec("/_cluster/reroute", Map.of(), null, null))); + } + + @Test + void constructorRejectsDisallowedArg() { + assertThrows( + IllegalArgumentException.class, + () -> + new RestCatalogSource( + client, new RestSpec("/_cluster/health", Map.of("bad", "x"), null, null))); + } + + @Test + void constructorRejectsNegativeCount() { + assertThrows( + IllegalArgumentException.class, + () -> new RestCatalogSource(client, new RestSpec("/_cat/indices", Map.of(), -1, null))); + } + + @Test + void constructorRejectsTimeoutArg() { + assertThrows( + IllegalArgumentException.class, + () -> + new RestCatalogSource(client, new RestSpec("/_cluster/health", Map.of(), null, "5s"))); + } + + @Test + void restRequestShapesResponseRows() { + Map health = new LinkedHashMap<>(); + health.put("status", "green"); + health.put("number_of_nodes", 1); + when(client.clusterHealth(any())).thenReturn(health); + + RestCatalogSource source = new RestCatalogSource(client, healthSpec()); + List rows = source.createRequest().search(); + assertEquals(1, rows.size()); + assertEquals("green", rows.get(0).tupleValue().get("status").stringValue()); + assertEquals(1, rows.get(0).tupleValue().get("number_of_nodes").integerValue()); + } + + @Test + void countTruncatesRows() { + Map idx1 = new LinkedHashMap<>(); + idx1.put("index", "a"); + Map idx2 = new LinkedHashMap<>(); + idx2.put("index", "b"); + when(client.catIndices(any())).thenReturn(List.of(idx1, idx2)); + + RestCatalogSource source = + new RestCatalogSource(client, new RestSpec("/_cat/indices", Map.of(), 1, null)); + List rows = source.createRequest().search(); + assertEquals(1, rows.size()); + assertEquals("a", rows.get(0).tupleValue().get("index").stringValue()); + } +} diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistryTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistryTest.java new file mode 100644 index 00000000000..c5a978bb495 --- /dev/null +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistryTest.java @@ -0,0 +1,290 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.rest; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.when; +import static org.opensearch.sql.data.type.ExprCoreType.INTEGER; +import static org.opensearch.sql.data.type.ExprCoreType.STRING; + +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.opensearch.client.OpenSearchClient; +import org.opensearch.sql.utils.SystemIndexUtils.RestSpec; + +@ExtendWith(MockitoExtension.class) +class RestEndpointRegistryTest { + + @Mock private OpenSearchClient client; + + @Test + void resolveAllowListedEndpoint() { + RestEndpointRegistry.Endpoint endpoint = RestEndpointRegistry.resolve("/_cluster/health"); + assertEquals("/_cluster/health", endpoint.getPath()); + assertEquals(STRING, endpoint.getSchema().get("status")); + assertEquals(INTEGER, endpoint.getSchema().get("number_of_nodes")); + } + + @Test + void resolveRejectsNonAllowListedEndpoint() { + // A mutating endpoint is simply absent from the registry and is refused here. + assertThrows( + IllegalArgumentException.class, () -> RestEndpointRegistry.resolve("/_cluster/reroute")); + assertThrows( + IllegalArgumentException.class, + () -> RestEndpointRegistry.resolve("/services/server/info")); + } + + @Test + void validateRejectsUnknownArg() { + RestSpec spec = new RestSpec("/_cluster/health", Map.of("not_allowed", "x"), null, null); + assertThrows(IllegalArgumentException.class, () -> RestEndpointRegistry.validate(spec)); + } + + @Test + void validateAcceptsAllowedArg() { + RestSpec spec = new RestSpec("/_cluster/health", Map.of("local", "true"), null, null); + RestEndpointRegistry.validate(spec); // no throw + } + + @Test + void catEndpointsRedactAddressesWhenRedactionEnabled() { + Map node = new LinkedHashMap<>(); + node.put("name", "ip-10-0-0-7"); + node.put("ip", "10.0.0.7"); + node.put("node_role", "dir"); + node.put("heap_percent", 44); + node.put("ram_percent", 95); + node.put("cpu", 2); + when(client.catNodes(any())).thenReturn(List.of(node)); + + RestEndpointRegistry.Endpoint endpoint = RestEndpointRegistry.resolve("/_cat/nodes"); + RestSpec spec = new RestSpec("/_cat/nodes", Map.of(), null, null); + + Map redacted = endpoint.toRows(client, spec, true).get(0).tupleValue(); + assertEquals("x.x.x.x", redacted.get("ip").stringValue()); + assertEquals("", redacted.get("name").stringValue()); + assertEquals(44, redacted.get("heap_percent").integerValue()); + + Map plain = endpoint.toRows(client, spec, false).get(0).tupleValue(); + assertEquals("10.0.0.7", plain.get("ip").stringValue()); + assertEquals("ip-10-0-0-7", plain.get("name").stringValue()); + } + + @Test + void catClusterManagerRedactsHostAndIp() { + Map row = new LinkedHashMap<>(); + row.put("id", "fWhl6_ZQTaSJD9cJ82Ln2w"); + row.put("host", "10.0.0.7"); + row.put("ip", "10.0.0.7"); + row.put("node", "71d03b567bb755839a73d437b2b066d4"); + when(client.catClusterManager(any())).thenReturn(List.of(row)); + + RestEndpointRegistry.Endpoint endpoint = RestEndpointRegistry.resolve("/_cat/cluster_manager"); + RestSpec spec = new RestSpec("/_cat/cluster_manager", Map.of(), null, null); + + Map redacted = endpoint.toRows(client, spec, true).get(0).tupleValue(); + assertEquals("x.x.x.x", redacted.get("host").stringValue()); + assertEquals("x.x.x.x", redacted.get("ip").stringValue()); + assertEquals("fWhl6_ZQTaSJD9cJ82Ln2w", redacted.get("id").stringValue()); + assertEquals("71d03b567bb755839a73d437b2b066d4", redacted.get("node").stringValue()); + } + + @Test + void nonCatEndpointNotRedactedEvenWhenEnabled() { + Map health = new LinkedHashMap<>(); + health.put("cluster_name", "10.0.0.7"); + health.put("status", "green"); + health.put("number_of_nodes", 3); + when(client.clusterHealth(any())).thenReturn(health); + + RestEndpointRegistry.Endpoint endpoint = RestEndpointRegistry.resolve("/_cluster/health"); + RestSpec spec = new RestSpec("/_cluster/health", Map.of(), null, null); + + Map row = endpoint.toRows(client, spec, true).get(0).tupleValue(); + assertEquals("10.0.0.7", row.get("cluster_name").stringValue()); + } + + @Test + void clusterSettingsMasksAvailabilityZoneInValue() { + Map setting = new LinkedHashMap<>(); + setting.put("setting", "cluster.routing.allocation.awareness.attributes"); + setting.put("value", "zone:us-east-1a"); + setting.put("tier", "persistent"); + when(client.clusterSettings(any())).thenReturn(List.of(setting)); + + RestEndpointRegistry.Endpoint endpoint = RestEndpointRegistry.resolve("/_cluster/settings"); + RestSpec spec = new RestSpec("/_cluster/settings", Map.of(), null, null); + + Map row = endpoint.toRows(client, spec, true).get(0).tupleValue(); + assertEquals("zone:xx-xxxxx-xx", row.get("value").stringValue()); + assertEquals( + "cluster.routing.allocation.awareness.attributes", row.get("setting").stringValue()); + assertEquals("persistent", row.get("tier").stringValue()); + } + + @Test + void validateRejectsDroppedLevelArg() { + // level was dropped (no-op against the fixed cluster-level health schema); now unknown. + RestSpec spec = new RestSpec("/_cluster/health", Map.of("level", "indices"), null, null); + IllegalArgumentException ex = + assertThrows(IllegalArgumentException.class, () -> RestEndpointRegistry.validate(spec)); + assertTrue(ex.getMessage().contains("does not accept arg")); + } + + @Test + void validateRejectsDroppedFlatSettingsArg() { + // flat_settings was dropped (redundant: settings are already flattened to dotted keys). + RestSpec spec = new RestSpec("/_cluster/settings", Map.of("flat_settings", "true"), null, null); + assertThrows(IllegalArgumentException.class, () -> RestEndpointRegistry.validate(spec)); + } + + @Test + void validateAcceptsValidArgValues() { + RestEndpointRegistry.validate( + new RestSpec("/_cat/indices", Map.of("health", "green"), null, null)); + RestEndpointRegistry.validate( + new RestSpec("/_resolve/index", Map.of("expand_wildcards", "open"), null, null)); + RestEndpointRegistry.validate( + new RestSpec("/_resolve/index", Map.of("expand_wildcards", "open,closed"), null, null)); + } + + @Test + void validateRejectsBadArgValue() { + IllegalArgumentException health = + assertThrows( + IllegalArgumentException.class, + () -> + RestEndpointRegistry.validate( + new RestSpec("/_cat/indices", Map.of("health", "purple"), null, null))); + assertTrue(health.getMessage().contains("unsupported value")); + + IllegalArgumentException local = + assertThrows( + IllegalArgumentException.class, + () -> + RestEndpointRegistry.validate( + new RestSpec("/_cluster/health", Map.of("local", "maybe"), null, null))); + assertTrue(local.getMessage().contains("unsupported value")); + + assertThrows( + IllegalArgumentException.class, + () -> + RestEndpointRegistry.validate( + new RestSpec( + "/_resolve/index", Map.of("expand_wildcards", "sideways"), null, null))); + } + + @Test + void resolveRejectsBlankEndpoint() { + IllegalArgumentException emptyEx = + assertThrows(IllegalArgumentException.class, () -> RestEndpointRegistry.resolve("")); + assertTrue(emptyEx.getMessage().contains("non-empty path")); + assertThrows(IllegalArgumentException.class, () -> RestEndpointRegistry.resolve(" ")); + assertThrows(IllegalArgumentException.class, () -> RestEndpointRegistry.resolve(null)); + } + + @Test + void validateRejectsNegativeCount() { + RestSpec spec = new RestSpec("/_cat/indices", Map.of(), -1, null); + IllegalArgumentException ex = + assertThrows(IllegalArgumentException.class, () -> RestEndpointRegistry.validate(spec)); + assertTrue(ex.getMessage().contains("non-negative")); + } + + @Test + void validateAcceptsZeroCount() { + RestSpec spec = new RestSpec("/_cat/indices", Map.of(), 0, null); + RestEndpointRegistry.validate(spec); // no throw: 0 is a valid limit + } + + @Test + void validateRejectsTimeoutArg() { + RestSpec spec = new RestSpec("/_cluster/health", Map.of(), null, "5s"); + IllegalArgumentException ex = + assertThrows(IllegalArgumentException.class, () -> RestEndpointRegistry.validate(spec)); + assertTrue(ex.getMessage().contains("timeout")); + } + + @Test + void coerceParsesNumericStringValues() { + // The cat JSON API returns numeric columns as strings; coerce must parse them. + Map health = new LinkedHashMap<>(); + health.put("status", "green"); + health.put("number_of_nodes", "3"); + when(client.clusterHealth(any())).thenReturn(health); + + RestEndpointRegistry.Endpoint endpoint = RestEndpointRegistry.resolve("/_cluster/health"); + List rows = + endpoint.toRows(client, new RestSpec("/_cluster/health", Map.of(), null, null)); + + assertEquals(3, rows.get(0).tupleValue().get("number_of_nodes").integerValue()); + } + + @Test + void coerceThrowsClearErrorOnUncoercibleValue() { + // A non-numeric value for an INTEGER column must surface a clear client error (HTTP 400), + // not a raw ClassCastException / NumberFormatException (HTTP 500). + Map health = new LinkedHashMap<>(); + health.put("status", "green"); + health.put("number_of_nodes", "not-a-number"); + when(client.clusterHealth(any())).thenReturn(health); + + RestEndpointRegistry.Endpoint endpoint = RestEndpointRegistry.resolve("/_cluster/health"); + IllegalArgumentException ex = + assertThrows( + IllegalArgumentException.class, + () -> endpoint.toRows(client, new RestSpec("/_cluster/health", Map.of(), null, null))); + assertTrue(ex.getMessage().contains("number_of_nodes")); + assertTrue(ex.getMessage().contains("not-a-number")); + } + + @Test + void clusterHealthRowsAreShapedToFixedSchema() { + Map health = new LinkedHashMap<>(); + health.put("cluster_name", "test-cluster"); + health.put("status", "green"); + health.put("number_of_nodes", 1); + when(client.clusterHealth(any())).thenReturn(health); + + RestEndpointRegistry.Endpoint endpoint = RestEndpointRegistry.resolve("/_cluster/health"); + List rows = + endpoint.toRows(client, new RestSpec("/_cluster/health", Map.of(), null, null)); + + assertEquals(1, rows.size()); + assertEquals("green", rows.get(0).tupleValue().get("status").stringValue()); + assertEquals(1, rows.get(0).tupleValue().get("number_of_nodes").integerValue()); + // a declared column the action did not return becomes null, never absent. + assertTrue(rows.get(0).tupleValue().get("relocating_shards").isNull()); + } + + @Test + void catIndicesRowsAreShapedToFixedSchema() { + Map idx = new LinkedHashMap<>(); + idx.put("index", "books"); + idx.put("health", "yellow"); + idx.put("pri", 1); + idx.put("rep", 1); + when(client.catIndices(any())).thenReturn(List.of(idx)); + + RestEndpointRegistry.Endpoint endpoint = RestEndpointRegistry.resolve("/_cat/indices"); + List rows = + endpoint.toRows(client, new RestSpec("/_cat/indices", Map.of(), null, null)); + + assertEquals(1, rows.size()); + assertEquals("books", rows.get(0).tupleValue().get("index").stringValue()); + assertEquals("yellow", rows.get(0).tupleValue().get("health").stringValue()); + } +} diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestResponseRedactorTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestResponseRedactorTest.java new file mode 100644 index 00000000000..3b038306da5 --- /dev/null +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestResponseRedactorTest.java @@ -0,0 +1,98 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.rest; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.junit.jupiter.api.Test; + +class RestResponseRedactorTest { + + @Test + void masksIpv4() { + assertEquals("x.x.x.x", RestResponseRedactor.redact("0.0.0.0")); + assertEquals("x.x.x.x", RestResponseRedactor.redact("255.255.255.255")); + assertEquals("x.x.x.x", RestResponseRedactor.redact("192.168.1.1")); + assertEquals("x.x.x.x", RestResponseRedactor.redact("1.2.3.4")); + assertEquals("x.x.x.x:9200", RestResponseRedactor.redact("10.0.0.1:9200")); + assertEquals("a x.x.x.x b x.x.x.x c", RestResponseRedactor.redact("a 10.0.0.1 b 172.16.5.4 c")); + } + + @Test + void doesNotMaskInvalidOrPartialIpv4() { + assertEquals("256.1.1.1", RestResponseRedactor.redact("256.1.1.1")); + assertEquals("1.2.3", RestResponseRedactor.redact("1.2.3")); + assertEquals("44", RestResponseRedactor.redact("44")); + } + + @Test + void masksEc2HostName() { + assertEquals("", RestResponseRedactor.redact("ip-10-0-0-1")); + assertEquals("", RestResponseRedactor.redact("ip-172-31-255-9")); + assertEquals("node here", RestResponseRedactor.redact("node ip-10-1-2-3 here")); + assertEquals("ip-256-0-0-1", RestResponseRedactor.redact("ip-256-0-0-1")); + } + + @Test + void masksFullIpv6() { + assertEquals("x.x.x.x", RestResponseRedactor.redact("fe80:0:0:0:0:0:0:1")); + assertEquals("x.x.x.x", RestResponseRedactor.redact("2001:0db8:85a3:0000:0000:8a2e:0370:7334")); + assertEquals("x.x.x.x", RestResponseRedactor.redact("FE80:0:0:0:0:0:0:1")); + } + + @Test + void masksCompressedIpv6() { + assertEquals("x.x.x.x", RestResponseRedactor.redact("::1")); + assertEquals("x.x.x.x", RestResponseRedactor.redact("fe80::1")); + assertEquals("x.x.x.x", RestResponseRedactor.redact("2001:db8::1")); + assertEquals("x.x.x.x", RestResponseRedactor.redact("2001:db8::8a2e:370:7334")); + } + + @Test + void masksInetAddress() { + assertEquals("inet[/x.x.x.x:9200]", RestResponseRedactor.redact("inet[/10.0.0.7:9200]")); + } + + @Test + void masksAvailabilityZones() { + assertEquals("xx-xxxxx-xx", RestResponseRedactor.redact("us-east-1a")); + assertEquals("xx-xxxxx-xx", RestResponseRedactor.redact("ap-southeast-2b")); + assertEquals("xx-xxxxx-xx", RestResponseRedactor.redact("eu-west-1c")); + assertEquals("xx-xxxxx-xx", RestResponseRedactor.redact("us-gov-west-1a")); + // Shape-based match covers regions not in any hard-coded list (e.g. mx-central-1). + assertEquals("xx-xxxxx-xx", RestResponseRedactor.redact("mx-central-1a")); + assertEquals( + "a xx-xxxxx-xx b xx-xxxxx-xx", RestResponseRedactor.redact("a us-east-1a b us-west-2b")); + } + + @Test + void maskAvailabilityZoneMasksOnlyZones() { + assertEquals("xx-xxxxx-xx", RestResponseRedactor.maskAvailabilityZone("us-east-1a")); + assertEquals("10.0.0.7", RestResponseRedactor.maskAvailabilityZone("10.0.0.7")); + assertEquals("ip-10-0-0-1", RestResponseRedactor.maskAvailabilityZone("ip-10-0-0-1")); + } + + @Test + void leavesNonAddressesIntact() { + assertEquals( + "e4e136ea81e27370ff73cf753ba22d39", + RestResponseRedactor.redact("e4e136ea81e27370ff73cf753ba22d39")); + assertEquals( + "data,ingest,remote_cluster_client", + RestResponseRedactor.redact("data,ingest,remote_cluster_client")); + assertEquals( + "x.x.x.x 44 95 imr - e4e136ea", + RestResponseRedactor.redact("10.0.0.7 44 95 imr - e4e136ea")); + } + + @Test + void handlesNullAndEmpty() { + assertEquals(null, RestResponseRedactor.redact(null)); + assertEquals("", RestResponseRedactor.redact("")); + assertEquals(null, RestResponseRedactor.maskAvailabilityZone(null)); + assertEquals("", RestResponseRedactor.maskAvailabilityZone("")); + } +} diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/system/OpenSearchSystemIndexTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/system/OpenSearchCatalogTableTest.java similarity index 76% rename from opensearch/src/test/java/org/opensearch/sql/opensearch/storage/system/OpenSearchSystemIndexTest.java rename to opensearch/src/test/java/org/opensearch/sql/opensearch/storage/system/OpenSearchCatalogTableTest.java index 0b0aa1ec521..df81225afbe 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/system/OpenSearchSystemIndexTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/system/OpenSearchCatalogTableTest.java @@ -32,8 +32,12 @@ import org.opensearch.sql.planner.physical.ProjectOperator; import org.opensearch.sql.storage.Table; +/** + * Covers the generic {@link OpenSearchCatalogTable} through a {@link SystemIndexCatalogSource}: + * schema delegation, the predefined-table contract, and the V2 physical path. + */ @ExtendWith(MockitoExtension.class) -class OpenSearchSystemIndexTest { +class OpenSearchCatalogTableTest { @Mock private OpenSearchClient client; @@ -41,36 +45,37 @@ class OpenSearchSystemIndexTest { @Mock private Settings settings; + private OpenSearchCatalogTable systemTable(String name) { + return new OpenSearchCatalogTable(new SystemIndexCatalogSource(client, name), settings); + } + @Test void testGetFieldTypesOfMetaTable() { - OpenSearchSystemIndex systemIndex = new OpenSearchSystemIndex(client, settings, TABLE_INFO); - final Map fieldTypes = systemIndex.getFieldTypes(); + final Map fieldTypes = systemTable(TABLE_INFO).getFieldTypes(); assertThat(fieldTypes, anyOf(hasEntry("TABLE_CAT", STRING))); } @Test void testGetFieldTypesOfMappingTable() { - OpenSearchSystemIndex systemIndex = - new OpenSearchSystemIndex(client, settings, mappingTable("test_index")); - final Map fieldTypes = systemIndex.getFieldTypes(); + final Map fieldTypes = + systemTable(mappingTable("test_index")).getFieldTypes(); assertThat(fieldTypes, anyOf(hasEntry("COLUMN_NAME", STRING))); } @Test void testIsExist() { - Table systemIndex = new OpenSearchSystemIndex(client, settings, TABLE_INFO); - assertTrue(systemIndex.exists()); + assertTrue(systemTable(TABLE_INFO).exists()); } @Test void testCreateTable() { - Table systemIndex = new OpenSearchSystemIndex(client, settings, TABLE_INFO); + Table systemIndex = systemTable(TABLE_INFO); assertThrows(UnsupportedOperationException.class, () -> systemIndex.create(ImmutableMap.of())); } @Test void implement() { - OpenSearchSystemIndex systemIndex = new OpenSearchSystemIndex(client, settings, TABLE_INFO); + OpenSearchCatalogTable systemIndex = systemTable(TABLE_INFO); NamedExpression projectExpr = named("TABLE_NAME", ref("TABLE_NAME", STRING)); final PhysicalPlan plan = diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java b/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java index ab8b923e3ae..48ad5c0b0a8 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java +++ b/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java @@ -197,6 +197,10 @@ public List getRestHandlers( Metrics.getInstance().registerDefaultMetrics(); + // Publish the node SettingsFilter so the in-cluster `rest '/_cluster/settings'` fetcher can + // redact filtered settings exactly as the native GET /_cluster/settings endpoint does. + org.opensearch.sql.opensearch.storage.rest.RestSettingsFilterHolder.set(settingsFilter); + return Arrays.asList( new RestPPLQueryAction(), new RestPPLGrammarAction(), diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestUnifiedQueryAction.java b/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestUnifiedQueryAction.java index 62f3dd0c346..943bae38dc6 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestUnifiedQueryAction.java +++ b/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestUnifiedQueryAction.java @@ -99,13 +99,14 @@ public boolean isAnalyticsIndex(String query, QueryType queryType) { .equals( IndicesService.CLUSTER_PLUGGABLE_DATAFORMAT_VALUE_SETTING.get( clusterService.getSettings()))) { - // Analytics engine can't serve system catalog; SHOW/DESCRIBE fall back to default pipeline + // Analytics engine serves neither the system catalog nor the rest command's reserved + // in-cluster source; both fall back to the default (Calcite) pipeline. try (UnifiedQueryContext context = buildParsingContext(queryType)) { - boolean systemCatalog = + boolean defaultPipeline = extractIndexName(query, queryType, context) - .map(RestUnifiedQueryAction::isSystemCatalog) + .map(name -> isSystemCatalog(name) || SystemIndexUtils.isRestSource(name)) .orElse(false); - return !systemCatalog; + return !defaultPipeline; } catch (Exception e) { // Check legacy-syntax SHOW/DESCRIBE; otherwise let AE handle and surface the error. return !isLegacySystemCatalogQuery(query); diff --git a/plugin/src/test/java/org/opensearch/sql/plugin/rest/RestUnifiedQueryActionTest.java b/plugin/src/test/java/org/opensearch/sql/plugin/rest/RestUnifiedQueryActionTest.java index 111597bb587..0cf87f0604e 100644 --- a/plugin/src/test/java/org/opensearch/sql/plugin/rest/RestUnifiedQueryActionTest.java +++ b/plugin/src/test/java/org/opensearch/sql/plugin/rest/RestUnifiedQueryActionTest.java @@ -178,6 +178,12 @@ public void describeStatementNotRoutedToAnalyticsEngineUnderClusterComposite() { assertFalse(action.isAnalyticsIndex("DESCRIBE TABLES LIKE 'parquet_logs'", QueryType.SQL)); } + @Test + public void restCommandNotRoutedToAnalyticsEngineUnderClusterComposite() { + enableClusterComposite(); + assertFalse(action.isAnalyticsIndex("| rest '/_cluster/health'", QueryType.PPL)); + } + @Test public void dataQueryStillRoutesToAnalyticsUnderClusterComposite() { enableClusterComposite(); diff --git a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 index 4bc69a8f295..197886342e7 100644 --- a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 @@ -13,6 +13,8 @@ options { caseInsensitive = true; } SEARCH: 'SEARCH'; DESCRIBE: 'DESCRIBE'; SHOW: 'SHOW'; +REST: 'REST'; +TIMEOUT: 'TIMEOUT'; EXPLAIN: 'EXPLAIN'; FROM: 'FROM'; WHERE: 'WHERE'; diff --git a/ppl/src/main/antlr/OpenSearchPPLParser.g4 b/ppl/src/main/antlr/OpenSearchPPLParser.g4 index 98f85b08282..8dca6a54ba0 100644 --- a/ppl/src/main/antlr/OpenSearchPPLParser.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLParser.g4 @@ -45,6 +45,7 @@ subSearch // commands pplCommands : describeCommand + | restCommand | showDataSourcesCommand | searchCommand | multisearchCommand @@ -103,6 +104,7 @@ commands commandName : SEARCH | DESCRIBE + | REST | SHOW | WHERE | FIELDS @@ -206,6 +208,16 @@ describeCommand : DESCRIBE tableSourceClause ; + +restCommand + : REST stringLiteral (restArgument)* + ; + +restArgument + : COUNT EQUAL integerLiteral + | TIMEOUT EQUAL stringLiteral + | ident EQUAL literalValue + ; showDataSourcesCommand : SHOW DATASOURCES ; @@ -1773,4 +1785,6 @@ searchableKeyWord | MAX_DEPTH | DEPTH_FIELD | EDGE + // rest command token, also usable as a free-text search term / identifier + | TIMEOUT ; diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java index 3741137f5a9..546450b8191 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java @@ -109,6 +109,7 @@ import org.opensearch.sql.ast.tree.Rename; import org.opensearch.sql.ast.tree.Replace; import org.opensearch.sql.ast.tree.ReplacePair; +import org.opensearch.sql.ast.tree.RestRelation; import org.opensearch.sql.ast.tree.Reverse; import org.opensearch.sql.ast.tree.Rex; import org.opensearch.sql.ast.tree.SPath; @@ -140,6 +141,7 @@ import org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParserBaseVisitor; import org.opensearch.sql.ppl.utils.ArgumentFactory; import org.opensearch.sql.ppl.utils.UnresolvedPlanHelper; +import org.opensearch.sql.utils.SystemIndexUtils; /** Class of building the AST. Refines the visit path and build the AST nodes */ public class AstBuilder extends OpenSearchPPLParserBaseVisitor { @@ -251,6 +253,37 @@ public UnresolvedPlan visitShowDataSourcesCommand( return new DescribeRelation(qualifiedName(DATASOURCES_TABLE_NAME)); } + /** + * Rest command.
+ * Leading command that reads an allow-listed, read-only in-cluster management endpoint + * (cluster/cat/nodes) as rows. The validated endpoint spec is encoded into a single reserved + * table name via {@link org.opensearch.sql.utils.SystemIndexUtils#restTable}; that name resolves + * through the storage engine to a REST source table on the Calcite path, mirroring how DESCRIBE + * resolves to a system index. Allow-list/authorization enforcement happens at source-table + * construction in the storage engine (it owns the transport actions and per-endpoint schemas). + */ + @Override + public UnresolvedPlan visitRestCommand(OpenSearchPPLParser.RestCommandContext ctx) { + String endpoint = StringUtils.unquoteText(ctx.stringLiteral().getText()); + LinkedHashMap args = new LinkedHashMap<>(); + Integer count = null; + String timeout = null; + for (OpenSearchPPLParser.RestArgumentContext arg : ctx.restArgument()) { + if (arg.COUNT() != null) { + count = Integer.parseInt(arg.integerLiteral().getText()); + } else if (arg.TIMEOUT() != null) { + timeout = StringUtils.unquoteText(arg.stringLiteral().getText()); + } else { + args.put( + StringUtils.unquoteIdentifier(arg.ident().getText()), + StringUtils.unquoteText(arg.literalValue().getText())); + } + } + String token = + SystemIndexUtils.restTable(new SystemIndexUtils.RestSpec(endpoint, args, count, timeout)); + return new RestRelation(new QualifiedName(token)); + } + /** Where command. */ @Override public UnresolvedPlan visitWhereCommand(WhereCommandContext ctx) { diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java b/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java index 4b75d444467..fbd447c4e0d 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java @@ -96,6 +96,7 @@ import org.opensearch.sql.ast.tree.Relation; import org.opensearch.sql.ast.tree.Rename; import org.opensearch.sql.ast.tree.Replace; +import org.opensearch.sql.ast.tree.RestRelation; import org.opensearch.sql.ast.tree.Reverse; import org.opensearch.sql.ast.tree.Rex; import org.opensearch.sql.ast.tree.SPath; @@ -122,6 +123,7 @@ import org.opensearch.sql.planner.logical.LogicalRemove; import org.opensearch.sql.planner.logical.LogicalRename; import org.opensearch.sql.planner.logical.LogicalSort; +import org.opensearch.sql.utils.SystemIndexUtils; /** Utility class to mask sensitive information in incoming PPL queries. */ public class PPLQueryDataAnonymizer extends AbstractNodeVisitor { @@ -165,6 +167,23 @@ public String visitExplain(Explain node, String context) { @Override public String visitRelation(Relation node, String context) { + if (node instanceof RestRelation) { + SystemIndexUtils.RestSpec spec = + SystemIndexUtils.decodeRestSpec(node.getTableQualifiedName().toString()); + StringBuilder sb = new StringBuilder("rest ").append(spec.getEndpoint()); + if (spec.getCount() != null) { + sb.append(" count=").append(MASK_LITERAL); + } + if (spec.getTimeout() != null) { + sb.append(" timeout=").append(MASK_LITERAL); + } + if (spec.getArgs() != null) { + for (String key : spec.getArgs().keySet()) { + sb.append(' ').append(key).append('=').append(MASK_LITERAL); + } + } + return sb.toString(); + } if (node instanceof DescribeRelation) { return StringUtils.format("describe %s", MASK_TABLE); } diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLRestTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLRestTest.java new file mode 100644 index 00000000000..da9424c9c8f --- /dev/null +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLRestTest.java @@ -0,0 +1,67 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.ppl.calcite; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; + +import org.junit.Test; +import org.opensearch.sql.ast.Node; +import org.opensearch.sql.ast.tree.Project; +import org.opensearch.sql.ast.tree.RestRelation; +import org.opensearch.sql.common.setting.Settings; +import org.opensearch.sql.ppl.antlr.PPLSyntaxParser; +import org.opensearch.sql.ppl.parser.AstBuilder; +import org.opensearch.sql.utils.SystemIndexUtils; + +/** + * Calcite-path coverage for the {@code rest} leading command at the parse / AST tier. + * + *

The {@code rest} row source resolves through {@code OpenSearchStorageEngine.getTable} -> + * {@code RestCatalogSource} -> {@code CalciteLogicalCatalogScan}, which lives in the {@code + * opensearch} module. This ppl-module Calcite harness binds a Calcite SCOTT schema rather than the + * OpenSearch storage engine, so the optimized {@code CalciteScannableCatalogScan} logical-plan + * assertion is exercised in {@code RestCatalogSourceTest} (source schema + request, unit) and + * {@code CalcitePPLRestIT} (schema + datarows on a live single-node cluster). This test pins the + * Calcite-facing contract that the ppl module owns: the grammar/AST rewrite of {@code rest} into a + * {@code RestRelation} carrying the validated, reserved-name-encoded endpoint spec that rides + * {@code visitRelation} exactly like {@code DESCRIBE}. + */ +public class CalcitePPLRestTest { + + private final PPLSyntaxParser parser = new PPLSyntaxParser(); + private final Settings settings = mock(Settings.class); + + private Node parse(String ppl) { + return new AstBuilder(ppl, settings).visit(parser.parse(ppl)); + } + + @Test + public void restHealthProjectsDeclaredColumns() { + Project project = + (Project) parse("| rest \"/_cluster/health\" | fields status, number_of_nodes"); + RestRelation rest = (RestRelation) project.getChild().get(0); + SystemIndexUtils.RestSpec spec = + SystemIndexUtils.decodeRestSpec(rest.getTableQualifiedName().toString()); + assertEquals("/_cluster/health", spec.getEndpoint()); + // downstream fields compose on top of the rest row source. + assertEquals(2, project.getProjectList().size()); + } + + @Test + public void restReservedNameRoundTrips() { + RestRelation rest = + (RestRelation) parse("| rest \"/_cat/indices\" count=10 timeout=\"5s\" health=\"green\""); + String reserved = rest.getTableQualifiedName().toString(); + assertTrue(SystemIndexUtils.isRestSource(reserved)); + SystemIndexUtils.RestSpec spec = SystemIndexUtils.decodeRestSpec(reserved); + assertEquals("/_cat/indices", spec.getEndpoint()); + assertEquals(Integer.valueOf(10), spec.getCount()); + assertEquals("5s", spec.getTimeout()); + assertEquals("green", spec.getArgs().get("health")); + } +} diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java index 9d70487c741..d57ca8a69bb 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java @@ -1939,4 +1939,32 @@ public void testJoinNoPrefixComparisonStaysCondition() { public void testJoinPrefixWithoutCriteriaKeywordIsSyntaxError() { assertThrows(SyntaxCheckException.class, () -> plan("source=t1 | inner join a t2")); } + + // rest command tests + + @Test + public void testRestCommand() { + org.opensearch.sql.ast.tree.Project project = + (org.opensearch.sql.ast.tree.Project) + plan("| rest \"/_cluster/health\" | fields status, number_of_nodes"); + org.opensearch.sql.ast.tree.RestRelation rest = + (org.opensearch.sql.ast.tree.RestRelation) project.getChild().get(0); + SystemIndexUtils.RestSpec spec = + SystemIndexUtils.decodeRestSpec(rest.getTableQualifiedName().toString()); + assertEquals("/_cluster/health", spec.getEndpoint()); + assertTrue(spec.getArgs().isEmpty()); + } + + @Test + public void testRestCommandWithArgs() { + org.opensearch.sql.ast.tree.RestRelation rest = + (org.opensearch.sql.ast.tree.RestRelation) + plan("| rest \"/_cluster/health\" count=5 timeout=\"30s\" level=\"indices\""); + SystemIndexUtils.RestSpec spec = + SystemIndexUtils.decodeRestSpec(rest.getTableQualifiedName().toString()); + assertEquals("/_cluster/health", spec.getEndpoint()); + assertEquals(Integer.valueOf(5), spec.getCount()); + assertEquals("30s", spec.getTimeout()); + assertEquals("indices", spec.getArgs().get("level")); + } } diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java index 6756cdc198a..21c333166f1 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java @@ -48,6 +48,18 @@ public void testPrometheusPPLCommand() { assertEquals("source=table", anonymize("source=prometheus.http_requests_process")); } + @Test + public void testRestCommand() { + assertEquals("rest /_cluster/health", anonymize("| rest \"/_cluster/health\"")); + } + + @Test + public void testRestCommandMasksArgValues() { + assertEquals( + "rest /_cluster/health count=*** timeout=*** level=***", + anonymize("| rest \"/_cluster/health\" count=5 timeout=\"30s\" level=\"indices\"")); + } + @Test public void testWhereCommand() { assertEquals("source=table | where identifier = ***", anonymize("search source=t | where a=1"));