From bf890b24838ecba6b323151b39043b4af0ba3526 Mon Sep 17 00:00:00 2001 From: Timothy Mugo Date: Wed, 26 Aug 2026 12:16:54 +0300 Subject: [PATCH 1/3] feat: enable superset stats API to fetch extra columns that are not part of the dimensions --- .../viz/gateway/services/StatsService.java | 50 ++++++++++++++++--- .../viz/gateway/services/Utils.java | 44 +++++++++++++++- .../src/main/resources/application-dev.yml | 2 +- 3 files changed, 86 insertions(+), 10 deletions(-) diff --git a/superset-proxy/src/main/java/org/devgateway/viz/gateway/services/StatsService.java b/superset-proxy/src/main/java/org/devgateway/viz/gateway/services/StatsService.java index 19f4c529..6c1e5dd7 100644 --- a/superset-proxy/src/main/java/org/devgateway/viz/gateway/services/StatsService.java +++ b/superset-proxy/src/main/java/org/devgateway/viz/gateway/services/StatsService.java @@ -42,7 +42,8 @@ public Object getStats(String datasetId, Map queryParams, String } Map labelMap = buildLabelMap(datasetResult); - JsonNode requestBody = buildSupersetDataRequest(datasetResult, datasetId, queryParams, groupsPath); + List extraColumns = resolveIncludeColumns(queryParams, datasetResult); + JsonNode requestBody = buildSupersetDataRequest(datasetResult, datasetId, queryParams, groupsPath, extraColumns); boolean force = "true".equals(queryParams.get("force")); @@ -67,12 +68,38 @@ public Object getStats(String datasetId, Map queryParams, String getGroupsArray(groupsPath), labelMap, isCached, - cachedAt); + cachedAt, + extraColumns); } logger.warn("Returning an Empty list"); return Collections.emptyList(); } + // Dimensions requested via includeColumns are returned as an extra per-row value without being + // part of the query breakdown. Validated against the dataset's real columns so a client-supplied + // name can never be interpolated into the SQL expression sent to Superset. + private List resolveIncludeColumns(Map queryParams, JsonNode datasetResult) { + String includeColumnsParam = queryParams.get("includeColumns"); + if (includeColumnsParam == null || includeColumnsParam.isBlank()) { + return Collections.emptyList(); + } + + Set knownColumns = extractAllColumns(datasetResult); + List resolved = new ArrayList<>(); + for (String name : includeColumnsParam.split(",")) { + String trimmed = name.trim(); + if (trimmed.isEmpty()) { + continue; + } + if (knownColumns.contains(trimmed)) { + resolved.add(trimmed); + } else { + logger.warn("includeColumns requested unknown column: " + trimmed); + } + } + return resolved; + } + private JsonNode getDatasetResult(String datasetId) { JsonNode dataset = superSetClient.fetchDataset(datasetId); if (dataset == null || !dataset.has("result")) { @@ -113,7 +140,7 @@ private Map buildLabelMap(JsonNode datasetResult) { return labels; } - private JsonNode buildSupersetDataRequest(JsonNode datasetResult, String datasetId, Map queryParams, String groupsPath) { + private JsonNode buildSupersetDataRequest(JsonNode datasetResult, String datasetId, Map queryParams, String groupsPath, List extraColumns) { List measuresArr = extractUniqueMeasures(datasetResult); Set filterableColumns = extractFilterableColumns(datasetResult); @@ -125,12 +152,12 @@ private JsonNode buildSupersetDataRequest(JsonNode datasetResult, String dataset String[] groupsArray = getGroupsArray(groupsPath); if (groupsArray.length > 0) { - Map queryForDimension1 = createQuery(new String[]{groupsArray[0]}, measuresArr, filterableColumns, queryParams); + Map queryForDimension1 = createQuery(new String[]{groupsArray[0]}, measuresArr, filterableColumns, queryParams, extraColumns); queries.add(queryForDimension1); } if (groupsArray.length > 1) { - Map queryForDimension2 = createQuery(groupsArray, measuresArr, filterableColumns, queryParams); + Map queryForDimension2 = createQuery(groupsArray, measuresArr, filterableColumns, queryParams, extraColumns); queries.add(queryForDimension2); } @@ -142,7 +169,7 @@ private JsonNode buildSupersetDataRequest(JsonNode datasetResult, String dataset } private Object transformData(JsonNode data, JsonNode dimensionsNode, JsonNode metricsNode, String[] dimensionsArray, - Map labelMap, boolean isCached, String cachedAt) { + Map labelMap, boolean isCached, String cachedAt, List extraColumns) { Map transformed = createTransformedData(); List> measuresList = new ArrayList<>(); List> typesList = new ArrayList<>(); @@ -157,6 +184,10 @@ private Object transformData(JsonNode data, JsonNode dimensionsNode, JsonNode me } for (JsonNode metric : metricsNode) { + // Extra-column adhoc metrics are objects, not plain metric name strings; skip them here. + if (!metric.isTextual()) { + continue; + } String metricName = metric.asText(); measuresList.add(createMeasure(metricName, labelMap.getOrDefault(metricName, metricName))); } @@ -169,6 +200,9 @@ private Object transformData(JsonNode data, JsonNode dimensionsNode, JsonNode me JsonNode overallData = data.get(data.size() - 1); if (overallData != null && overallData.has("data")) { for (JsonNode metric : metricsNode) { + if (!metric.isTextual()) { + continue; + } transformed.put(metric.asText(), overallData.get("data").get(0).get(metric.asText()).asDouble()); } @@ -189,7 +223,7 @@ private Object transformData(JsonNode data, JsonNode dimensionsNode, JsonNode me .ifPresent(type -> ((Set>) type.computeIfAbsent("items", k -> new HashSet<>())) .add(createItem(firstDim, row.get(firstDim).asText(), Constants.COLORS.get(0)))); - Map dataItem = createDataItem(firstDim, row.get(firstDim).asText(), metricsNode, row); + Map dataItem = createDataItem(firstDim, row.get(firstDim).asText(), metricsNode, row, extraColumns); ((List>) transformed.get("children")).add(dataItem); } } @@ -206,7 +240,7 @@ private Object transformData(JsonNode data, JsonNode dimensionsNode, JsonNode me .ifPresent(type -> ((Set>) type.computeIfAbsent("items", k -> new HashSet<>())) .add(createItem(secondDim, row.get(secondDim).asText(), Constants.COLORS.get(0)))); - Map dataItem = createDataItem(secondDim, row.get(secondDim).asText(), metricsNode, row); + Map dataItem = createDataItem(secondDim, row.get(secondDim).asText(), metricsNode, row, extraColumns); for (Map child : (List>) transformed.get("children")) { if (child.get("value").equals(row.get(firstDim).asText())) { ((List>) child.computeIfAbsent("children", k -> new ArrayList<>())).add(dataItem); diff --git a/superset-proxy/src/main/java/org/devgateway/viz/gateway/services/Utils.java b/superset-proxy/src/main/java/org/devgateway/viz/gateway/services/Utils.java index 37f2543d..5162a246 100644 --- a/superset-proxy/src/main/java/org/devgateway/viz/gateway/services/Utils.java +++ b/superset-proxy/src/main/java/org/devgateway/viz/gateway/services/Utils.java @@ -48,11 +48,22 @@ public static Map createQuery(String field, Set filterab public static Map createQuery(String[] groupArray, List measuresArr, Set filterableColumns, Map queryParams) { + return createQuery(groupArray, measuresArr, filterableColumns, queryParams, Collections.emptyList()); + } + + // extraColumns are dimensions requested via includeColumns: returned per-row + // without being added to "columns"/groupby, so they don't affect the query breakdown. + public static Map createQuery(String[] groupArray, List measuresArr, + Set filterableColumns, Map queryParams, List extraColumns) { SortedMap query = new TreeMap<>(); // must be sorted to ensure consistent order for caching purposes if (groupArray.length > 0) { query.put("columns", Arrays.asList(groupArray)); } - query.put("metrics", measuresArr); + List metrics = new ArrayList<>(measuresArr); + for (String column : extraColumns != null ? extraColumns : Collections.emptyList()) { + metrics.add(createExtraColumnMetric(column)); + } + query.put("metrics", metrics); query.put("row_limit", Constants.ROW_LIMIT); List> filters = new ArrayList<>(); @@ -76,6 +87,14 @@ public static Map createQuery(String[] groupArray, List return query; } + public static Map createExtraColumnMetric(String column) { + Map metric = new HashMap<>(); + metric.put("expressionType", "SQL"); + metric.put("sqlExpression", "MAX(" + column + ")"); + metric.put("label", column); + return metric; + } + public static Map createSupersetRequest(Map datasource, List> queries) { Map supersetRequest = new HashMap<>(); supersetRequest.put("datasource", datasource); @@ -138,15 +157,27 @@ public static Map createType(String dim) { } public static Map createDataItem(String dim, String value, JsonNode metricsNode, JsonNode row) { + return createDataItem(dim, value, metricsNode, row, Collections.emptyList()); + } + + public static Map createDataItem(String dim, String value, JsonNode metricsNode, JsonNode row, List extraColumns) { Map dataItem = new HashMap<>(); dataItem.put("type", dim); dataItem.put("value", value); for (JsonNode metric : metricsNode) { + // Extra-column adhoc metrics are objects, not plain metric name strings; skip them here. + if (!metric.isTextual()) { + continue; + } String metricName = metric.asText(); dataItem.put(metricName, row.hasNonNull(metricName) ? row.get(metricName).asDouble() : null); } + for (String column : extraColumns != null ? extraColumns : Collections.emptyList()) { + dataItem.put(column, row.hasNonNull(column) ? row.get(column).asText() : null); + } + return dataItem; } @@ -181,6 +212,17 @@ public static List extractUniqueMeasures(JsonNode result) { return measures; } + public static Set extractAllColumns(JsonNode result) { + Set columnNames = new LinkedHashSet<>(); + for (JsonNode column : result.get("columns")) { + String name = column.path("column_name").asText(null); + if (name != null && !name.isEmpty()) { + columnNames.add(name); + } + } + return columnNames; + } + public static Set extractFilterableColumns(JsonNode result) { Set columnNames = new LinkedHashSet<>(); for (JsonNode column : result.get("columns")) { diff --git a/superset-proxy/src/main/resources/application-dev.yml b/superset-proxy/src/main/resources/application-dev.yml index b187ad1a..27b7e951 100644 --- a/superset-proxy/src/main/resources/application-dev.yml +++ b/superset-proxy/src/main/resources/application-dev.yml @@ -22,7 +22,7 @@ spring: viz: superset: - url: https://superset.alive.dgstg.org + url: ${SUPERSET_URL:http://localhost:8088} warmup: top: 100 From 5b6f0c4ff94f5e34e9d9cc012120f061f315d984 Mon Sep 17 00:00:00 2001 From: Timothy Mugo Date: Wed, 26 Aug 2026 12:39:50 +0300 Subject: [PATCH 2/3] chore: update gh actions ci --- .github/workflows/release-template.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release-template.yml b/.github/workflows/release-template.yml index d5ec0b4d..c72797e1 100644 --- a/.github/workflows/release-template.yml +++ b/.github/workflows/release-template.yml @@ -83,8 +83,8 @@ jobs: with: github_token: ${{ secrets.GITHUB_TOKEN }} tag_prefix: ${{ inputs.project_name }}@v - release_branches: production,release/* - pre_release_branches: main + release_branches: __no-stable-release-branch__ + pre_release_branches: .* append_to_pre_release_tag: rc dry_run: "true" From f4bd70ddf0cae7fba38bbd3e0da62a4bbbdfea9e Mon Sep 17 00:00:00 2001 From: Timothy Mugo Gachengo <55899373+timothygachengo@users.noreply.github.com> Date: Wed, 26 Aug 2026 17:05:31 +0300 Subject: [PATCH 3/3] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- .../org/devgateway/viz/gateway/services/StatsService.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/superset-proxy/src/main/java/org/devgateway/viz/gateway/services/StatsService.java b/superset-proxy/src/main/java/org/devgateway/viz/gateway/services/StatsService.java index 6c1e5dd7..6a101935 100644 --- a/superset-proxy/src/main/java/org/devgateway/viz/gateway/services/StatsService.java +++ b/superset-proxy/src/main/java/org/devgateway/viz/gateway/services/StatsService.java @@ -85,19 +85,19 @@ private List resolveIncludeColumns(Map queryParams, Json } Set knownColumns = extractAllColumns(datasetResult); - List resolved = new ArrayList<>(); + Set resolvedSet = new LinkedHashSet<>(); for (String name : includeColumnsParam.split(",")) { String trimmed = name.trim(); if (trimmed.isEmpty()) { continue; } if (knownColumns.contains(trimmed)) { - resolved.add(trimmed); + resolvedSet.add(trimmed); } else { - logger.warn("includeColumns requested unknown column: " + trimmed); + logger.warn("includeColumns requested unknown column: {}", trimmed); } } - return resolved; + return new ArrayList<>(resolvedSet); } private JsonNode getDatasetResult(String datasetId) {