Skip to content

feat: enable superset stats API to fetch extra columns that are not p… #101

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
timothygachengo merged 3 commits into from
Aug 27, 2026
Merged

feat: enable superset stats API to fetch extra columns that are not p… #101

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
bf890b2
feat: enable superset stats API to fetch extra columns that are not p…
timothygachengo Aug 26, 2026
5b6f0c4
chore: update gh actions ci
timothygachengo Aug 26, 2026
f4bd70d
Potential fix for pull request finding
timothygachengo Aug 26, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions .github/workflows/release-template.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -83,8 +83,8 @@ jobs:
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
tag_prefix: ${{ inputs.project_name }}@v
release_branches: production,release/*
pre_release_branches: main
release_branches: __no-stable-release-branch__
pre_release_branches: .*
Comment thread
timothygachengo marked this conversation as resolved.
append_to_pre_release_tag: rc
dry_run: "true"

Expand Down
50 changes: 42 additions & 8 deletions superset-proxy/src/main/java/org/devgateway/viz/gateway/services/StatsService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,8 @@ public Object getStats(String datasetId, Map<String, String> queryParams, String
}

Map<String, String> labelMap = buildLabelMap(datasetResult);
JsonNode requestBody = buildSupersetDataRequest(datasetResult, datasetId, queryParams, groupsPath);
List<String> extraColumns = resolveIncludeColumns(queryParams, datasetResult);
JsonNode requestBody = buildSupersetDataRequest(datasetResult, datasetId, queryParams, groupsPath, extraColumns);

boolean force = "true".equals(queryParams.get("force"));

Expand All @@ -67,12 +68,38 @@ public Object getStats(String datasetId, Map<String, String> queryParams, String
getGroupsArray(groupsPath),
labelMap,
isCached,
cachedAt);
cachedAt,
extraColumns);
}
logger.warn("Returning an Empty list");
return Collections.emptyList();
}

// Dimensions requested via includeColumns are returned as an extra per-row value without being
// part of the query breakdown. Validated against the dataset's real columns so a client-supplied
// name can never be interpolated into the SQL expression sent to Superset.
private List<String> resolveIncludeColumns(Map<String, String> queryParams, JsonNode datasetResult) {
String includeColumnsParam = queryParams.get("includeColumns");
if (includeColumnsParam == null || includeColumnsParam.isBlank()) {
return Collections.emptyList();
}

Set<String> knownColumns = extractAllColumns(datasetResult);
Set<String> resolvedSet = new LinkedHashSet<>();
for (String name : includeColumnsParam.split(",")) {
String trimmed = name.trim();
if (trimmed.isEmpty()) {
continue;
}
if (knownColumns.contains(trimmed)) {
resolvedSet.add(trimmed);
} else {
logger.warn("includeColumns requested unknown column: {}", trimmed);
}
}
return new ArrayList<>(resolvedSet);
}

private JsonNode getDatasetResult(String datasetId) {
JsonNode dataset = superSetClient.fetchDataset(datasetId);
if (dataset == null || !dataset.has("result")) {
Expand Down Expand Up @@ -113,7 +140,7 @@ private Map<String, String> buildLabelMap(JsonNode datasetResult) {
return labels;
}

private JsonNode buildSupersetDataRequest(JsonNode datasetResult, String datasetId, Map<String, String> queryParams, String groupsPath) {
private JsonNode buildSupersetDataRequest(JsonNode datasetResult, String datasetId, Map<String, String> queryParams, String groupsPath, List<String> extraColumns) {

List<String> measuresArr = extractUniqueMeasures(datasetResult);
Set<String> filterableColumns = extractFilterableColumns(datasetResult);
Expand All @@ -125,12 +152,12 @@ private JsonNode buildSupersetDataRequest(JsonNode datasetResult, String dataset
String[] groupsArray = getGroupsArray(groupsPath);

if (groupsArray.length > 0) {
Map<String, Object> queryForDimension1 = createQuery(new String[]{groupsArray[0]}, measuresArr, filterableColumns, queryParams);
Map<String, Object> queryForDimension1 = createQuery(new String[]{groupsArray[0]}, measuresArr, filterableColumns, queryParams, extraColumns);
queries.add(queryForDimension1);
}

if (groupsArray.length > 1) {
Map<String, Object> queryForDimension2 = createQuery(groupsArray, measuresArr, filterableColumns, queryParams);
Map<String, Object> queryForDimension2 = createQuery(groupsArray, measuresArr, filterableColumns, queryParams, extraColumns);
queries.add(queryForDimension2);
}

Expand All @@ -142,7 +169,7 @@ private JsonNode buildSupersetDataRequest(JsonNode datasetResult, String dataset
}

private Object transformData(JsonNode data, JsonNode dimensionsNode, JsonNode metricsNode, String[] dimensionsArray,
Map<String, String> labelMap, boolean isCached, String cachedAt) {
Map<String, String> labelMap, boolean isCached, String cachedAt, List<String> extraColumns) {
Map<String, Object> transformed = createTransformedData();
List<Map<String, Object>> measuresList = new ArrayList<>();
List<Map<String, Object>> typesList = new ArrayList<>();
Expand All @@ -157,6 +184,10 @@ private Object transformData(JsonNode data, JsonNode dimensionsNode, JsonNode me
}

for (JsonNode metric : metricsNode) {
// Extra-column adhoc metrics are objects, not plain metric name strings; skip them here.
if (!metric.isTextual()) {
continue;
}
String metricName = metric.asText();
measuresList.add(createMeasure(metricName, labelMap.getOrDefault(metricName, metricName)));
}
Expand All @@ -169,6 +200,9 @@ private Object transformData(JsonNode data, JsonNode dimensionsNode, JsonNode me
JsonNode overallData = data.get(data.size() - 1);
if (overallData != null && overallData.has("data")) {
for (JsonNode metric : metricsNode) {
if (!metric.isTextual()) {
continue;
}
transformed.put(metric.asText(), overallData.get("data").get(0).get(metric.asText()).asDouble());
}

Expand All @@ -189,7 +223,7 @@ private Object transformData(JsonNode data, JsonNode dimensionsNode, JsonNode me
.ifPresent(type -> ((Set<Map<String, Object>>) type.computeIfAbsent("items", k -> new HashSet<>()))
.add(createItem(firstDim, row.get(firstDim).asText(), Constants.COLORS.get(0))));

Map<String, Object> dataItem = createDataItem(firstDim, row.get(firstDim).asText(), metricsNode, row);
Map<String, Object> dataItem = createDataItem(firstDim, row.get(firstDim).asText(), metricsNode, row, extraColumns);
((List<Map<String, Object>>) transformed.get("children")).add(dataItem);
}
}
Expand All @@ -206,7 +240,7 @@ private Object transformData(JsonNode data, JsonNode dimensionsNode, JsonNode me
.ifPresent(type -> ((Set<Map<String, Object>>) type.computeIfAbsent("items", k -> new HashSet<>()))
.add(createItem(secondDim, row.get(secondDim).asText(), Constants.COLORS.get(0))));

Map<String, Object> dataItem = createDataItem(secondDim, row.get(secondDim).asText(), metricsNode, row);
Map<String, Object> dataItem = createDataItem(secondDim, row.get(secondDim).asText(), metricsNode, row, extraColumns);
for (Map<String, Object> child : (List<Map<String, Object>>) transformed.get("children")) {
if (child.get("value").equals(row.get(firstDim).asText())) {
((List<Map<String, Object>>) child.computeIfAbsent("children", k -> new ArrayList<>())).add(dataItem);
Expand Down
44 changes: 43 additions & 1 deletion superset-proxy/src/main/java/org/devgateway/viz/gateway/services/Utils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,11 +48,22 @@ public static Map<String, Object> createQuery(String field, Set<String> filterab

public static Map<String, Object> createQuery(String[] groupArray, List<String> measuresArr,
Set<String> filterableColumns, Map<String, String> queryParams) {
return createQuery(groupArray, measuresArr, filterableColumns, queryParams, Collections.emptyList());
}

// extraColumns are dimensions requested via includeColumns: returned per-row
// without being added to "columns"/groupby, so they don't affect the query breakdown.
public static Map<String, Object> createQuery(String[] groupArray, List<String> measuresArr,
Set<String> filterableColumns, Map<String, String> queryParams, List<String> extraColumns) {
SortedMap<String, Object> query = new TreeMap<>(); // must be sorted to ensure consistent order for caching purposes
if (groupArray.length > 0) {
query.put("columns", Arrays.asList(groupArray));
}
query.put("metrics", measuresArr);
List<Object> metrics = new ArrayList<>(measuresArr);
for (String column : extraColumns != null ? extraColumns : Collections.<String>emptyList()) {
metrics.add(createExtraColumnMetric(column));
}
query.put("metrics", metrics);
query.put("row_limit", Constants.ROW_LIMIT);

List<Map<String, Object>> filters = new ArrayList<>();
Expand All @@ -76,6 +87,14 @@ public static Map<String, Object> createQuery(String[] groupArray, List<String>
return query;
}

public static Map<String, Object> createExtraColumnMetric(String column) {
Map<String, Object> metric = new HashMap<>();
metric.put("expressionType", "SQL");
metric.put("sqlExpression", "MAX(" + column + ")");
metric.put("label", column);
Comment thread
timothygachengo marked this conversation as resolved.
return metric;
}

public static Map<String, Object> createSupersetRequest(Map<String, Object> datasource, List<Map<String, Object>> queries) {
Map<String, Object> supersetRequest = new HashMap<>();
supersetRequest.put("datasource", datasource);
Expand Down Expand Up @@ -138,15 +157,27 @@ public static Map<String, Object> createType(String dim) {
}

public static Map<String, Object> createDataItem(String dim, String value, JsonNode metricsNode, JsonNode row) {
return createDataItem(dim, value, metricsNode, row, Collections.emptyList());
}

public static Map<String, Object> createDataItem(String dim, String value, JsonNode metricsNode, JsonNode row, List<String> extraColumns) {
Map<String, Object> dataItem = new HashMap<>();
dataItem.put("type", dim);
dataItem.put("value", value);

for (JsonNode metric : metricsNode) {
// Extra-column adhoc metrics are objects, not plain metric name strings; skip them here.
if (!metric.isTextual()) {
continue;
}
String metricName = metric.asText();
dataItem.put(metricName, row.hasNonNull(metricName) ? row.get(metricName).asDouble() : null);
}

for (String column : extraColumns != null ? extraColumns : Collections.<String>emptyList()) {
dataItem.put(column, row.hasNonNull(column) ? row.get(column).asText() : null);
}

return dataItem;
}

Expand Down Expand Up @@ -181,6 +212,17 @@ public static List<String> extractUniqueMeasures(JsonNode result) {
return measures;
}

public static Set<String> extractAllColumns(JsonNode result) {
Set<String> columnNames = new LinkedHashSet<>();
for (JsonNode column : result.get("columns")) {
String name = column.path("column_name").asText(null);
if (name != null && !name.isEmpty()) {
columnNames.add(name);
}
}
return columnNames;
}

public static Set<String> extractFilterableColumns(JsonNode result) {
Set<String> columnNames = new LinkedHashSet<>();
for (JsonNode column : result.get("columns")) {
Expand Down
2 changes: 1 addition & 1 deletion superset-proxy/src/main/resources/application-dev.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ spring:

viz:
superset:
url: https://superset.alive.dgstg.org
url: ${SUPERSET_URL:http://localhost:8088}
warmup:
top: 100

Expand Down