From bfb135c2a6e98a0d37df753f39e46157cf70f0fd Mon Sep 17 00:00:00 2001 From: John Hayes Date: Mon, 20 Jul 2026 16:44:52 +0100 Subject: [PATCH 1/7] Add Microsoft Purview plugin v1.4.3 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add dataSourceName field to Scans on Data Source stream, read directly from the API response rather than parsed from the scan id - Fix lastRunResult fields in dataSourceScans and scansForDataSource: status was scanRunStatus, times were nested under discoveryExecutionDetails, and runType was scanLevelType — all previously broken due to incorrect field name assumptions Co-Authored-By: Claude Sonnet 4.6 --- .../MicrosoftPurview/v1/configValidation.json | 18 ++ plugins/MicrosoftPurview/v1/custom_types.json | 16 ++ .../v1/dataStreams/accountAssetCount.json | 25 ++ .../dataStreams/assetCountByEntityType.json | 37 +++ .../v1/dataStreams/assetLineage.json | 72 +++++ .../v1/dataStreams/assetsByCollection.json | 42 +++ .../dataStreams/assetsByEntityTypeGlobal.json | 26 ++ .../dataStreams/classificationCoverage.json | 31 ++ .../v1/dataStreams/classificationTypes.json | 27 ++ .../collectionClassificationSummary.json | 36 +++ .../dataStreams/collectionPolicyCoverage.json | 34 +++ .../v1/dataStreams/collections.json | 72 +++++ .../v1/dataStreams/dataSourceScans.json | 75 +++++ .../v1/dataStreams/listDataSources.json | 82 ++++++ .../v1/dataStreams/metadataPolicies.json | 27 ++ .../v1/dataStreams/scanHistory.json | 85 ++++++ .../v1/dataStreams/scansForDataSource.json | 34 +++ .../dataStreams/scripts/accountAssetCount.js | 2 + .../scripts/assetCountByEntityType.js | 2 + .../v1/dataStreams/scripts/assetLineage.js | 27 ++ .../dataStreams/scripts/assetsByCollection.js | 9 + .../scripts/assetsByEntityTypeGlobal.js | 8 + .../scripts/classificationCoverage.js | 14 + .../scripts/classificationTypes.js | 6 + .../collectionClassificationSummary.js | 9 + .../scripts/collectionPolicyCoverage.js | 29 ++ .../v1/dataStreams/scripts/dataSourceScans.js | 21 ++ .../dataStreams/scripts/metadataPolicies.js | 11 + .../v1/dataStreams/scripts/searchAssets.js | 8 + .../v1/dataStreams/searchAssets.json | 58 ++++ .../collectionDashboard.dash.json | 196 +++++++++++++ .../dataSourceDashboard.dash.json | 115 ++++++++ .../v1/defaultContent/manifest.json | 7 + .../v1/defaultContent/overview.dash.json | 267 ++++++++++++++++++ .../v1/defaultContent/scopes.json | 32 +++ plugins/MicrosoftPurview/v1/docs/README.md | 66 +++++ plugins/MicrosoftPurview/v1/icon.svg | 30 ++ .../v1/indexDefinitions/default.json | 36 +++ plugins/MicrosoftPurview/v1/metadata.json | 70 +++++ plugins/MicrosoftPurview/v1/ui.json | 39 +++ 40 files changed, 1801 insertions(+) create mode 100644 plugins/MicrosoftPurview/v1/configValidation.json create mode 100644 plugins/MicrosoftPurview/v1/custom_types.json create mode 100644 plugins/MicrosoftPurview/v1/dataStreams/accountAssetCount.json create mode 100644 plugins/MicrosoftPurview/v1/dataStreams/assetCountByEntityType.json create mode 100644 plugins/MicrosoftPurview/v1/dataStreams/assetLineage.json create mode 100644 plugins/MicrosoftPurview/v1/dataStreams/assetsByCollection.json create mode 100644 plugins/MicrosoftPurview/v1/dataStreams/assetsByEntityTypeGlobal.json create mode 100644 plugins/MicrosoftPurview/v1/dataStreams/classificationCoverage.json create mode 100644 plugins/MicrosoftPurview/v1/dataStreams/classificationTypes.json create mode 100644 plugins/MicrosoftPurview/v1/dataStreams/collectionClassificationSummary.json create mode 100644 plugins/MicrosoftPurview/v1/dataStreams/collectionPolicyCoverage.json create mode 100644 plugins/MicrosoftPurview/v1/dataStreams/collections.json create mode 100644 plugins/MicrosoftPurview/v1/dataStreams/dataSourceScans.json create mode 100644 plugins/MicrosoftPurview/v1/dataStreams/listDataSources.json create mode 100644 plugins/MicrosoftPurview/v1/dataStreams/metadataPolicies.json create mode 100644 plugins/MicrosoftPurview/v1/dataStreams/scanHistory.json create mode 100644 plugins/MicrosoftPurview/v1/dataStreams/scansForDataSource.json create mode 100644 plugins/MicrosoftPurview/v1/dataStreams/scripts/accountAssetCount.js create mode 100644 plugins/MicrosoftPurview/v1/dataStreams/scripts/assetCountByEntityType.js create mode 100644 plugins/MicrosoftPurview/v1/dataStreams/scripts/assetLineage.js create mode 100644 plugins/MicrosoftPurview/v1/dataStreams/scripts/assetsByCollection.js create mode 100644 plugins/MicrosoftPurview/v1/dataStreams/scripts/assetsByEntityTypeGlobal.js create mode 100644 plugins/MicrosoftPurview/v1/dataStreams/scripts/classificationCoverage.js create mode 100644 plugins/MicrosoftPurview/v1/dataStreams/scripts/classificationTypes.js create mode 100644 plugins/MicrosoftPurview/v1/dataStreams/scripts/collectionClassificationSummary.js create mode 100644 plugins/MicrosoftPurview/v1/dataStreams/scripts/collectionPolicyCoverage.js create mode 100644 plugins/MicrosoftPurview/v1/dataStreams/scripts/dataSourceScans.js create mode 100644 plugins/MicrosoftPurview/v1/dataStreams/scripts/metadataPolicies.js create mode 100644 plugins/MicrosoftPurview/v1/dataStreams/scripts/searchAssets.js create mode 100644 plugins/MicrosoftPurview/v1/dataStreams/searchAssets.json create mode 100644 plugins/MicrosoftPurview/v1/defaultContent/collectionDashboard.dash.json create mode 100644 plugins/MicrosoftPurview/v1/defaultContent/dataSourceDashboard.dash.json create mode 100644 plugins/MicrosoftPurview/v1/defaultContent/manifest.json create mode 100644 plugins/MicrosoftPurview/v1/defaultContent/overview.dash.json create mode 100644 plugins/MicrosoftPurview/v1/defaultContent/scopes.json create mode 100644 plugins/MicrosoftPurview/v1/docs/README.md create mode 100644 plugins/MicrosoftPurview/v1/icon.svg create mode 100644 plugins/MicrosoftPurview/v1/indexDefinitions/default.json create mode 100644 plugins/MicrosoftPurview/v1/metadata.json create mode 100644 plugins/MicrosoftPurview/v1/ui.json diff --git a/plugins/MicrosoftPurview/v1/configValidation.json b/plugins/MicrosoftPurview/v1/configValidation.json new file mode 100644 index 00000000..e4c22a98 --- /dev/null +++ b/plugins/MicrosoftPurview/v1/configValidation.json @@ -0,0 +1,18 @@ +{ + "steps": [ + { + "displayName": "Authenticate and list collections", + "dataStream": { "name": "collections" }, + "required": true, + "error": "Could not authenticate or read collections. Check the account name, tenant/client ID, secret, and that the app registration has been added as Data Reader on the Purview root collection.", + "success": "Connected to Purview and read collections." + }, + { + "displayName": "List data sources", + "dataStream": { "name": "listDataSources" }, + "required": false, + "error": "Authenticated but could not list data sources. Add the app registration as Data Source Administrator on the Purview root collection.", + "success": "Data sources accessible." + } + ] +} diff --git a/plugins/MicrosoftPurview/v1/custom_types.json b/plugins/MicrosoftPurview/v1/custom_types.json new file mode 100644 index 00000000..09941497 --- /dev/null +++ b/plugins/MicrosoftPurview/v1/custom_types.json @@ -0,0 +1,16 @@ +[ + { + "name": "Purview Collection", + "sourceType": "Purview Collection", + "icon": "folder-tree", + "singular": "Collection", + "plural": "Collections" + }, + { + "name": "Purview Data Source", + "sourceType": "Purview Data Source", + "icon": "database", + "singular": "Data Source", + "plural": "Data Sources" + } +] diff --git a/plugins/MicrosoftPurview/v1/dataStreams/accountAssetCount.json b/plugins/MicrosoftPurview/v1/dataStreams/accountAssetCount.json new file mode 100644 index 00000000..dfd27ff7 --- /dev/null +++ b/plugins/MicrosoftPurview/v1/dataStreams/accountAssetCount.json @@ -0,0 +1,25 @@ +{ + "name": "accountAssetCount", + "displayName": "Total Asset Count", + "description": "Total number of assets cataloged across the whole Purview account", + "tags": ["Assets"], + "baseDataSourceName": "httpRequestUnscoped", + "config": { + "httpMethod": "post", + "endpointPath": "datamap/api/search/query", + "getArgs": [ + { "key": "api-version", "value": "2023-09-01" } + ], + "postBody": { + "keywords": null, + "limit": 1 + }, + "postRequestScript": "accountAssetCount.js" + }, + "matches": "none", + "metadata": [ + { "name": "label", "displayName": "Label", "role": "label" }, + { "name": "totalAssets", "displayName": "Total Assets", "shape": "number", "role": "value" } + ], + "timeframes": false +} diff --git a/plugins/MicrosoftPurview/v1/dataStreams/assetCountByEntityType.json b/plugins/MicrosoftPurview/v1/dataStreams/assetCountByEntityType.json new file mode 100644 index 00000000..ab1f3016 --- /dev/null +++ b/plugins/MicrosoftPurview/v1/dataStreams/assetCountByEntityType.json @@ -0,0 +1,37 @@ +{ + "name": "assetCountByEntityType", + "displayName": "Asset Count by Entity Type", + "description": "Count of assets in the selected collection grouped by entity type", + "tags": ["Assets"], + "baseDataSourceName": "httpRequestScopedSingle", + "config": { + "httpMethod": "post", + "endpointPath": "datamap/api/search/query", + "getArgs": [ + { "key": "api-version", "value": "2023-09-01" } + ], + "postBody": { + "keywords": null, + "limit": 1, + "filter": { + "collectionId": "{{object.collectionName}}" + }, + "facets": [ + { + "facet": "entityType", + "count": 50, + "sort": { "count": "desc" } + } + ] + }, + "pathToData": "@search.facets.entityType", + "postRequestScript": "assetCountByEntityType.js" + }, + "matches": { "sourceType": { "type": "oneOf", "values": ["Purview Collection"] } }, + "metadata": [ + { "name": "entityType", "displayName": "Entity Type", "role": "label" }, + { "name": "count", "displayName": "Assets", "shape": "number" } + ], + "timeframes": false, + "defaultShaping": { "sort": { "by": [["count", "desc"]] } } +} diff --git a/plugins/MicrosoftPurview/v1/dataStreams/assetLineage.json b/plugins/MicrosoftPurview/v1/dataStreams/assetLineage.json new file mode 100644 index 00000000..6dd965e4 --- /dev/null +++ b/plugins/MicrosoftPurview/v1/dataStreams/assetLineage.json @@ -0,0 +1,72 @@ +{ + "name": "assetLineage", + "displayName": "Asset Lineage", + "description": "Upstream and downstream lineage for an asset identified by its GUID", + "tags": ["Assets"], + "baseDataSourceName": "httpRequestUnscoped", + "ui": [ + { + "type": "text", + "name": "assetGuid", + "label": "Asset GUID", + "placeholder": "Asset GUID from the catalog", + "validation": { "required": true }, + "help": "The Purview asset's GUID. Copy it from the asset details page in the Purview portal, or from the **Asset GUID** column of the Search Assets stream." + }, + { + "type": "switch", + "name": "direction", + "label": "Direction", + "defaultValue": "BOTH", + "options": [ + { "value": "BOTH", "label": "Both" }, + { "value": "INPUT", "label": "Upstream" }, + { "value": "OUTPUT", "label": "Downstream" } + ] + }, + { + "type": "number", + "name": "depth", + "label": "Depth", + "defaultValue": 3, + "help": "Number of hops to traverse in each direction." + } + ], + "config": { + "httpMethod": "get", + "endpointPath": "datamap/api/atlas/v2/lineage/{{assetGuid}}", + "getArgs": [ + { "key": "direction", "value": "{{direction}}" }, + { "key": "depth", "value": "{{depth}}" } + ], + "postRequestScript": "assetLineage.js" + }, + "manualConfigApply": true, + "matches": "none", + "metadata": [ + { "name": "guid", "displayName": "GUID", "shape": "guid", "visible": false }, + { "name": "name", "displayName": "Name", "role": "label" }, + { "name": "typeName", "displayName": "Entity Type" }, + { "name": "qualifiedName", "displayName": "Qualified Name" }, + { + "name": "role", + "displayName": "Role", + "shape": [ + "state", + { + "map": { + "success": ["Base"], + "warning": ["Upstream"], + "unknown": ["Downstream"] + } + } + ] + }, + { "name": "status", "displayName": "Status" }, + { + "name": "classifications", + "displayName": "Classifications" + } + ], + "timeframes": false +} diff --git a/plugins/MicrosoftPurview/v1/dataStreams/assetsByCollection.json b/plugins/MicrosoftPurview/v1/dataStreams/assetsByCollection.json new file mode 100644 index 00000000..e0948dff --- /dev/null +++ b/plugins/MicrosoftPurview/v1/dataStreams/assetsByCollection.json @@ -0,0 +1,42 @@ +{ + "name": "assetsByCollection", + "displayName": "Assets in Collection", + "description": "Assets found in the selected Purview collection", + "tags": ["Assets"], + "baseDataSourceName": "httpRequestScopedSingle", + "config": { + "httpMethod": "post", + "endpointPath": "datamap/api/search/query", + "getArgs": [ + { "key": "api-version", "value": "2023-09-01" } + ], + "postBody": { + "keywords": null, + "limit": 100, + "filter": { + "collectionId": "{{object.collectionName}}" + }, + "orderby": [{ "name": "ASC" }] + }, + "postRequestScript": "assetsByCollection.js", + "paging": { + "mode": "token", + "in": { "realm": "payload", "path": "continuationToken" }, + "out": { "realm": "body", "path": "continuationToken" } + } + }, + "matches": { "sourceType": { "type": "oneOf", "values": ["Purview Collection"] } }, + "metadata": [ + { "name": "id", "displayName": "Asset GUID", "shape": "guid", "visible": false }, + { "name": "name", "displayName": "Name", "role": "label" }, + { "name": "qualifiedName", "displayName": "Qualified Name" }, + { "name": "entityType", "displayName": "Entity Type" }, + { "name": "assetType", "displayName": "Asset Type" }, + { "name": "classification", "displayName": "Classifications" }, + { "name": "term", "displayName": "Glossary Terms" }, + { "name": "label", "displayName": "Labels" }, + { "name": "description", "displayName": "Description" }, + { "name": "owner", "displayName": "Owner" } + ], + "timeframes": false +} diff --git a/plugins/MicrosoftPurview/v1/dataStreams/assetsByEntityTypeGlobal.json b/plugins/MicrosoftPurview/v1/dataStreams/assetsByEntityTypeGlobal.json new file mode 100644 index 00000000..0d52ae2c --- /dev/null +++ b/plugins/MicrosoftPurview/v1/dataStreams/assetsByEntityTypeGlobal.json @@ -0,0 +1,26 @@ +{ + "name": "assetsByEntityTypeGlobal", + "displayName": "Asset Count by Entity Type", + "description": "Count of assets grouped by entity type across the whole account", + "tags": ["Assets"], + "baseDataSourceName": "httpRequestUnscoped", + "config": { + "httpMethod": "post", + "endpointPath": "datamap/api/search/query", + "getArgs": [ + { "key": "api-version", "value": "2023-09-01" } + ], + "postBody": { + "keywords": null, + "limit": 1000 + }, + "postRequestScript": "assetsByEntityTypeGlobal.js" + }, + "matches": "none", + "metadata": [ + { "name": "entityType", "displayName": "Entity Type", "role": "label" }, + { "name": "count", "displayName": "Assets", "shape": "number" } + ], + "timeframes": false, + "defaultShaping": { "sort": { "by": [["count", "desc"]] } } +} diff --git a/plugins/MicrosoftPurview/v1/dataStreams/classificationCoverage.json b/plugins/MicrosoftPurview/v1/dataStreams/classificationCoverage.json new file mode 100644 index 00000000..d2a19f6a --- /dev/null +++ b/plugins/MicrosoftPurview/v1/dataStreams/classificationCoverage.json @@ -0,0 +1,31 @@ +{ + "name": "classificationCoverage", + "displayName": "Classification Coverage", + "description": "Count of assets carrying each classification across the whole account", + "tags": ["Classifications"], + "baseDataSourceName": "httpRequestUnscoped", + "config": { + "httpMethod": "post", + "endpointPath": "datamap/api/search/query", + "getArgs": [ + { "key": "api-version", "value": "2023-09-01" } + ], + "postBody": { + "keywords": null, + "limit": 1000 + }, + "postRequestScript": "classificationCoverage.js" + }, + "matches": "none", + "metadata": [ + { "name": "classification", "displayName": "Classification", "role": "label" }, + { "name": "assets", "displayName": "Assets", "shape": "number" }, + { + "name": "coverage", + "displayName": "% of Total Assets", + "shape": ["percent", { "asZeroToOne": true, "decimalPlaces": 1 }] + } + ], + "timeframes": false, + "defaultShaping": { "sort": { "by": [["assets", "desc"]] } } +} diff --git a/plugins/MicrosoftPurview/v1/dataStreams/classificationTypes.json b/plugins/MicrosoftPurview/v1/dataStreams/classificationTypes.json new file mode 100644 index 00000000..ec4d4b13 --- /dev/null +++ b/plugins/MicrosoftPurview/v1/dataStreams/classificationTypes.json @@ -0,0 +1,27 @@ +{ + "name": "classificationTypes", + "displayName": "Classification Types", + "description": "All classification type definitions in the Purview account", + "tags": ["Classifications"], + "baseDataSourceName": "httpRequestUnscoped", + "config": { + "httpMethod": "get", + "endpointPath": "datamap/api/atlas/v2/types/typedefs", + "getArgs": [ + { "key": "type", "value": "classification" } + ], + "postRequestScript": "classificationTypes.js" + }, + "matches": "none", + "metadata": [ + { "name": "name", "displayName": "Classification", "role": "label" }, + { "name": "description", "displayName": "Description" }, + { "name": "category", "displayName": "Category" }, + { "name": "typeVersion", "displayName": "Version" }, + { "name": "superTypes", "displayName": "Super Types" }, + { "name": "createdBy", "displayName": "Created By" }, + { "name": "createTime", "displayName": "Created", "shape": "date" }, + { "name": "guid", "displayName": "GUID", "shape": "guid", "visible": false } + ], + "timeframes": false +} diff --git a/plugins/MicrosoftPurview/v1/dataStreams/collectionClassificationSummary.json b/plugins/MicrosoftPurview/v1/dataStreams/collectionClassificationSummary.json new file mode 100644 index 00000000..92d91c02 --- /dev/null +++ b/plugins/MicrosoftPurview/v1/dataStreams/collectionClassificationSummary.json @@ -0,0 +1,36 @@ +{ + "name": "collectionClassificationSummary", + "displayName": "Classification Summary", + "description": "Classified vs unclassified asset counts for the selected collection", + "tags": ["Classifications"], + "baseDataSourceName": "httpRequestScopedSingle", + "config": { + "httpMethod": "post", + "endpointPath": "datamap/api/search/query", + "getArgs": [ + { "key": "api-version", "value": "2023-09-01" } + ], + "postBody": { + "keywords": null, + "limit": 1, + "filter": { + "collectionId": "{{object.collectionName}}" + }, + "facets": [ + { + "facet": "classification", + "count": 200, + "sort": { "count": "desc" } + } + ] + }, + "postRequestScript": "collectionClassificationSummary.js" + }, + "matches": { "sourceType": { "type": "oneOf", "values": ["Purview Collection"] } }, + "metadata": [ + { "name": "state", "displayName": "State", "shape": ["state", { "map": { "success": ["Classified"], "warning": ["Unclassified"] } }] }, + { "name": "label", "displayName": "Category", "role": "label" }, + { "name": "assets", "displayName": "Assets", "shape": "number", "role": "value" } + ], + "timeframes": false +} diff --git a/plugins/MicrosoftPurview/v1/dataStreams/collectionPolicyCoverage.json b/plugins/MicrosoftPurview/v1/dataStreams/collectionPolicyCoverage.json new file mode 100644 index 00000000..89cc9251 --- /dev/null +++ b/plugins/MicrosoftPurview/v1/dataStreams/collectionPolicyCoverage.json @@ -0,0 +1,34 @@ +{ + "name": "collectionPolicyCoverage", + "displayName": "Policy Coverage", + "description": "Metadata policies attached to the selected collection", + "tags": ["Policies"], + "baseDataSourceName": "httpRequestScopedSingle", + "config": { + "httpMethod": "get", + "endpointPath": "policyStore/metadataPolicies", + "getArgs": [ + { "key": "api-version", "value": "2021-07-01-preview" }, + { "key": "collectionName", "value": "{{object.collectionName}}" } + ], + "postRequestScript": "collectionPolicyCoverage.js" + }, + "matches": { "sourceType": { "type": "oneOf", "values": ["Purview Collection"] } }, + "metadata": [ + { "name": "id", "displayName": "Policy ID", "shape": "guid", "visible": false }, + { "name": "name", "displayName": "Policy", "role": "label" }, + { "name": "version", "displayName": "Version", "shape": "number" }, + { "name": "description", "displayName": "Description" }, + { "name": "decisionRuleCount", "displayName": "Decision Rules", "shape": "number" }, + { "name": "attributeRuleCount", "displayName": "Attribute Rules", "shape": "number" }, + { + "name": "state", + "displayName": "State", + "shape": [ + "state", + { "map": { "success": ["Attached"], "warning": ["No policy"] } } + ] + } + ], + "timeframes": false +} diff --git a/plugins/MicrosoftPurview/v1/dataStreams/collections.json b/plugins/MicrosoftPurview/v1/dataStreams/collections.json new file mode 100644 index 00000000..af86ac55 --- /dev/null +++ b/plugins/MicrosoftPurview/v1/dataStreams/collections.json @@ -0,0 +1,72 @@ +{ + "name": "collections", + "displayName": "Collections", + "description": "Collections (organizational units) in the Purview account", + "tags": ["Collections"], + "baseDataSourceName": "httpRequestUnscoped", + "config": { + "httpMethod": "get", + "endpointPath": "account/collections", + "getArgs": [ + { "key": "api-version", "value": "2019-11-01-preview" } + ], + "pathToData": "value", + "paging": { + "mode": "nextUrl", + "in": { "realm": "payload", "path": "nextLink" } + } + }, + "matches": "none", + "metadata": [ + { + "name": "name", + "displayName": "Collection ID", + "role": "sourceId", + "visible": false + }, + { + "name": "collectionName", + "displayName": "Collection", + "computed": true, + "valueExpression": "{{ $['name'] }}", + "visible": false + }, + { + "name": "friendlyName", + "displayName": "Friendly Name", + "visible": false + }, + { + "name": "displayName", + "displayName": "Name", + "computed": true, + "valueExpression": "{{ $['friendlyName'] || $['name'] }}", + "role": "label" + }, + { + "name": "description", + "displayName": "Description" + }, + { + "name": "parentCollection", + "visible": false + }, + { + "name": "parentCollectionName", + "computed": true, + "valueExpression": "{{ $['parentCollection'] && $['parentCollection'].referenceName ? $['parentCollection'].referenceName : '' }}", + "displayName": "Parent Collection" + }, + { + "name": "collectionProvisioningState", + "displayName": "State" + }, + { + "name": "sourceType", + "computed": true, + "valueExpression": "Purview Collection", + "visible": false + } + ], + "timeframes": false +} diff --git a/plugins/MicrosoftPurview/v1/dataStreams/dataSourceScans.json b/plugins/MicrosoftPurview/v1/dataStreams/dataSourceScans.json new file mode 100644 index 00000000..c0d0a906 --- /dev/null +++ b/plugins/MicrosoftPurview/v1/dataStreams/dataSourceScans.json @@ -0,0 +1,75 @@ +{ + "name": "dataSourceScans", + "displayName": "Scans on Data Source", + "description": "Scans configured on the selected data source with latest run status", + "tags": ["Data Sources"], + "baseDataSourceName": "httpRequestScopedSingle", + "config": { + "httpMethod": "get", + "endpointPath": "scan/datasources/{{object.rawId}}/scans", + "getArgs": [ + { "key": "api-version", "value": "2023-09-01" } + ], + "postRequestScript": "dataSourceScans.js", + "paging": { + "mode": "nextUrl", + "in": { "realm": "payload", "path": "nextLink" } + } + }, + "matches": { "sourceType": { "type": "oneOf", "values": ["Purview Data Source"] } }, + "metadata": [ + { "name": "name", "displayName": "Scan", "role": "label" }, + { "name": "dataSourceName", "displayName": "Data Source" }, + { "name": "kind", "displayName": "Kind" }, + { + "name": "properties.collection.referenceName", + "displayName": "Collection", + "visible": false + }, + { + "name": "collection", + "computed": true, + "valueExpression": "{{ $['properties.collection.referenceName'] || '' }}", + "displayName": "Collection" + }, + { + "name": "properties.scanRulesetName", + "displayName": "Ruleset" + }, + { + "name": "properties.scanRulesetType", + "displayName": "Ruleset Type" + }, + { + "name": "lastRunResult.status", + "displayName": "Last Status", + "shape": [ + "state", + { + "map": { + "success": ["Succeeded"], + "error": ["Failed", "TransientFailure"], + "warning": ["Canceled"], + "unknown": ["Accepted", "InProgress"] + } + } + ] + }, + { + "name": "lastRunResult.startTime", + "displayName": "Last Run Started", + "shape": "date" + }, + { + "name": "lastRunResult.endTime", + "displayName": "Last Run Ended", + "shape": "date" + }, + { + "name": "lastRunResult.scanLevelType", + "displayName": "Scan Level" + }, + { "name": "creationType", "displayName": "Creation Type" } + ], + "timeframes": false +} diff --git a/plugins/MicrosoftPurview/v1/dataStreams/listDataSources.json b/plugins/MicrosoftPurview/v1/dataStreams/listDataSources.json new file mode 100644 index 00000000..5725eb62 --- /dev/null +++ b/plugins/MicrosoftPurview/v1/dataStreams/listDataSources.json @@ -0,0 +1,82 @@ +{ + "name": "listDataSources", + "displayName": "Data Sources", + "description": "Registered data sources in the Purview account", + "tags": ["Data Sources"], + "baseDataSourceName": "httpRequestUnscoped", + "config": { + "httpMethod": "get", + "endpointPath": "scan/datasources", + "getArgs": [ + { "key": "api-version", "value": "2023-09-01" } + ], + "pathToData": "value", + "expandInnerObjects": true, + "paging": { + "mode": "nextUrl", + "in": { "realm": "payload", "path": "nextLink" } + } + }, + "matches": "none", + "metadata": [ + { "name": "name", "displayName": "Name", "role": "label" }, + { "name": "kind", "displayName": "Kind" }, + { + "name": "properties.collection.referenceName", + "displayName": "Collection", + "visible": false + }, + { + "name": "collection", + "computed": true, + "valueExpression": "{{ $['properties.collection.referenceName'] || '' }}", + "displayName": "Collection" + }, + { + "name": "properties.resourceGroup", + "displayName": "Resource Group", + "visible": false + }, + { + "name": "resourceGroup", + "computed": true, + "valueExpression": "{{ $['properties.resourceGroup'] || '' }}", + "displayName": "Resource Group" + }, + { + "name": "properties.subscriptionId", + "displayName": "Subscription", + "visible": false + }, + { + "name": "subscriptionId", + "computed": true, + "valueExpression": "{{ $['properties.subscriptionId'] || '' }}", + "displayName": "Subscription", + "visible": false + }, + { + "name": "subscriptionUrl", + "computed": true, + "valueExpression": "{{ $['properties.subscriptionId'] ? 'https://portal.azure.com/#resource/subscriptions/' + $['properties.subscriptionId'] : '' }}", + "displayName": "Subscription", + "shape": "url" + }, + { + "name": "properties.createdAt", + "displayName": "Created", + "shape": "date" + }, + { + "name": "creationType", + "displayName": "Creation Type" + }, + { + "name": "sourceType", + "computed": true, + "valueExpression": "Purview Data Source", + "visible": false + } + ], + "timeframes": false +} diff --git a/plugins/MicrosoftPurview/v1/dataStreams/metadataPolicies.json b/plugins/MicrosoftPurview/v1/dataStreams/metadataPolicies.json new file mode 100644 index 00000000..8e7f22e0 --- /dev/null +++ b/plugins/MicrosoftPurview/v1/dataStreams/metadataPolicies.json @@ -0,0 +1,27 @@ +{ + "name": "metadataPolicies", + "displayName": "Metadata Policies", + "description": "All metadata policies across the Purview account", + "tags": ["Policies"], + "baseDataSourceName": "httpRequestUnscoped", + "config": { + "httpMethod": "get", + "endpointPath": "policyStore/metadataPolicies", + "getArgs": [ + { "key": "api-version", "value": "2021-07-01-preview" } + ], + "postRequestScript": "metadataPolicies.js" + }, + "matches": "none", + "metadata": [ + { "name": "id", "displayName": "Policy ID", "shape": "guid", "visible": false }, + { "name": "name", "displayName": "Name", "role": "label" }, + { "name": "version", "displayName": "Version", "shape": "number" }, + { "name": "collection", "displayName": "Collection" }, + { "name": "description", "displayName": "Description" }, + { "name": "decisionRuleCount", "displayName": "Decision Rules", "shape": "number" }, + { "name": "attributeRuleCount", "displayName": "Attribute Rules", "shape": "number" } + ], + "timeframes": false, + "defaultShaping": { "sort": { "by": [["collection", "asc"]] } } +} diff --git a/plugins/MicrosoftPurview/v1/dataStreams/scanHistory.json b/plugins/MicrosoftPurview/v1/dataStreams/scanHistory.json new file mode 100644 index 00000000..d5d200e0 --- /dev/null +++ b/plugins/MicrosoftPurview/v1/dataStreams/scanHistory.json @@ -0,0 +1,85 @@ +{ + "name": "scanHistory", + "displayName": "Scan History", + "description": "Historical scan runs for a named scan on the selected data source", + "tags": ["Data Sources"], + "baseDataSourceName": "httpRequestScopedSingle", + "ui": [ + { + "type": "autocomplete", + "name": "scanName", + "label": "Scan Name", + "placeholder": "Select a scan", + "isMulti": false, + "allowCustomValues": false, + "validation": { "required": true }, + "data": { + "source": "dataStream", + "dataStreamName": "scansForDataSource", + "dataSourceConfig": {} + } + } + ], + "config": { + "httpMethod": "get", + "endpointPath": "scan/datasources/{{object.rawId}}/scans/{{scanName}}/runs", + "getArgs": [ + { "key": "api-version", "value": "2023-09-01" } + ], + "pathToData": "value", + "expandInnerObjects": true, + "paging": { + "mode": "nextUrl", + "in": { "realm": "payload", "path": "nextLink" } + } + }, + "matches": { "sourceType": { "type": "oneOf", "values": ["Purview Data Source"] } }, + "metadata": [ + { "name": "id", "displayName": "Run ID", "shape": "guid", "visible": false }, + { "name": "startTime", "displayName": "Started", "shape": "date", "role": "timestamp" }, + { "name": "endTime", "displayName": "Ended", "shape": "date" }, + { + "name": "status", + "displayName": "Status", + "shape": [ + "state", + { + "map": { + "success": ["Succeeded"], + "error": ["Failed", "TransientFailure"], + "warning": ["Canceled"], + "unknown": ["Accepted", "InProgress"] + } + } + ] + }, + { "name": "runType", "displayName": "Run Type" }, + { "name": "scanLevelType", "displayName": "Scan Level" }, + { + "name": "duration", + "displayName": "Duration", + "computed": true, + "valueExpression": "{{ $['startTime'] && $['endTime'] ? (new Date($['endTime']).getTime() - new Date($['startTime']).getTime()) / 1000 : null }}", + "shape": ["seconds", { "formatDuration": true, "decimalPlaces": 0 }] + }, + { + "name": "discoveryExecutionDetails.statistics.assets.discovered", + "displayName": "Assets Discovered", + "shape": "number" + }, + { + "name": "discoveryExecutionDetails.statistics.assets.classified", + "displayName": "Assets Classified", + "shape": "number" + }, + { + "name": "discoveryExecutionDetails.statistics.assets.failed", + "displayName": "Assets Failed", + "shape": "number" + }, + { "name": "errorMessage", "displayName": "Error", "visible": false }, + { "name": "dataSourceType", "displayName": "Data Source Type" } + ], + "timeframes": true, + "defaultShaping": { "sort": { "by": [["startTime", "desc"]] } } +} diff --git a/plugins/MicrosoftPurview/v1/dataStreams/scansForDataSource.json b/plugins/MicrosoftPurview/v1/dataStreams/scansForDataSource.json new file mode 100644 index 00000000..647c7fd7 --- /dev/null +++ b/plugins/MicrosoftPurview/v1/dataStreams/scansForDataSource.json @@ -0,0 +1,34 @@ +{ + "name": "scansForDataSource", + "displayName": "Scan Definitions", + "description": "Lists the scan definitions configured on a data source, with their last run date, sorted most-recent-first. Used to populate the Scan Name picker on the Scan History stream.", + "baseDataSourceName": "httpRequestScopedSingle", + "visibility": { "type": "hidden" }, + "config": { + "httpMethod": "get", + "endpointPath": "scan/datasources/{{object.rawId}}/scans", + "getArgs": [ + { "key": "api-version", "value": "2023-09-01" } + ], + "pathToData": "value", + "expandInnerObjects": true, + "paging": { + "mode": "nextUrl", + "in": { "realm": "payload", "path": "nextLink" } + } + }, + "matches": { "sourceType": { "type": "oneOf", "values": ["Purview Data Source"] } }, + "metadata": [ + { "name": "name", "displayName": "Scan Name", "role": "value", "visible": false }, + { "name": "lastRunResult.discoveryExecutionDetails.discoveryStartTime", "displayName": "Last Run Started", "shape": "date", "visible": false }, + { + "name": "scanLabel", + "displayName": "Scan", + "role": "label", + "computed": true, + "valueExpression": "{{ $['lastRunResult.discoveryExecutionDetails.discoveryStartTime'] ? $['name'] + ' — ' + new Date($['lastRunResult.discoveryExecutionDetails.discoveryStartTime']).toISOString().slice(0, 10) : $['name'] + ' — never run' }}" + } + ], + "timeframes": false, + "defaultShaping": { "sort": { "by": [["lastRunResult.discoveryExecutionDetails.discoveryStartTime", "desc"]] } } +} diff --git a/plugins/MicrosoftPurview/v1/dataStreams/scripts/accountAssetCount.js b/plugins/MicrosoftPurview/v1/dataStreams/scripts/accountAssetCount.js new file mode 100644 index 00000000..b0a26dc7 --- /dev/null +++ b/plugins/MicrosoftPurview/v1/dataStreams/scripts/accountAssetCount.js @@ -0,0 +1,2 @@ +const total = Number(data?.['@search.count']) || 0; +result = [{ label: 'Total Assets', totalAssets: total }]; diff --git a/plugins/MicrosoftPurview/v1/dataStreams/scripts/assetCountByEntityType.js b/plugins/MicrosoftPurview/v1/dataStreams/scripts/assetCountByEntityType.js new file mode 100644 index 00000000..4217db2e --- /dev/null +++ b/plugins/MicrosoftPurview/v1/dataStreams/scripts/assetCountByEntityType.js @@ -0,0 +1,2 @@ +const facets = data?.['@search.facets']?.entityType || []; +result = facets.map((f) => ({ entityType: f.value, count: Number(f.count) })); diff --git a/plugins/MicrosoftPurview/v1/dataStreams/scripts/assetLineage.js b/plugins/MicrosoftPurview/v1/dataStreams/scripts/assetLineage.js new file mode 100644 index 00000000..235589fb --- /dev/null +++ b/plugins/MicrosoftPurview/v1/dataStreams/scripts/assetLineage.js @@ -0,0 +1,27 @@ +const baseGuid = data?.baseEntityGuid; +const map = data?.guidEntityMap || {}; +const relations = data?.relations || []; + +const upstream = new Set(); +const downstream = new Set(); +for (const r of relations) { + if (r.toEntityId === baseGuid) upstream.add(r.fromEntityId); + if (r.fromEntityId === baseGuid) downstream.add(r.toEntityId); +} + +result = Object.entries(map).map(([guid, e]) => { + let role = 'Downstream'; + if (guid === baseGuid) role = 'Base'; + else if (upstream.has(guid)) role = 'Upstream'; + + const attrs = e.attributes || {}; + return { + guid, + name: attrs.name || e.displayText || guid, + typeName: e.typeName || '', + qualifiedName: attrs.qualifiedName || '', + role, + status: e.status || '', + classifications: (e.classificationNames || []).join(', '), + }; +}); diff --git a/plugins/MicrosoftPurview/v1/dataStreams/scripts/assetsByCollection.js b/plugins/MicrosoftPurview/v1/dataStreams/scripts/assetsByCollection.js new file mode 100644 index 00000000..9fcac4e9 --- /dev/null +++ b/plugins/MicrosoftPurview/v1/dataStreams/scripts/assetsByCollection.js @@ -0,0 +1,9 @@ +const arrayToText = (v) => Array.isArray(v) ? v.join(', ') : (v == null ? '' : String(v)); + +result = (data?.value || []).map((hit) => ({ + ...hit, + assetType: arrayToText(hit.assetType), + classification: arrayToText(hit.classification), + term: arrayToText(hit.term), + label: arrayToText(hit.label) +})); diff --git a/plugins/MicrosoftPurview/v1/dataStreams/scripts/assetsByEntityTypeGlobal.js b/plugins/MicrosoftPurview/v1/dataStreams/scripts/assetsByEntityTypeGlobal.js new file mode 100644 index 00000000..835db28c --- /dev/null +++ b/plugins/MicrosoftPurview/v1/dataStreams/scripts/assetsByEntityTypeGlobal.js @@ -0,0 +1,8 @@ +const counts = {}; +for (const item of (data?.value || [])) { + const et = item.entityType || 'unknown'; + counts[et] = (counts[et] || 0) + 1; +} +result = Object.entries(counts) + .map(([entityType, count]) => ({ entityType, count })) + .sort((a, b) => b.count - a.count); diff --git a/plugins/MicrosoftPurview/v1/dataStreams/scripts/classificationCoverage.js b/plugins/MicrosoftPurview/v1/dataStreams/scripts/classificationCoverage.js new file mode 100644 index 00000000..2ae69013 --- /dev/null +++ b/plugins/MicrosoftPurview/v1/dataStreams/scripts/classificationCoverage.js @@ -0,0 +1,14 @@ +const total = Number(data?.['@search.count']) || 0; +const counts = {}; +for (const item of (data?.value || [])) { + for (const c of (Array.isArray(item.classification) ? item.classification : [])) { + counts[c] = (counts[c] || 0) + 1; + } +} +result = Object.entries(counts) + .map(([classification, assets]) => ({ + classification, + assets, + coverage: total > 0 ? assets / total : 0 + })) + .sort((a, b) => b.assets - a.assets); diff --git a/plugins/MicrosoftPurview/v1/dataStreams/scripts/classificationTypes.js b/plugins/MicrosoftPurview/v1/dataStreams/scripts/classificationTypes.js new file mode 100644 index 00000000..2cb80236 --- /dev/null +++ b/plugins/MicrosoftPurview/v1/dataStreams/scripts/classificationTypes.js @@ -0,0 +1,6 @@ +const arrayToText = (v) => Array.isArray(v) ? v.join(', ') : (v == null ? '' : String(v)); + +result = (data?.classificationDefs || []).map((def) => ({ + ...def, + superTypes: arrayToText(def.superTypes) +})); diff --git a/plugins/MicrosoftPurview/v1/dataStreams/scripts/collectionClassificationSummary.js b/plugins/MicrosoftPurview/v1/dataStreams/scripts/collectionClassificationSummary.js new file mode 100644 index 00000000..82daace8 --- /dev/null +++ b/plugins/MicrosoftPurview/v1/dataStreams/scripts/collectionClassificationSummary.js @@ -0,0 +1,9 @@ +const total = Number(data?.['@search.count']) || 0; +const facets = data?.['@search.facets']?.classification || []; +const classified = facets.reduce((sum, f) => sum + (Number(f.count) || 0), 0); +const unclassified = Math.max(0, total - classified); + +result = [ + { state: 'Classified', label: 'Classified', assets: classified }, + { state: 'Unclassified', label: 'Unclassified', assets: unclassified }, +]; diff --git a/plugins/MicrosoftPurview/v1/dataStreams/scripts/collectionPolicyCoverage.js b/plugins/MicrosoftPurview/v1/dataStreams/scripts/collectionPolicyCoverage.js new file mode 100644 index 00000000..c69f513f --- /dev/null +++ b/plugins/MicrosoftPurview/v1/dataStreams/scripts/collectionPolicyCoverage.js @@ -0,0 +1,29 @@ +const policies = data?.values || data?.value || []; + +if (policies.length === 0) { + result = [ + { + id: null, + name: '(no policy)', + version: 0, + description: '', + decisionRuleCount: 0, + attributeRuleCount: 0, + state: 'No policy', + }, + ]; +} else { + result = policies.map((p) => ({ + id: p.id, + name: p.name, + version: Number(p.version) || 0, + description: p.properties?.description || '', + decisionRuleCount: Array.isArray(p.properties?.decisionRules) + ? p.properties.decisionRules.length + : 0, + attributeRuleCount: Array.isArray(p.properties?.attributeRules) + ? p.properties.attributeRules.length + : 0, + state: 'Attached', + })); +} diff --git a/plugins/MicrosoftPurview/v1/dataStreams/scripts/dataSourceScans.js b/plugins/MicrosoftPurview/v1/dataStreams/scripts/dataSourceScans.js new file mode 100644 index 00000000..97279c43 --- /dev/null +++ b/plugins/MicrosoftPurview/v1/dataStreams/scripts/dataSourceScans.js @@ -0,0 +1,21 @@ +const scans = data?.value || []; + +result = scans.map((scan) => { + const lr = scan.lastRunResult || {}; + const discovery = lr.discoveryExecutionDetails || {}; + const props = scan.properties || {}; + + return { + name: scan.name || '', + dataSourceName: scan.dataSourceName || '', + kind: scan.kind || '', + creationType: scan.creationType || '', + 'properties.collection.referenceName': props.collection?.referenceName || '', + 'properties.scanRulesetName': props.scanRulesetName || '', + 'properties.scanRulesetType': props.scanRulesetType || '', + 'lastRunResult.status': lr.scanRunStatus || '', + 'lastRunResult.startTime': discovery.discoveryStartTime || null, + 'lastRunResult.endTime': discovery.discoveryEndTime || null, + 'lastRunResult.scanLevelType': lr.scanLevelType || '', + }; +}); diff --git a/plugins/MicrosoftPurview/v1/dataStreams/scripts/metadataPolicies.js b/plugins/MicrosoftPurview/v1/dataStreams/scripts/metadataPolicies.js new file mode 100644 index 00000000..cb06aa3c --- /dev/null +++ b/plugins/MicrosoftPurview/v1/dataStreams/scripts/metadataPolicies.js @@ -0,0 +1,11 @@ +const policies = data?.values || data?.value || []; + +result = policies.map((p) => ({ + id: p.id, + name: p.name, + version: Number(p.version) || 0, + collection: p.properties?.collection?.referenceName || '', + description: p.properties?.description || '', + decisionRuleCount: Array.isArray(p.properties?.decisionRules) ? p.properties.decisionRules.length : 0, + attributeRuleCount: Array.isArray(p.properties?.attributeRules) ? p.properties.attributeRules.length : 0, +})); diff --git a/plugins/MicrosoftPurview/v1/dataStreams/scripts/searchAssets.js b/plugins/MicrosoftPurview/v1/dataStreams/scripts/searchAssets.js new file mode 100644 index 00000000..271e8d5b --- /dev/null +++ b/plugins/MicrosoftPurview/v1/dataStreams/scripts/searchAssets.js @@ -0,0 +1,8 @@ +const arrayToText = (v) => Array.isArray(v) ? v.join(', ') : (v == null ? '' : String(v)); + +result = (data?.value || []).map((hit) => ({ + ...hit, + assetType: arrayToText(hit.assetType), + classification: arrayToText(hit.classification), + term: arrayToText(hit.term) +})); diff --git a/plugins/MicrosoftPurview/v1/dataStreams/searchAssets.json b/plugins/MicrosoftPurview/v1/dataStreams/searchAssets.json new file mode 100644 index 00000000..4beef9d6 --- /dev/null +++ b/plugins/MicrosoftPurview/v1/dataStreams/searchAssets.json @@ -0,0 +1,58 @@ +{ + "name": "searchAssets", + "displayName": "Search Assets", + "description": "Search the Purview catalog with optional keyword, entity-type, and classification filters", + "tags": ["Assets"], + "baseDataSourceName": "httpRequestUnscoped", + "ui": [ + { + "type": "text", + "name": "keywords", + "label": "Keywords", + "placeholder": "Search keywords (optional)" + }, + { + "type": "text", + "name": "entityType", + "label": "Entity Type", + "placeholder": "e.g. azure_blob_path" + }, + { + "type": "text", + "name": "classification", + "label": "Classification", + "placeholder": "e.g. MICROSOFT.GOVERNMENT.US.SOCIAL_SECURITY_NUMBER" + }, + { + "type": "number", + "name": "limit", + "label": "Result Limit", + "defaultValue": 100 + } + ], + "config": { + "httpMethod": "post", + "endpointPath": "datamap/api/search/query", + "getArgs": [ + { "key": "api-version", "value": "2023-09-01" } + ], + "postBody": { + "keywords": "{{ keywords || null }}", + "limit": "{{ Number(limit) || 100 }}", + "filter": "{{ (entityType || classification) ? { and: [entityType ? { entityType: entityType } : null, classification ? { classification: classification } : null].filter(Boolean) } : null }}" + }, + "postRequestScript": "searchAssets.js" + }, + "manualConfigApply": true, + "matches": "none", + "metadata": [ + { "name": "id", "displayName": "Asset GUID", "shape": "guid", "visible": false }, + { "name": "name", "displayName": "Name", "role": "label" }, + { "name": "qualifiedName", "displayName": "Qualified Name" }, + { "name": "entityType", "displayName": "Entity Type" }, + { "name": "assetType", "displayName": "Asset Type" }, + { "name": "classification", "displayName": "Classifications" }, + { "name": "term", "displayName": "Glossary Terms" } + ], + "timeframes": false +} diff --git a/plugins/MicrosoftPurview/v1/defaultContent/collectionDashboard.dash.json b/plugins/MicrosoftPurview/v1/defaultContent/collectionDashboard.dash.json new file mode 100644 index 00000000..7c4c53e3 --- /dev/null +++ b/plugins/MicrosoftPurview/v1/defaultContent/collectionDashboard.dash.json @@ -0,0 +1,196 @@ +{ + "name": "Collection", + "schemaVersion": "1.5", + "variables": ["{{variables.Collection}}"], + "dashboard": { + "_type": "layout/grid", + "columns": 4, + "version": 1, + "contents": [ + { + "static": false, + "w": 1, + "moved": false, + "h": 4, + "x": 0, + "y": 0, + "i": "25f69cbd-2d27-4175-8273-75f70bea420d", + "z": 0, + "config": { + "_type": "tile/data-stream", + "title": "Properties", + "description": "", + "activePluginConfigIds": ["{{configId}}"], + "variables": ["{{variables.Collection}}"], + "dataStream": { "id": "datastream-properties" }, + "scope": { + "scope": "{{scopes.Collections}}", + "workspace": "{{workspaceId}}", + "variable": "{{variables.Collection}}" + }, + "visualisation": { + "type": "data-stream-table", + "config": { + "data-stream-table": { "transpose": true } + } + } + } + }, + { + "static": false, + "w": 3, + "moved": false, + "h": 4, + "x": 1, + "y": 0, + "i": "5c456e3c-637a-4aea-b0a6-318dbbf4641e", + "z": 0, + "config": { + "_type": "tile/data-stream", + "title": "Classification Summary", + "description": "", + "activePluginConfigIds": ["{{configId}}"], + "variables": ["{{variables.Collection}}"], + "dataStream": { + "id": "{{dataStreams.collectionClassificationSummary}}", + "name": "collectionClassificationSummary", + "pluginConfigId": "{{configId}}" + }, + "scope": { + "scope": "{{scopes.Collections}}", + "workspace": "{{workspaceId}}", + "variable": "{{variables.Collection}}" + }, + "visualisation": { + "type": "data-stream-donut-chart", + "config": { + "data-stream-donut-chart": { + "valueColumn": "assets", + "labelColumn": "label", + "showValuesAsPercentage": true, + "legendPosition": "auto", + "legendMode": "table" + } + } + } + } + }, + { + "static": false, + "w": 2, + "moved": false, + "h": 5, + "x": 0, + "y": 4, + "i": "3b4de95e-0982-4058-8965-d362452e4150", + "z": 0, + "config": { + "_type": "tile/data-stream", + "title": "Asset Count by Entity Type", + "description": "", + "activePluginConfigIds": ["{{configId}}"], + "variables": ["{{variables.Collection}}"], + "dataStream": { + "id": "{{dataStreams.assetCountByEntityType}}", + "name": "assetCountByEntityType", + "pluginConfigId": "{{configId}}" + }, + "scope": { + "scope": "{{scopes.Collections}}", + "workspace": "{{workspaceId}}", + "variable": "{{variables.Collection}}" + }, + "visualisation": { + "type": "data-stream-bar-chart", + "config": { + "data-stream-bar-chart": { + "xAxisData": "entityType", + "yAxisData": ["count"], + "xAxisGroup": "none", + "showLegend": false, + "showGrid": true, + "horizontalLayout": "horizontal", + "displayMode": "actual", + "showValue": true, + "grouping": false, + "range": { "type": "auto" } + } + } + } + } + }, + { + "static": false, + "w": 2, + "moved": false, + "h": 5, + "x": 2, + "y": 4, + "i": "f335dba3-f381-4154-89cd-46674611bc64", + "z": 0, + "config": { + "_type": "tile/data-stream", + "title": "Policy Coverage", + "description": "", + "activePluginConfigIds": ["{{configId}}"], + "variables": ["{{variables.Collection}}"], + "dataStream": { + "id": "{{dataStreams.collectionPolicyCoverage}}", + "name": "collectionPolicyCoverage", + "pluginConfigId": "{{configId}}" + }, + "scope": { + "scope": "{{scopes.Collections}}", + "workspace": "{{workspaceId}}", + "variable": "{{variables.Collection}}" + }, + "visualisation": { + "type": "data-stream-table", + "config": { + "data-stream-table": { + "transpose": false, + "columnOrder": ["state", "name", "version", "decisionRuleCount", "attributeRuleCount"] + } + } + } + } + }, + { + "static": false, + "w": 4, + "moved": false, + "h": 5, + "x": 0, + "y": 9, + "i": "1f931a39-24c6-416c-b564-75bb490bcf5c", + "z": 0, + "config": { + "_type": "tile/data-stream", + "title": "Assets in Collection", + "description": "", + "activePluginConfigIds": ["{{configId}}"], + "variables": ["{{variables.Collection}}"], + "dataStream": { + "id": "{{dataStreams.assetsByCollection}}", + "name": "assetsByCollection", + "pluginConfigId": "{{configId}}" + }, + "scope": { + "scope": "{{scopes.Collections}}", + "workspace": "{{workspaceId}}", + "variable": "{{variables.Collection}}" + }, + "visualisation": { + "type": "data-stream-table", + "config": { + "data-stream-table": { + "transpose": false, + "columnOrder": ["name", "entityType", "assetType", "classification", "term", "qualifiedName"] + } + } + } + } + } + ] + } +} diff --git a/plugins/MicrosoftPurview/v1/defaultContent/dataSourceDashboard.dash.json b/plugins/MicrosoftPurview/v1/defaultContent/dataSourceDashboard.dash.json new file mode 100644 index 00000000..5f944db3 --- /dev/null +++ b/plugins/MicrosoftPurview/v1/defaultContent/dataSourceDashboard.dash.json @@ -0,0 +1,115 @@ +{ + "name": "Data Source", + "schemaVersion": "1.5", + "variables": ["{{variables.DataSource}}"], + "dashboard": { + "_type": "layout/grid", + "columns": 4, + "version": 1, + "contents": [ + { + "static": false, + "w": 1, + "moved": false, + "h": 4, + "x": 0, + "y": 0, + "i": "0f4d8de5-c60c-4fa7-beb7-76260f18e504", + "z": 0, + "config": { + "_type": "tile/data-stream", + "title": "Properties", + "description": "", + "activePluginConfigIds": ["{{configId}}"], + "variables": ["{{variables.DataSource}}"], + "dataStream": { "id": "datastream-properties" }, + "scope": { + "scope": "{{scopes.DataSources}}", + "workspace": "{{workspaceId}}", + "variable": "{{variables.DataSource}}" + }, + "visualisation": { + "type": "data-stream-table", + "config": { + "data-stream-table": { "transpose": true } + } + } + } + }, + { + "static": false, + "w": 3, + "moved": false, + "h": 4, + "x": 1, + "y": 0, + "i": "07d8ec4e-404c-4925-94c1-01a1da2e9938", + "z": 0, + "config": { + "_type": "tile/data-stream", + "title": "Scans", + "description": "", + "activePluginConfigIds": ["{{configId}}"], + "variables": ["{{variables.DataSource}}"], + "dataStream": { + "id": "{{dataStreams.dataSourceScans}}", + "name": "dataSourceScans", + "pluginConfigId": "{{configId}}" + }, + "scope": { + "scope": "{{scopes.DataSources}}", + "workspace": "{{workspaceId}}", + "variable": "{{variables.DataSource}}" + }, + "visualisation": { + "type": "data-stream-table", + "config": { + "data-stream-table": { + "transpose": false, + "columnOrder": ["name", "lastRunResult.status", "lastRunResult.startTime", "lastRunResult.endTime", "properties.scanRulesetName", "kind"] + } + } + } + } + }, + { + "static": false, + "w": 4, + "moved": false, + "h": 4, + "x": 0, + "y": 4, + "i": "18f50b03-c364-481b-a341-67f63b5c844b", + "z": 0, + "config": { + "_type": "tile/data-stream", + "title": "Scan Status", + "description": "", + "activePluginConfigIds": ["{{configId}}"], + "variables": ["{{variables.DataSource}}"], + "dataStream": { + "id": "{{dataStreams.dataSourceScans}}", + "name": "dataSourceScans", + "pluginConfigId": "{{configId}}" + }, + "scope": { + "scope": "{{scopes.DataSources}}", + "workspace": "{{workspaceId}}", + "variable": "{{variables.DataSource}}" + }, + "visualisation": { + "type": "data-stream-blocks", + "config": { + "data-stream-blocks": { + "labelColumn": "name", + "stateColumn": "lastRunResult.status", + "linkColumn": "none", + "columns": 4 + } + } + } + } + } + ] + } +} diff --git a/plugins/MicrosoftPurview/v1/defaultContent/manifest.json b/plugins/MicrosoftPurview/v1/defaultContent/manifest.json new file mode 100644 index 00000000..6fdd2350 --- /dev/null +++ b/plugins/MicrosoftPurview/v1/defaultContent/manifest.json @@ -0,0 +1,7 @@ +{ + "items": [ + { "name": "overview", "type": "dashboard" }, + { "name": "collectionDashboard", "type": "dashboard" }, + { "name": "dataSourceDashboard", "type": "dashboard" } + ] +} diff --git a/plugins/MicrosoftPurview/v1/defaultContent/overview.dash.json b/plugins/MicrosoftPurview/v1/defaultContent/overview.dash.json new file mode 100644 index 00000000..b9dc4f10 --- /dev/null +++ b/plugins/MicrosoftPurview/v1/defaultContent/overview.dash.json @@ -0,0 +1,267 @@ +{ + "name": "Overview", + "schemaVersion": "1.5", + "dashboard": { + "_type": "layout/grid", + "columns": 4, + "version": 1, + "contents": [ + { + "static": false, + "w": 1, + "moved": false, + "h": 2, + "x": 0, + "y": 0, + "i": "3e1f06c3-5cda-4be5-b3e2-0eac7bf464e1", + "z": 0, + "config": { + "_type": "tile/data-stream", + "title": "Total Assets", + "description": "", + "activePluginConfigIds": ["{{configId}}"], + "dataStream": { + "id": "{{dataStreams.accountAssetCount}}", + "name": "accountAssetCount", + "pluginConfigId": "{{configId}}" + }, + "visualisation": { + "type": "data-stream-scalar", + "config": { + "data-stream-scalar": { + "value": "totalAssets", + "comparisonColumn": "none", + "label": "assets" + } + } + } + } + }, + { + "static": false, + "w": 1, + "moved": false, + "h": 2, + "x": 1, + "y": 0, + "i": "51587d73-b87d-4fd8-8c6c-9dc637bd691e", + "z": 0, + "config": { + "_type": "tile/data-stream", + "title": "Collections", + "description": "", + "activePluginConfigIds": ["{{configId}}"], + "dataStream": { + "id": "{{dataStreams.collections}}", + "name": "collections", + "pluginConfigId": "{{configId}}" + }, + "visualisation": { + "type": "data-stream-scalar", + "config": { + "data-stream-scalar": { + "value": { "type": "count" }, + "comparisonColumn": "none", + "label": "collections" + } + } + } + } + }, + { + "static": false, + "w": 1, + "moved": false, + "h": 2, + "x": 2, + "y": 0, + "i": "415693db-8ef0-41c0-a948-ea5ae0fef955", + "z": 0, + "config": { + "_type": "tile/data-stream", + "title": "Data Sources", + "description": "", + "activePluginConfigIds": ["{{configId}}"], + "dataStream": { + "id": "{{dataStreams.listDataSources}}", + "name": "listDataSources", + "pluginConfigId": "{{configId}}" + }, + "visualisation": { + "type": "data-stream-scalar", + "config": { + "data-stream-scalar": { + "value": { "type": "count" }, + "comparisonColumn": "none", + "label": "data sources" + } + } + } + } + }, + { + "static": false, + "w": 1, + "moved": false, + "h": 2, + "x": 3, + "y": 0, + "i": "0ac7deb9-c0cb-4aec-8031-6c91d95ab3c8", + "z": 0, + "config": { + "_type": "tile/data-stream", + "title": "Metadata Policies", + "description": "", + "activePluginConfigIds": ["{{configId}}"], + "dataStream": { + "id": "{{dataStreams.metadataPolicies}}", + "name": "metadataPolicies", + "pluginConfigId": "{{configId}}" + }, + "visualisation": { + "type": "data-stream-scalar", + "config": { + "data-stream-scalar": { + "value": { "type": "count" }, + "comparisonColumn": "none", + "label": "policies" + } + } + } + } + }, + { + "static": false, + "w": 2, + "moved": false, + "h": 4, + "x": 0, + "y": 2, + "i": "cae04063-e8b4-4557-9e25-3ad54cc16bc8", + "z": 0, + "config": { + "_type": "tile/data-stream", + "title": "Asset Count by Entity Type", + "description": "", + "activePluginConfigIds": ["{{configId}}"], + "dataStream": { + "id": "{{dataStreams.assetsByEntityTypeGlobal}}", + "name": "assetsByEntityTypeGlobal", + "pluginConfigId": "{{configId}}" + }, + "visualisation": { + "type": "data-stream-bar-chart", + "config": { + "data-stream-bar-chart": { + "xAxisData": "entityType", + "yAxisData": ["count"], + "xAxisGroup": "none", + "xAxisLabel": "", + "yAxisLabel": "", + "showXAxisLabel": false, + "showYAxisLabel": false, + "showLegend": false, + "showGrid": true, + "horizontalLayout": "horizontal", + "displayMode": "actual", + "showValue": true, + "grouping": false, + "range": { "type": "auto" } + } + } + } + } + }, + { + "static": false, + "w": 2, + "moved": false, + "h": 4, + "x": 2, + "y": 2, + "i": "898b191e-9d88-48d3-8964-de7e2f59d25e", + "z": 0, + "config": { + "_type": "tile/data-stream", + "title": "Classification Coverage", + "description": "", + "activePluginConfigIds": ["{{configId}}"], + "dataStream": { + "id": "{{dataStreams.classificationCoverage}}", + "name": "classificationCoverage", + "pluginConfigId": "{{configId}}" + }, + "visualisation": { + "type": "data-stream-table", + "config": { + "data-stream-table": { + "transpose": false, + "columnOrder": ["classification", "assets", "coverage"] + } + } + } + } + }, + { + "static": false, + "w": 4, + "moved": false, + "h": 4, + "x": 0, + "y": 6, + "i": "1d34b3f2-89f5-4df2-a280-713b65d94731", + "z": 0, + "config": { + "_type": "tile/data-stream", + "title": "Data Sources", + "description": "", + "activePluginConfigIds": ["{{configId}}"], + "dataStream": { + "id": "{{dataStreams.listDataSources}}", + "name": "listDataSources", + "pluginConfigId": "{{configId}}" + }, + "visualisation": { + "type": "data-stream-table", + "config": { + "data-stream-table": { + "transpose": false, + "columnOrder": ["name", "kind", "collection", "subscriptionUrl"] + } + } + } + } + }, + { + "static": false, + "w": 4, + "moved": false, + "h": 4, + "x": 0, + "y": 10, + "i": "c87c3519-81e9-48b3-831c-3bf19c8f21f3", + "z": 0, + "config": { + "_type": "tile/data-stream", + "title": "Metadata Policies", + "description": "", + "activePluginConfigIds": ["{{configId}}"], + "dataStream": { + "id": "{{dataStreams.metadataPolicies}}", + "name": "metadataPolicies", + "pluginConfigId": "{{configId}}" + }, + "visualisation": { + "type": "data-stream-table", + "config": { + "data-stream-table": { + "transpose": false, + "columnOrder": ["name", "collection", "version", "decisionRuleCount", "attributeRuleCount"] + } + } + } + } + } + ] + } +} diff --git a/plugins/MicrosoftPurview/v1/defaultContent/scopes.json b/plugins/MicrosoftPurview/v1/defaultContent/scopes.json new file mode 100644 index 00000000..a23303c4 --- /dev/null +++ b/plugins/MicrosoftPurview/v1/defaultContent/scopes.json @@ -0,0 +1,32 @@ +[ + { + "name": "Collections", + "matches": { + "sourceType": { + "type": "oneOf", + "values": ["Purview Collection"] + } + }, + "variable": { + "name": "Collection", + "type": "object", + "default": "none", + "allowMultipleSelection": false + } + }, + { + "name": "DataSources", + "matches": { + "sourceType": { + "type": "oneOf", + "values": ["Purview Data Source"] + } + }, + "variable": { + "name": "DataSource", + "type": "object", + "default": "none", + "allowMultipleSelection": false + } + } +] diff --git a/plugins/MicrosoftPurview/v1/docs/README.md b/plugins/MicrosoftPurview/v1/docs/README.md new file mode 100644 index 00000000..7f27595d --- /dev/null +++ b/plugins/MicrosoftPurview/v1/docs/README.md @@ -0,0 +1,66 @@ +# Microsoft Purview + +This plugin connects SquaredUp to Microsoft Purview and surfaces data governance posture across collections, data sources, scans, classifications, policies, and lineage. + +## What it does + +- Imports Purview **Collections**, **Data Sources**, and **Scans** into the SquaredUp graph so they can be used as objects, drilldowns, and dashboard variables. +- Provides data streams for: + - Assets — searchable per collection, with breakdowns by entity type and classification + - Classification coverage — what proportion of assets carry each classification + - Scan status and run history — per data source, per scan, and global recent runs + - Policy coverage — metadata policies attached to collections + - Lineage — upstream and downstream lineage for a given asset +- Ships with out-of-the-box dashboards: an overall **Overview** plus per-object **Collection**, **Data Source**, and **Scan** perspectives. + +## Prerequisites + +You need a Microsoft Entra ID (Azure AD) **app registration** with a **client secret** and at least the **Data Reader** role on the Purview root collection. For scan endpoints, **Data Source Administrator** is required. + +### 1. Register an application in Microsoft Entra ID + +1. In the Azure portal, go to **Microsoft Entra ID → App registrations → New registration**. +2. Give it a name (e.g. `SquaredUp Purview Reader`) and register it (no redirect URI needed). +3. From the **Overview** page, copy the **Application (client) ID** and **Directory (tenant) ID**. +4. Go to **Certificates & secrets → Client secrets → New client secret**. Copy the **Value** immediately — it's only shown once. + +### 2. Grant the application access to your Purview account + +Purview uses **collection-level role assignments**, not Microsoft Graph API permissions. Grant the app registration access inside Purview itself. + +1. Open the **Microsoft Purview governance portal** for your account. +2. Go to **Data Map → Collections → Root collection → Role assignments**. +3. Add the app registration to: + - **Data Reader** (required — read assets, classifications, lineage, search) + - **Data Source Administrator** (required — list data sources, scans, and scan history) + - **Policy Author** or **Collection Administrator** is **not** needed for read-only use, but the app must be able to **read** metadata policies. The Data Source Administrator role is sufficient for this. + +Role assignments at the root collection are inherited by all child collections. + +## Configuration fields + +| Field | What it is | Where to find it | Required | +| --------------------------- | ----------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------- | -------- | +| Purview account name | The short name of your Purview account — the bit before `.purview.azure.com`. For `contoso-pv.purview.azure.com`, enter `contoso-pv` | Azure portal → your Purview account → **Overview** (the "Name" field) | Yes | +| Directory (tenant) ID | Your Entra ID tenant ID | Entra ID → **Overview** → Tenant ID | Yes | +| Application (client) ID | The app registration's client ID | App registration → **Overview** → Application (client) ID | Yes | +| Client secret | The secret value generated in step 1 | App registration → **Certificates & secrets** | Yes | + +## What gets indexed + +| Object type | Represents | +| ---------------------- | --------------------------------------------------------------------------------------------------------- | +| **Purview Collection** | An organisational unit in your Purview account. The root collection is named after the account itself. | +| **Purview Data Source** | A registered data source — e.g. an Azure SQL database, ADLS Gen2 account, Snowflake instance, Amazon S3 bucket. | +| **Purview Scan** | A configured scan attached to a data source. | + +Assets (tables, files, columns) are **not** indexed because a single Purview account can hold millions of them. They are queried on-demand by the data streams instead. + +## Known limitations + +- **Asset cardinality** — large estates can return millions of assets. The asset search streams page through results but cap at the API's per-call limit of 1000 rows per page. Use the keyword and entity-type filters to narrow results. +- **Lineage depth** — the lineage stream walks up to a configurable depth (default 3 hops) in each direction; very deep lineage graphs are truncated by the API. +- **Policy coverage** — only **metadata policies** are surfaced. DLP and sensitivity-label policies live in Microsoft Graph and are not in scope for this plugin. +- **Permissions are not Graph permissions** — granting roles in Entra ID alone does nothing. The app registration must be added to Purview collection role assignments. +- **Account name format** — the account name field is the bare account name only. Do **not** include `https://` or `.purview.azure.com`. +- **API throttling** — the classic Purview Data Plane has per-account rate limits. Imports can take several minutes for large estates. diff --git a/plugins/MicrosoftPurview/v1/icon.svg b/plugins/MicrosoftPurview/v1/icon.svg new file mode 100644 index 00000000..b8fcf842 --- /dev/null +++ b/plugins/MicrosoftPurview/v1/icon.svg @@ -0,0 +1,30 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/plugins/MicrosoftPurview/v1/indexDefinitions/default.json b/plugins/MicrosoftPurview/v1/indexDefinitions/default.json new file mode 100644 index 00000000..5927c073 --- /dev/null +++ b/plugins/MicrosoftPurview/v1/indexDefinitions/default.json @@ -0,0 +1,36 @@ +{ + "steps": [ + { + "name": "collections", + "dataStream": { "name": "collections" }, + "timeframe": "none", + "objectMapping": { + "id": "name", + "name": "displayName", + "type": "sourceType", + "properties": [ + "collectionName", + { "parentCollection": "parentCollectionName" }, + "description" + ] + } + }, + { + "name": "dataSources", + "dataStream": { "name": "listDataSources" }, + "timeframe": "none", + "objectMapping": { + "id": "name", + "name": "name", + "type": "sourceType", + "properties": [ + "kind", + "collection", + "endpoint", + "resourceGroup", + "subscriptionId" + ] + } + } + ] +} diff --git a/plugins/MicrosoftPurview/v1/metadata.json b/plugins/MicrosoftPurview/v1/metadata.json new file mode 100644 index 00000000..d5fdfa25 --- /dev/null +++ b/plugins/MicrosoftPurview/v1/metadata.json @@ -0,0 +1,70 @@ +{ + "name": "microsoft-purview", + "displayName": "Microsoft Purview", + "version": "1.4.3", + "author": { + "name": "John Hayes", + "type": "community" + }, + "description": "Visualize Microsoft Purview data governance: assets by collection, classification coverage, scan status and history, policy coverage, and lineage.", + "category": "Security", + "type": "hybrid", + "schemaVersion": "2.0", + "keywords": [ + "purview", + "microsoft", + "azure", + "data governance", + "data catalog", + "lineage", + "classification", + "scan", + "data map" + ], + "objectTypes": [ + "Purview Collection", + "Purview Data Source" + ], + "links": [ + { + "category": "documentation", + "url": "https://github.com/squaredup/plugins/blob/main/plugins/MicrosoftPurview/v1/docs/README.md", + "label": "Help adding this plugin" + }, + { + "category": "source", + "url": "https://github.com/squaredup/plugins/tree/main/plugins/MicrosoftPurview/v1", + "label": "Repository" + } + ], + "base": { + "plugin": "WebAPI", + "majorVersion": "1", + "config": { + "baseUrl": "https://{{accountName}}.purview.azure.com/", + "authMode": "oauth2", + "oauth2GrantType": "clientCredentials", + "oauth2TokenUrl": "https://login.microsoftonline.com/{{tenantId}}/oauth2/v2.0/token", + "oauth2ClientId": "{{clientId}}", + "oauth2ClientSecret": "{{clientSecret}}", + "oauth2ClientSecretLocationDuringAuth": "body", + "oauth2Scope": "https://purview.azure.net/.default", + "oauth2TokenExtraHeaders": [ + { + "key": "Content-Type", + "value": "application/x-www-form-urlencoded" + } + ], + "oauth2TokenExtraArgs": [ + { + "key": "grant_type", + "value": "client_credentials" + }, + { + "key": "scope", + "value": "https://purview.azure.net/.default" + } + ] + } + } +} diff --git a/plugins/MicrosoftPurview/v1/ui.json b/plugins/MicrosoftPurview/v1/ui.json new file mode 100644 index 00000000..7c205444 --- /dev/null +++ b/plugins/MicrosoftPurview/v1/ui.json @@ -0,0 +1,39 @@ +[ + { + "type": "text", + "name": "accountName", + "label": "Purview account name", + "placeholder": "e.g. contoso-pv", + "help": "The short name of your Purview account — the part before `.purview.azure.com`. Do **not** include `https://` or the full domain.", + "validation": { + "required": true + } + }, + { + "type": "text", + "name": "tenantId", + "label": "Directory (tenant) ID", + "placeholder": "Enter a directory (tenant) ID", + "validation": { + "required": true + } + }, + { + "type": "text", + "name": "clientId", + "label": "Application (client) ID", + "placeholder": "Enter an application (client) ID", + "validation": { + "required": true + } + }, + { + "type": "password", + "name": "clientSecret", + "label": "Client secret", + "placeholder": "Enter a client secret", + "validation": { + "required": true + } + } +] From 02b33887e656530a5f4ceb055e287021f0978383 Mon Sep 17 00:00:00 2001 From: John Hayes Date: Thu, 23 Jul 2026 14:38:18 +0100 Subject: [PATCH 2/7] Update Microsoft Purview plugin to v1.5.0 - Fix classification summary (count from assets instead of missing facet count field) - Add Classified Assets stream (all classifications, one row per asset-classification pair) - Add Assets with Custom Classifications stream (non-Microsoft classifications only) - Add Custom Classifications stream (account-wide, available on Collection and Data Source) - Update Scans on Data Source stream with dataSourceName field and postRequestScript - Add Duration (s) numeric column to Scan History for average queries - Restrict Scan History to single object selection - Remove Collection and Data Source out-of-the-box dashboards Co-Authored-By: Claude Sonnet 4.6 --- .../collectionClassificationSummary.json | 16 +- .../collectionClassifiedAssets.json | 35 ++++ .../collectionCustomClassifiedAssets.json | 35 ++++ .../v1/dataStreams/customClassifications.json | 25 +++ .../v1/dataStreams/dataSourceScans.json | 31 +-- .../v1/dataStreams/scanHistory.json | 9 + .../v1/dataStreams/scansForDataSource.json | 6 +- .../collectionClassificationSummary.js | 4 +- .../scripts/collectionClassifiedAssets.js | 17 ++ .../collectionCustomClassifiedAssets.js | 17 ++ .../scripts/customClassifications.js | 12 ++ .../v1/dataStreams/scripts/dataSourceScans.js | 6 - .../collectionDashboard.dash.json | 196 ------------------ .../dataSourceDashboard.dash.json | 115 ---------- .../v1/defaultContent/manifest.json | 4 +- plugins/MicrosoftPurview/v1/metadata.json | 2 +- 16 files changed, 165 insertions(+), 365 deletions(-) create mode 100644 plugins/MicrosoftPurview/v1/dataStreams/collectionClassifiedAssets.json create mode 100644 plugins/MicrosoftPurview/v1/dataStreams/collectionCustomClassifiedAssets.json create mode 100644 plugins/MicrosoftPurview/v1/dataStreams/customClassifications.json create mode 100644 plugins/MicrosoftPurview/v1/dataStreams/scripts/collectionClassifiedAssets.js create mode 100644 plugins/MicrosoftPurview/v1/dataStreams/scripts/collectionCustomClassifiedAssets.js create mode 100644 plugins/MicrosoftPurview/v1/dataStreams/scripts/customClassifications.js delete mode 100644 plugins/MicrosoftPurview/v1/defaultContent/collectionDashboard.dash.json delete mode 100644 plugins/MicrosoftPurview/v1/defaultContent/dataSourceDashboard.dash.json diff --git a/plugins/MicrosoftPurview/v1/dataStreams/collectionClassificationSummary.json b/plugins/MicrosoftPurview/v1/dataStreams/collectionClassificationSummary.json index 92d91c02..153814c4 100644 --- a/plugins/MicrosoftPurview/v1/dataStreams/collectionClassificationSummary.json +++ b/plugins/MicrosoftPurview/v1/dataStreams/collectionClassificationSummary.json @@ -12,17 +12,15 @@ ], "postBody": { "keywords": null, - "limit": 1, + "limit": 1000, "filter": { "collectionId": "{{object.collectionName}}" - }, - "facets": [ - { - "facet": "classification", - "count": 200, - "sort": { "count": "desc" } - } - ] + } + }, + "paging": { + "mode": "token", + "in": { "realm": "payload", "path": "continuationToken" }, + "out": { "realm": "body", "path": "continuationToken" } }, "postRequestScript": "collectionClassificationSummary.js" }, diff --git a/plugins/MicrosoftPurview/v1/dataStreams/collectionClassifiedAssets.json b/plugins/MicrosoftPurview/v1/dataStreams/collectionClassifiedAssets.json new file mode 100644 index 00000000..0490d8fd --- /dev/null +++ b/plugins/MicrosoftPurview/v1/dataStreams/collectionClassifiedAssets.json @@ -0,0 +1,35 @@ +{ + "name": "collectionClassifiedAssets", + "displayName": "Classified Assets", + "description": "Assets with classifications in the selected collection, one row per asset-classification pair", + "tags": ["Classifications"], + "baseDataSourceName": "httpRequestScopedSingle", + "config": { + "httpMethod": "post", + "endpointPath": "datamap/api/search/query", + "getArgs": [ + { "key": "api-version", "value": "2023-09-01" } + ], + "postBody": { + "keywords": null, + "limit": 1000, + "filter": { + "collectionId": "{{object.collectionName}}" + } + }, + "paging": { + "mode": "token", + "in": { "realm": "payload", "path": "continuationToken" }, + "out": { "realm": "body", "path": "continuationToken" } + }, + "postRequestScript": "collectionClassifiedAssets.js" + }, + "matches": { "sourceType": { "type": "oneOf", "values": ["Purview Collection"] } }, + "metadata": [ + { "name": "assetName", "displayName": "Asset", "role": "label" }, + { "name": "entityType", "displayName": "Entity Type" }, + { "name": "collectionId", "displayName": "Collection", "visible": false }, + { "name": "classification", "displayName": "Classification" } + ], + "timeframes": false +} diff --git a/plugins/MicrosoftPurview/v1/dataStreams/collectionCustomClassifiedAssets.json b/plugins/MicrosoftPurview/v1/dataStreams/collectionCustomClassifiedAssets.json new file mode 100644 index 00000000..1570fa13 --- /dev/null +++ b/plugins/MicrosoftPurview/v1/dataStreams/collectionCustomClassifiedAssets.json @@ -0,0 +1,35 @@ +{ + "name": "collectionCustomClassifiedAssets", + "displayName": "Assets with Custom Classifications", + "description": "Assets using custom (non-Microsoft) classifications in the selected collection, one row per asset-classification pair", + "tags": ["Classifications"], + "baseDataSourceName": "httpRequestScopedSingle", + "config": { + "httpMethod": "post", + "endpointPath": "datamap/api/search/query", + "getArgs": [ + { "key": "api-version", "value": "2023-09-01" } + ], + "postBody": { + "keywords": null, + "limit": 1000, + "filter": { + "collectionId": "{{object.collectionName}}" + } + }, + "paging": { + "mode": "token", + "in": { "realm": "payload", "path": "continuationToken" }, + "out": { "realm": "body", "path": "continuationToken" } + }, + "postRequestScript": "collectionCustomClassifiedAssets.js" + }, + "matches": { "sourceType": { "type": "oneOf", "values": ["Purview Collection"] } }, + "metadata": [ + { "name": "assetName", "displayName": "Asset", "role": "label" }, + { "name": "entityType", "displayName": "Entity Type" }, + { "name": "collectionId", "displayName": "Collection", "visible": false }, + { "name": "classification", "displayName": "Classification" } + ], + "timeframes": false +} diff --git a/plugins/MicrosoftPurview/v1/dataStreams/customClassifications.json b/plugins/MicrosoftPurview/v1/dataStreams/customClassifications.json new file mode 100644 index 00000000..d636e650 --- /dev/null +++ b/plugins/MicrosoftPurview/v1/dataStreams/customClassifications.json @@ -0,0 +1,25 @@ +{ + "name": "customClassifications", + "displayName": "Custom Classifications", + "description": "Custom classification definitions for this Purview account", + "tags": ["Classifications"], + "baseDataSourceName": "httpRequestScopedSingle", + "config": { + "httpMethod": "get", + "endpointPath": "datamap/api/atlas/v2/types/typedefs", + "getArgs": [ + { "key": "api-version", "value": "2023-09-01" } + ], + "postRequestScript": "customClassifications.js" + }, + "matches": { "sourceType": { "type": "oneOf", "values": ["Purview Collection", "Purview Data Source"] } }, + "metadata": [ + { "name": "name", "displayName": "Classification", "role": "label" }, + { "name": "description", "displayName": "Description" }, + { "name": "createdBy", "displayName": "Created By" }, + { "name": "createTime", "displayName": "Created", "shape": "date" }, + { "name": "updatedBy", "displayName": "Updated By" }, + { "name": "updateTime", "displayName": "Updated", "shape": "date" } + ], + "timeframes": false +} diff --git a/plugins/MicrosoftPurview/v1/dataStreams/dataSourceScans.json b/plugins/MicrosoftPurview/v1/dataStreams/dataSourceScans.json index c0d0a906..edd4ed39 100644 --- a/plugins/MicrosoftPurview/v1/dataStreams/dataSourceScans.json +++ b/plugins/MicrosoftPurview/v1/dataStreams/dataSourceScans.json @@ -1,7 +1,7 @@ { "name": "dataSourceScans", "displayName": "Scans on Data Source", - "description": "Scans configured on the selected data source with latest run status", + "description": "Scan definitions configured on the selected data source", "tags": ["Data Sources"], "baseDataSourceName": "httpRequestScopedSingle", "config": { @@ -40,35 +40,6 @@ "name": "properties.scanRulesetType", "displayName": "Ruleset Type" }, - { - "name": "lastRunResult.status", - "displayName": "Last Status", - "shape": [ - "state", - { - "map": { - "success": ["Succeeded"], - "error": ["Failed", "TransientFailure"], - "warning": ["Canceled"], - "unknown": ["Accepted", "InProgress"] - } - } - ] - }, - { - "name": "lastRunResult.startTime", - "displayName": "Last Run Started", - "shape": "date" - }, - { - "name": "lastRunResult.endTime", - "displayName": "Last Run Ended", - "shape": "date" - }, - { - "name": "lastRunResult.scanLevelType", - "displayName": "Scan Level" - }, { "name": "creationType", "displayName": "Creation Type" } ], "timeframes": false diff --git a/plugins/MicrosoftPurview/v1/dataStreams/scanHistory.json b/plugins/MicrosoftPurview/v1/dataStreams/scanHistory.json index d5d200e0..8faa2c22 100644 --- a/plugins/MicrosoftPurview/v1/dataStreams/scanHistory.json +++ b/plugins/MicrosoftPurview/v1/dataStreams/scanHistory.json @@ -33,9 +33,11 @@ "in": { "realm": "payload", "path": "nextLink" } } }, + "objectLimit": 1, "matches": { "sourceType": { "type": "oneOf", "values": ["Purview Data Source"] } }, "metadata": [ { "name": "id", "displayName": "Run ID", "shape": "guid", "visible": false }, + { "name": "parentId", "displayName": "Scan Definition ID", "shape": "guid", "visible": false }, { "name": "startTime", "displayName": "Started", "shape": "date", "role": "timestamp" }, { "name": "endTime", "displayName": "Ended", "shape": "date" }, { @@ -62,6 +64,13 @@ "valueExpression": "{{ $['startTime'] && $['endTime'] ? (new Date($['endTime']).getTime() - new Date($['startTime']).getTime()) / 1000 : null }}", "shape": ["seconds", { "formatDuration": true, "decimalPlaces": 0 }] }, + { + "name": "durationSeconds", + "displayName": "Duration (s)", + "computed": true, + "valueExpression": "{{ $['startTime'] && $['endTime'] ? (new Date($['endTime']).getTime() - new Date($['startTime']).getTime()) / 1000 : null }}", + "shape": "number" + }, { "name": "discoveryExecutionDetails.statistics.assets.discovered", "displayName": "Assets Discovered", diff --git a/plugins/MicrosoftPurview/v1/dataStreams/scansForDataSource.json b/plugins/MicrosoftPurview/v1/dataStreams/scansForDataSource.json index 647c7fd7..c3224eb9 100644 --- a/plugins/MicrosoftPurview/v1/dataStreams/scansForDataSource.json +++ b/plugins/MicrosoftPurview/v1/dataStreams/scansForDataSource.json @@ -20,15 +20,15 @@ "matches": { "sourceType": { "type": "oneOf", "values": ["Purview Data Source"] } }, "metadata": [ { "name": "name", "displayName": "Scan Name", "role": "value", "visible": false }, - { "name": "lastRunResult.discoveryExecutionDetails.discoveryStartTime", "displayName": "Last Run Started", "shape": "date", "visible": false }, + { "name": "lastRunResult.startTime", "displayName": "Last Run Started", "shape": "date", "visible": false }, { "name": "scanLabel", "displayName": "Scan", "role": "label", "computed": true, - "valueExpression": "{{ $['lastRunResult.discoveryExecutionDetails.discoveryStartTime'] ? $['name'] + ' — ' + new Date($['lastRunResult.discoveryExecutionDetails.discoveryStartTime']).toISOString().slice(0, 10) : $['name'] + ' — never run' }}" + "valueExpression": "{{ $['lastRunResult.startTime'] ? $['name'] + ' — ' + new Date($['lastRunResult.startTime']).toISOString().slice(0, 10) : $['name'] + ' — never run' }}" } ], "timeframes": false, - "defaultShaping": { "sort": { "by": [["lastRunResult.discoveryExecutionDetails.discoveryStartTime", "desc"]] } } + "defaultShaping": { "sort": { "by": [["lastRunResult.startTime", "desc"]] } } } diff --git a/plugins/MicrosoftPurview/v1/dataStreams/scripts/collectionClassificationSummary.js b/plugins/MicrosoftPurview/v1/dataStreams/scripts/collectionClassificationSummary.js index 82daace8..447ca8cf 100644 --- a/plugins/MicrosoftPurview/v1/dataStreams/scripts/collectionClassificationSummary.js +++ b/plugins/MicrosoftPurview/v1/dataStreams/scripts/collectionClassificationSummary.js @@ -1,6 +1,6 @@ +const assets = data?.value || []; const total = Number(data?.['@search.count']) || 0; -const facets = data?.['@search.facets']?.classification || []; -const classified = facets.reduce((sum, f) => sum + (Number(f.count) || 0), 0); +const classified = assets.filter(a => (a.classification?.length ?? 0) > 0).length; const unclassified = Math.max(0, total - classified); result = [ diff --git a/plugins/MicrosoftPurview/v1/dataStreams/scripts/collectionClassifiedAssets.js b/plugins/MicrosoftPurview/v1/dataStreams/scripts/collectionClassifiedAssets.js new file mode 100644 index 00000000..9b5b7dac --- /dev/null +++ b/plugins/MicrosoftPurview/v1/dataStreams/scripts/collectionClassifiedAssets.js @@ -0,0 +1,17 @@ +const assets = data?.value || []; +const rows = []; + +for (const asset of assets) { + const classifications = asset.classification || []; + if (classifications.length === 0) continue; + for (const cls of classifications) { + rows.push({ + assetName: asset.name || '', + entityType: asset.entityType || '', + collectionId: asset.collectionId || '', + classification: cls, + }); + } +} + +result = rows; diff --git a/plugins/MicrosoftPurview/v1/dataStreams/scripts/collectionCustomClassifiedAssets.js b/plugins/MicrosoftPurview/v1/dataStreams/scripts/collectionCustomClassifiedAssets.js new file mode 100644 index 00000000..b3c00200 --- /dev/null +++ b/plugins/MicrosoftPurview/v1/dataStreams/scripts/collectionCustomClassifiedAssets.js @@ -0,0 +1,17 @@ +const assets = data?.value || []; +const rows = []; + +for (const asset of assets) { + const classifications = (asset.classification || []).filter(c => !c.startsWith('MICROSOFT.')); + if (classifications.length === 0) continue; + for (const cls of classifications) { + rows.push({ + assetName: asset.name || '', + entityType: asset.entityType || '', + collectionId: asset.collectionId || '', + classification: cls, + }); + } +} + +result = rows; diff --git a/plugins/MicrosoftPurview/v1/dataStreams/scripts/customClassifications.js b/plugins/MicrosoftPurview/v1/dataStreams/scripts/customClassifications.js new file mode 100644 index 00000000..6ef568a2 --- /dev/null +++ b/plugins/MicrosoftPurview/v1/dataStreams/scripts/customClassifications.js @@ -0,0 +1,12 @@ +const defs = data?.classificationDefs || []; + +result = defs + .filter(c => !c.name?.startsWith('MICROSOFT.')) + .map(c => ({ + name: c.name || '', + description: c.description || '', + createdBy: c.createdBy || '', + createTime: c.createTime ? new Date(c.createTime).toISOString() : null, + updatedBy: c.updatedBy || '', + updateTime: c.updateTime ? new Date(c.updateTime).toISOString() : null, + })); diff --git a/plugins/MicrosoftPurview/v1/dataStreams/scripts/dataSourceScans.js b/plugins/MicrosoftPurview/v1/dataStreams/scripts/dataSourceScans.js index 97279c43..e2d144dd 100644 --- a/plugins/MicrosoftPurview/v1/dataStreams/scripts/dataSourceScans.js +++ b/plugins/MicrosoftPurview/v1/dataStreams/scripts/dataSourceScans.js @@ -1,8 +1,6 @@ const scans = data?.value || []; result = scans.map((scan) => { - const lr = scan.lastRunResult || {}; - const discovery = lr.discoveryExecutionDetails || {}; const props = scan.properties || {}; return { @@ -13,9 +11,5 @@ result = scans.map((scan) => { 'properties.collection.referenceName': props.collection?.referenceName || '', 'properties.scanRulesetName': props.scanRulesetName || '', 'properties.scanRulesetType': props.scanRulesetType || '', - 'lastRunResult.status': lr.scanRunStatus || '', - 'lastRunResult.startTime': discovery.discoveryStartTime || null, - 'lastRunResult.endTime': discovery.discoveryEndTime || null, - 'lastRunResult.scanLevelType': lr.scanLevelType || '', }; }); diff --git a/plugins/MicrosoftPurview/v1/defaultContent/collectionDashboard.dash.json b/plugins/MicrosoftPurview/v1/defaultContent/collectionDashboard.dash.json deleted file mode 100644 index 7c4c53e3..00000000 --- a/plugins/MicrosoftPurview/v1/defaultContent/collectionDashboard.dash.json +++ /dev/null @@ -1,196 +0,0 @@ -{ - "name": "Collection", - "schemaVersion": "1.5", - "variables": ["{{variables.Collection}}"], - "dashboard": { - "_type": "layout/grid", - "columns": 4, - "version": 1, - "contents": [ - { - "static": false, - "w": 1, - "moved": false, - "h": 4, - "x": 0, - "y": 0, - "i": "25f69cbd-2d27-4175-8273-75f70bea420d", - "z": 0, - "config": { - "_type": "tile/data-stream", - "title": "Properties", - "description": "", - "activePluginConfigIds": ["{{configId}}"], - "variables": ["{{variables.Collection}}"], - "dataStream": { "id": "datastream-properties" }, - "scope": { - "scope": "{{scopes.Collections}}", - "workspace": "{{workspaceId}}", - "variable": "{{variables.Collection}}" - }, - "visualisation": { - "type": "data-stream-table", - "config": { - "data-stream-table": { "transpose": true } - } - } - } - }, - { - "static": false, - "w": 3, - "moved": false, - "h": 4, - "x": 1, - "y": 0, - "i": "5c456e3c-637a-4aea-b0a6-318dbbf4641e", - "z": 0, - "config": { - "_type": "tile/data-stream", - "title": "Classification Summary", - "description": "", - "activePluginConfigIds": ["{{configId}}"], - "variables": ["{{variables.Collection}}"], - "dataStream": { - "id": "{{dataStreams.collectionClassificationSummary}}", - "name": "collectionClassificationSummary", - "pluginConfigId": "{{configId}}" - }, - "scope": { - "scope": "{{scopes.Collections}}", - "workspace": "{{workspaceId}}", - "variable": "{{variables.Collection}}" - }, - "visualisation": { - "type": "data-stream-donut-chart", - "config": { - "data-stream-donut-chart": { - "valueColumn": "assets", - "labelColumn": "label", - "showValuesAsPercentage": true, - "legendPosition": "auto", - "legendMode": "table" - } - } - } - } - }, - { - "static": false, - "w": 2, - "moved": false, - "h": 5, - "x": 0, - "y": 4, - "i": "3b4de95e-0982-4058-8965-d362452e4150", - "z": 0, - "config": { - "_type": "tile/data-stream", - "title": "Asset Count by Entity Type", - "description": "", - "activePluginConfigIds": ["{{configId}}"], - "variables": ["{{variables.Collection}}"], - "dataStream": { - "id": "{{dataStreams.assetCountByEntityType}}", - "name": "assetCountByEntityType", - "pluginConfigId": "{{configId}}" - }, - "scope": { - "scope": "{{scopes.Collections}}", - "workspace": "{{workspaceId}}", - "variable": "{{variables.Collection}}" - }, - "visualisation": { - "type": "data-stream-bar-chart", - "config": { - "data-stream-bar-chart": { - "xAxisData": "entityType", - "yAxisData": ["count"], - "xAxisGroup": "none", - "showLegend": false, - "showGrid": true, - "horizontalLayout": "horizontal", - "displayMode": "actual", - "showValue": true, - "grouping": false, - "range": { "type": "auto" } - } - } - } - } - }, - { - "static": false, - "w": 2, - "moved": false, - "h": 5, - "x": 2, - "y": 4, - "i": "f335dba3-f381-4154-89cd-46674611bc64", - "z": 0, - "config": { - "_type": "tile/data-stream", - "title": "Policy Coverage", - "description": "", - "activePluginConfigIds": ["{{configId}}"], - "variables": ["{{variables.Collection}}"], - "dataStream": { - "id": "{{dataStreams.collectionPolicyCoverage}}", - "name": "collectionPolicyCoverage", - "pluginConfigId": "{{configId}}" - }, - "scope": { - "scope": "{{scopes.Collections}}", - "workspace": "{{workspaceId}}", - "variable": "{{variables.Collection}}" - }, - "visualisation": { - "type": "data-stream-table", - "config": { - "data-stream-table": { - "transpose": false, - "columnOrder": ["state", "name", "version", "decisionRuleCount", "attributeRuleCount"] - } - } - } - } - }, - { - "static": false, - "w": 4, - "moved": false, - "h": 5, - "x": 0, - "y": 9, - "i": "1f931a39-24c6-416c-b564-75bb490bcf5c", - "z": 0, - "config": { - "_type": "tile/data-stream", - "title": "Assets in Collection", - "description": "", - "activePluginConfigIds": ["{{configId}}"], - "variables": ["{{variables.Collection}}"], - "dataStream": { - "id": "{{dataStreams.assetsByCollection}}", - "name": "assetsByCollection", - "pluginConfigId": "{{configId}}" - }, - "scope": { - "scope": "{{scopes.Collections}}", - "workspace": "{{workspaceId}}", - "variable": "{{variables.Collection}}" - }, - "visualisation": { - "type": "data-stream-table", - "config": { - "data-stream-table": { - "transpose": false, - "columnOrder": ["name", "entityType", "assetType", "classification", "term", "qualifiedName"] - } - } - } - } - } - ] - } -} diff --git a/plugins/MicrosoftPurview/v1/defaultContent/dataSourceDashboard.dash.json b/plugins/MicrosoftPurview/v1/defaultContent/dataSourceDashboard.dash.json deleted file mode 100644 index 5f944db3..00000000 --- a/plugins/MicrosoftPurview/v1/defaultContent/dataSourceDashboard.dash.json +++ /dev/null @@ -1,115 +0,0 @@ -{ - "name": "Data Source", - "schemaVersion": "1.5", - "variables": ["{{variables.DataSource}}"], - "dashboard": { - "_type": "layout/grid", - "columns": 4, - "version": 1, - "contents": [ - { - "static": false, - "w": 1, - "moved": false, - "h": 4, - "x": 0, - "y": 0, - "i": "0f4d8de5-c60c-4fa7-beb7-76260f18e504", - "z": 0, - "config": { - "_type": "tile/data-stream", - "title": "Properties", - "description": "", - "activePluginConfigIds": ["{{configId}}"], - "variables": ["{{variables.DataSource}}"], - "dataStream": { "id": "datastream-properties" }, - "scope": { - "scope": "{{scopes.DataSources}}", - "workspace": "{{workspaceId}}", - "variable": "{{variables.DataSource}}" - }, - "visualisation": { - "type": "data-stream-table", - "config": { - "data-stream-table": { "transpose": true } - } - } - } - }, - { - "static": false, - "w": 3, - "moved": false, - "h": 4, - "x": 1, - "y": 0, - "i": "07d8ec4e-404c-4925-94c1-01a1da2e9938", - "z": 0, - "config": { - "_type": "tile/data-stream", - "title": "Scans", - "description": "", - "activePluginConfigIds": ["{{configId}}"], - "variables": ["{{variables.DataSource}}"], - "dataStream": { - "id": "{{dataStreams.dataSourceScans}}", - "name": "dataSourceScans", - "pluginConfigId": "{{configId}}" - }, - "scope": { - "scope": "{{scopes.DataSources}}", - "workspace": "{{workspaceId}}", - "variable": "{{variables.DataSource}}" - }, - "visualisation": { - "type": "data-stream-table", - "config": { - "data-stream-table": { - "transpose": false, - "columnOrder": ["name", "lastRunResult.status", "lastRunResult.startTime", "lastRunResult.endTime", "properties.scanRulesetName", "kind"] - } - } - } - } - }, - { - "static": false, - "w": 4, - "moved": false, - "h": 4, - "x": 0, - "y": 4, - "i": "18f50b03-c364-481b-a341-67f63b5c844b", - "z": 0, - "config": { - "_type": "tile/data-stream", - "title": "Scan Status", - "description": "", - "activePluginConfigIds": ["{{configId}}"], - "variables": ["{{variables.DataSource}}"], - "dataStream": { - "id": "{{dataStreams.dataSourceScans}}", - "name": "dataSourceScans", - "pluginConfigId": "{{configId}}" - }, - "scope": { - "scope": "{{scopes.DataSources}}", - "workspace": "{{workspaceId}}", - "variable": "{{variables.DataSource}}" - }, - "visualisation": { - "type": "data-stream-blocks", - "config": { - "data-stream-blocks": { - "labelColumn": "name", - "stateColumn": "lastRunResult.status", - "linkColumn": "none", - "columns": 4 - } - } - } - } - } - ] - } -} diff --git a/plugins/MicrosoftPurview/v1/defaultContent/manifest.json b/plugins/MicrosoftPurview/v1/defaultContent/manifest.json index 6fdd2350..1c0385b4 100644 --- a/plugins/MicrosoftPurview/v1/defaultContent/manifest.json +++ b/plugins/MicrosoftPurview/v1/defaultContent/manifest.json @@ -1,7 +1,5 @@ { "items": [ - { "name": "overview", "type": "dashboard" }, - { "name": "collectionDashboard", "type": "dashboard" }, - { "name": "dataSourceDashboard", "type": "dashboard" } + { "name": "overview", "type": "dashboard" } ] } diff --git a/plugins/MicrosoftPurview/v1/metadata.json b/plugins/MicrosoftPurview/v1/metadata.json index d5fdfa25..94ca868c 100644 --- a/plugins/MicrosoftPurview/v1/metadata.json +++ b/plugins/MicrosoftPurview/v1/metadata.json @@ -1,7 +1,7 @@ { "name": "microsoft-purview", "displayName": "Microsoft Purview", - "version": "1.4.3", + "version": "1.5.0", "author": { "name": "John Hayes", "type": "community" From 2992e3cbb4e6992fe0f42e372c37dc6106717289 Mon Sep 17 00:00:00 2001 From: John Hayes Date: Thu, 23 Jul 2026 16:53:22 +0100 Subject: [PATCH 3/7] Update Overview dashboard layout to v1.5.1 Resize Classification Coverage and Metadata Policies tiles to 2x2 and stack them on the right side of the Asset Count by Entity Type tile. Co-Authored-By: Claude Sonnet 4.6 --- .../v1/defaultContent/overview.dash.json | 10 +++++----- plugins/MicrosoftPurview/v1/metadata.json | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/plugins/MicrosoftPurview/v1/defaultContent/overview.dash.json b/plugins/MicrosoftPurview/v1/defaultContent/overview.dash.json index b9dc4f10..a0e34139 100644 --- a/plugins/MicrosoftPurview/v1/defaultContent/overview.dash.json +++ b/plugins/MicrosoftPurview/v1/defaultContent/overview.dash.json @@ -176,7 +176,7 @@ "static": false, "w": 2, "moved": false, - "h": 4, + "h": 2, "x": 2, "y": 2, "i": "898b191e-9d88-48d3-8964-de7e2f59d25e", @@ -234,11 +234,11 @@ }, { "static": false, - "w": 4, + "w": 2, "moved": false, - "h": 4, - "x": 0, - "y": 10, + "h": 2, + "x": 2, + "y": 4, "i": "c87c3519-81e9-48b3-831c-3bf19c8f21f3", "z": 0, "config": { diff --git a/plugins/MicrosoftPurview/v1/metadata.json b/plugins/MicrosoftPurview/v1/metadata.json index 94ca868c..af4267de 100644 --- a/plugins/MicrosoftPurview/v1/metadata.json +++ b/plugins/MicrosoftPurview/v1/metadata.json @@ -1,7 +1,7 @@ { "name": "microsoft-purview", "displayName": "Microsoft Purview", - "version": "1.5.0", + "version": "1.5.1", "author": { "name": "John Hayes", "type": "community" From 3952610b2059272736df10cdda1754287f0e121d Mon Sep 17 00:00:00 2001 From: John Hayes Date: Thu, 23 Jul 2026 18:10:05 +0100 Subject: [PATCH 4/7] Add Governance Domains and Glossary Terms streams to v1.6.2 - Add Governance Domains stream (Unified Catalog API, account-wide) - Add Glossary Terms stream scoped to Purview Governance Domain objects - Index Governance Domains as a new object type - Strip HTML from description fields in both streams - Update README with Unified Catalog permission requirements - Add Purview Governance Domain to objectTypes and What gets indexed Co-Authored-By: Claude Sonnet 4.6 --- .../v1/dataStreams/glossaryTerms.json | 33 +++++++++++++++++++ .../v1/dataStreams/governanceDomains.json | 33 +++++++++++++++++++ .../v1/dataStreams/scripts/glossaryTerms.js | 14 ++++++++ .../dataStreams/scripts/governanceDomains.js | 16 +++++++++ plugins/MicrosoftPurview/v1/docs/README.md | 17 +++++++++- .../v1/indexDefinitions/default.json | 15 +++++++++ plugins/MicrosoftPurview/v1/metadata.json | 5 +-- 7 files changed, 130 insertions(+), 3 deletions(-) create mode 100644 plugins/MicrosoftPurview/v1/dataStreams/glossaryTerms.json create mode 100644 plugins/MicrosoftPurview/v1/dataStreams/governanceDomains.json create mode 100644 plugins/MicrosoftPurview/v1/dataStreams/scripts/glossaryTerms.js create mode 100644 plugins/MicrosoftPurview/v1/dataStreams/scripts/governanceDomains.js diff --git a/plugins/MicrosoftPurview/v1/dataStreams/glossaryTerms.json b/plugins/MicrosoftPurview/v1/dataStreams/glossaryTerms.json new file mode 100644 index 00000000..7808fd79 --- /dev/null +++ b/plugins/MicrosoftPurview/v1/dataStreams/glossaryTerms.json @@ -0,0 +1,33 @@ +{ + "name": "glossaryTerms", + "displayName": "Glossary Terms", + "description": "Glossary terms defined in the selected governance domain", + "tags": ["Governance"], + "baseDataSourceName": "httpRequestScopedSingle", + "objectLimit": 1, + "config": { + "httpMethod": "get", + "endpointPath": "datagovernance/catalog/terms", + "getArgs": [ + { "key": "api-version", "value": "2026-03-20-preview" }, + { "key": "domainId", "value": "{{object.rawId}}" } + ], + "postRequestScript": "glossaryTerms.js", + "paging": { + "mode": "nextUrl", + "in": { "realm": "payload", "path": "nextLink" } + } + }, + "matches": { "sourceType": { "type": "oneOf", "values": ["Purview Governance Domain"] } }, + "metadata": [ + { "name": "id", "displayName": "ID", "shape": "guid", "visible": false }, + { "name": "name", "displayName": "Term", "role": "label" }, + { "name": "description", "displayName": "Description" }, + { "name": "status", "displayName": "Status" }, + { "name": "acronyms", "displayName": "Acronyms" }, + { "name": "parentId", "displayName": "Parent Term ID", "shape": "guid", "visible": false }, + { "name": "createdAt", "displayName": "Created", "shape": "date" }, + { "name": "lastModifiedAt", "displayName": "Last Modified", "shape": "date" } + ], + "timeframes": false +} diff --git a/plugins/MicrosoftPurview/v1/dataStreams/governanceDomains.json b/plugins/MicrosoftPurview/v1/dataStreams/governanceDomains.json new file mode 100644 index 00000000..c3d85d73 --- /dev/null +++ b/plugins/MicrosoftPurview/v1/dataStreams/governanceDomains.json @@ -0,0 +1,33 @@ +{ + "name": "governanceDomains", + "displayName": "Governance Domains", + "description": "Governance domains defined in the Purview Unified Catalog", + "tags": ["Governance"], + "baseDataSourceName": "httpRequestUnscoped", + "config": { + "httpMethod": "get", + "endpointPath": "datagovernance/catalog/businessdomains", + "getArgs": [ + { "key": "api-version", "value": "2026-03-20-preview" } + ], + "postRequestScript": "governanceDomains.js", + "paging": { + "mode": "nextUrl", + "in": { "realm": "payload", "path": "nextLink" } + } + }, + "matches": "none", + "metadata": [ + { "name": "id", "displayName": "ID", "shape": "guid", "visible": false }, + { "name": "name", "displayName": "Domain", "role": "label" }, + { "name": "description", "displayName": "Description" }, + { "name": "type", "displayName": "Type" }, + { "name": "status", "displayName": "Status" }, + { "name": "parentId", "displayName": "Parent ID", "shape": "guid", "visible": false }, + { "name": "isRestricted", "displayName": "Restricted" }, + { "name": "createdAt", "displayName": "Created", "shape": "date" }, + { "name": "lastModifiedAt", "displayName": "Last Modified", "shape": "date" }, + { "name": "sourceType", "displayName": "Source Type", "visible": false } + ], + "timeframes": false +} diff --git a/plugins/MicrosoftPurview/v1/dataStreams/scripts/glossaryTerms.js b/plugins/MicrosoftPurview/v1/dataStreams/scripts/glossaryTerms.js new file mode 100644 index 00000000..30f64fba --- /dev/null +++ b/plugins/MicrosoftPurview/v1/dataStreams/scripts/glossaryTerms.js @@ -0,0 +1,14 @@ +const stripHtml = s => (s || '').replace(/<[^>]*>/g, '').replace(/ /g, ' ').trim() || null; + +const terms = data?.value || []; + +result = terms.map(term => ({ + id: term.id || '', + name: term.name || '', + description: stripHtml(term.description), + status: term.status || '', + acronyms: (term.acronyms || []).join(', ') || null, + parentId: term.parentId || null, + createdAt: term.systemData?.createdAt || null, + lastModifiedAt: term.systemData?.lastModifiedAt || null, +})); diff --git a/plugins/MicrosoftPurview/v1/dataStreams/scripts/governanceDomains.js b/plugins/MicrosoftPurview/v1/dataStreams/scripts/governanceDomains.js new file mode 100644 index 00000000..95736ab8 --- /dev/null +++ b/plugins/MicrosoftPurview/v1/dataStreams/scripts/governanceDomains.js @@ -0,0 +1,16 @@ +const stripHtml = s => (s || '').replace(/<[^>]*>/g, '').replace(/ /g, ' ').trim() || null; + +const domains = data?.value || []; + +result = domains.map(d => ({ + id: d.id || '', + name: d.name || '', + description: stripHtml(d.description), + type: d.type || '', + status: d.status || '', + parentId: d.parentId || null, + isRestricted: d.isRestricted ?? null, + createdAt: d.systemData?.createdAt || null, + lastModifiedAt: d.systemData?.lastModifiedAt || null, + sourceType: 'Purview Governance Domain', +})); diff --git a/plugins/MicrosoftPurview/v1/docs/README.md b/plugins/MicrosoftPurview/v1/docs/README.md index 7f27595d..af6f6dc2 100644 --- a/plugins/MicrosoftPurview/v1/docs/README.md +++ b/plugins/MicrosoftPurview/v1/docs/README.md @@ -8,10 +8,13 @@ This plugin connects SquaredUp to Microsoft Purview and surfaces data governance - Provides data streams for: - Assets — searchable per collection, with breakdowns by entity type and classification - Classification coverage — what proportion of assets carry each classification + - Classifications — custom and built-in classifications applied to assets - Scan status and run history — per data source, per scan, and global recent runs - Policy coverage — metadata policies attached to collections - Lineage — upstream and downstream lineage for a given asset -- Ships with out-of-the-box dashboards: an overall **Overview** plus per-object **Collection**, **Data Source**, and **Scan** perspectives. + - Governance domains — domains defined in the Unified Catalog + - Glossary terms — terms defined within a governance domain +- Ships with an out-of-the-box **Overview** dashboard. ## Prerequisites @@ -37,6 +40,17 @@ Purview uses **collection-level role assignments**, not Microsoft Graph API perm Role assignments at the root collection are inherited by all child collections. +### 3. Grant access to the Unified Catalog (governance domains and glossary terms) + +The **Governance Domains** and **Glossary Terms** streams use the Purview Unified Catalog API, which has its own permission model separate from Data Map collection roles. + +1. Open the **Microsoft Purview portal** and go to **Unified Catalog → Catalog Management → Governance Domains**. +2. Select the governance domain you want to expose. +3. Go to the **Roles** tab. +4. Add the app registration to the **Data Catalog Reader** role. + +Repeat for each governance domain you want to surface. Without this role, the Governance Domains and Glossary Terms streams will return an "Unauthorized" error even if all Data Map roles are correctly assigned. + ## Configuration fields | Field | What it is | Where to find it | Required | @@ -53,6 +67,7 @@ Role assignments at the root collection are inherited by all child collections. | **Purview Collection** | An organisational unit in your Purview account. The root collection is named after the account itself. | | **Purview Data Source** | A registered data source — e.g. an Azure SQL database, ADLS Gen2 account, Snowflake instance, Amazon S3 bucket. | | **Purview Scan** | A configured scan attached to a data source. | +| **Purview Governance Domain** | A governance domain defined in the Unified Catalog. Requires the Data Catalog Reader role — see step 3 above. | Assets (tables, files, columns) are **not** indexed because a single Purview account can hold millions of them. They are queried on-demand by the data streams instead. diff --git a/plugins/MicrosoftPurview/v1/indexDefinitions/default.json b/plugins/MicrosoftPurview/v1/indexDefinitions/default.json index 5927c073..ec30fb99 100644 --- a/plugins/MicrosoftPurview/v1/indexDefinitions/default.json +++ b/plugins/MicrosoftPurview/v1/indexDefinitions/default.json @@ -31,6 +31,21 @@ "subscriptionId" ] } + }, + { + "name": "governanceDomains", + "dataStream": { "name": "governanceDomains" }, + "timeframe": "none", + "objectMapping": { + "id": "id", + "name": "name", + "type": "sourceType", + "properties": [ + "description", + "type", + "status" + ] + } } ] } diff --git a/plugins/MicrosoftPurview/v1/metadata.json b/plugins/MicrosoftPurview/v1/metadata.json index af4267de..ee06798b 100644 --- a/plugins/MicrosoftPurview/v1/metadata.json +++ b/plugins/MicrosoftPurview/v1/metadata.json @@ -1,7 +1,7 @@ { "name": "microsoft-purview", "displayName": "Microsoft Purview", - "version": "1.5.1", + "version": "1.6.2", "author": { "name": "John Hayes", "type": "community" @@ -23,7 +23,8 @@ ], "objectTypes": [ "Purview Collection", - "Purview Data Source" + "Purview Data Source", + "Purview Governance Domain" ], "links": [ { From d6234e4b58bb7e824b8b838c2fd43bc012d13fca Mon Sep 17 00:00:00 2001 From: John Hayes Date: Tue, 28 Jul 2026 17:15:32 +0100 Subject: [PATCH 5/7] Address CodeRabbit review comments on Microsoft Purview plugin v1.7.0 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add GUID regex validation to assetGuid field; add min/max constraints to depth field (assetLineage) - Add token-based pagination to assetsByEntityTypeGlobal and classificationCoverage so all pages are aggregated before counting - Fix classificationTypes type filter to uppercase CLASSIFICATION per API spec - Replace mustache valueExpression computed fields with .map() postRequestScripts in collections and listDataSources; remove expandInnerObjects from listDataSources - Fix dataSourceScans to emit collection directly from props in the script, removing dotted-path computed field pattern - Add tags to scansForDataSource (was missing mandatory tags property) - Detect @search.count.approximate in accountAssetCount to avoid presenting estimated counts as exact - Upgrade assetLineage traversal from single-hop to multi-hop BFS so all reachable upstream/downstream nodes are correctly classified - Expose category and typeVersion in customClassifications script and metadata - Add example placeholder and 1–1000 clamping to searchAssets keywords and limit fields - Index Purview Scans: add scan step to default.json, emit sourceType from dataSourceScans script, add Purview Scan to objectTypes - Add GUID-format placeholders and help text to tenantId and clientId fields in ui.json Co-Authored-By: Claude Sonnet 4.6 --- .../v1/dataStreams/assetLineage.json | 6 +- .../dataStreams/assetsByEntityTypeGlobal.json | 7 +- .../dataStreams/classificationCoverage.json | 7 +- .../v1/dataStreams/classificationTypes.json | 2 +- .../v1/dataStreams/collections.json | 58 +++-------------- .../v1/dataStreams/customClassifications.json | 2 + .../v1/dataStreams/dataSourceScans.json | 25 ++----- .../v1/dataStreams/listDataSources.json | 65 +++---------------- .../v1/dataStreams/scansForDataSource.json | 1 + .../dataStreams/scripts/accountAssetCount.js | 3 +- .../v1/dataStreams/scripts/assetLineage.js | 42 +++++++++++- .../v1/dataStreams/scripts/collections.js | 10 +++ .../scripts/customClassifications.js | 2 + .../v1/dataStreams/scripts/dataSourceScans.js | 3 +- .../v1/dataStreams/scripts/listDataSources.js | 17 +++++ .../v1/dataStreams/searchAssets.json | 7 +- .../v1/indexDefinitions/default.json | 15 +++++ plugins/MicrosoftPurview/v1/metadata.json | 3 +- plugins/MicrosoftPurview/v1/ui.json | 7 +- 19 files changed, 141 insertions(+), 141 deletions(-) create mode 100644 plugins/MicrosoftPurview/v1/dataStreams/scripts/collections.js create mode 100644 plugins/MicrosoftPurview/v1/dataStreams/scripts/listDataSources.js diff --git a/plugins/MicrosoftPurview/v1/dataStreams/assetLineage.json b/plugins/MicrosoftPurview/v1/dataStreams/assetLineage.json index 6dd965e4..a19f0120 100644 --- a/plugins/MicrosoftPurview/v1/dataStreams/assetLineage.json +++ b/plugins/MicrosoftPurview/v1/dataStreams/assetLineage.json @@ -10,7 +10,10 @@ "name": "assetGuid", "label": "Asset GUID", "placeholder": "Asset GUID from the catalog", - "validation": { "required": true }, + "validation": { + "required": true, + "pattern": "^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$" + }, "help": "The Purview asset's GUID. Copy it from the asset details page in the Purview portal, or from the **Asset GUID** column of the Search Assets stream." }, { @@ -29,6 +32,7 @@ "name": "depth", "label": "Depth", "defaultValue": 3, + "validation": { "min": 0, "max": 10 }, "help": "Number of hops to traverse in each direction." } ], diff --git a/plugins/MicrosoftPurview/v1/dataStreams/assetsByEntityTypeGlobal.json b/plugins/MicrosoftPurview/v1/dataStreams/assetsByEntityTypeGlobal.json index 0d52ae2c..0110d20b 100644 --- a/plugins/MicrosoftPurview/v1/dataStreams/assetsByEntityTypeGlobal.json +++ b/plugins/MicrosoftPurview/v1/dataStreams/assetsByEntityTypeGlobal.json @@ -14,7 +14,12 @@ "keywords": null, "limit": 1000 }, - "postRequestScript": "assetsByEntityTypeGlobal.js" + "postRequestScript": "assetsByEntityTypeGlobal.js", + "paging": { + "mode": "token", + "in": { "realm": "payload", "path": "continuationToken" }, + "out": { "realm": "body", "path": "continuationToken" } + } }, "matches": "none", "metadata": [ diff --git a/plugins/MicrosoftPurview/v1/dataStreams/classificationCoverage.json b/plugins/MicrosoftPurview/v1/dataStreams/classificationCoverage.json index d2a19f6a..4d4de443 100644 --- a/plugins/MicrosoftPurview/v1/dataStreams/classificationCoverage.json +++ b/plugins/MicrosoftPurview/v1/dataStreams/classificationCoverage.json @@ -14,7 +14,12 @@ "keywords": null, "limit": 1000 }, - "postRequestScript": "classificationCoverage.js" + "postRequestScript": "classificationCoverage.js", + "paging": { + "mode": "token", + "in": { "realm": "payload", "path": "continuationToken" }, + "out": { "realm": "body", "path": "continuationToken" } + } }, "matches": "none", "metadata": [ diff --git a/plugins/MicrosoftPurview/v1/dataStreams/classificationTypes.json b/plugins/MicrosoftPurview/v1/dataStreams/classificationTypes.json index ec4d4b13..935428e3 100644 --- a/plugins/MicrosoftPurview/v1/dataStreams/classificationTypes.json +++ b/plugins/MicrosoftPurview/v1/dataStreams/classificationTypes.json @@ -8,7 +8,7 @@ "httpMethod": "get", "endpointPath": "datamap/api/atlas/v2/types/typedefs", "getArgs": [ - { "key": "type", "value": "classification" } + { "key": "type", "value": "CLASSIFICATION" } ], "postRequestScript": "classificationTypes.js" }, diff --git a/plugins/MicrosoftPurview/v1/dataStreams/collections.json b/plugins/MicrosoftPurview/v1/dataStreams/collections.json index af86ac55..76275cd0 100644 --- a/plugins/MicrosoftPurview/v1/dataStreams/collections.json +++ b/plugins/MicrosoftPurview/v1/dataStreams/collections.json @@ -11,6 +11,7 @@ { "key": "api-version", "value": "2019-11-01-preview" } ], "pathToData": "value", + "postRequestScript": "collections.js", "paging": { "mode": "nextUrl", "in": { "realm": "payload", "path": "nextLink" } @@ -18,55 +19,14 @@ }, "matches": "none", "metadata": [ - { - "name": "name", - "displayName": "Collection ID", - "role": "sourceId", - "visible": false - }, - { - "name": "collectionName", - "displayName": "Collection", - "computed": true, - "valueExpression": "{{ $['name'] }}", - "visible": false - }, - { - "name": "friendlyName", - "displayName": "Friendly Name", - "visible": false - }, - { - "name": "displayName", - "displayName": "Name", - "computed": true, - "valueExpression": "{{ $['friendlyName'] || $['name'] }}", - "role": "label" - }, - { - "name": "description", - "displayName": "Description" - }, - { - "name": "parentCollection", - "visible": false - }, - { - "name": "parentCollectionName", - "computed": true, - "valueExpression": "{{ $['parentCollection'] && $['parentCollection'].referenceName ? $['parentCollection'].referenceName : '' }}", - "displayName": "Parent Collection" - }, - { - "name": "collectionProvisioningState", - "displayName": "State" - }, - { - "name": "sourceType", - "computed": true, - "valueExpression": "Purview Collection", - "visible": false - } + { "name": "name", "displayName": "Collection ID", "role": "sourceId", "visible": false }, + { "name": "collectionName", "displayName": "Collection", "visible": false }, + { "name": "friendlyName", "displayName": "Friendly Name", "visible": false }, + { "name": "displayName", "displayName": "Name", "role": "label" }, + { "name": "description", "displayName": "Description" }, + { "name": "parentCollectionName", "displayName": "Parent Collection" }, + { "name": "collectionProvisioningState", "displayName": "State" }, + { "name": "sourceType", "displayName": "Source Type", "visible": false } ], "timeframes": false } diff --git a/plugins/MicrosoftPurview/v1/dataStreams/customClassifications.json b/plugins/MicrosoftPurview/v1/dataStreams/customClassifications.json index d636e650..cb8522ba 100644 --- a/plugins/MicrosoftPurview/v1/dataStreams/customClassifications.json +++ b/plugins/MicrosoftPurview/v1/dataStreams/customClassifications.json @@ -16,6 +16,8 @@ "metadata": [ { "name": "name", "displayName": "Classification", "role": "label" }, { "name": "description", "displayName": "Description" }, + { "name": "category", "displayName": "Category" }, + { "name": "typeVersion", "displayName": "Version" }, { "name": "createdBy", "displayName": "Created By" }, { "name": "createTime", "displayName": "Created", "shape": "date" }, { "name": "updatedBy", "displayName": "Updated By" }, diff --git a/plugins/MicrosoftPurview/v1/dataStreams/dataSourceScans.json b/plugins/MicrosoftPurview/v1/dataStreams/dataSourceScans.json index edd4ed39..0233c263 100644 --- a/plugins/MicrosoftPurview/v1/dataStreams/dataSourceScans.json +++ b/plugins/MicrosoftPurview/v1/dataStreams/dataSourceScans.json @@ -21,26 +21,11 @@ { "name": "name", "displayName": "Scan", "role": "label" }, { "name": "dataSourceName", "displayName": "Data Source" }, { "name": "kind", "displayName": "Kind" }, - { - "name": "properties.collection.referenceName", - "displayName": "Collection", - "visible": false - }, - { - "name": "collection", - "computed": true, - "valueExpression": "{{ $['properties.collection.referenceName'] || '' }}", - "displayName": "Collection" - }, - { - "name": "properties.scanRulesetName", - "displayName": "Ruleset" - }, - { - "name": "properties.scanRulesetType", - "displayName": "Ruleset Type" - }, - { "name": "creationType", "displayName": "Creation Type" } + { "name": "collection", "displayName": "Collection" }, + { "name": "properties.scanRulesetName", "displayName": "Ruleset" }, + { "name": "properties.scanRulesetType", "displayName": "Ruleset Type" }, + { "name": "creationType", "displayName": "Creation Type" }, + { "name": "sourceType", "displayName": "Source Type", "visible": false } ], "timeframes": false } diff --git a/plugins/MicrosoftPurview/v1/dataStreams/listDataSources.json b/plugins/MicrosoftPurview/v1/dataStreams/listDataSources.json index 5725eb62..d50577f4 100644 --- a/plugins/MicrosoftPurview/v1/dataStreams/listDataSources.json +++ b/plugins/MicrosoftPurview/v1/dataStreams/listDataSources.json @@ -11,7 +11,7 @@ { "key": "api-version", "value": "2023-09-01" } ], "pathToData": "value", - "expandInnerObjects": true, + "postRequestScript": "listDataSources.js", "paging": { "mode": "nextUrl", "in": { "realm": "payload", "path": "nextLink" } @@ -21,62 +21,13 @@ "metadata": [ { "name": "name", "displayName": "Name", "role": "label" }, { "name": "kind", "displayName": "Kind" }, - { - "name": "properties.collection.referenceName", - "displayName": "Collection", - "visible": false - }, - { - "name": "collection", - "computed": true, - "valueExpression": "{{ $['properties.collection.referenceName'] || '' }}", - "displayName": "Collection" - }, - { - "name": "properties.resourceGroup", - "displayName": "Resource Group", - "visible": false - }, - { - "name": "resourceGroup", - "computed": true, - "valueExpression": "{{ $['properties.resourceGroup'] || '' }}", - "displayName": "Resource Group" - }, - { - "name": "properties.subscriptionId", - "displayName": "Subscription", - "visible": false - }, - { - "name": "subscriptionId", - "computed": true, - "valueExpression": "{{ $['properties.subscriptionId'] || '' }}", - "displayName": "Subscription", - "visible": false - }, - { - "name": "subscriptionUrl", - "computed": true, - "valueExpression": "{{ $['properties.subscriptionId'] ? 'https://portal.azure.com/#resource/subscriptions/' + $['properties.subscriptionId'] : '' }}", - "displayName": "Subscription", - "shape": "url" - }, - { - "name": "properties.createdAt", - "displayName": "Created", - "shape": "date" - }, - { - "name": "creationType", - "displayName": "Creation Type" - }, - { - "name": "sourceType", - "computed": true, - "valueExpression": "Purview Data Source", - "visible": false - } + { "name": "collection", "displayName": "Collection", "visible": false }, + { "name": "resourceGroup", "displayName": "Resource Group" }, + { "name": "subscriptionId", "displayName": "Subscription", "visible": false }, + { "name": "subscriptionUrl", "displayName": "Subscription", "shape": "url" }, + { "name": "createdAt", "displayName": "Created", "shape": "date" }, + { "name": "creationType", "displayName": "Creation Type" }, + { "name": "sourceType", "displayName": "Source Type", "visible": false } ], "timeframes": false } diff --git a/plugins/MicrosoftPurview/v1/dataStreams/scansForDataSource.json b/plugins/MicrosoftPurview/v1/dataStreams/scansForDataSource.json index c3224eb9..9bd598ee 100644 --- a/plugins/MicrosoftPurview/v1/dataStreams/scansForDataSource.json +++ b/plugins/MicrosoftPurview/v1/dataStreams/scansForDataSource.json @@ -2,6 +2,7 @@ "name": "scansForDataSource", "displayName": "Scan Definitions", "description": "Lists the scan definitions configured on a data source, with their last run date, sorted most-recent-first. Used to populate the Scan Name picker on the Scan History stream.", + "tags": ["Data Sources"], "baseDataSourceName": "httpRequestScopedSingle", "visibility": { "type": "hidden" }, "config": { diff --git a/plugins/MicrosoftPurview/v1/dataStreams/scripts/accountAssetCount.js b/plugins/MicrosoftPurview/v1/dataStreams/scripts/accountAssetCount.js index b0a26dc7..88c30e84 100644 --- a/plugins/MicrosoftPurview/v1/dataStreams/scripts/accountAssetCount.js +++ b/plugins/MicrosoftPurview/v1/dataStreams/scripts/accountAssetCount.js @@ -1,2 +1,3 @@ +const isApproximate = data?.['@search.count.approximate'] === true; const total = Number(data?.['@search.count']) || 0; -result = [{ label: 'Total Assets', totalAssets: total }]; +result = [{ label: isApproximate ? 'Total Assets (approx)' : 'Total Assets', totalAssets: total }]; diff --git a/plugins/MicrosoftPurview/v1/dataStreams/scripts/assetLineage.js b/plugins/MicrosoftPurview/v1/dataStreams/scripts/assetLineage.js index 235589fb..9df3b51b 100644 --- a/plugins/MicrosoftPurview/v1/dataStreams/scripts/assetLineage.js +++ b/plugins/MicrosoftPurview/v1/dataStreams/scripts/assetLineage.js @@ -2,11 +2,47 @@ const baseGuid = data?.baseEntityGuid; const map = data?.guidEntityMap || {}; const relations = data?.relations || []; +// Build directed adjacency: inEdges[node] = sources feeding into node (upstream) +// outEdges[node] = destinations fed from node (downstream) +const inEdges = {}; +const outEdges = {}; +for (const r of relations) { + if (!inEdges[r.toEntityId]) inEdges[r.toEntityId] = []; + inEdges[r.toEntityId].push(r.fromEntityId); + if (!outEdges[r.fromEntityId]) outEdges[r.fromEntityId] = []; + outEdges[r.fromEntityId].push(r.toEntityId); +} + +// BFS to collect all reachable upstream nodes across multiple hops const upstream = new Set(); +let frontier = [baseGuid]; +while (frontier.length > 0) { + const next = []; + for (const node of frontier) { + for (const src of (inEdges[node] || [])) { + if (src !== baseGuid && !upstream.has(src)) { + upstream.add(src); + next.push(src); + } + } + } + frontier = next; +} + +// BFS to collect all reachable downstream nodes across multiple hops const downstream = new Set(); -for (const r of relations) { - if (r.toEntityId === baseGuid) upstream.add(r.fromEntityId); - if (r.fromEntityId === baseGuid) downstream.add(r.toEntityId); +frontier = [baseGuid]; +while (frontier.length > 0) { + const next = []; + for (const node of frontier) { + for (const dst of (outEdges[node] || [])) { + if (dst !== baseGuid && !downstream.has(dst)) { + downstream.add(dst); + next.push(dst); + } + } + } + frontier = next; } result = Object.entries(map).map(([guid, e]) => { diff --git a/plugins/MicrosoftPurview/v1/dataStreams/scripts/collections.js b/plugins/MicrosoftPurview/v1/dataStreams/scripts/collections.js new file mode 100644 index 00000000..3d72782f --- /dev/null +++ b/plugins/MicrosoftPurview/v1/dataStreams/scripts/collections.js @@ -0,0 +1,10 @@ +result = (data?.value || []).map(c => ({ + name: c.name || '', + collectionName: c.name || '', + friendlyName: c.friendlyName || '', + displayName: c.friendlyName || c.name || '', + description: c.description || '', + parentCollectionName: c.parentCollection?.referenceName || '', + collectionProvisioningState: c.collectionProvisioningState || '', + sourceType: 'Purview Collection', +})); diff --git a/plugins/MicrosoftPurview/v1/dataStreams/scripts/customClassifications.js b/plugins/MicrosoftPurview/v1/dataStreams/scripts/customClassifications.js index 6ef568a2..c335fd53 100644 --- a/plugins/MicrosoftPurview/v1/dataStreams/scripts/customClassifications.js +++ b/plugins/MicrosoftPurview/v1/dataStreams/scripts/customClassifications.js @@ -5,6 +5,8 @@ result = defs .map(c => ({ name: c.name || '', description: c.description || '', + category: c.category || '', + typeVersion: c.typeVersion || '', createdBy: c.createdBy || '', createTime: c.createTime ? new Date(c.createTime).toISOString() : null, updatedBy: c.updatedBy || '', diff --git a/plugins/MicrosoftPurview/v1/dataStreams/scripts/dataSourceScans.js b/plugins/MicrosoftPurview/v1/dataStreams/scripts/dataSourceScans.js index e2d144dd..045178ea 100644 --- a/plugins/MicrosoftPurview/v1/dataStreams/scripts/dataSourceScans.js +++ b/plugins/MicrosoftPurview/v1/dataStreams/scripts/dataSourceScans.js @@ -7,9 +7,10 @@ result = scans.map((scan) => { name: scan.name || '', dataSourceName: scan.dataSourceName || '', kind: scan.kind || '', + collection: props.collection?.referenceName || '', creationType: scan.creationType || '', - 'properties.collection.referenceName': props.collection?.referenceName || '', 'properties.scanRulesetName': props.scanRulesetName || '', 'properties.scanRulesetType': props.scanRulesetType || '', + sourceType: 'Purview Scan', }; }); diff --git a/plugins/MicrosoftPurview/v1/dataStreams/scripts/listDataSources.js b/plugins/MicrosoftPurview/v1/dataStreams/scripts/listDataSources.js new file mode 100644 index 00000000..e6d43f1d --- /dev/null +++ b/plugins/MicrosoftPurview/v1/dataStreams/scripts/listDataSources.js @@ -0,0 +1,17 @@ +result = (data?.value || []).map(item => { + const props = item.properties || {}; + const subscriptionId = props.subscriptionId || ''; + return { + name: item.name || '', + kind: item.kind || '', + collection: props.collection?.referenceName || '', + resourceGroup: props.resourceGroup || '', + subscriptionId, + subscriptionUrl: subscriptionId + ? 'https://portal.azure.com/#resource/subscriptions/' + subscriptionId + : '', + createdAt: props.createdAt || null, + creationType: item.creationType || '', + sourceType: 'Purview Data Source', + }; +}); diff --git a/plugins/MicrosoftPurview/v1/dataStreams/searchAssets.json b/plugins/MicrosoftPurview/v1/dataStreams/searchAssets.json index 4beef9d6..2f1f660d 100644 --- a/plugins/MicrosoftPurview/v1/dataStreams/searchAssets.json +++ b/plugins/MicrosoftPurview/v1/dataStreams/searchAssets.json @@ -9,7 +9,7 @@ "type": "text", "name": "keywords", "label": "Keywords", - "placeholder": "Search keywords (optional)" + "placeholder": "e.g. sales report" }, { "type": "text", @@ -27,7 +27,8 @@ "type": "number", "name": "limit", "label": "Result Limit", - "defaultValue": 100 + "defaultValue": 100, + "validation": { "min": 1, "max": 1000 } } ], "config": { @@ -38,7 +39,7 @@ ], "postBody": { "keywords": "{{ keywords || null }}", - "limit": "{{ Number(limit) || 100 }}", + "limit": "{{ Math.min(Math.max(Number(limit) || 100, 1), 1000) }}", "filter": "{{ (entityType || classification) ? { and: [entityType ? { entityType: entityType } : null, classification ? { classification: classification } : null].filter(Boolean) } : null }}" }, "postRequestScript": "searchAssets.js" diff --git a/plugins/MicrosoftPurview/v1/indexDefinitions/default.json b/plugins/MicrosoftPurview/v1/indexDefinitions/default.json index ec30fb99..569074dc 100644 --- a/plugins/MicrosoftPurview/v1/indexDefinitions/default.json +++ b/plugins/MicrosoftPurview/v1/indexDefinitions/default.json @@ -46,6 +46,21 @@ "status" ] } + }, + { + "name": "scans", + "dataStream": { "name": "dataSourceScans" }, + "timeframe": "none", + "objectMapping": { + "id": "name", + "name": "name", + "type": "sourceType", + "properties": [ + "kind", + "collection", + "dataSourceName" + ] + } } ] } diff --git a/plugins/MicrosoftPurview/v1/metadata.json b/plugins/MicrosoftPurview/v1/metadata.json index ee06798b..8056468e 100644 --- a/plugins/MicrosoftPurview/v1/metadata.json +++ b/plugins/MicrosoftPurview/v1/metadata.json @@ -1,7 +1,7 @@ { "name": "microsoft-purview", "displayName": "Microsoft Purview", - "version": "1.6.2", + "version": "1.7.0", "author": { "name": "John Hayes", "type": "community" @@ -24,6 +24,7 @@ "objectTypes": [ "Purview Collection", "Purview Data Source", + "Purview Scan", "Purview Governance Domain" ], "links": [ diff --git a/plugins/MicrosoftPurview/v1/ui.json b/plugins/MicrosoftPurview/v1/ui.json index 7c205444..86d28406 100644 --- a/plugins/MicrosoftPurview/v1/ui.json +++ b/plugins/MicrosoftPurview/v1/ui.json @@ -13,7 +13,8 @@ "type": "text", "name": "tenantId", "label": "Directory (tenant) ID", - "placeholder": "Enter a directory (tenant) ID", + "placeholder": "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx", + "help": "Found in Microsoft Entra ID → Overview → Directory (tenant) ID.", "validation": { "required": true } @@ -22,7 +23,8 @@ "type": "text", "name": "clientId", "label": "Application (client) ID", - "placeholder": "Enter an application (client) ID", + "placeholder": "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx", + "help": "Found in your app registration → Overview → Application (client) ID.", "validation": { "required": true } @@ -32,6 +34,7 @@ "name": "clientSecret", "label": "Client secret", "placeholder": "Enter a client secret", + "help": "The secret value generated in your app registration → Certificates & secrets.", "validation": { "required": true } From 39d17a1cd0abff6b57465ee573f6624c29aa03b6 Mon Sep 17 00:00:00 2001 From: John Hayes Date: Tue, 28 Jul 2026 17:32:41 +0100 Subject: [PATCH 6/7] Update readme to remove incorrect reference to Scans being indexed --- plugins/MicrosoftPurview/v1/docs/README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/plugins/MicrosoftPurview/v1/docs/README.md b/plugins/MicrosoftPurview/v1/docs/README.md index af6f6dc2..c8fbe08c 100644 --- a/plugins/MicrosoftPurview/v1/docs/README.md +++ b/plugins/MicrosoftPurview/v1/docs/README.md @@ -65,8 +65,7 @@ Repeat for each governance domain you want to surface. Without this role, the Go | Object type | Represents | | ---------------------- | --------------------------------------------------------------------------------------------------------- | | **Purview Collection** | An organisational unit in your Purview account. The root collection is named after the account itself. | -| **Purview Data Source** | A registered data source — e.g. an Azure SQL database, ADLS Gen2 account, Snowflake instance, Amazon S3 bucket. | -| **Purview Scan** | A configured scan attached to a data source. | +| **Purview Data Source** | A registered data source — e.g. an Azure SQL database, ADLS Gen2 account, Snowflake instance, Amazon S3 bucket. | | | **Purview Governance Domain** | A governance domain defined in the Unified Catalog. Requires the Data Catalog Reader role — see step 3 above. | Assets (tables, files, columns) are **not** indexed because a single Purview account can hold millions of them. They are queried on-demand by the data streams instead. From 368c78d1031dc470ca9dd13b04a03d8898aee01e Mon Sep 17 00:00:00 2001 From: John Hayes Date: Tue, 28 Jul 2026 17:41:23 +0100 Subject: [PATCH 7/7] Use composite scan ID to ensure globally unique indexed scan objects Scan names alone are not unique across data sources (e.g. two sources can both have a scan named "DefaultScan"). Emit a composite id of dataSourceName/name from the script, declare it as sourceId in the metadata, and map id in the index definition objectMapping so scans from different data sources never collide. Co-Authored-By: Claude Sonnet 4.6 --- plugins/MicrosoftPurview/v1/dataStreams/dataSourceScans.json | 1 + .../MicrosoftPurview/v1/dataStreams/scripts/dataSourceScans.js | 1 + plugins/MicrosoftPurview/v1/indexDefinitions/default.json | 2 +- plugins/MicrosoftPurview/v1/metadata.json | 2 +- 4 files changed, 4 insertions(+), 2 deletions(-) diff --git a/plugins/MicrosoftPurview/v1/dataStreams/dataSourceScans.json b/plugins/MicrosoftPurview/v1/dataStreams/dataSourceScans.json index 0233c263..b2d0fcfd 100644 --- a/plugins/MicrosoftPurview/v1/dataStreams/dataSourceScans.json +++ b/plugins/MicrosoftPurview/v1/dataStreams/dataSourceScans.json @@ -18,6 +18,7 @@ }, "matches": { "sourceType": { "type": "oneOf", "values": ["Purview Data Source"] } }, "metadata": [ + { "name": "id", "displayName": "Scan ID", "role": "sourceId", "visible": false }, { "name": "name", "displayName": "Scan", "role": "label" }, { "name": "dataSourceName", "displayName": "Data Source" }, { "name": "kind", "displayName": "Kind" }, diff --git a/plugins/MicrosoftPurview/v1/dataStreams/scripts/dataSourceScans.js b/plugins/MicrosoftPurview/v1/dataStreams/scripts/dataSourceScans.js index 045178ea..b1e75191 100644 --- a/plugins/MicrosoftPurview/v1/dataStreams/scripts/dataSourceScans.js +++ b/plugins/MicrosoftPurview/v1/dataStreams/scripts/dataSourceScans.js @@ -4,6 +4,7 @@ result = scans.map((scan) => { const props = scan.properties || {}; return { + id: (scan.dataSourceName || '') + '/' + (scan.name || ''), name: scan.name || '', dataSourceName: scan.dataSourceName || '', kind: scan.kind || '', diff --git a/plugins/MicrosoftPurview/v1/indexDefinitions/default.json b/plugins/MicrosoftPurview/v1/indexDefinitions/default.json index 569074dc..1488fd61 100644 --- a/plugins/MicrosoftPurview/v1/indexDefinitions/default.json +++ b/plugins/MicrosoftPurview/v1/indexDefinitions/default.json @@ -52,7 +52,7 @@ "dataStream": { "name": "dataSourceScans" }, "timeframe": "none", "objectMapping": { - "id": "name", + "id": "id", "name": "name", "type": "sourceType", "properties": [ diff --git a/plugins/MicrosoftPurview/v1/metadata.json b/plugins/MicrosoftPurview/v1/metadata.json index 8056468e..a129fdcb 100644 --- a/plugins/MicrosoftPurview/v1/metadata.json +++ b/plugins/MicrosoftPurview/v1/metadata.json @@ -1,7 +1,7 @@ { "name": "microsoft-purview", "displayName": "Microsoft Purview", - "version": "1.7.0", + "version": "1.7.1", "author": { "name": "John Hayes", "type": "community"