From d26d9be36b9a32902421e43339d4a08979e7ba1c Mon Sep 17 00:00:00 2001 From: Yannick BETOU Date: Thu, 20 Aug 2026 15:58:17 +0200 Subject: [PATCH 1/3] add cursor based pagination --- custom-recipes/api-connect/Cobuild.md | 2 +- custom-recipes/api-connect/recipe.json | 17 +++++++++ .../api-connect_dataset/connector.json | 17 +++++++++ python-lib/dku_utils.py | 1 + python-lib/pagination.py | 38 +++++++++++++++++-- python-lib/rest_api_client.py | 4 ++ 6 files changed, 75 insertions(+), 4 deletions(-) diff --git a/custom-recipes/api-connect/Cobuild.md b/custom-recipes/api-connect/Cobuild.md index 5685154..fb18c22 100644 --- a/custom-recipes/api-connect/Cobuild.md +++ b/custom-recipes/api-connect/Cobuild.md @@ -4,4 +4,4 @@ - API Connect automatically injects an API-key preset into the header or query parameter configured by the user in that preset. Do not add the API key again to request fields. - Use the dot-separated `extraction_key` when response rows are nested under a JSON path. - Keep `raw_output=true` when each response item should be preserved as raw JSON instead of flattened into columns. -- Select the pagination mechanism from the target API's documentation; do not infer one from the endpoint shape. Relative next-page URLs require `next_page_url_base`, page pagination requires `extraction_key`, and offset or page pagination requires `skip_key`. +- Select the pagination mechanism from the target API's documentation; do not infer one from the endpoint shape. Relative next-page URLs require `next_page_url_base`, page pagination requires `extraction_key`, offset or page pagination requires `skip_key`, cursor pagination requires `cursor_next_token_path` and `cursor_query_param`. diff --git a/custom-recipes/api-connect/recipe.json b/custom-recipes/api-connect/recipe.json index d18e921..e72faa1 100644 --- a/custom-recipes/api-connect/recipe.json +++ b/custom-recipes/api-connect/recipe.json @@ -222,6 +222,7 @@ "selectChoices":[ {"value": "na", "label": "No pagination"}, {"value": "next_page", "label": "Next page URL provided"}, + {"value": "cursor", "label": "Cursor pagination"}, {"value": "offset", "label": "Offset pagination"}, {"value": "page", "label": "Per page"} ] @@ -279,6 +280,22 @@ "defaultValue": null, "visibilityCondition": "model.pagination_type=='page'" }, + { + "name": "cursor_next_token_path", + "label": "Key to next cursor token", + "description": "Dot separated key path to next cursor token", + "type": "STRING", + "defaultValue": null, + "visibilityCondition": "model.pagination_type=='cursor'" + }, + { + "name": "cursor_query_param", + "label": "Query parameter", + "description": "Name of the url parameter that will contain the cursor token in following calls", + "type": "STRING", + "defaultValue": null, + "visibilityCondition": "model.pagination_type=='cursor'" + }, { "type": "SEPARATOR", "label": "Advanced" diff --git a/python-connectors/api-connect_dataset/connector.json b/python-connectors/api-connect_dataset/connector.json index db18a68..1532630 100644 --- a/python-connectors/api-connect_dataset/connector.json +++ b/python-connectors/api-connect_dataset/connector.json @@ -178,6 +178,7 @@ "selectChoices":[ {"value": "na", "label": "No pagination"}, {"value": "next_page", "label": "Next page URL provided"}, + {"value": "cursor", "label": "Cursor pagination"}, {"value": "offset", "label": "Offset pagination"}, {"value": "page", "label": "Per page"} ] @@ -235,6 +236,22 @@ "defaultValue": null, "visibilityCondition": "model.pagination_type=='page'" }, + { + "name": "cursor_next_token_path", + "label": "Key to next cursor token", + "description": "Dot separated key path to next cursor token", + "type": "STRING", + "defaultValue": null, + "visibilityCondition": "model.pagination_type=='cursor'" + }, + { + "name": "cursor_query_param", + "label": "Query parameter", + "description": "Name of the url parameter that will contain the cursor token in following calls", + "type": "STRING", + "defaultValue": null, + "visibilityCondition": "model.pagination_type=='cursor'" + }, { "type": "SEPARATOR", "label": "Advanced" diff --git a/python-lib/dku_utils.py b/python-lib/dku_utils.py index 692300a..4da6bca 100644 --- a/python-lib/dku_utils.py +++ b/python-lib/dku_utils.py @@ -39,6 +39,7 @@ def get_endpoint_parameters(configuration): "requests_per_minute", "pagination_type", "next_page_url_key", "is_next_page_url_relative", "next_page_url_base", + "cursor_next_token_path", "cursor_query_param", "top_key", "skip_key", "maximum_number_rows", "use_mtls", "mtls_certificate_path", "mtls_key_path", "force_csv_parameters", "csv_delimiter", diff --git a/python-lib/pagination.py b/python-lib/pagination.py index 252c6b6..f105cfa 100644 --- a/python-lib/pagination.py +++ b/python-lib/pagination.py @@ -12,6 +12,9 @@ def __init__(self): self.next_page_url_base = None self.skip_key = None self.next_page_url = None + self.cursor_next_token_path = None + self.cursor_query_param = None + self.cursor_query_param_value = None self.records_to_skip = None self.pagination_type = "" self.counting_key = None @@ -25,7 +28,8 @@ def __init__(self): self.update_next_page = self.update_next_page_default def configure_paging(self, config=None, skip_key=None, - next_page_key=None, next_page_url_base=None, pagination_type="na"): + next_page_key=None, next_page_url_base=None, + cursor_next_token_path=None, cursor_query_param=None, pagination_type="na"): config = {} if config is None else config self.pagination_type = config.get("pagination_type", pagination_type) if self.pagination_type == "next_page": @@ -34,10 +38,13 @@ def configure_paging(self, config=None, skip_key=None, if next_page_url_base: next_page_url_base = next_page_url_base.strip('/') self.next_page_url_base = next_page_url_base + elif self.pagination_type == "cursor": + self.cursor_next_token_path = config.get("cursor_next_token_path", cursor_next_token_path) + self.cursor_query_param = cursor_query_param elif self.pagination_type in ["offset", "page"]: self.skip_key = config.get("skip_key", skip_key) - logger.info("configure_paging: self.pagination_type='{}', self.next_page_key='{}', self.next_page_url_base='{}', self.skip_key='{}'".format( - self.pagination_type, self.next_page_key, self.next_page_url_base, self.skip_key + logger.info("configure_paging: self.pagination_type='{}', self.next_page_key='{}', self.next_page_url_base='{}', self.skip_key='{}', self.cursor_next_token_path='{}', self.query_param='{}'".format( + self.pagination_type, self.next_page_key, self.next_page_url_base, self.skip_key, self.cursor_next_token_path, self.cursor_query_param )) if self.pagination_type == "next_page": self.update_next_page = self.update_next_page_link @@ -45,6 +52,8 @@ def configure_paging(self, config=None, skip_key=None, self.update_next_page = self.update_next_page_offset elif self.pagination_type == "page": self.update_next_page = self.update_next_page_per_page + elif self.pagination_type == "cursor": + self.update_next_page = self.update_next_page_cursor else: self.update_next_page = self.update_next_page_default @@ -127,6 +136,16 @@ def update_next_page_link(self, data, response_links=None): next_link, self.next_page_url, self.params_must_be_blanked, self.next_page_number, self.counter )) + def update_next_page_cursor(self, data, response_links=None): + self.is_first_batch = False + self.counter += 1 + self.cursor_query_param_value = None + self.data_is_list = False + + if self.cursor_next_token_path and self.cursor_query_param: + self.cursor_query_param_value = extract_key_using_json_path(data, self.cursor_next_token_path) + logger.info("update_next_page_cursor:{}".format({self.cursor_query_param_value})) + def update_next_page_default(self, data, response_links=None): self.is_first_batch = False @@ -145,6 +164,15 @@ def has_next_page(self): ret )) return ret + if self.pagination_type == "cursor": + ret = (self.cursor_query_param_value is not None) and (self.cursor_query_param_value != "") + logger.info("has_next_page:cursor_next_token_path={} cursor_query_param={} cursor_query_param_value={} -> {}".format( + self.cursor_next_token_path, + self.cursor_query_param, + self.cursor_query_param_value, + ret + )) + return ret if self.pagination_type in ["page", "offset"]: if self.counting_key: # There is a counting key and we already know the last batch was not empty @@ -168,6 +196,10 @@ def get_params(self): ret.update({ self.skip_key: self.next_page_number if self.pagination_type == "page" else self.records_to_skip }) + if self.cursor_query_param: + ret.update({ + self.cursor_query_param: self.cursor_query_param_value + }) return ret def get_next_page_url(self): diff --git a/python-lib/rest_api_client.py b/python-lib/rest_api_client.py index d57e5d4..4fbd7ac 100644 --- a/python-lib/rest_api_client.py +++ b/python-lib/rest_api_client.py @@ -89,6 +89,8 @@ def __init__(self, credential, secure_credentials, endpoint, custom_key_values={ next_page_url_base = endpoint.get("next_page_url_base", None) if is_next_page_url_relative else None next_page_url_base = format_template(next_page_url_base, **self.presets_variables) skip_key = endpoint.get("skip_key") + cursor_next_token_path = endpoint.get("cursor_next_token_path") + cursor_query_param = endpoint.get("cursor_query_param") pagination_type = endpoint.get("pagination_type", "na") if pagination_type == "next_page" and is_next_page_url_relative and not next_page_url_base: raise RestAPIClientError("Pagination's 'Next page URL' is relative but no 'Base URL to next page' has been set") @@ -96,6 +98,8 @@ def __init__(self, credential, secure_credentials, endpoint, custom_key_values={ skip_key=skip_key, next_page_key=next_page_url_key, next_page_url_base=next_page_url_base, + cursor_next_token_path=cursor_next_token_path, + cursor_query_param=cursor_query_param, pagination_type=pagination_type ) self.last_interaction = None From 1ead0f07e7bc7cf034bcd2ecf14eb55b81addc8c Mon Sep 17 00:00:00 2001 From: Yannick BETOU Date: Thu, 20 Aug 2026 16:00:30 +0200 Subject: [PATCH 2/3] bump --- CHANGELOG.md | 5 +++++ plugin.json | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 932dd21..7065ee6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # Changelog + +## Version 1.6.1 - Enhancement release - 2026-08-20 + +- Adding support for cursor based pagination + ## Version 1.6.0 - Enhancement release - 2026-08-19 - Added supported Python versions: 3.12, 3.13, 3.14 diff --git a/plugin.json b/plugin.json index f947286..f1f86df 100644 --- a/plugin.json +++ b/plugin.json @@ -1,6 +1,6 @@ { "id": "api-connect", - "version": "1.6.0", + "version": "1.6.1", "meta": { "label": "API Connect", "description": "Retrieve data from any REST API", From 84846922bda8bcde6d3479fcfb18b73de2f62510 Mon Sep 17 00:00:00 2001 From: Yannick BETOU Date: Fri, 21 Aug 2026 09:27:33 +0200 Subject: [PATCH 3/3] add cusrsor initial token --- custom-recipes/api-connect/Cobuild.md | 2 +- custom-recipes/api-connect/recipe.json | 8 ++++++++ .../api-connect_dataset/connector.json | 8 ++++++++ python-lib/dku_utils.py | 2 +- python-lib/pagination.py | 14 +++++++++----- python-lib/rest_api_client.py | 2 ++ 6 files changed, 29 insertions(+), 7 deletions(-) diff --git a/custom-recipes/api-connect/Cobuild.md b/custom-recipes/api-connect/Cobuild.md index fb18c22..bcca278 100644 --- a/custom-recipes/api-connect/Cobuild.md +++ b/custom-recipes/api-connect/Cobuild.md @@ -4,4 +4,4 @@ - API Connect automatically injects an API-key preset into the header or query parameter configured by the user in that preset. Do not add the API key again to request fields. - Use the dot-separated `extraction_key` when response rows are nested under a JSON path. - Keep `raw_output=true` when each response item should be preserved as raw JSON instead of flattened into columns. -- Select the pagination mechanism from the target API's documentation; do not infer one from the endpoint shape. Relative next-page URLs require `next_page_url_base`, page pagination requires `extraction_key`, offset or page pagination requires `skip_key`, cursor pagination requires `cursor_next_token_path` and `cursor_query_param`. +- Select the pagination mechanism from the target API's documentation; do not infer one from the endpoint shape. Relative next-page URLs require `next_page_url_base`, page pagination requires `extraction_key`, offset or page pagination requires `skip_key`, cursor pagination requires `cursor_next_token_path` and `cursor_query_param`, and can optionally use `cursor_initial_token` to start from a known cursor. diff --git a/custom-recipes/api-connect/recipe.json b/custom-recipes/api-connect/recipe.json index e72faa1..5972ab9 100644 --- a/custom-recipes/api-connect/recipe.json +++ b/custom-recipes/api-connect/recipe.json @@ -296,6 +296,14 @@ "defaultValue": null, "visibilityCondition": "model.pagination_type=='cursor'" }, + { + "name": "cursor_initial_token", + "label": "Initial cursor value", + "description": "Optional value sent in the query parameter on the first request", + "type": "STRING", + "defaultValue": null, + "visibilityCondition": "model.pagination_type=='cursor'" + }, { "type": "SEPARATOR", "label": "Advanced" diff --git a/python-connectors/api-connect_dataset/connector.json b/python-connectors/api-connect_dataset/connector.json index 1532630..252dfec 100644 --- a/python-connectors/api-connect_dataset/connector.json +++ b/python-connectors/api-connect_dataset/connector.json @@ -252,6 +252,14 @@ "defaultValue": null, "visibilityCondition": "model.pagination_type=='cursor'" }, + { + "name": "cursor_initial_token", + "label": "Initial cursor value", + "description": "Optional value sent in the query parameter on the first request", + "type": "STRING", + "defaultValue": null, + "visibilityCondition": "model.pagination_type=='cursor'" + }, { "type": "SEPARATOR", "label": "Advanced" diff --git a/python-lib/dku_utils.py b/python-lib/dku_utils.py index 4da6bca..aa9c1da 100644 --- a/python-lib/dku_utils.py +++ b/python-lib/dku_utils.py @@ -39,7 +39,7 @@ def get_endpoint_parameters(configuration): "requests_per_minute", "pagination_type", "next_page_url_key", "is_next_page_url_relative", "next_page_url_base", - "cursor_next_token_path", "cursor_query_param", + "cursor_next_token_path", "cursor_query_param", "cursor_initial_token", "top_key", "skip_key", "maximum_number_rows", "use_mtls", "mtls_certificate_path", "mtls_key_path", "force_csv_parameters", "csv_delimiter", diff --git a/python-lib/pagination.py b/python-lib/pagination.py index f105cfa..0cd4dbf 100644 --- a/python-lib/pagination.py +++ b/python-lib/pagination.py @@ -14,6 +14,7 @@ def __init__(self): self.next_page_url = None self.cursor_next_token_path = None self.cursor_query_param = None + self.cursor_initial_token = None self.cursor_query_param_value = None self.records_to_skip = None self.pagination_type = "" @@ -29,7 +30,8 @@ def __init__(self): def configure_paging(self, config=None, skip_key=None, next_page_key=None, next_page_url_base=None, - cursor_next_token_path=None, cursor_query_param=None, pagination_type="na"): + cursor_next_token_path=None, cursor_query_param=None, + cursor_initial_token=None, pagination_type="na"): config = {} if config is None else config self.pagination_type = config.get("pagination_type", pagination_type) if self.pagination_type == "next_page": @@ -40,11 +42,12 @@ def configure_paging(self, config=None, skip_key=None, self.next_page_url_base = next_page_url_base elif self.pagination_type == "cursor": self.cursor_next_token_path = config.get("cursor_next_token_path", cursor_next_token_path) - self.cursor_query_param = cursor_query_param + self.cursor_query_param = config.get("cursor_query_param", cursor_query_param) + self.cursor_initial_token = config.get("cursor_initial_token", cursor_initial_token) elif self.pagination_type in ["offset", "page"]: self.skip_key = config.get("skip_key", skip_key) - logger.info("configure_paging: self.pagination_type='{}', self.next_page_key='{}', self.next_page_url_base='{}', self.skip_key='{}', self.cursor_next_token_path='{}', self.query_param='{}'".format( - self.pagination_type, self.next_page_key, self.next_page_url_base, self.skip_key, self.cursor_next_token_path, self.cursor_query_param + logger.info("configure_paging: self.pagination_type='{}', self.next_page_key='{}', self.next_page_url_base='{}', self.skip_key='{}', self.cursor_next_token_path='{}', self.cursor_query_param='{}', self.cursor_initial_token='{}'".format( + self.pagination_type, self.next_page_key, self.next_page_url_base, self.skip_key, self.cursor_next_token_path, self.cursor_query_param, self.cursor_initial_token )) if self.pagination_type == "next_page": self.update_next_page = self.update_next_page_link @@ -66,6 +69,7 @@ def reset_paging(self, counting_key=None, url=None): else: self.next_page_number = 0 self.next_page_url = url + self.cursor_query_param_value = self.cursor_initial_token self.is_last_batch_empty = False self.is_first_batch = True self.is_paging_started = True @@ -196,7 +200,7 @@ def get_params(self): ret.update({ self.skip_key: self.next_page_number if self.pagination_type == "page" else self.records_to_skip }) - if self.cursor_query_param: + if self.cursor_query_param and self.cursor_query_param_value not in [None, ""]: ret.update({ self.cursor_query_param: self.cursor_query_param_value }) diff --git a/python-lib/rest_api_client.py b/python-lib/rest_api_client.py index 4fbd7ac..1f18251 100644 --- a/python-lib/rest_api_client.py +++ b/python-lib/rest_api_client.py @@ -91,6 +91,7 @@ def __init__(self, credential, secure_credentials, endpoint, custom_key_values={ skip_key = endpoint.get("skip_key") cursor_next_token_path = endpoint.get("cursor_next_token_path") cursor_query_param = endpoint.get("cursor_query_param") + cursor_initial_token = format_template(endpoint.get("cursor_initial_token"), **self.presets_variables) pagination_type = endpoint.get("pagination_type", "na") if pagination_type == "next_page" and is_next_page_url_relative and not next_page_url_base: raise RestAPIClientError("Pagination's 'Next page URL' is relative but no 'Base URL to next page' has been set") @@ -100,6 +101,7 @@ def __init__(self, credential, secure_credentials, endpoint, custom_key_values={ next_page_url_base=next_page_url_base, cursor_next_token_path=cursor_next_token_path, cursor_query_param=cursor_query_param, + cursor_initial_token=cursor_initial_token, pagination_type=pagination_type ) self.last_interaction = None