diff --git a/CHANGELOG.md b/CHANGELOG.md index 3f93957..0f520b9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## Version 1.6.1 - Enhancement release - 2026-08-20 + +- Adding support for cursor based pagination + ## Version 1.6.0 - Enhancement release - 2026-08-20 - Added supported Python versions: 3.12, 3.13, 3.14 diff --git a/custom-recipes/api-connect/Cobuild.md b/custom-recipes/api-connect/Cobuild.md index 5685154..bcca278 100644 --- a/custom-recipes/api-connect/Cobuild.md +++ b/custom-recipes/api-connect/Cobuild.md @@ -4,4 +4,4 @@ - API Connect automatically injects an API-key preset into the header or query parameter configured by the user in that preset. Do not add the API key again to request fields. - Use the dot-separated `extraction_key` when response rows are nested under a JSON path. - Keep `raw_output=true` when each response item should be preserved as raw JSON instead of flattened into columns. -- Select the pagination mechanism from the target API's documentation; do not infer one from the endpoint shape. Relative next-page URLs require `next_page_url_base`, page pagination requires `extraction_key`, and offset or page pagination requires `skip_key`. +- Select the pagination mechanism from the target API's documentation; do not infer one from the endpoint shape. Relative next-page URLs require `next_page_url_base`, page pagination requires `extraction_key`, offset or page pagination requires `skip_key`, cursor pagination requires `cursor_next_token_path` and `cursor_query_param`, and can optionally use `cursor_initial_token` to start from a known cursor. diff --git a/custom-recipes/api-connect/recipe.json b/custom-recipes/api-connect/recipe.json index d18e921..5972ab9 100644 --- a/custom-recipes/api-connect/recipe.json +++ b/custom-recipes/api-connect/recipe.json @@ -222,6 +222,7 @@ "selectChoices":[ {"value": "na", "label": "No pagination"}, {"value": "next_page", "label": "Next page URL provided"}, + {"value": "cursor", "label": "Cursor pagination"}, {"value": "offset", "label": "Offset pagination"}, {"value": "page", "label": "Per page"} ] @@ -279,6 +280,30 @@ "defaultValue": null, "visibilityCondition": "model.pagination_type=='page'" }, + { + "name": "cursor_next_token_path", + "label": "Key to next cursor token", + "description": "Dot separated key path to next cursor token", + "type": "STRING", + "defaultValue": null, + "visibilityCondition": "model.pagination_type=='cursor'" + }, + { + "name": "cursor_query_param", + "label": "Query parameter", + "description": "Name of the url parameter that will contain the cursor token in following calls", + "type": "STRING", + "defaultValue": null, + "visibilityCondition": "model.pagination_type=='cursor'" + }, + { + "name": "cursor_initial_token", + "label": "Initial cursor value", + "description": "Optional value sent in the query parameter on the first request", + "type": "STRING", + "defaultValue": null, + "visibilityCondition": "model.pagination_type=='cursor'" + }, { "type": "SEPARATOR", "label": "Advanced" diff --git a/plugin.json b/plugin.json index f947286..f1f86df 100644 --- a/plugin.json +++ b/plugin.json @@ -1,6 +1,6 @@ { "id": "api-connect", - "version": "1.6.0", + "version": "1.6.1", "meta": { "label": "API Connect", "description": "Retrieve data from any REST API", diff --git a/python-connectors/api-connect_dataset/connector.json b/python-connectors/api-connect_dataset/connector.json index db18a68..252dfec 100644 --- a/python-connectors/api-connect_dataset/connector.json +++ b/python-connectors/api-connect_dataset/connector.json @@ -178,6 +178,7 @@ "selectChoices":[ {"value": "na", "label": "No pagination"}, {"value": "next_page", "label": "Next page URL provided"}, + {"value": "cursor", "label": "Cursor pagination"}, {"value": "offset", "label": "Offset pagination"}, {"value": "page", "label": "Per page"} ] @@ -235,6 +236,30 @@ "defaultValue": null, "visibilityCondition": "model.pagination_type=='page'" }, + { + "name": "cursor_next_token_path", + "label": "Key to next cursor token", + "description": "Dot separated key path to next cursor token", + "type": "STRING", + "defaultValue": null, + "visibilityCondition": "model.pagination_type=='cursor'" + }, + { + "name": "cursor_query_param", + "label": "Query parameter", + "description": "Name of the url parameter that will contain the cursor token in following calls", + "type": "STRING", + "defaultValue": null, + "visibilityCondition": "model.pagination_type=='cursor'" + }, + { + "name": "cursor_initial_token", + "label": "Initial cursor value", + "description": "Optional value sent in the query parameter on the first request", + "type": "STRING", + "defaultValue": null, + "visibilityCondition": "model.pagination_type=='cursor'" + }, { "type": "SEPARATOR", "label": "Advanced" diff --git a/python-lib/dku_utils.py b/python-lib/dku_utils.py index 692300a..aa9c1da 100644 --- a/python-lib/dku_utils.py +++ b/python-lib/dku_utils.py @@ -39,6 +39,7 @@ def get_endpoint_parameters(configuration): "requests_per_minute", "pagination_type", "next_page_url_key", "is_next_page_url_relative", "next_page_url_base", + "cursor_next_token_path", "cursor_query_param", "cursor_initial_token", "top_key", "skip_key", "maximum_number_rows", "use_mtls", "mtls_certificate_path", "mtls_key_path", "force_csv_parameters", "csv_delimiter", diff --git a/python-lib/pagination.py b/python-lib/pagination.py index 252c6b6..0cd4dbf 100644 --- a/python-lib/pagination.py +++ b/python-lib/pagination.py @@ -12,6 +12,10 @@ def __init__(self): self.next_page_url_base = None self.skip_key = None self.next_page_url = None + self.cursor_next_token_path = None + self.cursor_query_param = None + self.cursor_initial_token = None + self.cursor_query_param_value = None self.records_to_skip = None self.pagination_type = "" self.counting_key = None @@ -25,7 +29,9 @@ def __init__(self): self.update_next_page = self.update_next_page_default def configure_paging(self, config=None, skip_key=None, - next_page_key=None, next_page_url_base=None, pagination_type="na"): + next_page_key=None, next_page_url_base=None, + cursor_next_token_path=None, cursor_query_param=None, + cursor_initial_token=None, pagination_type="na"): config = {} if config is None else config self.pagination_type = config.get("pagination_type", pagination_type) if self.pagination_type == "next_page": @@ -34,10 +40,14 @@ def configure_paging(self, config=None, skip_key=None, if next_page_url_base: next_page_url_base = next_page_url_base.strip('/') self.next_page_url_base = next_page_url_base + elif self.pagination_type == "cursor": + self.cursor_next_token_path = config.get("cursor_next_token_path", cursor_next_token_path) + self.cursor_query_param = config.get("cursor_query_param", cursor_query_param) + self.cursor_initial_token = config.get("cursor_initial_token", cursor_initial_token) elif self.pagination_type in ["offset", "page"]: self.skip_key = config.get("skip_key", skip_key) - logger.info("configure_paging: self.pagination_type='{}', self.next_page_key='{}', self.next_page_url_base='{}', self.skip_key='{}'".format( - self.pagination_type, self.next_page_key, self.next_page_url_base, self.skip_key + logger.info("configure_paging: self.pagination_type='{}', self.next_page_key='{}', self.next_page_url_base='{}', self.skip_key='{}', self.cursor_next_token_path='{}', self.cursor_query_param='{}', self.cursor_initial_token='{}'".format( + self.pagination_type, self.next_page_key, self.next_page_url_base, self.skip_key, self.cursor_next_token_path, self.cursor_query_param, self.cursor_initial_token )) if self.pagination_type == "next_page": self.update_next_page = self.update_next_page_link @@ -45,6 +55,8 @@ def configure_paging(self, config=None, skip_key=None, self.update_next_page = self.update_next_page_offset elif self.pagination_type == "page": self.update_next_page = self.update_next_page_per_page + elif self.pagination_type == "cursor": + self.update_next_page = self.update_next_page_cursor else: self.update_next_page = self.update_next_page_default @@ -57,6 +69,7 @@ def reset_paging(self, counting_key=None, url=None): else: self.next_page_number = 0 self.next_page_url = url + self.cursor_query_param_value = self.cursor_initial_token self.is_last_batch_empty = False self.is_first_batch = True self.is_paging_started = True @@ -127,6 +140,16 @@ def update_next_page_link(self, data, response_links=None): next_link, self.next_page_url, self.params_must_be_blanked, self.next_page_number, self.counter )) + def update_next_page_cursor(self, data, response_links=None): + self.is_first_batch = False + self.counter += 1 + self.cursor_query_param_value = None + self.data_is_list = False + + if self.cursor_next_token_path and self.cursor_query_param: + self.cursor_query_param_value = extract_key_using_json_path(data, self.cursor_next_token_path) + logger.info("update_next_page_cursor:{}".format({self.cursor_query_param_value})) + def update_next_page_default(self, data, response_links=None): self.is_first_batch = False @@ -145,6 +168,15 @@ def has_next_page(self): ret )) return ret + if self.pagination_type == "cursor": + ret = (self.cursor_query_param_value is not None) and (self.cursor_query_param_value != "") + logger.info("has_next_page:cursor_next_token_path={} cursor_query_param={} cursor_query_param_value={} -> {}".format( + self.cursor_next_token_path, + self.cursor_query_param, + self.cursor_query_param_value, + ret + )) + return ret if self.pagination_type in ["page", "offset"]: if self.counting_key: # There is a counting key and we already know the last batch was not empty @@ -168,6 +200,10 @@ def get_params(self): ret.update({ self.skip_key: self.next_page_number if self.pagination_type == "page" else self.records_to_skip }) + if self.cursor_query_param and self.cursor_query_param_value not in [None, ""]: + ret.update({ + self.cursor_query_param: self.cursor_query_param_value + }) return ret def get_next_page_url(self): diff --git a/python-lib/rest_api_client.py b/python-lib/rest_api_client.py index d57e5d4..1f18251 100644 --- a/python-lib/rest_api_client.py +++ b/python-lib/rest_api_client.py @@ -89,6 +89,9 @@ def __init__(self, credential, secure_credentials, endpoint, custom_key_values={ next_page_url_base = endpoint.get("next_page_url_base", None) if is_next_page_url_relative else None next_page_url_base = format_template(next_page_url_base, **self.presets_variables) skip_key = endpoint.get("skip_key") + cursor_next_token_path = endpoint.get("cursor_next_token_path") + cursor_query_param = endpoint.get("cursor_query_param") + cursor_initial_token = format_template(endpoint.get("cursor_initial_token"), **self.presets_variables) pagination_type = endpoint.get("pagination_type", "na") if pagination_type == "next_page" and is_next_page_url_relative and not next_page_url_base: raise RestAPIClientError("Pagination's 'Next page URL' is relative but no 'Base URL to next page' has been set") @@ -96,6 +99,9 @@ def __init__(self, credential, secure_credentials, endpoint, custom_key_values={ skip_key=skip_key, next_page_key=next_page_url_key, next_page_url_base=next_page_url_base, + cursor_next_token_path=cursor_next_token_path, + cursor_query_param=cursor_query_param, + cursor_initial_token=cursor_initial_token, pagination_type=pagination_type ) self.last_interaction = None