From 36ab6c41c76e85cb7936cb5b10355c464b139a8b Mon Sep 17 00:00:00 2001 From: SergioLangaritaBenitez Date: Tue, 7 Oct 2025 10:48:14 +0200 Subject: [PATCH 01/11] pagination option added --- oscar_python/client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/oscar_python/client.py b/oscar_python/client.py index 62a87b2..5305235 100644 --- a/oscar_python/client.py +++ b/oscar_python/client.py @@ -205,8 +205,8 @@ def get_job_logs(self, svc, job): return utils.make_request(self, _LOGS_PATH+"/"+svc+"/"+job, _GET) """ List a service jobs """ - def list_jobs(self, svc): - return utils.make_request(self, _LOGS_PATH+"/"+svc, _GET) + def list_jobs(self, svc, page=""): + return utils.make_request(self, _LOGS_PATH+"/"+svc+"?page="+page, _GET) """ Remove a service job """ def remove_job(self, svc, job): From 174b356fff92955c9eb02d192a1e93ff37034891 Mon Sep 17 00:00:00 2001 From: SergioLangaritaBenitez Date: Tue, 7 Oct 2025 17:00:01 +0200 Subject: [PATCH 02/11] adapt fdl scipt path --- oscar_python/client.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/oscar_python/client.py b/oscar_python/client.py index 5305235..8cc8e1c 100644 --- a/oscar_python/client.py +++ b/oscar_python/client.py @@ -132,7 +132,9 @@ def _check_fdl_definition(self, fdl_path): except KeyError as err: raise Exception("FDL clusterID does not match current clusterID: {0}".format(err)) try: - with open(svc["script"]) as s: + fdl_directory = os.path.dirname(fdl_path) + script_path = fdl_directory + "/" + svc["script"] + with open(script_path) as s: svc["script"] = s.read() except IOError: raise Exception("Couldn't read script") From 8e11b5f04d59a38c4c5daad05d3275a885dafc62 Mon Sep 17 00:00:00 2001 From: SergioLangaritaBenitez Date: Wed, 8 Oct 2025 08:49:16 +0200 Subject: [PATCH 03/11] absolute path posibility added --- oscar_python/client.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/oscar_python/client.py b/oscar_python/client.py index 8cc8e1c..3f3ebb5 100644 --- a/oscar_python/client.py +++ b/oscar_python/client.py @@ -132,8 +132,11 @@ def _check_fdl_definition(self, fdl_path): except KeyError as err: raise Exception("FDL clusterID does not match current clusterID: {0}".format(err)) try: - fdl_directory = os.path.dirname(fdl_path) - script_path = fdl_directory + "/" + svc["script"] + if os.path.isabs(svc["script"]): + script_path = svc["script"] + else: + fdl_directory = os.path.dirname(fdl_path) + script_path = fdl_directory + "/" + svc["script"] with open(script_path) as s: svc["script"] = s.read() except IOError: From a8141f60691754351feaf9f7caa77f30b1946102 Mon Sep 17 00:00:00 2001 From: SergioLangaritaBenitez Date: Wed, 8 Oct 2025 09:18:05 +0200 Subject: [PATCH 04/11] client_id as optional argument --- oscar_python/_oidc.py | 4 ++-- oscar_python/client.py | 7 ++++++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/oscar_python/_oidc.py b/oscar_python/_oidc.py index f094bb7..34bb1ba 100644 --- a/oscar_python/_oidc.py +++ b/oscar_python/_oidc.py @@ -77,14 +77,14 @@ def is_access_token_expired(token): return True @staticmethod - def refresh_access_token(refresh_token, scopes, token_endpoint): + def refresh_access_token(refresh_token, scopes, token_endpoint, client_id='token-portal'): """ Refresh the access token using the refresh token """ data = { 'grant_type': 'refresh_token', 'refresh_token': refresh_token, - 'client_id': 'token-portal', + 'client_id': client_id, 'scope': ' '.join(scopes) } diff --git a/oscar_python/client.py b/oscar_python/client.py index 3f3ebb5..034a628 100644 --- a/oscar_python/client.py +++ b/oscar_python/client.py @@ -72,6 +72,9 @@ def oidc_client(self, options): self.token_endpoint = options.get('token_endpoint', _DEFAULT_TOKEN_ENDPOINT) self.ssl = bool(options['ssl']) + self.client_id = options.get('client_id') + if self.client_id is None: + self.client_id = 'token-portal' def set_auth_type(self, options): if 'user' in options: @@ -91,7 +94,8 @@ def get_access_token(self): if self.refresh_token and OIDC.is_access_token_expired(self.oidc_token): self.oidc_token = OIDC.refresh_access_token(self.refresh_token, self.scopes, - self.token_endpoint) + self.token_endpoint, + self.client_id) return self.oidc_token """ Creates a generic storage client to interact with the storage providers @@ -137,6 +141,7 @@ def _check_fdl_definition(self, fdl_path): else: fdl_directory = os.path.dirname(fdl_path) script_path = fdl_directory + "/" + svc["script"] + print(script_path) with open(script_path) as s: svc["script"] = s.read() except IOError: From c53e1cdf648464708689d32a6e4147508d27f47f Mon Sep 17 00:00:00 2001 From: SergioLangaritaBenitez Date: Wed, 8 Oct 2025 09:27:29 +0200 Subject: [PATCH 05/11] client_id default value --- oscar_python/_oidc.py | 2 +- oscar_python/client.py | 6 +++--- tests/test_oidc.py | 3 ++- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/oscar_python/_oidc.py b/oscar_python/_oidc.py index 34bb1ba..d035033 100644 --- a/oscar_python/_oidc.py +++ b/oscar_python/_oidc.py @@ -77,7 +77,7 @@ def is_access_token_expired(token): return True @staticmethod - def refresh_access_token(refresh_token, scopes, token_endpoint, client_id='token-portal'): + def refresh_access_token(refresh_token, scopes, token_endpoint, client_id): """ Refresh the access token using the refresh token """ diff --git a/oscar_python/client.py b/oscar_python/client.py index 034a628..22c5596 100644 --- a/oscar_python/client.py +++ b/oscar_python/client.py @@ -41,6 +41,7 @@ # Default values for OIDC refresh token using EGI CheckIn _DEFAULT_SCOPES = ['openid', 'email', 'profile', 'voperson_id', 'eduperson_entitlement'] _DEFAULT_TOKEN_ENDPOINT = 'https://aai.egi.eu/auth/realms/egi/protocol/openid-connect/token' +_DEFAULT_CLIENT_ID = 'token-portal' class Client(DefaultClient): @@ -72,9 +73,8 @@ def oidc_client(self, options): self.token_endpoint = options.get('token_endpoint', _DEFAULT_TOKEN_ENDPOINT) self.ssl = bool(options['ssl']) - self.client_id = options.get('client_id') - if self.client_id is None: - self.client_id = 'token-portal' + self.client_id = options.get('client_id', + _DEFAULT_CLIENT_ID) def set_auth_type(self, options): if 'user' in options: diff --git a/tests/test_oidc.py b/tests/test_oidc.py index ceb4877..f699369 100644 --- a/tests/test_oidc.py +++ b/tests/test_oidc.py @@ -24,7 +24,8 @@ def test_refresh_access_token(): mock_post.return_value = mock_response access_token = OIDC.refresh_access_token("old_refresh_token", ["openid", "profile", "email"], - "http://test.com/token") + "http://test.com/token", + "token-portal") assert access_token == "new_access_token" mock_post.assert_called_once_with( From 71488d61e9f2fa1eacb5f0226a448c40b3606763 Mon Sep 17 00:00:00 2001 From: SergioLangaritaBenitez Date: Wed, 8 Oct 2025 09:42:24 +0200 Subject: [PATCH 06/11] Update readme --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 286aacb..360646f 100644 --- a/README.md +++ b/README.md @@ -77,6 +77,7 @@ and `scopes`: 'refresh_token':'token', 'scopes': ["openid", "profile", "email"], 'token_endpoint': "http://issuer.com/token", + 'client_id': "your_client_id" 'ssl':'True'} client = Client(options = options_oidc_auth) @@ -201,6 +202,8 @@ logs = client.get_job_logs("service_name", "job_id") # returns an http response ``` python # get a list of jobs in a service log_list = client.list_jobs("service_name") # returns an http response +# to get more jobs use the page parameter +log_list = client.list_jobs("service_name",page="token_to_next_page") # returns an http response ``` **remove_job** From 79dc7805047f468793052a2f36ed65396ed30942 Mon Sep 17 00:00:00 2001 From: SergioLangaritaBenitez Date: Fri, 10 Oct 2025 09:45:37 +0200 Subject: [PATCH 07/11] clean code and set default values --- oscar_python/_oidc.py | 2 +- oscar_python/client.py | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/oscar_python/_oidc.py b/oscar_python/_oidc.py index d035033..b6321de 100644 --- a/oscar_python/_oidc.py +++ b/oscar_python/_oidc.py @@ -77,7 +77,7 @@ def is_access_token_expired(token): return True @staticmethod - def refresh_access_token(refresh_token, scopes, token_endpoint, client_id): + def refresh_access_token(refresh_token, scopes, token_endpoint, client_id="token-portal"): """ Refresh the access token using the refresh token """ diff --git a/oscar_python/client.py b/oscar_python/client.py index 22c5596..834ee45 100644 --- a/oscar_python/client.py +++ b/oscar_python/client.py @@ -140,11 +140,10 @@ def _check_fdl_definition(self, fdl_path): script_path = svc["script"] else: fdl_directory = os.path.dirname(fdl_path) - script_path = fdl_directory + "/" + svc["script"] - print(script_path) + script_path = os.path.join(fdl_directory, svc['script']) with open(script_path) as s: svc["script"] = s.read() - except IOError: + except IOError as e: raise Exception("Couldn't read script") # cpu parameter has to be string on the request From 51478075ab0d32dcad0820b1d280fe52acb61965 Mon Sep 17 00:00:00 2001 From: SergioLangaritaBenitez Date: Fri, 10 Oct 2025 10:11:53 +0200 Subject: [PATCH 08/11] Updating versions/notes for 1.3.3-beta1 release --- test.py | 34 ++++++++++++++++++++++++++++++++++ version.py | 2 +- 2 files changed, 35 insertions(+), 1 deletion(-) create mode 100644 test.py diff --git a/test.py b/test.py new file mode 100644 index 0000000..cc680d3 --- /dev/null +++ b/test.py @@ -0,0 +1,34 @@ +from build.lib.oscar_python.client import Client + +""" +options_basic_auth = {'cluster_id':'oscar-cluster-keycloak', + 'endpoint':'https://determined-cray3.im.grycap.net', + 'refresh_token': 'eyJhbGciOiJIUzUxMiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICI1OTU3ODBiOC0xYjYxLTRjMjgtOWZjNS1kMGU0ZTg0Yjk0MTgifQ.eyJleHAiOjE3Njg1NDY2MjAsImlhdCI6MTc1OTkwNjYyMCwianRpIjoiMmMzOGFiYWQtYzVhNC00MzM5LTljZjAtY2M1OGNhMDlkNjQ2IiwiaXNzIjoiaHR0cHM6Ly9rZXljbG9hay5ncnljYXAubmV0L3JlYWxtcy9ncnljYXAiLCJhdWQiOiJodHRwczovL2tleWNsb2FrLmdyeWNhcC5uZXQvcmVhbG1zL2dyeWNhcCIsInN1YiI6IjJiZjkyMzU3LThhYTAtNGYyMC1iYjg2LTkyMDdkYzU0MTBhOCIsInR5cCI6IlJlZnJlc2giLCJhenAiOiJvc2Nhci1rZXljbG9hay1jbGllbnQiLCJzaWQiOiIyMGU4OGJiMS04NDA5LTQ0YzEtOTNiNy1kN2VlMmRmMzA0NmQiLCJzY29wZSI6ImVtYWlsIGFjciB3ZWItb3JpZ2lucyByb2xlcyBvcGVuaWQgYmFzaWMgcHJvZmlsZSJ9.t8pu6d0lx7e5rMiIoPBbnbF7-6iXHxS4GxWacB0hYVwu_ttdeGW2cnU2bcpfn_mjeVx56ewj1GZZvG8lNBV2cg', + 'scopes': ["openid", "profile", "email"], + 'token_endpoint': "https://keycloak.grycap.net/realms/grycap/protocol/openid-connect/token", + 'client_id': "oscar-keycloak-client", + 'ssl':'True'} + + +options_basic_auth = {'cluster_id':'oscar-cluster-egi', + 'endpoint':'https://determined-cray3.im.grycap.net', + 'shortname':'keycloakgrycap', + 'ssl':'True'} +""" +options_basic_auth = {'cluster_id':'oscar-cluster-keycloak', + 'endpoint':'https://determined-cray3.im.grycap.net', + 'shortname':'slangarita', + 'ssl':'True'} + +client = Client(options = options_basic_auth) +#services = client.list_services() # returns an http response or an HTTPError +client.create_service("/home/slangarita/Work/oscar/examples/cowsay/cowsay.yaml") +client.remove_service("cowsay") +client.create_service("/home/slangarita/Work/oscar/examples/cowsay/cowsay_abs.yaml") +client.remove_service("cowsay") +client.create_service("../oscar/examples/cowsay/cowsay.yaml") +client.remove_service("cowsay") +#print(services.json()) +#jobs= client.list_jobs("grayify","") +#print(jobs.json()) + diff --git a/version.py b/version.py index 4aff280..5062a27 100644 --- a/version.py +++ b/version.py @@ -14,4 +14,4 @@ """Stores the package version.""" -__version__ = '1.3.2' +__version__ = '1.3.3-beta1' From 461646e691a3836c6ead9445957052e49d6978be Mon Sep 17 00:00:00 2001 From: SergioLangaritaBenitez Date: Fri, 10 Oct 2025 10:12:06 +0200 Subject: [PATCH 09/11] Updating versions/notes for 1.3.3-beta1 release --- test.py | 34 ---------------------------------- 1 file changed, 34 deletions(-) delete mode 100644 test.py diff --git a/test.py b/test.py deleted file mode 100644 index cc680d3..0000000 --- a/test.py +++ /dev/null @@ -1,34 +0,0 @@ -from build.lib.oscar_python.client import Client - -""" -options_basic_auth = {'cluster_id':'oscar-cluster-keycloak', - 'endpoint':'https://determined-cray3.im.grycap.net', - 'refresh_token': 'eyJhbGciOiJIUzUxMiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICI1OTU3ODBiOC0xYjYxLTRjMjgtOWZjNS1kMGU0ZTg0Yjk0MTgifQ.eyJleHAiOjE3Njg1NDY2MjAsImlhdCI6MTc1OTkwNjYyMCwianRpIjoiMmMzOGFiYWQtYzVhNC00MzM5LTljZjAtY2M1OGNhMDlkNjQ2IiwiaXNzIjoiaHR0cHM6Ly9rZXljbG9hay5ncnljYXAubmV0L3JlYWxtcy9ncnljYXAiLCJhdWQiOiJodHRwczovL2tleWNsb2FrLmdyeWNhcC5uZXQvcmVhbG1zL2dyeWNhcCIsInN1YiI6IjJiZjkyMzU3LThhYTAtNGYyMC1iYjg2LTkyMDdkYzU0MTBhOCIsInR5cCI6IlJlZnJlc2giLCJhenAiOiJvc2Nhci1rZXljbG9hay1jbGllbnQiLCJzaWQiOiIyMGU4OGJiMS04NDA5LTQ0YzEtOTNiNy1kN2VlMmRmMzA0NmQiLCJzY29wZSI6ImVtYWlsIGFjciB3ZWItb3JpZ2lucyByb2xlcyBvcGVuaWQgYmFzaWMgcHJvZmlsZSJ9.t8pu6d0lx7e5rMiIoPBbnbF7-6iXHxS4GxWacB0hYVwu_ttdeGW2cnU2bcpfn_mjeVx56ewj1GZZvG8lNBV2cg', - 'scopes': ["openid", "profile", "email"], - 'token_endpoint': "https://keycloak.grycap.net/realms/grycap/protocol/openid-connect/token", - 'client_id': "oscar-keycloak-client", - 'ssl':'True'} - - -options_basic_auth = {'cluster_id':'oscar-cluster-egi', - 'endpoint':'https://determined-cray3.im.grycap.net', - 'shortname':'keycloakgrycap', - 'ssl':'True'} -""" -options_basic_auth = {'cluster_id':'oscar-cluster-keycloak', - 'endpoint':'https://determined-cray3.im.grycap.net', - 'shortname':'slangarita', - 'ssl':'True'} - -client = Client(options = options_basic_auth) -#services = client.list_services() # returns an http response or an HTTPError -client.create_service("/home/slangarita/Work/oscar/examples/cowsay/cowsay.yaml") -client.remove_service("cowsay") -client.create_service("/home/slangarita/Work/oscar/examples/cowsay/cowsay_abs.yaml") -client.remove_service("cowsay") -client.create_service("../oscar/examples/cowsay/cowsay.yaml") -client.remove_service("cowsay") -#print(services.json()) -#jobs= client.list_jobs("grayify","") -#print(jobs.json()) - From 1a058bbae621fbb708ab782c61a8edb82d632767 Mon Sep 17 00:00:00 2001 From: SergioLangaritaBenitez Date: Mon, 12 Jan 2026 12:55:50 +0100 Subject: [PATCH 10/11] use token of the user instead of the service --- oscar_python/client.py | 2 ++ version.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/oscar_python/client.py b/oscar_python/client.py index 834ee45..c4786a4 100644 --- a/oscar_python/client.py +++ b/oscar_python/client.py @@ -198,6 +198,8 @@ def remove_service(self, name): return utils.make_request(self, _SVC_PATH+"/"+name, _DELETE) def _get_token(self, svc): + if self._AUTH_TYPE != 'basicauth': + return self.get_access_token() service = utils.make_request(self, _SVC_PATH+"/"+svc, _GET) service = json.loads(service.text) return service["token"] diff --git a/version.py b/version.py index 5062a27..867222d 100644 --- a/version.py +++ b/version.py @@ -14,4 +14,4 @@ """Stores the package version.""" -__version__ = '1.3.3-beta1' +__version__ = '1.3.4b1' From 8978171199c470de447cb0cbb49602515b70b228 Mon Sep 17 00:00:00 2001 From: SergioLangaritaBenitez Date: Mon, 25 May 2026 12:50:30 +0200 Subject: [PATCH 11/11] feat: implement missing API client methods and add unit tests - Add 21 new methods to client.py (health, deployments, volumes, buckets, metrics, quotas, system_logs) - Add load_config utility function in _utils.py - Fix make_request UnboundLocalError on POST/PUT without data - Add 29 unit tests across test_client.py and test_utils.py - Fix all style issues (flake8 clean across project) --- oscar_python/_providers/_s3.py | 2 +- oscar_python/_utils.py | 26 ++++- oscar_python/client.py | 126 ++++++++++++++++++++-- oscar_python/local_test.py | 2 +- oscar_python/storage.py | 7 +- tests/test_client.py | 188 ++++++++++++++++++++++++++++++++- tests/test_default_client.py | 8 +- tests/test_storage.py | 6 +- tests/test_utils.py | 69 +++++++++++- tests/test_webdav.py | 2 +- 10 files changed, 414 insertions(+), 22 deletions(-) diff --git a/oscar_python/_providers/_s3.py b/oscar_python/_providers/_s3.py index 176ceb3..b5a4a32 100644 --- a/oscar_python/_providers/_s3.py +++ b/oscar_python/_providers/_s3.py @@ -59,7 +59,7 @@ def upload_file(self, local_path, remote_path): bucket_name = remote_path.split('/')[0] file_key = remote_path.split('/', 1)[1] file_name = local_path.split('/')[-1] - print("Uploading to bucket '{0}' with key '{1}'".format(bucket_name,file_key)) + print("Uploading to bucket '{0}' with key '{1}'".format(bucket_name, file_key)) with open(local_path, 'rb') as data: try: self.client.upload_fileobj(data, bucket_name, file_key + "/" + file_name) diff --git a/oscar_python/_utils.py b/oscar_python/_utils.py index d159963..945d8c4 100644 --- a/oscar_python/_utils.py +++ b/oscar_python/_utils.py @@ -13,6 +13,7 @@ # limitations under the License. import base64 +import json import os import requests import liboidcagent as agent @@ -32,14 +33,16 @@ def make_request(c, path, method, **kwargs): url = c.endpoint+path if method in ["post", "put"]: - if "token" in kwargs.keys() and kwargs["token"]: + if "token" in kwargs.keys() and kwargs["token"]: headers = get_headers_with_token(kwargs["token"]) + req_kwargs = {"headers": headers, "verify": c.ssl, "timeout": timeout} if "data" in kwargs.keys() and kwargs["data"]: - result = requests.request(method, url, headers=headers, verify=c.ssl, data=kwargs["data"], timeout=timeout) + req_kwargs["data"] = kwargs["data"] + result = requests.request(method, url, **req_kwargs) else: result = requests.request(method, url, headers=headers, verify=c.ssl, timeout=timeout) - if "handle" in kwargs.keys() and kwargs["handle"] == False: + if "handle" in kwargs.keys() and kwargs["handle"] is False: return result result.raise_for_status() @@ -94,7 +97,7 @@ def decode_b64(b64_str, file_out): except ValueError: print('Error decoding output: Invalid base64 string.') except OSError: - print('Error decoding output: Failed to write decoded data to file.') + print('Error decoding output: Failed to write decoded data to file.') def encode_input(data): @@ -111,6 +114,21 @@ def encode_input(data): return base64.b64encode(message_bytes) +def load_config(config_path, cluster_name="default"): + with open(config_path) as f: + config = json.load(f) + cluster = config["clusters"][cluster_name] + opts = { + "cluster_id": cluster_name, + "endpoint": cluster["endpoint"], + "ssl": cluster.get("ssl", True), + } + for key in ("user", "password", "shortname", "oidc_token", "refresh_token"): + if key in cluster: + opts[key] = cluster[key] + return opts + + def decode_output(output, file_path): if isBase64(output): decode_b64(output, file_path) diff --git a/oscar_python/client.py b/oscar_python/client.py index c4786a4..8c74e8d 100644 --- a/oscar_python/client.py +++ b/oscar_python/client.py @@ -27,7 +27,12 @@ _SVC_PATH = "/system/services" _LOGS_PATH = "/system/logs" _RUN_PATH = "/run" -_STATUS_PATH="/system/status" +_STATUS_PATH = "/system/status" +_HEALTH_PATH = "/health" +_VOLUMES_PATH = "/system/volumes" +_BUCKETS_PATH = "/system/buckets" +_METRICS_PATH = "/system/metrics" +_QUOTAS_USER_PATH = "/system/quotas/user" # _JOB_PATH = "/job" @@ -101,13 +106,13 @@ def get_access_token(self): """ Creates a generic storage client to interact with the storage providers defined on a specific service of the refered OSCAR cluster """ def create_storage_client(self, svc=None): - if svc != None: + if svc is not None: return Storage( client_obj=self, svc_name=svc) else: return Storage( client_obj=self) - + """ Function to get cluster info """ def get_cluster_info(self): return utils.make_request(self, _INFO_PATH, _GET) @@ -137,17 +142,18 @@ def _check_fdl_definition(self, fdl_path): raise Exception("FDL clusterID does not match current clusterID: {0}".format(err)) try: if os.path.isabs(svc["script"]): - script_path = svc["script"] + script_path = svc["script"] else: fdl_directory = os.path.dirname(fdl_path) script_path = os.path.join(fdl_directory, svc['script']) with open(script_path) as s: svc["script"] = s.read() - except IOError as e: + except IOError: raise Exception("Couldn't read script") # cpu parameter has to be string on the request - if type(svc["cpu"]) is int or type(svc["cpu"]) is float: svc["cpu"] = str(svc["cpu"]) + if type(svc["cpu"]) is int or type(svc["cpu"]) is float: + svc["cpu"] = str(svc["cpu"]) except ValueError as err: print(err) @@ -226,3 +232,111 @@ def remove_job(self, svc, job): """ Remove all service jobs """ def remove_all_jobs(self, svc): return utils.make_request(self, _LOGS_PATH+"/"+svc, _DELETE) + + """ Check cluster health """ + def health_check(self): + return utils.make_request(self, _HEALTH_PATH, _GET) + + """ Get deployment status of a service """ + def get_deployment_status(self, name): + return utils.make_request(self, _SVC_PATH + "/" + name + "/deployment", _GET) + + """ Get deployment logs of a service """ + def get_deployment_logs(self, name): + return utils.make_request(self, _SVC_PATH + "/" + name + "/deployment/logs", _GET) + + """ List all managed volumes """ + def list_volumes(self): + return utils.make_request(self, _VOLUMES_PATH, _GET) + + """ Create a new managed volume """ + def create_volume(self, name, size): + data = json.dumps({"name": name, "size": size}) + return utils.make_request(self, _VOLUMES_PATH, _POST, data=data) + + """ Get a specific managed volume """ + def get_volume(self, name): + return utils.make_request(self, _VOLUMES_PATH + "/" + name, _GET) + + """ Delete a managed volume """ + def delete_volume(self, name): + return utils.make_request(self, _VOLUMES_PATH + "/" + name, _DELETE) + + """ Create a bucket """ + def create_bucket(self, name, visibility="private", allowed_users=None): + data = json.dumps({ + "bucket_name": name, + "visibility": visibility, + "allowed_users": allowed_users or [] + }) + return utils.make_request(self, _BUCKETS_PATH, _POST, data=data) + + """ Update a bucket """ + def update_bucket(self, name, visibility, allowed_users=None): + data = json.dumps({ + "bucket_name": name, + "visibility": visibility, + "allowed_users": allowed_users or [] + }) + return utils.make_request(self, _BUCKETS_PATH, _PUT, data=data) + + """ List all buckets """ + def list_buckets(self): + return utils.make_request(self, _BUCKETS_PATH, _GET) + + """ Get a specific bucket """ + def get_bucket(self, name): + return utils.make_request(self, _BUCKETS_PATH + "/" + name, _GET) + + """ Delete a bucket """ + def delete_bucket(self, name): + return utils.make_request(self, _BUCKETS_PATH + "/" + name, _DELETE) + + """ Get a presigned URL for a bucket file """ + def presign_bucket(self, name, object_key, operation="download", expires=0, content_type="", extra_headers=None): + path = _BUCKETS_PATH + "/" + name + "/presign" + data = json.dumps({ + "object_key": object_key, + "operation": operation, + "expires": expires, + "content_type": content_type, + "extra_headers": extra_headers or {}, + }) + return utils.make_request(self, path, _POST, data=data) + + """ Get system logs (admin only) """ + def get_system_logs(self, timestamps=False, previous=False): + path = _LOGS_PATH + params = [] + if timestamps: + params.append("timestamps=true") + if previous: + params.append("previous=true") + if params: + path += "?" + "&".join(params) + return utils.make_request(self, path, _GET) + + """ Get metrics summary """ + def get_metrics_summary(self): + return utils.make_request(self, _METRICS_PATH, _GET) + + """ Get metrics breakdown """ + def get_metrics_breakdown(self, group_by="service"): + return utils.make_request(self, _METRICS_PATH + "/breakdown?group_by=" + group_by, _GET) + + """ Get metrics for a specific service """ + def get_service_metrics(self, service_name): + return utils.make_request(self, _METRICS_PATH + "/" + service_name, _GET) + + """ Get own quota """ + def get_own_quota(self): + return utils.make_request(self, _QUOTAS_USER_PATH, _GET) + + """ Get quota for a specific user """ + def get_user_quota(self, user_id): + return utils.make_request(self, _QUOTAS_USER_PATH + "/" + user_id, _GET) + + """ Update quota for a user """ + def update_user_quota(self, user_id, cpu, memory): + data = json.dumps({"cpu": cpu, "memory": memory}) + return utils.make_request(self, _QUOTAS_USER_PATH + "/" + user_id, _PUT, data=data) diff --git a/oscar_python/local_test.py b/oscar_python/local_test.py index 0cad50a..6c0d77d 100644 --- a/oscar_python/local_test.py +++ b/oscar_python/local_test.py @@ -1,6 +1,6 @@ from client import Client -client = Client("oscar-gpu-cluster","https://focused-boyd8.im.grycap.net", "oscar", "oscar123", True) +client = Client("oscar-gpu-cluster", "https://focused-boyd8.im.grycap.net", "oscar", "oscar123", True) res = client.remove_service("cowsay") if res: diff --git a/oscar_python/storage.py b/oscar_python/storage.py index 4584ec0..e0653f2 100644 --- a/oscar_python/storage.py +++ b/oscar_python/storage.py @@ -31,10 +31,10 @@ # TODO check returns from functions class Storage: - def __init__(self, client_obj, svc_name = None) -> None: + def __init__(self, client_obj, svc_name=None) -> None: self.client_obj = client_obj self.storage_providers = {} - if svc_name != None: + if svc_name is not None: self.svc_name = svc_name self._store_provider_from_service() self._store_default_minio_provider() @@ -46,13 +46,12 @@ def _store_provider_from_service(self): """ Function to store the user credentials for the default MinIO provider """ def _store_default_minio_provider(self): - config = utils.make_request(self.client_obj, _CONFIG_PATH + "/" , _GET) + config = utils.make_request(self.client_obj, _CONFIG_PATH + "/", _GET) if _MINIO in self.storage_providers: self.storage_providers[_MINIO]["default"] = json.loads(config.text)["minio_provider"] else: default = {"default": json.loads(config.text)["minio_provider"]} self.storage_providers[_MINIO] = default - """ Function to retreive credentials of a specific storage provider """ def _get_provider_creds(self, provider, provider_name): diff --git a/tests/test_client.py b/tests/test_client.py index 6d38181..16ba450 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -107,7 +107,12 @@ def test_create_service_from_dict(options): def test_create_service_from_file(options): client = Client(options) - service_definition = "functions:\n oscar:\n - test_cluster:\n name: test_service\n script: test_script\n cpu: 1" + service_definition = ( + "functions:\n oscar:\n - test_cluster:\n" + " name: test_service\n" + " script: test_script\n" + " cpu: 1" + ) service_file = "path/to/service.yaml" with patch('os.path.isfile', return_value=True), \ patch('builtins.open', mock_open(read_data=service_definition)), \ @@ -136,3 +141,184 @@ def test_remove_service(options): with patch('oscar_python._utils.make_request') as mock_request: client.remove_service("test_service") mock_request.assert_called_once_with(client, "/system/services/test_service", "delete") + + +def test_health_check(options): + client = Client(options) + with patch('oscar_python._utils.make_request') as mock_request: + client.health_check() + mock_request.assert_called_once_with(client, "/health", "get") + + +def test_get_deployment_status(options): + client = Client(options) + with patch('oscar_python._utils.make_request') as mock_request: + client.get_deployment_status("test_service") + mock_request.assert_called_once_with(client, "/system/services/test_service/deployment", "get") + + +def test_get_deployment_logs(options): + client = Client(options) + with patch('oscar_python._utils.make_request') as mock_request: + client.get_deployment_logs("test_service") + mock_request.assert_called_once_with(client, "/system/services/test_service/deployment/logs", "get") + + +def test_list_volumes(options): + client = Client(options) + with patch('oscar_python._utils.make_request') as mock_request: + client.list_volumes() + mock_request.assert_called_once_with(client, "/system/volumes", "get") + + +def test_create_volume(options): + client = Client(options) + with patch('oscar_python._utils.make_request') as mock_request: + client.create_volume("test_vol", "1Gi") + mock_request.assert_called_once_with(client, "/system/volumes", "post", + data=json.dumps({"name": "test_vol", "size": "1Gi"})) + + +def test_get_volume(options): + client = Client(options) + with patch('oscar_python._utils.make_request') as mock_request: + client.get_volume("test_vol") + mock_request.assert_called_once_with(client, "/system/volumes/test_vol", "get") + + +def test_delete_volume(options): + client = Client(options) + with patch('oscar_python._utils.make_request') as mock_request: + client.delete_volume("test_vol") + mock_request.assert_called_once_with(client, "/system/volumes/test_vol", "delete") + + +def test_list_buckets(options): + client = Client(options) + with patch('oscar_python._utils.make_request') as mock_request: + client.list_buckets() + mock_request.assert_called_once_with(client, "/system/buckets", "get") + + +def test_get_bucket(options): + client = Client(options) + with patch('oscar_python._utils.make_request') as mock_request: + client.get_bucket("test_bucket") + mock_request.assert_called_once_with(client, "/system/buckets/test_bucket", "get") + + +def test_create_bucket(options): + client = Client(options) + with patch('oscar_python._utils.make_request') as mock_request: + client.create_bucket("test_bucket") + mock_request.assert_called_once_with( + client, "/system/buckets", "post", + data=json.dumps({"bucket_name": "test_bucket", + "visibility": "private", + "allowed_users": []})) + + +def test_create_bucket_with_visibility(options): + client = Client(options) + with patch('oscar_python._utils.make_request') as mock_request: + client.create_bucket("test_bucket", visibility="public", allowed_users=["user1"]) + mock_request.assert_called_once_with( + client, "/system/buckets", "post", + data=json.dumps({"bucket_name": "test_bucket", + "visibility": "public", + "allowed_users": ["user1"]})) + + +def test_update_bucket(options): + client = Client(options) + with patch('oscar_python._utils.make_request') as mock_request: + client.update_bucket("test_bucket", "public", ["user1"]) + mock_request.assert_called_once_with( + client, "/system/buckets", "put", + data=json.dumps({"bucket_name": "test_bucket", + "visibility": "public", + "allowed_users": ["user1"]})) + + +def test_delete_bucket(options): + client = Client(options) + with patch('oscar_python._utils.make_request') as mock_request: + client.delete_bucket("test_bucket") + mock_request.assert_called_once_with(client, "/system/buckets/test_bucket", "delete") + + +def test_presign_bucket(options): + client = Client(options) + with patch('oscar_python._utils.make_request') as mock_request: + client.presign_bucket("test_bucket", "file.txt", operation="upload", expires=3600) + mock_request.assert_called_once_with( + client, "/system/buckets/test_bucket/presign", "post", + data=json.dumps({"object_key": "file.txt", + "operation": "upload", + "expires": 3600, + "content_type": "", + "extra_headers": {}})) + + +def test_get_system_logs(options): + client = Client(options) + with patch('oscar_python._utils.make_request') as mock_request: + client.get_system_logs() + mock_request.assert_called_once_with(client, "/system/logs", "get") + + +def test_get_system_logs_with_flags(options): + client = Client(options) + with patch('oscar_python._utils.make_request') as mock_request: + client.get_system_logs(timestamps=True, previous=True) + mock_request.assert_called_once_with(client, "/system/logs?timestamps=true&previous=true", "get") + + +def test_get_metrics_summary(options): + client = Client(options) + with patch('oscar_python._utils.make_request') as mock_request: + client.get_metrics_summary() + mock_request.assert_called_once_with(client, "/system/metrics", "get") + + +def test_get_metrics_breakdown(options): + client = Client(options) + with patch('oscar_python._utils.make_request') as mock_request: + client.get_metrics_breakdown("user") + mock_request.assert_called_once_with(client, "/system/metrics/breakdown?group_by=user", "get") + + +def test_get_metrics_breakdown_default(options): + client = Client(options) + with patch('oscar_python._utils.make_request') as mock_request: + client.get_metrics_breakdown() + mock_request.assert_called_once_with(client, "/system/metrics/breakdown?group_by=service", "get") + + +def test_get_service_metrics(options): + client = Client(options) + with patch('oscar_python._utils.make_request') as mock_request: + client.get_service_metrics("test_service") + mock_request.assert_called_once_with(client, "/system/metrics/test_service", "get") + + +def test_get_own_quota(options): + client = Client(options) + with patch('oscar_python._utils.make_request') as mock_request: + client.get_own_quota() + mock_request.assert_called_once_with(client, "/system/quotas/user", "get") + + +def test_get_user_quota(options): + client = Client(options) + with patch('oscar_python._utils.make_request') as mock_request: + client.get_user_quota("test_user") + mock_request.assert_called_once_with(client, "/system/quotas/user/test_user", "get") + + +def test_update_user_quota(options): + client = Client(options) + with patch('oscar_python._utils.make_request') as mock_request: + client.update_user_quota("test_user", "2", "4Gi") + mock_request.assert_called_once_with(client, "/system/quotas/user/test_user", "put", + data=json.dumps({"cpu": "2", "memory": "4Gi"})) diff --git a/tests/test_default_client.py b/tests/test_default_client.py index 0ec71bc..74428e1 100644 --- a/tests/test_default_client.py +++ b/tests/test_default_client.py @@ -25,7 +25,9 @@ def test_run_service_with_input_and_token(mock_decode_output, mock_encode_input, response = client.run_service("test_service", input="test_input", token="test_token", output="output_file", timeout=30) mock_encode_input.assert_called_once_with("test_input") - mock_make_request.assert_called_once_with(client, _RUN_PATH+"/test_service", _POST, data="encoded_input", token="test_token", timeout=30) + mock_make_request.assert_called_once_with( + client, _RUN_PATH+"/test_service", _POST, + data="encoded_input", token="test_token", timeout=30) mock_decode_output.assert_called_once_with("response_text", "output_file") assert response == mock_response @@ -41,7 +43,9 @@ def test_run_service_with_input_no_token(mock_encode_input, mock_make_request, c response = client.run_service("test_service", input="test_input") mock_encode_input.assert_called_once_with("test_input") - mock_make_request.assert_called_once_with(client, _RUN_PATH+"/test_service", _POST, data="encoded_input", token="test_token", timeout=None) + mock_make_request.assert_called_once_with( + client, _RUN_PATH+"/test_service", _POST, + data="encoded_input", token="test_token", timeout=None) assert response == mock_response diff --git a/tests/test_storage.py b/tests/test_storage.py index 0f3318d..2d42e33 100644 --- a/tests/test_storage.py +++ b/tests/test_storage.py @@ -1,3 +1,4 @@ +import json import pytest from unittest.mock import MagicMock, patch from oscar_python.storage import Storage @@ -11,7 +12,10 @@ def mock_client_obj(): @pytest.fixture def storage(mock_client_obj): mock_response = MagicMock() - mock_response.text = '{"minio_provider": {"access_key": "key","secret_key": "secret", "endpoint": "http://test.endpoint", "region": "us-east-1", "verify": false}}' + mock_response.text = json.dumps({ + "minio_provider": {"access_key": "key", "secret_key": "secret", + "endpoint": "http://test.endpoint", + "region": "us-east-1", "verify": False}}) with patch('oscar_python._utils.make_request', return_value=mock_response): return Storage(mock_client_obj) diff --git a/tests/test_utils.py b/tests/test_utils.py index b636fef..dcc66dc 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,4 +1,6 @@ import base64 +import json +import pytest from unittest.mock import patch, MagicMock, mock_open import oscar_python._utils as utils @@ -74,7 +76,72 @@ class MockClient: mock_request.return_value.raise_for_status = MagicMock() response = utils.make_request(c, "/test", "post", data="test_data", token="test_token") assert response.status_code == 200 - mock_request.assert_called_once_with("post", "http://test.com/test", headers={"Authorization": "Bearer test_token"}, verify=True, data="test_data", timeout=60) + mock_request.assert_called_once_with( + "post", "http://test.com/test", + headers={"Authorization": "Bearer test_token"}, + verify=True, data="test_data", timeout=60) + + +def test_load_config(tmp_path): + config_file = tmp_path / "config.json" + config_file.write_text(json.dumps({ + "clusters": { + "default": { + "endpoint": "http://test.cluster", + "user": "test_user", + "password": "test_pass", + "ssl": False + } + } + })) + opts = utils.load_config(str(config_file)) + assert opts["cluster_id"] == "default" + assert opts["endpoint"] == "http://test.cluster" + assert opts["user"] == "test_user" + assert opts["password"] == "test_pass" + assert opts["ssl"] is False + + +def test_load_config_with_oidc(tmp_path): + config_file = tmp_path / "config.json" + config_file.write_text(json.dumps({ + "clusters": { + "oidc_cluster": { + "endpoint": "http://oidc.cluster", + "oidc_token": "test_token", + "ssl": True + } + } + })) + opts = utils.load_config(str(config_file), "oidc_cluster") + assert opts["cluster_id"] == "oidc_cluster" + assert opts["endpoint"] == "http://oidc.cluster" + assert "user" not in opts + assert opts["oidc_token"] == "test_token" + assert opts["ssl"] is True + + +def test_load_config_missing_cluster(tmp_path): + config_file = tmp_path / "config.json" + config_file.write_text(json.dumps({"clusters": {}})) + with pytest.raises(KeyError): + utils.load_config(str(config_file), "nonexistent") + + +def test_load_config_minimal(tmp_path): + config_file = tmp_path / "config.json" + config_file.write_text(json.dumps({ + "clusters": { + "minimal": { + "endpoint": "http://minimal.cluster" + } + } + })) + opts = utils.load_config(str(config_file), "minimal") + assert opts["endpoint"] == "http://minimal.cluster" + assert opts["ssl"] is True + for key in ("user", "password", "shortname", "oidc_token", "refresh_token"): + assert key not in opts def test_make_request_get(): diff --git a/tests/test_webdav.py b/tests/test_webdav.py index ba321ae..50d8526 100644 --- a/tests/test_webdav.py +++ b/tests/test_webdav.py @@ -34,7 +34,7 @@ def test_webdav_download_file(webdav): webdav.client = MagicMock(["download_sync"]) webdav.client.download_sync.return_value = None - with patch("builtins.open", mock_open()) as mock_file: + with patch("builtins.open", mock_open()): webdav.download_file('local_path', 'remote_path/file.txt') webdav.client.download_sync.assert_called_with('remote_path/file.txt', 'local_path/file.txt')