From 4668161e56335b6543acfc64a97c40c0d1972b03 Mon Sep 17 00:00:00 2001 From: natthan-pigoux Date: Wed, 12 Aug 2026 14:22:41 +0200 Subject: [PATCH 1/2] chore: drop boto for signurlarity --- setup.cfg | 3 +-- src/DIRAC/Resources/Storage/S3Storage.py | 18 +++++++----------- 2 files changed, 8 insertions(+), 13 deletions(-) diff --git a/setup.cfg b/setup.cfg index 6c9e1c1b2bd..e3f2892348f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -24,8 +24,6 @@ package_dir= =src packages = find: install_requires = - boto3>=1.35 - botocore>=1.35 cachetools certifi cwltool @@ -55,6 +53,7 @@ install_requires = pytz requests rucio-clients >=34.4.2 + signurlarity >=0.3.1 sqlalchemy typing_extensions >=4.3.0 Authlib >=1.0.0.a2 diff --git a/src/DIRAC/Resources/Storage/S3Storage.py b/src/DIRAC/Resources/Storage/S3Storage.py index ac20e294c34..41e275d224c 100644 --- a/src/DIRAC/Resources/Storage/S3Storage.py +++ b/src/DIRAC/Resources/Storage/S3Storage.py @@ -1,5 +1,4 @@ # https://docs.aws.amazon.com/AmazonS3/latest/API/API_Operations.html -# https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3.html#S3.Client.list_objects """ Configuration of an S3 storage Like others, but in protocol S3 add: @@ -17,22 +16,19 @@ import copy import errno import functools - import os -import requests - -import boto3 -from botocore.exceptions import ClientError +import requests +from signurlarity.client import Client +from signurlarity.exceptions import SignurlarityError -from DIRAC import S_OK, S_ERROR, gLogger +from DIRAC import S_ERROR, S_OK, gLogger from DIRAC.Core.Utilities.Adler import fileAdler from DIRAC.Core.Utilities.DErrno import cmpError from DIRAC.Core.Utilities.Pfn import pfnparse from DIRAC.DataManagementSystem.Client.S3GatewayClient import S3GatewayClient from DIRAC.Resources.Storage.StorageBase import StorageBase - LOG = gLogger.getSubLogger(__name__) @@ -106,7 +102,7 @@ def __init__(self, storageName, parameters): endpoint_url = f"{proto}://{parameters['Host']}:{port}" self.bucketName = parameters["Path"] - self.s3_client = boto3.client( + self.s3_client = Client( "s3", endpoint_url=endpoint_url, aws_access_key_id=aws_access_key_id, @@ -234,7 +230,7 @@ def _direct_exists(self, urls): try: self.s3_client.head_object(Bucket=self.bucketName, Key=key) successful[key] = True - except ClientError as exp: + except SignurlarityError as exp: if exp.response["Error"]["Code"] == "404": successful[key] = False else: @@ -773,7 +769,7 @@ def createPresignedUrl(self, urls, s3_method, expiration=3600): ) successful[key] = response - except ClientError as e: + except SignurlarityError as e: log.debug(e) failed[key] = repr(e) From 47219ac9fa66d0d5bde4e13ce3647e2e630ec0c4 Mon Sep 17 00:00:00 2001 From: natthan-pigoux Date: Wed, 12 Aug 2026 15:14:41 +0200 Subject: [PATCH 2/2] fix: method names and exceptions --- src/DIRAC/Resources/Storage/S3Storage.py | 25 ++++++++++-------------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/src/DIRAC/Resources/Storage/S3Storage.py b/src/DIRAC/Resources/Storage/S3Storage.py index 41e275d224c..4b0788020a0 100644 --- a/src/DIRAC/Resources/Storage/S3Storage.py +++ b/src/DIRAC/Resources/Storage/S3Storage.py @@ -20,7 +20,7 @@ import requests from signurlarity.client import Client -from signurlarity.exceptions import SignurlarityError +from signurlarity.exceptions import NoSuchBucketError, NoSuchKeyError, PresignError, SignurlarityError from DIRAC import S_ERROR, S_OK, gLogger from DIRAC.Core.Utilities.Adler import fileAdler @@ -103,7 +103,6 @@ def __init__(self, storageName, parameters): self.bucketName = parameters["Path"] self.s3_client = Client( - "s3", endpoint_url=endpoint_url, aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key, @@ -230,12 +229,9 @@ def _direct_exists(self, urls): try: self.s3_client.head_object(Bucket=self.bucketName, Key=key) successful[key] = True - except SignurlarityError as exp: - if exp.response["Error"]["Code"] == "404": - successful[key] = False - else: - failed[key] = repr(exp) - except Exception as exp: + except (NoSuchBucketError, NoSuchKeyError): + successful[key] = False + except PresignError as exp: failed[key] = repr(exp) resDict = {"Failed": failed, "Successful": successful} @@ -615,12 +611,11 @@ def _direct_removeFile(self, urls): # the @_extractKeyFromS3Path transformed URL into keys keys = urls - for key in keys: - try: - self.s3_client.delete_object(Bucket=self.bucketName, Key=key) - successful[key] = True - except Exception as exp: - failed[key] = repr(exp) + try: + self.s3_client.delete_objects(Bucket=self.bucketName, Delete={"Objects": keys, "Quiet": True}) + successful = True + except PresignError as exp: + failed = repr(exp) return S_OK({"Failed": failed, "Successful": successful}) @@ -638,7 +633,7 @@ def _presigned_removeFile(self, urls): failed = {} successful = {} - res = self.S3GatewayClient.createPresignedUrl(self.name, "delete_object", urls) + res = self.S3GatewayClient.createPresignedUrl(self.name, "delete_objects", urls) if not res["OK"]: return res