diff --git a/setup.cfg b/setup.cfg index 6c9e1c1b2bd..e3f2892348f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -24,8 +24,6 @@ package_dir= =src packages = find: install_requires = - boto3>=1.35 - botocore>=1.35 cachetools certifi cwltool @@ -55,6 +53,7 @@ install_requires = pytz requests rucio-clients >=34.4.2 + signurlarity >=0.3.1 sqlalchemy typing_extensions >=4.3.0 Authlib >=1.0.0.a2 diff --git a/src/DIRAC/Resources/Storage/S3Storage.py b/src/DIRAC/Resources/Storage/S3Storage.py index ac20e294c34..4b0788020a0 100644 --- a/src/DIRAC/Resources/Storage/S3Storage.py +++ b/src/DIRAC/Resources/Storage/S3Storage.py @@ -1,5 +1,4 @@ # https://docs.aws.amazon.com/AmazonS3/latest/API/API_Operations.html -# https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3.html#S3.Client.list_objects """ Configuration of an S3 storage Like others, but in protocol S3 add: @@ -17,22 +16,19 @@ import copy import errno import functools - import os -import requests - -import boto3 -from botocore.exceptions import ClientError +import requests +from signurlarity.client import Client +from signurlarity.exceptions import NoSuchBucketError, NoSuchKeyError, PresignError, SignurlarityError -from DIRAC import S_OK, S_ERROR, gLogger +from DIRAC import S_ERROR, S_OK, gLogger from DIRAC.Core.Utilities.Adler import fileAdler from DIRAC.Core.Utilities.DErrno import cmpError from DIRAC.Core.Utilities.Pfn import pfnparse from DIRAC.DataManagementSystem.Client.S3GatewayClient import S3GatewayClient from DIRAC.Resources.Storage.StorageBase import StorageBase - LOG = gLogger.getSubLogger(__name__) @@ -106,8 +102,7 @@ def __init__(self, storageName, parameters): endpoint_url = f"{proto}://{parameters['Host']}:{port}" self.bucketName = parameters["Path"] - self.s3_client = boto3.client( - "s3", + self.s3_client = Client( endpoint_url=endpoint_url, aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key, @@ -234,12 +229,9 @@ def _direct_exists(self, urls): try: self.s3_client.head_object(Bucket=self.bucketName, Key=key) successful[key] = True - except ClientError as exp: - if exp.response["Error"]["Code"] == "404": - successful[key] = False - else: - failed[key] = repr(exp) - except Exception as exp: + except (NoSuchBucketError, NoSuchKeyError): + successful[key] = False + except PresignError as exp: failed[key] = repr(exp) resDict = {"Failed": failed, "Successful": successful} @@ -619,12 +611,11 @@ def _direct_removeFile(self, urls): # the @_extractKeyFromS3Path transformed URL into keys keys = urls - for key in keys: - try: - self.s3_client.delete_object(Bucket=self.bucketName, Key=key) - successful[key] = True - except Exception as exp: - failed[key] = repr(exp) + try: + self.s3_client.delete_objects(Bucket=self.bucketName, Delete={"Objects": keys, "Quiet": True}) + successful = True + except PresignError as exp: + failed = repr(exp) return S_OK({"Failed": failed, "Successful": successful}) @@ -642,7 +633,7 @@ def _presigned_removeFile(self, urls): failed = {} successful = {} - res = self.S3GatewayClient.createPresignedUrl(self.name, "delete_object", urls) + res = self.S3GatewayClient.createPresignedUrl(self.name, "delete_objects", urls) if not res["OK"]: return res @@ -773,7 +764,7 @@ def createPresignedUrl(self, urls, s3_method, expiration=3600): ) successful[key] = response - except ClientError as e: + except SignurlarityError as e: log.debug(e) failed[key] = repr(e)