diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java index e7e78b812063..188bd65559ff 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java @@ -314,6 +314,7 @@ public final class OzoneConsts { public static final String S3_SETSECRET_USER = "S3SetSecretUser"; public static final String S3_REVOKESECRET_USER = "S3RevokeSecretUser"; public static final String S3_REVOKESTSTOKEN_USER = "S3RevokeSTSTokenUser"; + public static final String S3_STS_TEMP_ACCESS_KEY_ID = "tempAccessKeyId"; public static final String RENAMED_KEYS_MAP = "renamedKeysMap"; public static final String UNRENAMED_KEYS_MAP = "unRenamedKeysMap"; public static final String MULTIPART_UPLOAD_PART_NUMBER = "partNumber"; diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml index c72c351402e3..db79aa505f01 100644 --- a/hadoop-hdds/common/src/main/resources/ozone-default.xml +++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml @@ -5254,9 +5254,10 @@ 3h OZONE, OM, PERFORMANCE, SECURITY - A background job that periodically checks revoked STS token entries and - deletes ones that have existed for 12 hours. This entry controls the interval of this - cleanup check. Unit could be defined with postfix (ns,ms,s,m,h,d). + A background service that periodically scans the s3RevokedStsTokenTable and deletes + revocation entries whose cutoff is older than the maximum STS token lifetime (12 hours). + This property controls how often the cleanup service runs. Unit could be defined with + postfix (ns,ms,s,m,h,d). diff --git a/hadoop-hdds/docs/content/design/ozone-sts.md b/hadoop-hdds/docs/content/design/ozone-sts.md index 8c7d5cd3b44b..931531172a51 100644 --- a/hadoop-hdds/docs/content/design/ozone-sts.md +++ b/hadoop-hdds/docs/content/design/ozone-sts.md @@ -41,8 +41,9 @@ solutions that want to aggregate data across multiple cloud providers. # 3. How Ozone STS Works -The initial implementation of Ozone STS supports only the [AssumeRole](https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRole.html) -API from the AWS specification. A new STS endpoint on port `9880` (port `9881` for https) will be created to service STS requests in the S3 Gateway at the root path (`/`). +The initial implementation of Ozone STS supports the [AssumeRole](https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRole.html) +and [GetCallerIdentity](https://docs.aws.amazon.com/STS/latest/APIReference/API_GetCallerIdentity.html) +APIs from the AWS specification. A new STS endpoint on port `9880` (port `9881` for https) will be created to service STS requests in the S3 Gateway at the root path (`/`). We use a separate port for STS to align with AWS so we don't have conflicts at a later time. This means we have: - Admin port for Ozone specific S3 admin operations - STS port for STS APIs, analogous to AWS' separate STS endpoint @@ -66,6 +67,11 @@ return value of the AssumeRole call will be temporary credentials consisting of an IAM policy is specified, the temporary credential will have the permissions comprising the intersection of the role permissions and the IAM policy permissions. **Note:** If the IAM policy is specified and does not grant any permissions, then the generated temporary credentials won't have any permissions and will essentially be useless. +- [GetCallerIdentity](https://docs.aws.amazon.com/STS/latest/APIReference/API_GetCallerIdentity.html) returns the account, +ARN, and user ID for the caller credentials used to sign the request. Ozone uses a static account ID of `123456789012`. +For permanent S3 credentials, `UserId` is the resolved Kerberos principal and `Arn` is `arn:aws:iam::123456789012:user/` +where `` is the short username of the Kerberos principal. For STS temporary credentials, `UserId` is +the `AssumedRoleId` and `Arn` is the assumed-role user ARN from the session token. ## 3.2 Limitations in AssumeRole API Support @@ -139,17 +145,24 @@ was included with the AssumeRole request, the String return value will also incl would further limit the scope of the permissions, resources and actions granted by the role in Ranger, such that the temporary credential will have the permissions and actions comprising the intersection of the role permissions and actions and the sessionPolicy permissions and actions. - HMAC-SHA256 signature - used to ensure the sessionToken was created by Ozone and was not altered since it was created. +- creation time of the token (via `OMTokenProto#issueDate`, exposed as `STSTokenIdentifier#getCreationTime()`) - expiration time of the token (via `ShortLivedTokenIdentifier#getExpiry()`) - UUID of the OzoneManager secret key used to sign the sessionToken and encrypt the secretAccessKey (via `ShortLivedTokenIdentifier#getSecretKeyId()`) ## 3.5 STS Token Revocation In the rare event temporary credentials need to be revoked (ex. for security reasons), a table in the OzoneManager RocksDB will be created -to store revoked tokens, and a command-line utility will be created to add tokens to the table. A background cleaner service -will be created to run every 3 hours to delete revoked tokens that have been in the table for more than 12 hours. The -input parameter for the command-line utility will be the sessionToken - this value is returned in plain text as a result -of the AssumeRole call (mentioned above). In this way, specific STS tokens can be revoked as opposed to all tokens. Furthermore, -AWS doesn't have a standard API to revoke tokens therefore we are creating our own system. +to store revocation cutoffs per originalAccessKeyId, and a command-line utility will be created to add entries to the table. +A background cleaner service will be created to run every 3 hours to delete revocation entries whose cutoff is more than 12 hours old. + +The command-line utility accepts only `originalAccessKeyId`. The OM stores revocations by keying the table on +`originalAccessKeyId` and storing the revocation cutoff time in milliseconds as the value. When the command is issued, +all STS tokens created by that `originalAccessKeyId` whose signed `creationTime` is strictly before the cutoff are +revoked. Tokens created at or after the cutoff remain valid. + +Before writing a revocation entry, the OM verifies that `originalAccessKeyId` corresponds to a real Kerberos identity by +checking that an S3 secret exists for it. This prevents bogus entries from filling the table. Non-admins may only +revoke their own `originalAccessKeyId`; S3 and tenant admins may revoke other principals. Additionally, if the Kerberos identity of the user that created the STS token is revoked via the `ozone s3 revokesecret` command, then all the existing and unexpired STS tokens that user created will be revoked. @@ -221,7 +234,8 @@ created in Ranger as per the Prerequisites above. originalAccessKeyId in the session token and perform the following checks: - Ensure that if the accessKeyId starts with "ASIA", that a sessionToken was included in the `x-amz-security-token` header - Ensure the sessionToken is not expired - - Ensure the sessionToken is not revoked via a `keyMayExist` check in OzoneManager RocksDB + - Ensure the STS credentials are not revoked by looking up the revocation cutoff for the token's originalAccessKeyId + and comparing it against the token's signed creationTime - Validate the HMAC-SHA256 signature in the sessionToken - Decrypt the secretAccessKey from the sessionToken and validate the AWS signature - Authorize the call with either RangerOzoneAuthorizer or OzoneNativeAuthorizer diff --git a/hadoop-ozone/cli-shell/src/main/java/org/apache/hadoop/ozone/shell/s3/RevokeSTSTokenHandler.java b/hadoop-ozone/cli-shell/src/main/java/org/apache/hadoop/ozone/shell/s3/RevokeSTSTokenHandler.java index 274304217f86..9cd715d581a1 100644 --- a/hadoop-ozone/cli-shell/src/main/java/org/apache/hadoop/ozone/shell/s3/RevokeSTSTokenHandler.java +++ b/hadoop-ozone/cli-shell/src/main/java/org/apache/hadoop/ozone/shell/s3/RevokeSTSTokenHandler.java @@ -29,18 +29,18 @@ /** * Executes revocation of STS tokens. * - *

This command marks the specified STS token as revoked by adding it to the OM's revoked STS token table. - * Subsequent S3 requests using the same session token will be rejected once the revocation - * state has propagated.

+ *

This command records a revocation cutoff for the given original access key ID in the OM's + * revoked STS token table. Subsequent S3 requests using STS tokens created before that cutoff + * will be rejected once the revocation state has propagated.

*/ @Command(name = "revokeststoken", - description = "Revoke S3 STS token for the given session token") + description = "Revoke S3 STS tokens for the given original access key ID") public class RevokeSTSTokenHandler extends S3Handler { - @Option(names = "-t", + @Option(names = {"-o", "--original-access-key-id"}, required = true, - description = "STS session token") - private String sessionToken; + description = "Original long-lived access key ID whose STS tokens should be revoked") + private String originalAccessKeyId; @Option(names = "-y", description = "Continue without interactive user confirmation") @@ -56,8 +56,8 @@ protected void execute(OzoneClient client, OzoneAddress address) throws IOException { if (!yes) { - out().print("Enter 'y' to confirm STS token revocation for sessionToken '" + - sessionToken + "': "); + out().print( + "Enter 'y' to confirm STS token revocation for originalAccessKeyId '" + originalAccessKeyId + "': "); out().flush(); final Scanner scanner = new Scanner(new InputStreamReader(System.in, StandardCharsets.UTF_8)); final String confirmation = scanner.next().trim().toLowerCase(); @@ -67,7 +67,7 @@ protected void execute(OzoneClient client, OzoneAddress address) } } - client.getObjectStore().revokeSTSToken(sessionToken); - out().println("STS token revoked for sessionToken '" + sessionToken + "'."); + client.getObjectStore().revokeSTSToken(originalAccessKeyId); + out().println("STS tokens revoked for originalAccessKeyId '" + originalAccessKeyId + "'."); } } diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java index bd045ef04e03..32af4cacd4f3 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java @@ -36,6 +36,7 @@ import org.apache.hadoop.ozone.client.protocol.ClientProtocol; import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.helpers.AssumeRoleResponseInfo; +import org.apache.hadoop.ozone.om.helpers.CallerIdentityInfo; import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.DeleteTenantState; import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; @@ -813,12 +814,21 @@ public AssumeRoleResponseInfo assumeRole(String roleArn, String roleSessionName, } /** - * Revokes an STS token. - * @param sessionToken The STS sessionToken + * Returns the caller identity for the current S3-authenticated request. + * @return CallerIdentityInfo containing account, arn, and userId + * @throws IOException if an error occurs during the GetCallerIdentity operation + */ + public CallerIdentityInfo getCallerIdentity() throws IOException { + return proxy.getCallerIdentity(); + } + + /** + * Revokes STS tokens for the given original access key ID. + * @param originalAccessKeyId The original long-lived access key ID whose STS tokens to revoke * @throws IOException if an error occurs while revoking the STS token */ - public void revokeSTSToken(String sessionToken) throws IOException { - proxy.revokeSTSToken(sessionToken); + public void revokeSTSToken(String originalAccessKeyId) throws IOException { + proxy.revokeSTSToken(originalAccessKeyId); } /** diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java index 807cd2757cfc..73f099879bc6 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java @@ -48,6 +48,7 @@ import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.helpers.AssumeRoleResponseInfo; +import org.apache.hadoop.ozone.om.helpers.CallerIdentityInfo; import org.apache.hadoop.ozone.om.helpers.DeleteTenantState; import org.apache.hadoop.ozone.om.helpers.ErrorInfo; import org.apache.hadoop.ozone.om.helpers.LeaseKeyInfo; @@ -1648,11 +1649,18 @@ AssumeRoleResponseInfo assumeRole(String roleArn, String roleSessionName, int du String awsIamSessionPolicy, String requestId) throws IOException; /** - * Revokes an STS token. - * @param sessionToken The STS sessionToken + * Returns the caller identity for the current S3-authenticated request. + * @return CallerIdentityInfo containing account, arn, and userId + * @throws IOException if an error occurs during the GetCallerIdentity operation + */ + CallerIdentityInfo getCallerIdentity() throws IOException; + + /** + * Revokes STS tokens for the given original access key ID. + * @param originalAccessKeyId The original long-lived access key ID whose STS tokens to revoke * @throws IOException if an error occurs while revoking the STS token */ - void revokeSTSToken(String sessionToken) throws IOException; + void revokeSTSToken(String originalAccessKeyId) throws IOException; /** * Gets the lifecycle configuration information. diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java index 733c915dcd03..22fad1944429 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java @@ -130,6 +130,7 @@ import org.apache.hadoop.ozone.om.OmConfig; import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.helpers.AssumeRoleResponseInfo; +import org.apache.hadoop.ozone.om.helpers.CallerIdentityInfo; import org.apache.hadoop.ozone.om.helpers.BasicOmKeyInfo; import org.apache.hadoop.ozone.om.helpers.BucketEncryptionKeyInfo; import org.apache.hadoop.ozone.om.helpers.BucketLayout; @@ -3022,8 +3023,13 @@ public AssumeRoleResponseInfo assumeRole(String roleArn, String roleSessionName, } @Override - public void revokeSTSToken(String sessionToken) throws IOException { - ozoneManagerClient.revokeSTSToken(sessionToken); + public CallerIdentityInfo getCallerIdentity() throws IOException { + return ozoneManagerClient.getCallerIdentity(); + } + + @Override + public void revokeSTSToken(String originalAccessKeyId) throws IOException { + ozoneManagerClient.revokeSTSToken(originalAccessKeyId); } @Override diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java index 5da80215fad8..31530900cf2c 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java @@ -238,6 +238,7 @@ public static boolean isReadOnly(OMRequest omRequest) { case FinalizeUpgradeProgress: case PrepareStatus: case GetS3VolumeContext: + case GetCallerIdentity: case ListTenant: case TenantGetUserInfo: case TenantListUser: @@ -383,6 +384,7 @@ public static boolean shouldSendToFollower(OMRequest omRequest) { case FinalizeUpgradeProgress: case PrepareStatus: case GetS3VolumeContext: + case GetCallerIdentity: case ListTenant: case TenantGetUserInfo: case TenantListUser: diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/CallerIdentityInfo.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/CallerIdentityInfo.java new file mode 100644 index 000000000000..5a014f19fcfd --- /dev/null +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/CallerIdentityInfo.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om.helpers; + +import java.util.Objects; +import net.jcip.annotations.Immutable; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.GetCallerIdentityResponse; + +/** + * Utility class to handle GetCallerIdentityResponse protobuf message. + */ +@Immutable +public class CallerIdentityInfo { + + private final String account; + private final String arn; + private final String userId; + + public CallerIdentityInfo(String account, String arn, String userId) { + this.account = account; + this.arn = arn; + this.userId = userId; + } + + public String getAccount() { + return account; + } + + public String getArn() { + return arn; + } + + public String getUserId() { + return userId; + } + + public static CallerIdentityInfo fromProtobuf(GetCallerIdentityResponse response) { + return new CallerIdentityInfo(response.getAccount(), response.getArn(), response.getUserId()); + } + + public GetCallerIdentityResponse getProtobuf() { + return GetCallerIdentityResponse.newBuilder() + .setAccount(account) + .setArn(arn) + .setUserId(userId) + .build(); + } + + @Override + public String toString() { + return "CallerIdentityInfo{" + "account='" + account + "', arn='" + arn + "', userId='" + userId + "'}"; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + + if (o == null || getClass() != o.getClass()) { + return false; + } + + final CallerIdentityInfo that = (CallerIdentityInfo) o; + return Objects.equals(account, that.account) && Objects.equals(arn, that.arn) && + Objects.equals(userId, that.userId); + } + + @Override + public int hashCode() { + return Objects.hash(account, arn, userId); + } +} diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/S3STSUtils.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/S3STSUtils.java index 763c8fe9bfa4..17bf084f9b31 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/S3STSUtils.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/S3STSUtils.java @@ -40,9 +40,47 @@ public final class S3STSUtils { // AWS limit for session policy is 2048 characters public static final int MAX_SESSION_POLICY_LENGTH = 2048; + public static final String STS_TOKEN_PREFIX = "ASIA"; + public static final String STS_ACCESS_KEY_ID_ALLOWED_CHARS = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"; + public static final int STS_ACCESS_KEY_ID_ALLOWED_CHARS_LENGTH = STS_ACCESS_KEY_ID_ALLOWED_CHARS.length(); + public static final int STS_ACCESS_KEY_ID_RANDOM_LENGTH = 20; + + public static final String OZONE_STATIC_ACCOUNT_ID = "123456789012"; + private S3STSUtils() { } + /** + * Builds an IAM user ARN for the given Kerberos short name. + */ + public static String toIamUserArn(String kerberosShortName) { + return "arn:aws:iam::" + OZONE_STATIC_ACCOUNT_ID + ":user/" + kerberosShortName; + } + + /** + * Resolves the caller identity for GetCallerIdentity with permanent S3 credentials. + * + * @param resolvedPrincipal full Kerberos principal of the caller + * @param kerberosShortName short username + * @return caller identity with account, arn, and userId + */ + public static CallerIdentityInfo resolveCallerIdentityForPermanentCredentials(String resolvedPrincipal, + String kerberosShortName) { + return new CallerIdentityInfo(OZONE_STATIC_ACCOUNT_ID, toIamUserArn(kerberosShortName), resolvedPrincipal); + } + + /** + * Resolves the caller identity for GetCallerIdentity with temporary STS credentials. + * + * @param assumedRoleId assumed role ID from the STS token + * @param assumedRoleUserArn assumed role user ARN from the STS token + * @return caller identity with account, arn, and userId + */ + public static CallerIdentityInfo resolveCallerIdentityForStsCredentials(String assumedRoleId, + String assumedRoleUserArn) { + return new CallerIdentityInfo(OZONE_STATIC_ACCOUNT_ID, assumedRoleUserArn, assumedRoleId); + } + /** * Adds standard AssumeRole audit params. */ diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java index 604669487c68..7cea19d7a909 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java @@ -30,6 +30,7 @@ import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.helpers.AssumeRoleResponseInfo; +import org.apache.hadoop.ozone.om.helpers.CallerIdentityInfo; import org.apache.hadoop.ozone.om.helpers.DBUpdates; import org.apache.hadoop.ozone.om.helpers.DeleteTenantState; import org.apache.hadoop.ozone.om.helpers.ErrorInfo; @@ -1336,11 +1337,20 @@ default AssumeRoleResponseInfo assumeRole(String roleArn, String roleSessionName } /** - * Revokes an STS token. - * @param sessionToken The STS sessionToken + * Returns the caller identity for the current S3-authenticated request. + * @return CallerIdentityInfo containing account, arn, and userId + * @throws IOException if an error occurs during the GetCallerIdentity operation + */ + default CallerIdentityInfo getCallerIdentity() throws IOException { + throw new UnsupportedOperationException("OzoneManager does not require this to be implemented"); + } + + /** + * Revokes STS tokens for the given original access key ID. + * @param originalAccessKeyId The original long-lived access key ID whose STS tokens to revoke * @throws IOException if an error occurs while revoking the STS token */ - default void revokeSTSToken(String sessionToken) throws IOException { + default void revokeSTSToken(String originalAccessKeyId) throws IOException { throw new UnsupportedOperationException("OzoneManager does not require this to be implemented"); } } diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java index c60bc60db700..b5a9e3a2bbcf 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java @@ -60,6 +60,7 @@ import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.helpers.AssumeRoleResponseInfo; import org.apache.hadoop.ozone.om.helpers.BasicOmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.CallerIdentityInfo; import org.apache.hadoop.ozone.om.helpers.DBUpdates; import org.apache.hadoop.ozone.om.helpers.DeleteTenantState; import org.apache.hadoop.ozone.om.helpers.ErrorInfo; @@ -2981,10 +2982,19 @@ public AssumeRoleResponseInfo assumeRole(String roleArn, String roleSessionName, } @Override - public void revokeSTSToken(String sessionToken) throws IOException { + public CallerIdentityInfo getCallerIdentity() throws IOException { + final OMRequest omRequest = createOMRequest(Type.GetCallerIdentity) + .setGetCallerIdentityRequest(OzoneManagerProtocolProtos.GetCallerIdentityRequest.newBuilder().build()) + .build(); + + return CallerIdentityInfo.fromProtobuf(handleError(submitRequest(omRequest)).getGetCallerIdentityResponse()); + } + + @Override + public void revokeSTSToken(String originalAccessKeyId) throws IOException { final OzoneManagerProtocolProtos.RevokeSTSTokenRequest request = OzoneManagerProtocolProtos.RevokeSTSTokenRequest.newBuilder() - .setSessionToken(sessionToken) + .setOriginalAccessKeyId(originalAccessKeyId) .build(); final OMRequest omRequest = createOMRequest(Type.RevokeSTSToken) diff --git a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestCallerIdentityInfo.java b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestCallerIdentityInfo.java new file mode 100644 index 000000000000..f6bc0334acfe --- /dev/null +++ b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestCallerIdentityInfo.java @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om.helpers; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.GetCallerIdentityResponse; +import org.junit.jupiter.api.Test; + +/** + * Test CallerIdentityInfo. + */ +public class TestCallerIdentityInfo { + + private static final String ACCOUNT = "123456789012"; + private static final String ARN = "arn:aws:iam::123456789012:user/om"; + private static final String USER_ID = "om/polarisclient@root.comops.site"; + + @Test + public void testConstructor() { + final CallerIdentityInfo identity = new CallerIdentityInfo(ACCOUNT, ARN, USER_ID); + + assertEquals(ACCOUNT, identity.getAccount()); + assertEquals(ARN, identity.getArn()); + assertEquals(USER_ID, identity.getUserId()); + } + + @Test + public void testProtobufConversion() { + final CallerIdentityInfo identity = new CallerIdentityInfo(ACCOUNT, ARN, USER_ID); + + final GetCallerIdentityResponse proto = identity.getProtobuf(); + + assertNotNull(proto); + assertEquals(ACCOUNT, proto.getAccount()); + assertEquals(ARN, proto.getArn()); + assertEquals(USER_ID, proto.getUserId()); + } + + @Test + public void testFromProtobuf() { + final GetCallerIdentityResponse proto = GetCallerIdentityResponse.newBuilder() + .setAccount(ACCOUNT) + .setArn(ARN) + .setUserId(USER_ID) + .build(); + + final CallerIdentityInfo identity = CallerIdentityInfo.fromProtobuf(proto); + + assertEquals(ACCOUNT, identity.getAccount()); + assertEquals(ARN, identity.getArn()); + assertEquals(USER_ID, identity.getUserId()); + } + + @Test + public void testProtobufRoundTrip() { + final CallerIdentityInfo original = new CallerIdentityInfo(ACCOUNT, ARN, USER_ID); + + final CallerIdentityInfo recovered = CallerIdentityInfo.fromProtobuf(original.getProtobuf()); + + assertEquals(original, recovered); + } + + @Test + public void testEqualsAndHashCodeWithIdenticalObjects() { + final CallerIdentityInfo identity1 = new CallerIdentityInfo(ACCOUNT, ARN, USER_ID); + final CallerIdentityInfo identity2 = new CallerIdentityInfo(ACCOUNT, ARN, USER_ID); + + assertEquals(identity1, identity2); + assertEquals(identity1.hashCode(), identity2.hashCode()); + } + + @Test + public void testNotEqualsWithDifferentArn() { + final CallerIdentityInfo identity1 = new CallerIdentityInfo(ACCOUNT, ARN, USER_ID); + final CallerIdentityInfo identity2 = new CallerIdentityInfo( + ACCOUNT, "arn:aws:iam::123456789012:user/other", USER_ID); + + assertNotEquals(identity1, identity2); + assertNotEquals(identity1.hashCode(), identity2.hashCode()); + } + + @Test + public void testToString() { + final CallerIdentityInfo identity = new CallerIdentityInfo(ACCOUNT, ARN, USER_ID); + + assertEquals( + "CallerIdentityInfo{account='123456789012', arn='" + ARN + "', userId='" + USER_ID + "'}", identity.toString()); + } +} diff --git a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestS3STSUtilsCallerIdentity.java b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestS3STSUtilsCallerIdentity.java new file mode 100644 index 000000000000..f991bd7bffdc --- /dev/null +++ b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestS3STSUtilsCallerIdentity.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om.helpers; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.junit.jupiter.api.Test; + +/** + * Test caller identity resolution helpers in S3STSUtils. + */ +public class TestS3STSUtilsCallerIdentity { + + private static final String PRINCIPAL = "om/polarisclient@example.com"; + private static final String KERBEROS_SHORT_NAME = "om"; + private static final String ASSUMED_ROLE_ID = "AROATEST123456789:testsess"; + private static final String ASSUMED_ROLE_USER_ARN = + "arn:aws:sts::123456789012:assumed-role/test-role/testsess"; + + @Test + public void testToIamUserArn() { + assertEquals("arn:aws:iam::123456789012:user/om", S3STSUtils.toIamUserArn(KERBEROS_SHORT_NAME)); + } + + @Test + public void testResolveCallerIdentityForPermanentCredentials() { + final CallerIdentityInfo identity = S3STSUtils.resolveCallerIdentityForPermanentCredentials( + PRINCIPAL, KERBEROS_SHORT_NAME); + + assertEquals(S3STSUtils.OZONE_STATIC_ACCOUNT_ID, identity.getAccount()); + assertEquals("arn:aws:iam::123456789012:user/om", identity.getArn()); + assertEquals(PRINCIPAL, identity.getUserId()); + } + + @Test + public void testResolveCallerIdentityForStsCredentials() { + final CallerIdentityInfo identity = S3STSUtils.resolveCallerIdentityForStsCredentials( + ASSUMED_ROLE_ID, ASSUMED_ROLE_USER_ARN); + + assertEquals(S3STSUtils.OZONE_STATIC_ACCOUNT_ID, identity.getAccount()); + assertEquals(ASSUMED_ROLE_USER_ARN, identity.getArn()); + assertEquals(ASSUMED_ROLE_ID, identity.getUserId()); + } +} diff --git a/hadoop-ozone/dist/src/main/smoketest/security/ozone-secure-sts.resource b/hadoop-ozone/dist/src/main/smoketest/security/ozone-secure-sts.resource index 19cb6f4e2022..02c16a9418c2 100644 --- a/hadoop-ozone/dist/src/main/smoketest/security/ozone-secure-sts.resource +++ b/hadoop-ozone/dist/src/main/smoketest/security/ozone-secure-sts.resource @@ -103,10 +103,14 @@ Assume Role And Get Temporary Credentials ${stsAccessKeyId} = Execute echo '${json}' | jq -r '.Credentials.AccessKeyId' ${stsSecretKey} = Execute echo '${json}' | jq -r '.Credentials.SecretAccessKey' ${stsSessionToken} = Execute echo '${json}' | jq -r '.Credentials.SessionToken' + ${assumedRoleId} = Execute echo '${json}' | jq -r '.AssumedRoleUser.AssumedRoleId' + ${assumedRoleUserArn} = Execute echo '${json}' | jq -r '.AssumedRoleUser.Arn' Should Start With ${stsAccessKeyId} ASIA Set Global Variable ${STS_ACCESS_KEY_ID} ${stsAccessKeyId} Set Global Variable ${STS_SECRET_KEY} ${stsSecretKey} Set Global Variable ${STS_SESSION_TOKEN} ${stsSessionToken} + Set Global Variable ${STS_ASSUMED_ROLE_ID} ${assumedRoleId} + Set Global Variable ${STS_ASSUMED_ROLE_USER_ARN} ${assumedRoleUserArn} ${expected_duration} = Set Variable ${duration_seconds} # Ensure the expected duration defaults to 3600 seconds (1 hour) if not specified @@ -328,3 +332,36 @@ Assert Listed Keys Should Equal Sort List ${expected_sorted} ${actual_list} = Evaluate json.loads($actual_keys_json) modules=json Lists Should Be Equal ${actual_list} ${expected_sorted} + +Get Caller Identity + [Arguments] ${profile} + ${json} = Execute aws sts get-caller-identity --endpoint-url ${STS_ENDPOINT_URL} --output json --profile ${profile} + ${account} = Execute echo '${json}' | jq -r '.Account' + ${arn} = Execute echo '${json}' | jq -r '.Arn' + ${userId} = Execute echo '${json}' | jq -r '.UserId' + [Return] ${json} ${account} ${arn} ${userId} + +Get Caller Identity Using Curl + # AWS CLI always sends Version=2011-06-15 and rejects unknown flags, so use curl to test version validation. + # curl 7.76.1's --aws-sigv4 omits the port from the signed canonical "host" header (it signs "host:s3g" + # while sending "Host: s3g:9880"), so Ozone's SigV4 validation for the non-default STS port rejects it. + # Send an explicit Host header without the port so the sent and signed host values match. + # This should also work for newer versions of curl as well + [Arguments] ${perm_access_key_id} ${perm_secret_key} ${api_version}=2011-06-15 ${extra_curl_params}=${EMPTY} + ${sts_host} = Evaluate urllib.parse.urlparse("${STS_ENDPOINT_URL}").hostname modules=urllib.parse + ${cmd} = Set Variable curl --silent --show-error --include --request POST --aws-sigv4 "aws:amz:us-east-1:sts" --user '${perm_access_key_id}:${perm_secret_key}' --header "Host: ${sts_host}" --header "Content-Type: application/x-www-form-urlencoded" --data-urlencode "Action=GetCallerIdentity" + ${cmd} = Set Variable If '${api_version}' != '${EMPTY}' ${cmd} --data-urlencode "Version=${api_version}" ${cmd} + ${cmd} = Set Variable If '${extra_curl_params}' != '${EMPTY}' ${cmd} ${extra_curl_params} ${cmd} + ${cmd} = Set Variable ${cmd} ${STS_ENDPOINT_URL} + ${output} = Execute And Ignore Error ${cmd} + [Return] ${output} + +Get Caller Identity Should Fail + [Arguments] ${expected_error_contains} ${api_version}=${EMPTY} + ${output} = Get Caller Identity Using Curl ${PERMANENT_ACCESS_KEY_ID} ${PERMANENT_SECRET_KEY} api_version=${api_version} + Should Contain ${output} ${expected_error_contains} + @{http_codes} = Get Regexp Matches ${output} (?m)^HTTP/[0-9.]+ ([0-9]{3}) 1 + ${code_count} = Get Length ${http_codes} + Should Be True ${code_count} > 0 Expected to find an HTTP status code in curl output, but none was found. + ${http_code} = Get From List ${http_codes} -1 + Should Be Equal As Strings ${http_code} 400 diff --git a/hadoop-ozone/dist/src/main/smoketest/security/ozone-secure-sts.robot b/hadoop-ozone/dist/src/main/smoketest/security/ozone-secure-sts.robot index c0362d747048..b62a1053f5a7 100644 --- a/hadoop-ozone/dist/src/main/smoketest/security/ozone-secure-sts.robot +++ b/hadoop-ozone/dist/src/main/smoketest/security/ozone-secure-sts.robot @@ -66,6 +66,7 @@ ${ACTION_MATCHES_PUTOBJECT_CREATE_WRITE_ROLE_ARN} arn:aws:iam::123456789012:rol ${ACTION_MATCHES_GETOBJECT_PUTOBJECT_ROLE_ARN} arn:aws:iam::123456789012:role/${ACTION_MATCHES_GETOBJECT_PUTOBJECT_ROLE} ${ACTION_MATCHES_UPLOADPARTCOPY_EXPECTED_OWNER_ROLE_ARN} arn:aws:iam::123456789012:role/${ACTION_MATCHES_UPLOADPARTCOPY_EXPECTED_OWNER_ROLE} ${ACTION_MATCHES_GET_STAR_READ_ROLE_ARN} arn:aws:iam::123456789012:role/${ACTION_MATCHES_GET_STAR_READ_ROLE} +${TEST_USER_ADMIN} testuser ${TEST_USER_NON_ADMIN} testuser2 @{ICEBERG_OBJECT_KEYS} file1.txt file1again.txt folder/pepper.txt folder/salt.txt userA/userA.txt userB/userB.txt userAfile.txt @{ICEBERG_LISTABLE_OBJECT_KEYS_OBS} file1.txt file1again.txt folder/pepper.txt folder/salt.txt userA/userA.txt userB/userB.txt userAfile.txt zeroByteFile zeroByteFolder/ @@ -254,6 +255,17 @@ Configure STS Profile With Bogus Credential Part Configure STS Profile ${STS_ACCESS_KEY_ID} ${STS_SECRET_KEY} bogusSessionToken END +Verify STS Token Revocation And Post Revocation Assume Role + [Arguments] ${bucket} ${role_arn} ${revoker_user} ${revoker_keytab} + Assume Role And Configure STS Profile perm_access_key_id=${PERMANENT_ACCESS_KEY_ID} perm_secret_key=${PERMANENT_SECRET_KEY} role_arn=${role_arn} + Get Object Should Succeed ${bucket} ${ICEBERG_BUCKET_TESTFILE} + Kinit test user ${revoker_user} ${revoker_keytab} + ${output} = Execute ozone s3 revokeststoken -o ${PERMANENT_ACCESS_KEY_ID} -y ${OM_HA_PARAM} + Should Contain ${output} STS tokens revoked for originalAccessKeyId + Get Object Should Fail ${bucket} ${ICEBERG_BUCKET_TESTFILE} AccessDenied + Assume Role And Configure STS Profile perm_access_key_id=${PERMANENT_ACCESS_KEY_ID} perm_secret_key=${PERMANENT_SECRET_KEY} role_arn=${role_arn} + Get Object Should Succeed ${bucket} ${ICEBERG_BUCKET_TESTFILE} + *** Test Cases *** Create User in Ranger ${user_json} = Set Variable { "loginId": "${ICEBERG_SVC_CATALOG_USER}", "name": "${ICEBERG_SVC_CATALOG_USER}", "password": "Password123", "firstName": "Iceberg REST", "lastName": "Catalog", "emailAddress": "${ICEBERG_SVC_CATALOG_USER}@example.com", "userRoleList": ["ROLE_USER"], "userPermList": [ { "moduleId": 1, "isAllowed": 1 }, { "moduleId": 3, "isAllowed": 1 }, { "moduleId": 7, "isAllowed": 1 } ] } @@ -558,27 +570,32 @@ Verify Token Revocation via CLI FOR ${bucket} ${role_arn} IN ... ${ICEBERG_BUCKET_OBS} ${ICEBERG_ALL_ACCESS_ROLE_OBS_ARN} ... ${ICEBERG_BUCKET_FSO} ${ICEBERG_ALL_ACCESS_ROLE_FSO_ARN} - Assume Role And Configure STS Profile perm_access_key_id=${PERMANENT_ACCESS_KEY_ID} perm_secret_key=${PERMANENT_SECRET_KEY} role_arn=${role_arn} - ${output} = Execute ozone s3 revokeststoken -t ${STS_SESSION_TOKEN} -y ${OM_HA_PARAM} - Should Contain ${output} STS token revoked for sessionToken - # Trying to use the token for even get-object should now fail. - Get Object Should Fail ${bucket} ${ICEBERG_BUCKET_TESTFILE} AccessDenied + # Owner of the original access key can revoke the STS token. + Verify STS Token Revocation And Post Revocation Assume Role ${bucket} ${role_arn} ${ICEBERG_SVC_CATALOG_USER} ${ICEBERG_SVC_CATALOG_USER}.keytab + # S3 admin can also revoke an STS token owned by another user. + Verify STS Token Revocation And Post Revocation Assume Role ${bucket} ${role_arn} ${TEST_USER_ADMIN} ${TEST_USER_ADMIN}.keytab END Non-Admin Cannot Revoke STS Token FOR ${role_arn} IN ${ICEBERG_ALL_ACCESS_ROLE_OBS_ARN} ${ICEBERG_ALL_ACCESS_ROLE_FSO_ARN} # Create a token first. Assume Role And Get Temporary Credentials perm_access_key_id=${PERMANENT_ACCESS_KEY_ID} perm_secret_key=${PERMANENT_SECRET_KEY} role_arn=${role_arn} - ${token_to_revoke} = Set Variable ${STS_SESSION_TOKEN} # Kinit as non-admin user. Kinit test user ${TEST_USER_NON_ADMIN} ${TEST_USER_NON_ADMIN}.keytab # Try to revoke - should give USER_MISMATCH error. - ${output} = Execute And Ignore Error ozone s3 revokeststoken -t ${token_to_revoke} -y ${OM_HA_PARAM} + ${output} = Execute And Ignore Error ozone s3 revokeststoken -o ${PERMANENT_ACCESS_KEY_ID} -y ${OM_HA_PARAM} Should Contain ${output} USER_MISMATCH END +Revoke STS Token Should Fail For Unknown Original Access Key Id + # Revoking a bogus originalAccessKeyId must fail before writing to the revocation table. + Kinit test user ${TEST_USER_ADMIN} ${TEST_USER_ADMIN}.keytab + ${output} = Execute And Ignore Error ozone s3 revokeststoken -o bogus-original-access-key-id -y ${OM_HA_PARAM} + Should Contain ${output} ACCESS_ID_NOT_FOUND + Should Contain ${output} does not exist + List Objects V1 and V2 IAM Session Policy Matrix for OBS and FSO Kinit test user ${ICEBERG_SVC_CATALOG_USER} ${ICEBERG_SVC_CATALOG_USER}.keytab @@ -613,6 +630,13 @@ Tampered STS Token Service, Policy, or Signature Must Fail Get Object Should Fail ${ICEBERG_BUCKET_OBS} ${ICEBERG_BUCKET_TESTFILE} AccessDenied Put Object Should Fail ${ICEBERG_BUCKET_OBS} ${ICEBERG_BUCKET_TESTFILE} AccessDenied + # Exercise malformed token decoding with incorrect token structure. Unlike the earlier + # bogusSessionToken credential-part check, this literal decodes to a negative Writable + # length and covers unchecked decoder failures such as NegativeArraySizeException. + Configure STS Profile ${STS_ACCESS_KEY_ID} ${STS_SECRET_KEY} not-a-valid-token + Get Object Should Fail ${ICEBERG_BUCKET_OBS} ${ICEBERG_BUCKET_TESTFILE} AccessDenied + Put Object Should Fail ${ICEBERG_BUCKET_OBS} ${ICEBERG_BUCKET_TESTFILE} AccessDenied + Assume Role Session Policy With Multiple Buckets Should Access All Buckets ${session_policy} = Set Variable {"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"s3:GetObject","Resource":"arn:aws:s3:::${ICEBERG_BUCKET_OBS}/*"},{"Effect":"Allow","Action":"s3:GetObject","Resource":"arn:aws:s3:::${ICEBERG_BUCKET_FSO}/*"}]} Assume Role And Get Temporary Credentials policy_json=${session_policy} perm_access_key_id=${PERMANENT_ACCESS_KEY_ID} perm_secret_key=${PERMANENT_SECRET_KEY} role_arn=${ICEBERG_MULTI_BUCKET_ROLE_ARN} @@ -1240,6 +1264,38 @@ STS session policy containing only GetObject must deny DeleteObjects ${output} = Execute aws s3api --endpoint-url ${S3G_ENDPOINT_URL} delete-bucket --bucket ${bucket} --profile sts Should Not Contain ${output} AccessDenied +Get Caller Identity With Permanent Credentials Should Succeed + Configure AWS Profile permanent ${PERMANENT_ACCESS_KEY_ID} ${PERMANENT_SECRET_KEY} + ${json} ${account} ${arn} ${userId} = Get Caller Identity permanent + Should Be Equal ${account} 123456789012 + ${principal} = Execute klist | awk '/Default principal/ {print $3}' + Should Be Equal ${userId} ${principal} + Should Be Equal ${arn} arn:aws:iam::123456789012:user/${ICEBERG_SVC_CATALOG_USER} + +Get Caller Identity With STS Credentials Should Succeed + Assume Role And Configure STS Profile perm_access_key_id=${PERMANENT_ACCESS_KEY_ID} perm_secret_key=${PERMANENT_SECRET_KEY} role_arn=${ICEBERG_ALL_ACCESS_ROLE_OBS_ARN} + ${json} ${account} ${arn} ${userId} = Get Caller Identity sts + Should Be Equal ${account} 123456789012 + Should Be Equal ${userId} ${STS_ASSUMED_ROLE_ID} + Should Be Equal ${arn} ${STS_ASSUMED_ROLE_USER_ARN} + +Get Caller Identity Ignores Extra Parameters + # curl 7.76.1 produces an invalid SigV4 signature for STS GET requests with --get --data-urlencode. + ${extra_curl_params} = Set Variable --data-urlencode "RoleArn=${ICEBERG_ALL_ACCESS_ROLE_OBS_ARN}" --data-urlencode "RoleSessionName=${ROLE_SESSION_NAME}" --data-urlencode "DurationSeconds=3600" + ${output} = Get Caller Identity Using Curl ${PERMANENT_ACCESS_KEY_ID} ${PERMANENT_SECRET_KEY} extra_curl_params=${extra_curl_params} + Should Contain ${output} 123456789012 + @{http_codes} = Get Regexp Matches ${output} (?m)^HTTP/[0-9.]+ ([0-9]{3}) 1 + ${code_count} = Get Length ${http_codes} + Should Be True ${code_count} > 0 Expected to find an HTTP status code in curl output, but none was found. + ${http_code} = Get From List ${http_codes} -1 + Should Be Equal As Strings ${http_code} 200 + +Get Caller Identity Rejects Missing Version + Get Caller Identity Should Fail InvalidAction + +Get Caller Identity Rejects Invalid Version + Get Caller Identity Should Fail InvalidAction api_version=2020-01-01 + Expired STS temporary credentials must return ExpiredToken on S3 APIs # Increase timeout to account for 15 minute STS token expiration plus the time to execute the api calls [Timeout] 25 minutes diff --git a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto index c06d54209a0b..062a92d20fa9 100644 --- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto +++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto @@ -169,6 +169,7 @@ enum Type { AssumeRole = 153; RevokeSTSToken = 154; DeleteRevokedSTSTokens = 155; + GetCallerIdentity = 156; } enum SafeMode { @@ -333,6 +334,7 @@ message OMRequest { optional RevokeSTSTokenRequest revokeSTSTokenRequest = 155; optional DeleteRevokedSTSTokensRequest deleteRevokedSTSTokensRequest = 156; optional UpdateAssumeRoleRequest updateAssumeRoleRequest = 157; + optional GetCallerIdentityRequest getCallerIdentityRequest = 158; } message OMResponse { @@ -481,6 +483,7 @@ message OMResponse { optional AssumeRoleResponse assumeRoleResponse = 153; optional RevokeSTSTokenResponse revokeSTSTokenResponse = 154; optional DeleteRevokedSTSTokensResponse deleteRevokedSTSTokensResponse = 155; + optional GetCallerIdentityResponse getCallerIdentityResponse = 156; } enum Status { @@ -1600,6 +1603,8 @@ message OMTokenProto { optional string originalAccessKeyId = 18; optional string secretAccessKey = 19; optional string sessionPolicy = 20; + optional string assumedRoleId = 21; + optional string assumedRoleUserArn = 22; } message SecretKeyProto { @@ -2534,23 +2539,34 @@ message UpdateAssumeRoleRequest { } message RevokeSTSTokenRequest { - required string sessionToken = 1; + required string originalAccessKeyId = 1; + // Leader-generated revocation cutoff, replicated across OMs in HA mode. + optional uint64 revocationTimeMillis = 2; } message RevokeSTSTokenResponse { } /** - This will contain a list of revoked STS session tokens whose entries should be removed from + This will contain a list of originalAccessKeyIds whose revocation entries should be removed from the s3RevokedStsTokenTable. */ message DeleteRevokedSTSTokensRequest { - repeated string sessionToken = 1; + repeated string originalAccessKeyId = 1; } message DeleteRevokedSTSTokensResponse { } +message GetCallerIdentityRequest { +} + +message GetCallerIdentityResponse { + optional string account = 1; + optional string arn = 2; + optional string userId = 3; +} + enum ReadConsistencyProto { // Unspecified consistency, the read consistency behavior is decided // by the OM diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java index 97f6cf920365..f1a7a51d3e26 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java @@ -540,7 +540,7 @@ protected void initializeOmTables(CacheType cacheType, compactionLogTable = initializer.get(OMDBDefinition.COMPACTION_LOG_TABLE_DEF); - // sessionToken -> insertionTimeMillis + // originalAccessKeyId -> revocationTimeMillis // FULL_CACHE keeps revocations in memory as there are not expected to be many s3RevokedStsTokenTable = initializer.get( OMDBDefinition.S3_REVOKED_STS_TOKEN_TABLE_DEF, cacheType); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/codec/OMDBDefinition.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/codec/OMDBDefinition.java index 2e99871e17c4..08600bb99504 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/codec/OMDBDefinition.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/codec/OMDBDefinition.java @@ -60,7 +60,7 @@ * | userTable | /user :- UserVolumeInfo | * | dTokenTable | OzoneTokenID :- renew_time | * | s3SecretTable | s3g_access_key_id :- s3Secret | - * | s3RevokedStsTokenTable | sts_session_token :- insertionTimeMillis | + * | s3RevokedStsTokenTable | originalAccessKeyId :- revocationTimeMillis | * |------------------------------------------------------------------------| * } * @@ -169,7 +169,10 @@ public final class OMDBDefinition extends DBDefinition.WithMap { S3SecretValue.getCodec()); public static final String S3_REVOKED_STS_TOKEN_TABLE = "s3RevokedStsTokenTable"; - /** s3RevokedStsTokenTable: sts_session_token :- insertionTimeMillis.*/ + /** + * s3RevokedStsTokenTable: originalAccessKeyId :- revocationTimeMillis. + * The value is the revocation cutoff in milliseconds. + */ public static final DBColumnFamilyDefinition S3_REVOKED_STS_TOKEN_TABLE_DEF = new DBColumnFamilyDefinition<>(S3_REVOKED_STS_TOKEN_TABLE, StringCodec.get(), diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerStateMachine.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerStateMachine.java index 3db85f508051..a5e4154b43d0 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerStateMachine.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerStateMachine.java @@ -25,6 +25,7 @@ import com.google.common.base.Preconditions; import com.google.common.util.concurrent.ThreadFactoryBuilder; import java.io.IOException; +import java.time.Instant; import java.util.ArrayList; import java.util.Collection; import java.util.LinkedHashMap; @@ -683,15 +684,22 @@ OMResponse runCommand(OMRequest request, TermIndex termIndex) { if (s3Auth.hasSessionToken() && !s3Auth.getSessionToken().isEmpty()) { // ThreadLocal carries session policy for OmMetadataReader + // Use Instant.MAX for creationTime so a future revocation check on this ThreadLocal + // identifier never treats the token as issued before a stored cutoff. final STSTokenIdentifier rehydratedTokenIdentifier = new STSTokenIdentifier( - s3Auth.hasResolvedStsTempAccessKeyId() ? s3Auth.getResolvedStsTempAccessKeyId() : "", - s3Auth.hasResolvedStsOriginalAccessKeyId() ? s3Auth.getResolvedStsOriginalAccessKeyId() : "", - s3Auth.hasResolvedStsRoleArn() ? s3Auth.getResolvedStsRoleArn() : "", - java.time.Instant.MAX, // ensure it deterministically is not expired - "", // no secretAccessKey needed - s3Auth.hasResolvedStsSessionPolicy() ? s3Auth.getResolvedStsSessionPolicy() : "", - null // no encryption key needed - ); + STSTokenIdentifier.Params.newBuilder() + .setTempAccessKeyId( + s3Auth.hasResolvedStsTempAccessKeyId() ? s3Auth.getResolvedStsTempAccessKeyId() : "") + .setOriginalAccessKeyId( + s3Auth.hasResolvedStsOriginalAccessKeyId() ? s3Auth.getResolvedStsOriginalAccessKeyId() : "") + .setRoleArn(s3Auth.hasResolvedStsRoleArn() ? s3Auth.getResolvedStsRoleArn() : "") + .setCreationTime(Instant.MAX) + .setExpiry(Instant.MAX) // ensure it deterministically is not expired + .setSecretAccessKey("") // no secretAccessKey needed + .setSessionPolicy( + s3Auth.hasResolvedStsSessionPolicy() ? s3Auth.getResolvedStsSessionPolicy() : "") + .setEncryptionKey(null) // no encryption key needed + .build()); OzoneManager.setStsTokenIdentifier(rehydratedTokenIdentifier); isStsThreadLocalSet = true; } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/OMClientRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/OMClientRequest.java index 6b9c6698cf9f..29abe0eb8eec 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/OMClientRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/OMClientRequest.java @@ -46,10 +46,10 @@ import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OMAuditLogger; import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils; +import org.apache.hadoop.ozone.om.helpers.S3STSUtils; import org.apache.hadoop.ozone.om.lock.OMLockDetails; import org.apache.hadoop.ozone.om.protocolPB.grpc.GrpcClientConstants; import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerRatisUtils; -import org.apache.hadoop.ozone.om.request.s3.security.S3AssumeRoleRequest; import org.apache.hadoop.ozone.om.response.OMClientResponse; import org.apache.hadoop.ozone.om.upgrade.OMLayoutVersionManager; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; @@ -226,7 +226,7 @@ public OzoneManagerProtocolProtos.UserInfo getUserInfo() throws IOException { // falling back to accessId if session token not present. if (omRequest.hasS3Authentication()) { final String accessKeyId = omRequest.getS3Authentication().getAccessId(); - if (accessKeyId.startsWith(S3AssumeRoleRequest.STS_TOKEN_PREFIX) && + if (accessKeyId.startsWith(S3STSUtils.STS_TOKEN_PREFIX) && !omRequest.getS3Authentication().hasSessionToken()) { throw new IOException("Error with STS token", new AuthenticationException( "Missing session token for accessKeyId: " + accessKeyId)); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/security/S3AssumeRoleRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/security/S3AssumeRoleRequest.java index 4efd18b4b327..788ba74b3c56 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/security/S3AssumeRoleRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/security/S3AssumeRoleRequest.java @@ -17,6 +17,10 @@ package org.apache.hadoop.ozone.om.request.s3.security; +import static org.apache.hadoop.ozone.om.helpers.S3STSUtils.STS_ACCESS_KEY_ID_ALLOWED_CHARS; +import static org.apache.hadoop.ozone.om.helpers.S3STSUtils.STS_ACCESS_KEY_ID_ALLOWED_CHARS_LENGTH; +import static org.apache.hadoop.ozone.om.helpers.S3STSUtils.STS_ACCESS_KEY_ID_RANDOM_LENGTH; +import static org.apache.hadoop.ozone.om.helpers.S3STSUtils.STS_TOKEN_PREFIX; import static org.apache.hadoop.ozone.security.acl.AssumeRoleRequest.OzoneGrant; import com.google.common.annotations.VisibleForTesting; @@ -31,6 +35,7 @@ import java.util.Set; import org.apache.hadoop.hdds.scm.client.HddsClientUtils; import org.apache.hadoop.ipc_.ProtobufRpcEngine; +import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.audit.AuditLogger; import org.apache.hadoop.ozone.audit.OMAction; import org.apache.hadoop.ozone.om.OzoneAclUtils; @@ -48,6 +53,7 @@ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.AssumeRoleResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.UpdateAssumeRoleRequest; +import org.apache.hadoop.ozone.security.STSTokenSecretManager; import org.apache.hadoop.ozone.security.acl.iam.IamSessionPolicyResolver; import org.apache.hadoop.security.UserGroupInformation; @@ -70,16 +76,12 @@ public class S3AssumeRoleRequest extends OMClientRequest { SECURE_RANDOM = secureRandom; } - private static final int STS_ACCESS_KEY_ID_LENGTH = 20; private static final int STS_SECRET_ACCESS_KEY_LENGTH = 40; private static final int STS_ROLE_ID_LENGTH = 16; private static final String ASSUME_ROLE_ID_PREFIX = "AROA"; - private static final String CHARS_FOR_ACCESS_KEY_IDS = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"; - private static final int CHARS_FOR_ACCESS_KEY_IDS_LENGTH = CHARS_FOR_ACCESS_KEY_IDS.length(); - private static final String CHARS_FOR_SECRET_ACCESS_KEYS = CHARS_FOR_ACCESS_KEY_IDS + + private static final String CHARS_FOR_SECRET_ACCESS_KEYS = STS_ACCESS_KEY_ID_ALLOWED_CHARS + "abcdefghijklmnopqrstuvwxyz/+"; private static final int CHARS_FOR_SECRET_ACCESS_KEYS_LENGTH = CHARS_FOR_SECRET_ACCESS_KEYS.length(); - public static final String STS_TOKEN_PREFIX = "ASIA"; private final Clock clock; @@ -103,11 +105,13 @@ public OMRequest preExecute(OzoneManager ozoneManager) throws IOException { // Generate temporary AWS credentials using cryptographically strong SecureRandom final String tempAccessKeyId = STS_TOKEN_PREFIX + generateSecureRandomStringUsingChars( - CHARS_FOR_ACCESS_KEY_IDS, CHARS_FOR_ACCESS_KEY_IDS_LENGTH, STS_ACCESS_KEY_ID_LENGTH); + STS_ACCESS_KEY_ID_ALLOWED_CHARS, STS_ACCESS_KEY_ID_ALLOWED_CHARS_LENGTH, + STS_ACCESS_KEY_ID_RANDOM_LENGTH); final String secretAccessKey = generateSecureRandomStringUsingChars( CHARS_FOR_SECRET_ACCESS_KEYS, CHARS_FOR_SECRET_ACCESS_KEYS_LENGTH, STS_SECRET_ACCESS_KEY_LENGTH); final String roleId = ASSUME_ROLE_ID_PREFIX + generateSecureRandomStringUsingChars( - CHARS_FOR_ACCESS_KEY_IDS, CHARS_FOR_ACCESS_KEY_IDS_LENGTH, STS_ROLE_ID_LENGTH); + STS_ACCESS_KEY_ID_ALLOWED_CHARS, STS_ACCESS_KEY_ID_ALLOWED_CHARS_LENGTH, + STS_ROLE_ID_LENGTH); // Build UpdateAssumeRoleRequest with leader-generated credentials final UpdateAssumeRoleRequest.Builder updateAssumeRoleRequestBuilder = @@ -171,18 +175,27 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut "S3AssumeRoleRequest does not have S3 authentication", OMException.ResultCodes.INVALID_REQUEST); } - // Generate session token using leader-generated credentials - final String sessionToken = generateSessionToken( - targetRoleName, omRequest, ozoneManager, assumeRoleRequest, secretAccessKey, tempAccessKeyId); - // Generate AssumedRoleId for response using leader-generated roleId final String assumedRoleId = roleId + ":" + roleSessionName; + final String assumedRoleUserArn = S3STSUtils.toAssumedRoleUserArn(roleArn, roleSessionName); + + // Generate session token using leader-generated credentials + final String sessionToken = generateSessionToken(GenerateSessionTokenParams.newBuilder() + .setTargetRoleName(targetRoleName) + .setOmRequest(omRequest) + .setOzoneManager(ozoneManager) + .setAssumeRoleRequest(assumeRoleRequest) + .setSecretAccessKey(secretAccessKey) + .setTempAccessKeyId(tempAccessKeyId) + .setAssumedRoleId(assumedRoleId) + .setAssumedRoleUserArn(assumedRoleUserArn) + .build()); // Calculate expiration of session token final long expirationEpochSeconds = clock.instant().plusSeconds(durationSeconds).getEpochSecond(); // Add tempAccessKeyId to the log so it can be determined which permanent user created the tempAccessKeyId - auditMap.put("tempAccessKeyId", tempAccessKeyId); + auditMap.put(OzoneConsts.S3_STS_TEMP_ACCESS_KEY_ID, tempAccessKeyId); final AssumeRoleResponse.Builder responseBuilder = AssumeRoleResponse.newBuilder() .setAccessKeyId(tempAccessKeyId) @@ -216,9 +229,10 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut /** * Generates session token using components from the AssumeRoleRequest. */ - private String generateSessionToken(String targetRoleName, OMRequest omRequest, - OzoneManager ozoneManager, AssumeRoleRequest assumeRoleRequest, String secretAccessKey, - String tempAccessKeyId) throws IOException { + private String generateSessionToken(GenerateSessionTokenParams params) throws IOException { + final OzoneManager ozoneManager = params.getOzoneManager(); + final OMRequest omRequest = params.getOmRequest(); + final AssumeRoleRequest assumeRoleRequest = params.getAssumeRoleRequest(); InetAddress remoteIp = ProtobufRpcEngine.Server.getRemoteIp(); if (remoteIp == null) { @@ -239,11 +253,136 @@ private String generateSessionToken(String targetRoleName, OMRequest omRequest, final String roleArn = assumeRoleRequest.getRoleArn(); final String sessionPolicy = getSessionPolicy( ozoneManager, originalAccessKeyId, assumeRoleRequest.getAwsIamSessionPolicy(), hostName, remoteIp, ugi, - targetRoleName); + params.getTargetRoleName()); return ozoneManager.getSTSTokenSecretManager().createSTSTokenString( - tempAccessKeyId, originalAccessKeyId, roleArn, assumeRoleRequest.getDurationSeconds(), secretAccessKey, - sessionPolicy, clock); + STSTokenSecretManager.CreateSTSTokenParams.newBuilder() + .setTempAccessKeyId(params.getTempAccessKeyId()) + .setOriginalAccessKeyId(originalAccessKeyId) + .setRoleArn(roleArn) + .setDurationSeconds(assumeRoleRequest.getDurationSeconds()) + .setSecretAccessKey(params.getSecretAccessKey()) + .setSessionPolicy(sessionPolicy) + .setAssumedRoleId(params.getAssumedRoleId()) + .setAssumedRoleUserArn(params.getAssumedRoleUserArn()) + .setClock(clock) + .build()); + } + + /** + * Parameters for {@link #generateSessionToken(GenerateSessionTokenParams)}. + */ + private static final class GenerateSessionTokenParams { + private final String targetRoleName; + private final OMRequest omRequest; + private final OzoneManager ozoneManager; + private final AssumeRoleRequest assumeRoleRequest; + private final String secretAccessKey; + private final String tempAccessKeyId; + private final String assumedRoleId; + private final String assumedRoleUserArn; + + private GenerateSessionTokenParams(Builder builder) { + this.targetRoleName = builder.targetRoleName; + this.omRequest = builder.omRequest; + this.ozoneManager = builder.ozoneManager; + this.assumeRoleRequest = builder.assumeRoleRequest; + this.secretAccessKey = builder.secretAccessKey; + this.tempAccessKeyId = builder.tempAccessKeyId; + this.assumedRoleId = builder.assumedRoleId; + this.assumedRoleUserArn = builder.assumedRoleUserArn; + } + + static Builder newBuilder() { + return new Builder(); + } + + String getTargetRoleName() { + return targetRoleName; + } + + OMRequest getOmRequest() { + return omRequest; + } + + OzoneManager getOzoneManager() { + return ozoneManager; + } + + AssumeRoleRequest getAssumeRoleRequest() { + return assumeRoleRequest; + } + + String getSecretAccessKey() { + return secretAccessKey; + } + + String getTempAccessKeyId() { + return tempAccessKeyId; + } + + String getAssumedRoleId() { + return assumedRoleId; + } + + String getAssumedRoleUserArn() { + return assumedRoleUserArn; + } + + private static final class Builder { + private String targetRoleName; + private OMRequest omRequest; + private OzoneManager ozoneManager; + private AssumeRoleRequest assumeRoleRequest; + private String secretAccessKey; + private String tempAccessKeyId; + private String assumedRoleId; + private String assumedRoleUserArn; + + Builder setTargetRoleName(String value) { + this.targetRoleName = value; + return this; + } + + Builder setOmRequest(OMRequest value) { + this.omRequest = value; + return this; + } + + Builder setOzoneManager(OzoneManager value) { + this.ozoneManager = value; + return this; + } + + Builder setAssumeRoleRequest(AssumeRoleRequest value) { + this.assumeRoleRequest = value; + return this; + } + + Builder setSecretAccessKey(String value) { + this.secretAccessKey = value; + return this; + } + + Builder setTempAccessKeyId(String value) { + this.tempAccessKeyId = value; + return this; + } + + Builder setAssumedRoleId(String value) { + this.assumedRoleId = value; + return this; + } + + Builder setAssumedRoleUserArn(String value) { + this.assumedRoleUserArn = value; + return this; + } + + GenerateSessionTokenParams build() { + return new GenerateSessionTokenParams(this); + } + } } /** diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/security/S3DeleteRevokedSTSTokensRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/security/S3DeleteRevokedSTSTokensRequest.java index f41b20353a83..81558ec58504 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/security/S3DeleteRevokedSTSTokensRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/security/S3DeleteRevokedSTSTokensRequest.java @@ -35,6 +35,7 @@ /** * Handles DeleteRevokedSTSTokens requests submitted by {@link RevokedSTSTokenCleanupService}. + * Each request contains originalAccessKeyIds to remove from the revocation table. */ public class S3DeleteRevokedSTSTokensRequest extends OMClientRequest { @@ -62,8 +63,8 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut final DeleteRevokedSTSTokensRequest request = getOmRequest().getDeleteRevokedSTSTokensRequest(); final OMResponse.Builder omResponse = OmResponseUtil.getOMResponseBuilder(getOmRequest()); - final List sessionTokens = request.getSessionTokenList(); - return new S3DeleteRevokedSTSTokensResponse(sessionTokens, omResponse.build()); + final List originalAccessKeyIds = request.getOriginalAccessKeyIdList(); + return new S3DeleteRevokedSTSTokensResponse(originalAccessKeyIds, omResponse.build()); } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/security/S3RevokeSTSTokenRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/security/S3RevokeSTSTokenRequest.java index 52a92d8a5560..02e6cac1b3d4 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/security/S3RevokeSTSTokenRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/security/S3RevokeSTSTokenRequest.java @@ -17,26 +17,30 @@ package org.apache.hadoop.ozone.om.request.s3.security; +import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.ACCESS_ID_NOT_FOUND; +import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.INTERNAL_ERROR; +import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.INVALID_REQUEST; + import java.io.IOException; import java.time.Clock; import java.time.ZoneOffset; import java.util.HashMap; import java.util.Map; +import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.hdds.utils.db.cache.CacheKey; import org.apache.hadoop.hdds.utils.db.cache.CacheValue; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.audit.OMAction; import org.apache.hadoop.ozone.om.OzoneManager; +import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.execution.flowcontrol.ExecutionContext; import org.apache.hadoop.ozone.om.request.OMClientRequest; import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; import org.apache.hadoop.ozone.om.response.OMClientResponse; import org.apache.hadoop.ozone.om.response.s3.security.S3RevokeSTSTokenResponse; -import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; -import org.apache.hadoop.ozone.security.STSSecurityUtil; -import org.apache.hadoop.ozone.security.STSTokenIdentifier; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.RevokeSTSTokenRequest; import org.apache.hadoop.security.UserGroupInformation; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -44,10 +48,14 @@ /** * Handles S3RevokeSTSTokenRequest request. * - *

This request marks an STS session token as revoked by inserting - * it into the {@code s3RevokedStsTokenTable}. Subsequent S3 requests - * authenticated with the same STS session token will be rejected when the - * revocation state has propagated.

+ *

The client submits {@link RevokeSTSTokenRequest} with {@code originalAccessKeyId} only. On the + * leader, {@code preExecute} captures the revocation cutoff in {@code revocationTimeMillis} and + * replicates the updated request through Ratis so every OM applies the same cutoff.

+ * + *

This request records a revocation cutoff for the given {@code originalAccessKeyId} in the + * {@code s3RevokedStsTokenTable}. Subsequent S3 requests authenticated with STS tokens whose + * {@code creationTime} is strictly before the cutoff will be rejected when the revocation state + * has propagated.

*/ public class S3RevokeSTSTokenRequest extends OMClientRequest { @@ -61,48 +69,89 @@ public S3RevokeSTSTokenRequest(OMRequest omRequest) { @Override public OMRequest preExecute(OzoneManager ozoneManager) throws IOException { final OMRequest omRequest = super.preExecute(ozoneManager); - final OzoneManagerProtocolProtos.RevokeSTSTokenRequest revokeReq = - omRequest.getRevokeSTSTokenRequest(); + final RevokeSTSTokenRequest revokeReq = omRequest.getRevokeSTSTokenRequest(); + validateRevokeRequestFields(revokeReq); - // Get the original (long-lived) access key id from the session token - // and enforce the same permission model that is used for S3 secret + // Use the original (long-lived) access key ID from the request and enforce + // the same permission model that is used for S3 secret // operations (get/set/revoke). Only the owner of the original access // key (i.e. the creator of the STS token) or an S3 / tenant admin is allowed // to revoke its temporary STS credentials. - final String sessionToken = revokeReq.getSessionToken(); - final STSTokenIdentifier stsTokenIdentifier = STSSecurityUtil.constructValidateAndDecryptSTSToken( - sessionToken, ozoneManager.getSecretKeyClient(), CLOCK); - final String originalAccessKeyId = stsTokenIdentifier.getOriginalAccessKeyId(); + final String originalAccessKeyId = revokeReq.getOriginalAccessKeyId(); final UserGroupInformation ugi = S3SecretRequestHelper.getOrCreateUgi(originalAccessKeyId); S3SecretRequestHelper.checkAccessIdSecretOpPermission(ozoneManager, ugi, originalAccessKeyId); - return omRequest; + if (!ozoneManager.getS3SecretManager().hasS3Secret(originalAccessKeyId)) { + throw new OMException("originalAccessKeyId does not exist: " + originalAccessKeyId, ACCESS_ID_NOT_FOUND); + } + + final long revocationTimeMillis = CLOCK.millis(); + final RevokeSTSTokenRequest updatedRevokeReq = revokeReq.toBuilder() + .setRevocationTimeMillis(revocationTimeMillis) + .build(); + + return omRequest.toBuilder() + .setRevokeSTSTokenRequest(updatedRevokeReq) + .build(); } @Override public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, ExecutionContext context) { final OMResponse.Builder omResponse = OmResponseUtil.getOMResponseBuilder(getOmRequest()); + IOException exception = null; + OMClientResponse omClientResponse; + final Map auditMap = new HashMap<>(); - final OzoneManagerProtocolProtos.RevokeSTSTokenRequest revokeReq = getOmRequest().getRevokeSTSTokenRequest(); - final String sessionToken = revokeReq.getSessionToken(); + try { + final RevokeSTSTokenRequest revokeReq = validateReplicatedRevokeRequestFields(getOmRequest()); + final String originalAccessKeyId = revokeReq.getOriginalAccessKeyId(); + auditMap.put(OzoneConsts.S3_REVOKESTSTOKEN_USER, originalAccessKeyId); + final long revocationTimeMillis = revokeReq.getRevocationTimeMillis(); - // All actual DB mutations are done in the response's addToDBBatch(). - final OMClientResponse omClientResponse = new S3RevokeSTSTokenResponse( - sessionToken, omResponse.build()); + // All actual DB mutations are done in the response's addToDBBatch(). + omClientResponse = new S3RevokeSTSTokenResponse(originalAccessKeyId, revocationTimeMillis, omResponse.build()); - // Audit log - final Map auditMap = new HashMap<>(); - final OzoneManagerProtocolProtos.UserInfo userInfo = getOmRequest().getUserInfo(); - auditMap.put(OzoneConsts.S3_REVOKESTSTOKEN_USER, userInfo.getUserName()); - markForAudit(ozoneManager.getAuditLogger(), buildAuditMessage( - OMAction.REVOKE_STS_TOKEN, auditMap, null, userInfo)); + // Update the cache immediately so subsequent validation checks see the revocation + ozoneManager.getMetadataManager().getS3RevokedStsTokenTable().addCacheEntry( + new CacheKey<>(originalAccessKeyId), CacheValue.get(context.getIndex(), revocationTimeMillis)); - // Update the cache immediately so subsequent validation checks see the revocation - ozoneManager.getMetadataManager().getS3RevokedStsTokenTable().addCacheEntry( - new CacheKey<>(sessionToken), CacheValue.get(context.getIndex(), CLOCK.millis())); + LOG.info( + "Marked STS tokens as revoked for originalAccessKeyId={} with cutoff time {}.", + originalAccessKeyId, revocationTimeMillis); + } catch (IOException ex) { + exception = ex; + omClientResponse = new S3RevokeSTSTokenResponse(null, 0L, createErrorOMResponse(omResponse, ex)); + } - LOG.info("Marked STS session token '{}' as revoked.", sessionToken); + // Audit log + markForAudit( + ozoneManager.getAuditLogger(), buildAuditMessage( + OMAction.REVOKE_STS_TOKEN, auditMap, exception, getOmRequest().getUserInfo())); return omClientResponse; } + + private static void validateRevokeRequestFields(RevokeSTSTokenRequest revokeReq) throws OMException { + final String originalAccessKeyId = revokeReq.getOriginalAccessKeyId(); + if (StringUtils.isEmpty(originalAccessKeyId)) { + throw new OMException("originalAccessKeyId is required for STS token revocation", INVALID_REQUEST); + } + if (revokeReq.hasRevocationTimeMillis()) { + throw new OMException("revocationTimeMillis must not be set by client", INVALID_REQUEST); + } + } + + private static RevokeSTSTokenRequest validateReplicatedRevokeRequestFields(OMRequest omRequest) throws OMException { + if (!omRequest.hasRevokeSTSTokenRequest()) { + throw new OMException("revokeSTSTokenRequest is required for STS token revocation", INTERNAL_ERROR); + } + final RevokeSTSTokenRequest revokeReq = omRequest.getRevokeSTSTokenRequest(); + if (StringUtils.isEmpty(revokeReq.getOriginalAccessKeyId())) { + throw new OMException("originalAccessKeyId is required for STS token revocation", INTERNAL_ERROR); + } + if (!revokeReq.hasRevocationTimeMillis()) { + throw new OMException("revocationTimeMillis is required for STS token revocation", INTERNAL_ERROR); + } + return revokeReq; + } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/security/S3DeleteRevokedSTSTokensResponse.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/security/S3DeleteRevokedSTSTokensResponse.java index cb44e7f466d9..a1b255689de5 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/security/S3DeleteRevokedSTSTokensResponse.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/security/S3DeleteRevokedSTSTokensResponse.java @@ -36,16 +36,16 @@ @CleanupTableInfo(cleanupTables = {S3_REVOKED_STS_TOKEN_TABLE}) public class S3DeleteRevokedSTSTokensResponse extends OMClientResponse { - private final List sessionTokens; + private final List originalAccessKeyIds; - public S3DeleteRevokedSTSTokensResponse(List sessionTokens, @Nonnull OMResponse omResponse) { + public S3DeleteRevokedSTSTokensResponse(List originalAccessKeyIds, @Nonnull OMResponse omResponse) { super(omResponse); - this.sessionTokens = sessionTokens; + this.originalAccessKeyIds = originalAccessKeyIds; } @Override public void addToDBBatch(OMMetadataManager omMetadataManager, BatchOperation batchOperation) throws IOException { - if (sessionTokens == null || sessionTokens.isEmpty()) { + if (originalAccessKeyIds == null || originalAccessKeyIds.isEmpty()) { return; } if (!getOMResponse().hasStatus() || getOMResponse().getStatus() != OK) { @@ -57,8 +57,8 @@ public void addToDBBatch(OMMetadataManager omMetadataManager, BatchOperation bat return; } - for (String sessionToken : sessionTokens) { - table.deleteWithBatch(batchOperation, sessionToken); + for (String originalAccessKeyId : originalAccessKeyIds) { + table.deleteWithBatch(batchOperation, originalAccessKeyId); } } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/security/S3RevokeSTSTokenResponse.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/security/S3RevokeSTSTokenResponse.java index 5b1a8cf3b019..db9233357ed2 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/security/S3RevokeSTSTokenResponse.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/security/S3RevokeSTSTokenResponse.java @@ -22,8 +22,6 @@ import jakarta.annotation.Nonnull; import java.io.IOException; -import java.time.Clock; -import java.time.ZoneOffset; import org.apache.hadoop.hdds.utils.db.BatchOperation; import org.apache.hadoop.hdds.utils.db.Table; import org.apache.hadoop.ozone.om.OMMetadataManager; @@ -37,22 +35,23 @@ @CleanupTableInfo(cleanupTables = {S3_REVOKED_STS_TOKEN_TABLE}) public class S3RevokeSTSTokenResponse extends OMClientResponse { - private static final Clock CLOCK = Clock.system(ZoneOffset.UTC); + private final String originalAccessKeyId; + private final long revocationTimeMillis; - private final String sessionToken; - - public S3RevokeSTSTokenResponse(String sessionToken, @Nonnull OMResponse omResponse) { + public S3RevokeSTSTokenResponse(String originalAccessKeyId, long revocationTimeMillis, + @Nonnull OMResponse omResponse) { super(omResponse); - this.sessionToken = sessionToken; + this.originalAccessKeyId = originalAccessKeyId; + this.revocationTimeMillis = revocationTimeMillis; } @Override public void addToDBBatch(OMMetadataManager omMetadataManager, BatchOperation batchOperation) throws IOException { - if (sessionToken != null && getOMResponse().hasStatus() && getOMResponse().getStatus() == OK) { + if (originalAccessKeyId != null && getOMResponse().hasStatus() && getOMResponse().getStatus() == OK) { final Table table = omMetadataManager.getS3RevokedStsTokenTable(); if (table != null) { - // Store insertionTimeMillis as value - table.putWithBatch(batchOperation, sessionToken, CLOCK.millis()); + // Store revocationTimeMillis as value + table.putWithBatch(batchOperation, originalAccessKeyId, revocationTimeMillis); } } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/RevokedSTSTokenCleanupService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/RevokedSTSTokenCleanupService.java index 3d9668d6469c..c627f6a21cb7 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/RevokedSTSTokenCleanupService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/RevokedSTSTokenCleanupService.java @@ -37,6 +37,7 @@ import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.OMMetadataManager; import org.apache.hadoop.ozone.om.OzoneManager; +import org.apache.hadoop.ozone.om.helpers.S3STSUtils; import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerRatisUtils; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.DeleteRevokedSTSTokensRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; @@ -57,7 +58,8 @@ public class RevokedSTSTokenCleanupService extends BackgroundService { // Use a single thread private static final int REVOKED_STS_TOKEN_CLEANER_CORE_POOL_SIZE = 1; private static final Clock CLOCK = Clock.system(ZoneOffset.UTC); - private static final long CLEANUP_THRESHOLD = 12 * 60 * 60 * 1000L; // 12 hours in milliseconds + // Keep revocation entries until max STS token lifetime after the cutoff was captured. + private static final long CLEANUP_THRESHOLD = TimeUnit.SECONDS.toMillis(S3STSUtils.MAX_DURATION_SECONDS); // 12 hours private final OzoneManager ozoneManager; private final OMMetadataManager metadataManager; @@ -124,7 +126,7 @@ private boolean shouldRun() { return !suspended.get() && ozoneManager.isLeaderReady(); } - private class RevokedSTSTokenCleanupTask implements BackgroundTask { + private final class RevokedSTSTokenCleanupTask implements BackgroundTask { @Override public BackgroundTaskResult call() throws Exception { @@ -143,17 +145,17 @@ public BackgroundTaskResult call() throws Exception { iterator.seekToFirst(); while (iterator.hasNext()) { final Table.KeyValue entry = iterator.next(); - final String sessionToken = entry.getKey(); - final Long initialCreationTimeMillis = entry.getValue(); + final String originalAccessKeyId = entry.getKey(); + final Long revocationTimeMillis = entry.getValue(); - if (shouldCleanup(initialCreationTimeMillis)) { - // Calculate the size this token would add to the protobuf message. + if (shouldCleanup(revocationTimeMillis)) { + // Calculate the size this originalAccessKeyId would add to the protobuf message. // Make a copy of the batch to do the size check final List batchCopyWithCandidate = new ArrayList<>(batch); - batchCopyWithCandidate.add(sessionToken); + batchCopyWithCandidate.add(originalAccessKeyId); int batchWithCandidateSize = getBatchSerializedSize(batchCopyWithCandidate); - // If adding this token would exceed the limit, submit the current batch + // If adding this originalAccessKeyId would exceed the limit, submit the current batch if (batchWithCandidateSize > ratisByteLimit) { if (!batch.isEmpty()) { if (submitCleanupRequest(batch)) { @@ -163,22 +165,22 @@ public BackgroundTaskResult call() throws Exception { } batch.clear(); - // Re-calculate the size of the candidate token alone in an empty batch + // Re-calculate the size of the candidate key alone in an empty batch // to check if it exceeds the limit by itself. final List singleCandidateBatch = new ArrayList<>(); - singleCandidateBatch.add(sessionToken); + singleCandidateBatch.add(originalAccessKeyId); batchWithCandidateSize = getBatchSerializedSize(singleCandidateBatch); } - // Check if the single token exceeds the limit (either strictly single or after flush) + // Check if the single key exceeds the limit (either strictly single or after flush) if (batchWithCandidateSize > ratisByteLimit) { LOG.error( - "Single revoked STS Token size ({}) would exceed the ratisByteLimit ({}). SessionToken " + - "initialCreationTimeMillis: {}", batchWithCandidateSize, ratisByteLimit, initialCreationTimeMillis); + "Single originalAccessKeyId entry size ({}) would exceed the ratisByteLimit ({}). " + + "revocationTimeMillis: {}", batchWithCandidateSize, ratisByteLimit, revocationTimeMillis); continue; } } - batch.add(sessionToken); + batch.add(originalAccessKeyId); } } } catch (IOException e) { @@ -213,16 +215,16 @@ public BackgroundTaskResult call() throws Exception { } /** - * Returns true if the given STS session token has been in the table past the cleanup threshold. + * Returns true if the revocation cutoff is older than the cleanup threshold. */ - private boolean shouldCleanup(long initialCreationTimeMillis) { + private boolean shouldCleanup(long revocationTimeMillis) { final long now = CLOCK.millis(); - if (now - initialCreationTimeMillis > CLEANUP_THRESHOLD) { + if (now - revocationTimeMillis > CLEANUP_THRESHOLD) { if (LOG.isDebugEnabled()) { LOG.debug( - "Revoked STS token entry created at {} is older than 12 hours, will clean up. Current time: {}", - initialCreationTimeMillis, now); + "Revoked STS token cutoff at {} is older than {} ms, will clean up. Current time: {}", + revocationTimeMillis, CLEANUP_THRESHOLD, now); } return true; } @@ -230,11 +232,11 @@ private boolean shouldCleanup(long initialCreationTimeMillis) { } /** - * Builds and submits an OMRequest to delete the provided revoked STS token(s). + * Builds and submits an OMRequest to delete the provided originalAccessKeyId revocation entries. */ - private boolean submitCleanupRequest(List sessionTokens) { + private boolean submitCleanupRequest(List originalAccessKeyIds) { final DeleteRevokedSTSTokensRequest request = DeleteRevokedSTSTokensRequest.newBuilder() - .addAllSessionToken(sessionTokens) + .addAllOriginalAccessKeyId(originalAccessKeyIds) .build(); final OMRequest omRequest = OMRequest.newBuilder() @@ -254,9 +256,9 @@ private boolean submitCleanupRequest(List sessionTokens) { } } - private int getBatchSerializedSize(List sessionTokenBatch) { + private int getBatchSerializedSize(List originalAccessKeyIdBatch) { final DeleteRevokedSTSTokensRequest request = DeleteRevokedSTSTokensRequest.newBuilder() - .addAllSessionToken(sessionTokenBatch) + .addAllOriginalAccessKeyId(originalAccessKeyIdBatch) .build(); return request.getSerializedSize(); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java index 21afb1e42f70..0ff432318565 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java @@ -61,6 +61,7 @@ import org.apache.hadoop.hdds.scm.protocolPB.OzonePBHelper; import org.apache.hadoop.hdds.utils.FaultInjector; import org.apache.hadoop.ozone.OzoneAcl; +import org.apache.hadoop.ozone.om.OzoneAclUtils; import org.apache.hadoop.ozone.om.OzoneManager; import org.apache.hadoop.ozone.om.OzoneManagerPrepareState; import org.apache.hadoop.ozone.om.exceptions.OMException; @@ -86,6 +87,7 @@ import org.apache.hadoop.ozone.om.helpers.OpenKeySession; import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus; import org.apache.hadoop.ozone.om.helpers.OzoneFileStatusLight; +import org.apache.hadoop.ozone.om.helpers.S3STSUtils; import org.apache.hadoop.ozone.om.helpers.ServiceInfo; import org.apache.hadoop.ozone.om.helpers.ServiceInfoEx; import org.apache.hadoop.ozone.om.helpers.SnapshotDiffJob; @@ -113,6 +115,7 @@ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.FinalizeUpgradeProgressResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.GetBucketTaggingRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.GetBucketTaggingResponse; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.GetCallerIdentityResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.GetFileStatusRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.GetFileStatusResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.GetKeyInfoRequest; @@ -171,11 +174,13 @@ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.TenantListUserResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type; import org.apache.hadoop.ozone.request.validation.RequestProcessingPhase; +import org.apache.hadoop.ozone.security.STSTokenIdentifier; import org.apache.hadoop.ozone.security.acl.OzoneObjInfo; import org.apache.hadoop.ozone.snapshot.ListSnapshotResponse; import org.apache.hadoop.ozone.upgrade.UpgradeFinalization.StatusAndMessages; import org.apache.hadoop.ozone.util.PayloadUtils; import org.apache.hadoop.ozone.util.ProtobufUtils; +import org.apache.hadoop.security.UserGroupInformation; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -316,6 +321,9 @@ public OMResponse handleReadRequest(OMRequest request) { getS3VolumeContext(); responseBuilder.setGetS3VolumeContextResponse(s3VolumeContextResponse); break; + case GetCallerIdentity: + responseBuilder.setGetCallerIdentityResponse(getCallerIdentity()); + break; case TenantGetUserInfo: impl.checkS3MultiTenancyEnabled(); TenantGetUserInfoResponse getUserInfoResponse = tenantGetUserInfo( @@ -1448,6 +1456,22 @@ private GetS3VolumeContextResponse getS3VolumeContext() return impl.getS3VolumeContext().getProtobuf(); } + private GetCallerIdentityResponse getCallerIdentity() throws OMException { + impl.checkS3STSEnabled(); + if (OzoneManager.getS3Auth() == null) { + throw new OMException( + "GetCallerIdentity does not have S3 authentication", OMException.ResultCodes.INVALID_REQUEST); + } + final STSTokenIdentifier stsTokenIdentifier = OzoneManager.getStsTokenIdentifier(); + if (stsTokenIdentifier != null) { + return S3STSUtils.resolveCallerIdentityForStsCredentials( + stsTokenIdentifier.getAssumedRoleId(), stsTokenIdentifier.getAssumedRoleUserArn()).getProtobuf(); + } + final String resolvedPrincipal = OzoneAclUtils.accessIdToUserPrincipal(OzoneManager.getS3AuthEffectiveAccessId()); + final String kerberosShortName = UserGroupInformation.createRemoteUser(resolvedPrincipal).getShortUserName(); + return S3STSUtils.resolveCallerIdentityForPermanentCredentials(resolvedPrincipal, kerberosShortName).getProtobuf(); + } + @DisallowedUntilLayoutVersion(FILESYSTEM_SNAPSHOT) private SnapshotDiffResponse snapshotDiff( SnapshotDiffRequest snapshotDiffRequest) throws IOException { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/S3SecurityUtil.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/S3SecurityUtil.java index 08ac1f2bee11..6612fff2bad8 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/S3SecurityUtil.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/S3SecurityUtil.java @@ -75,8 +75,10 @@ public static void validateS3Credential(OMRequest omRequest, token, ozoneManager.getSecretKeyClient(), CLOCK); // Ensure the token is not revoked - if (isRevokedStsToken(token, ozoneManager)) { - LOG.info("Session token has been revoked: {}, {}", stsTokenIdentifier.getTempAccessKeyId(), token); + if (isRevokedStsToken(stsTokenIdentifier, ozoneManager)) { + LOG.info( + "STS token has been revoked for originalAccessKeyId={}, tempAccessKeyId={}", + stsTokenIdentifier.getOriginalAccessKeyId(), stsTokenIdentifier.getTempAccessKeyId()); throw new OMException("STS token has been revoked", REVOKED_TOKEN); } @@ -157,11 +159,12 @@ private static void validateSTSTokenAwsSignature(STSTokenIdentifier stsTokenIden } /** - * Returns true if the STS session token is present in the revoked STS token table. + * Returns true if the STS token was created before the revocation cutoff for its originalAccessKeyId. */ - private static boolean isRevokedStsToken(String sessionToken, OzoneManager ozoneManager) + private static boolean isRevokedStsToken(STSTokenIdentifier stsTokenIdentifier, OzoneManager ozoneManager) throws OMException { try { + final String originalAccessKeyId = stsTokenIdentifier.getOriginalAccessKeyId(); final OMMetadataManager metadataManager = ozoneManager.getMetadataManager(); if (metadataManager == null) { final String msg = "Could not determine STS revocation: metadataManager is null"; @@ -176,7 +179,9 @@ private static boolean isRevokedStsToken(String sessionToken, OzoneManager ozone throw new OMException(msg, INTERNAL_ERROR); } - return revokedStsTokenTable.getIfExist(sessionToken) != null; + final Long revocationTimeMillis = revokedStsTokenTable.getIfExist(originalAccessKeyId); + return revocationTimeMillis != null + && stsTokenIdentifier.getCreationTime().toEpochMilli() < revocationTimeMillis; } catch (Exception e) { final String msg = "Could not determine STS revocation because of Exception: " + e.getMessage(); LOG.warn(msg, e); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/STSSecurityUtil.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/STSSecurityUtil.java index 2212ad6db797..41aefeee7682 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/STSSecurityUtil.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/STSSecurityUtil.java @@ -154,10 +154,14 @@ private static ManagedSecretKey getValidatedSecretKey(UUID secretKeyId, SecretKe private static Token decodeTokenFromString(String encodedToken) throws SecretManager.InvalidToken { final Token token = new Token<>(); + // token.decodeFromUrlString() only declares IOException, but deserialization can throw + // unchecked exceptions (e.g. NegativeArraySizeException) when malformed input decodes to a + // negative byte-array length. Map those to InvalidToken (via catching RuntimeException) + // instead of failing the OM request. try { token.decodeFromUrlString(encodedToken); return token; - } catch (IOException e) { + } catch (IOException | RuntimeException e) { throw new SecretManager.InvalidToken("Failed to decode STS token string: " + e); } } @@ -180,6 +184,9 @@ static void ensureEssentialFieldsArePresentInToken(STSTokenIdentifier stsTokenId if (StringUtils.isEmpty(stsTokenIdentifier.getSecretAccessKey())) { throw new SecretManager.InvalidToken("Invalid STS token - secretAccessKey is null/empty"); } + if (stsTokenIdentifier.getCreationTime() == null) { + throw new SecretManager.InvalidToken("Invalid STS token - creationTime is null"); + } } /** diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/STSTokenIdentifier.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/STSTokenIdentifier.java index 8c13aac51905..9535be2f030c 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/STSTokenIdentifier.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/STSTokenIdentifier.java @@ -46,6 +46,9 @@ public class STSTokenIdentifier extends ShortLivedTokenIdentifier { private String originalAccessKeyId; private String secretAccessKey; private String sessionPolicy; + private String assumedRoleId; + private String assumedRoleUserArn; + private Instant creationTime; // Encryption key derived from ManagedSecretKey for this token private transient byte[] encryptionKey; @@ -63,23 +66,161 @@ public STSTokenIdentifier() { /** * Create a new STS token identifier with encryption support. * - * @param tempAccessKeyId the temporary access key ID (owner) - * @param originalAccessKeyId the original long-lived access key ID that created this token - * @param roleArn the ARN of the assumed role - * @param expiry the token expiration time - * @param secretAccessKey the secret access key associated with the temporary access key ID - * @param sessionPolicy an optional opaque identifier that further limits the scope of - * the permissions granted by the role - * @param encryptionKey the key bytes for encrypting sensitive fields + * @param params the STS token creation parameters */ - public STSTokenIdentifier(String tempAccessKeyId, String originalAccessKeyId, String roleArn, Instant expiry, - String secretAccessKey, String sessionPolicy, byte[] encryptionKey) { - super(tempAccessKeyId, expiry); - this.originalAccessKeyId = originalAccessKeyId; - this.roleArn = roleArn; - this.secretAccessKey = secretAccessKey; - this.sessionPolicy = sessionPolicy; - this.encryptionKey = encryptionKey != null ? encryptionKey.clone() : null; + public STSTokenIdentifier(Params params) { + super(params.getTempAccessKeyId(), params.getExpiry()); + this.originalAccessKeyId = params.getOriginalAccessKeyId(); + this.roleArn = params.getRoleArn(); + this.creationTime = params.getCreationTime(); + this.secretAccessKey = params.getSecretAccessKey(); + this.sessionPolicy = params.getSessionPolicy(); + this.assumedRoleId = params.getAssumedRoleId(); + this.assumedRoleUserArn = params.getAssumedRoleUserArn(); + this.encryptionKey = params.getEncryptionKey(); // already cloned via Params + } + + /** + * Parameters for constructing an {@link STSTokenIdentifier}. + */ + public static final class Params { + private final String tempAccessKeyId; + private final String originalAccessKeyId; + private final String roleArn; + private final Instant creationTime; + private final Instant expiry; + private final String secretAccessKey; + private final String sessionPolicy; + private final String assumedRoleId; + private final String assumedRoleUserArn; + private final byte[] encryptionKey; + + private Params(Builder builder) { + this.tempAccessKeyId = builder.tempAccessKeyId; + this.originalAccessKeyId = builder.originalAccessKeyId; + this.roleArn = builder.roleArn; + this.creationTime = builder.creationTime; + this.expiry = builder.expiry; + this.secretAccessKey = builder.secretAccessKey; + this.sessionPolicy = builder.sessionPolicy; + this.assumedRoleId = builder.assumedRoleId; + this.assumedRoleUserArn = builder.assumedRoleUserArn; + this.encryptionKey = builder.encryptionKey; + } + + public static Builder newBuilder() { + return new Builder(); + } + + public String getTempAccessKeyId() { + return tempAccessKeyId; + } + + public String getOriginalAccessKeyId() { + return originalAccessKeyId; + } + + public String getRoleArn() { + return roleArn; + } + + public Instant getCreationTime() { + return creationTime; + } + + public Instant getExpiry() { + return expiry; + } + + public String getSecretAccessKey() { + return secretAccessKey; + } + + public String getSessionPolicy() { + return sessionPolicy; + } + + public String getAssumedRoleId() { + return assumedRoleId; + } + + public String getAssumedRoleUserArn() { + return assumedRoleUserArn; + } + + public byte[] getEncryptionKey() { + return encryptionKey != null ? encryptionKey.clone() : null; + } + + /** + * Builder for {@link Params}. + */ + public static final class Builder { + private String tempAccessKeyId; + private String originalAccessKeyId; + private String roleArn; + private Instant creationTime; + private Instant expiry; + private String secretAccessKey; + private String sessionPolicy; + private String assumedRoleId; + private String assumedRoleUserArn; + private byte[] encryptionKey; + + public Builder setTempAccessKeyId(String value) { + this.tempAccessKeyId = value; + return this; + } + + public Builder setOriginalAccessKeyId(String value) { + this.originalAccessKeyId = value; + return this; + } + + public Builder setRoleArn(String value) { + this.roleArn = value; + return this; + } + + public Builder setCreationTime(Instant value) { + this.creationTime = value; + return this; + } + + public Builder setExpiry(Instant value) { + this.expiry = value; + return this; + } + + public Builder setSecretAccessKey(String value) { + this.secretAccessKey = value; + return this; + } + + public Builder setSessionPolicy(String value) { + this.sessionPolicy = value; + return this; + } + + public Builder setAssumedRoleId(String value) { + this.assumedRoleId = value; + return this; + } + + public Builder setAssumedRoleUserArn(String value) { + this.assumedRoleUserArn = value; + return this; + } + + public Builder setEncryptionKey(byte[] value) { + this.encryptionKey = value != null ? value.clone() : null; + return this; + } + + public Params build() { + return new Params(this); + } + } } @Override @@ -123,13 +264,16 @@ public OMTokenProto toProtoBuf() { builder .setType(OMTokenProto.Type.S3_STS_TOKEN) + .setIssueDate(creationTime.toEpochMilli()) .setMaxDate(getExpiry().toEpochMilli()) .setOwner(getOwnerId() != null ? getOwnerId() : "") .setAccessKeyId(getOwnerId() != null ? getOwnerId() : "") .setOriginalAccessKeyId(originalAccessKeyId != null ? originalAccessKeyId : "") .setRoleArn(roleArn != null ? roleArn : "") .setSecretAccessKey(secretAccessKey != null ? encryptSensitiveField(secretAccessKey) : "") - .setSessionPolicy(sessionPolicy != null ? sessionPolicy : ""); + .setSessionPolicy(sessionPolicy != null ? sessionPolicy : "") + .setAssumedRoleId(assumedRoleId != null ? assumedRoleId : "") + .setAssumedRoleUserArn(assumedRoleUserArn != null ? assumedRoleUserArn : ""); return builder.build(); } @@ -146,6 +290,9 @@ public void fromProtoBuf(OMTokenProto token) throws IOException { setOwnerId(token.getOwner()); setExpiry(Instant.ofEpochMilli(token.getMaxDate())); + if (token.hasIssueDate()) { + this.creationTime = Instant.ofEpochMilli(token.getIssueDate()); + } if (token.hasOriginalAccessKeyId()) { this.originalAccessKeyId = token.getOriginalAccessKeyId(); } @@ -169,6 +316,12 @@ public void fromProtoBuf(OMTokenProto token) throws IOException { if (token.hasSessionPolicy()) { this.sessionPolicy = token.getSessionPolicy(); } + if (token.hasAssumedRoleId()) { + this.assumedRoleId = token.getAssumedRoleId(); + } + if (token.hasAssumedRoleUserArn()) { + this.assumedRoleUserArn = token.getAssumedRoleUserArn(); + } } /** @@ -244,6 +397,18 @@ public String getSessionPolicy() { return sessionPolicy; } + public String getAssumedRoleId() { + return assumedRoleId; + } + + public String getAssumedRoleUserArn() { + return assumedRoleUserArn; + } + + public Instant getCreationTime() { + return creationTime; + } + public void setEncryptionKey(byte[] encryptionKey) { this.encryptionKey = encryptionKey.clone(); } @@ -265,13 +430,15 @@ public boolean equals(Object o) { final STSTokenIdentifier that = (STSTokenIdentifier) o; return Objects.equals(roleArn, that.roleArn) && Objects.equals(secretAccessKey, that.secretAccessKey) && Objects.equals(originalAccessKeyId, that.originalAccessKeyId) && - Objects.equals(sessionPolicy, that.sessionPolicy); + Objects.equals(sessionPolicy, that.sessionPolicy) && Objects.equals(assumedRoleId, that.assumedRoleId) && + Objects.equals(assumedRoleUserArn, that.assumedRoleUserArn) && Objects.equals(creationTime, that.creationTime); } @Override public int hashCode() { return Objects.hash( - super.hashCode(), roleArn, secretAccessKey, originalAccessKeyId, sessionPolicy); + super.hashCode(), roleArn, secretAccessKey, originalAccessKeyId, sessionPolicy, assumedRoleId, + assumedRoleUserArn, creationTime); } @Override @@ -279,7 +446,8 @@ public String toString() { // Intentionally left off secretAccessKey return "STSTokenIdentifier{" + "tempAccessKeyId='" + getOwnerId() + "'" + ", originalAccessKeyId='" + originalAccessKeyId + "', roleArn='" + roleArn + "'" + - ", expiry='" + getExpiry() + "', secretKeyId='" + getSecretKeyId() + "'" + - ", sessionPolicy='" + sessionPolicy + "'}"; + ", assumedRoleId='" + assumedRoleId + "', assumedRoleUserArn='" + assumedRoleUserArn + "'" + + ", creationTime='" + creationTime + "', expiry='" + getExpiry() + "', secretKeyId='" + getSecretKeyId() + + "', sessionPolicy='" + sessionPolicy + "'}"; } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/STSTokenSecretManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/STSTokenSecretManager.java index f72b1892de85..be952f108f3b 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/STSTokenSecretManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/STSTokenSecretManager.java @@ -17,9 +17,11 @@ package org.apache.hadoop.ozone.security; +import com.google.common.base.Preconditions; import java.io.IOException; import java.time.Clock; import java.time.Instant; +import java.util.Objects; import org.apache.hadoop.hdds.annotation.InterfaceAudience; import org.apache.hadoop.hdds.annotation.InterfaceStability; import org.apache.hadoop.hdds.security.symmetric.ManagedSecretKey; @@ -65,6 +67,15 @@ public STSTokenSecretManager(SecretKeySignerClient secretKeyClient) { public Token generateToken(STSTokenIdentifier tokenIdentifier) { final ManagedSecretKey secretKey = secretKeyClient.getCurrentSecretKey(); tokenIdentifier.setSecretKeyId(secretKey.getId()); + return generateToken(tokenIdentifier, secretKey); + } + + private Token generateToken(STSTokenIdentifier tokenIdentifier, ManagedSecretKey secretKey) { + Objects.requireNonNull( + tokenIdentifier.getSecretKeyId(), "secretKeyId must be set on the token identifier before signing"); + Preconditions.checkArgument( + secretKey.getId().equals(tokenIdentifier.getSecretKeyId()), "secretKeyId on the token identifier " + + "must match the signing secret key"); final byte[] identifierBytes = tokenIdentifier.getBytes(); final byte[] password = secretKey.sign(identifierBytes); return new Token<>(identifierBytes, password, tokenIdentifier.getKind(), new Text(tokenIdentifier.getService())); @@ -73,33 +84,168 @@ public Token generateToken(STSTokenIdentifier tokenIdentifie /** * Create an STS token and return it as an encoded string. * - * @param tempAccessKeyId the temporary access key ID - * @param originalAccessKeyId the original long-lived access key ID - * @param roleArn the ARN of the assumed role - * @param durationSeconds how long the token should be valid for - * @param secretAccessKey the secret access key associated with the temporary access key ID - * @param sessionPolicy an optional opaque identifier that further limits the scope of - * the permissions granted by the role - * @param clock the system clock + * @param params the STS token creation parameters * @return base64 encoded token string */ - public String createSTSTokenString(String tempAccessKeyId, String originalAccessKeyId, String roleArn, - int durationSeconds, String secretAccessKey, String sessionPolicy, Clock clock) throws IOException { - final Instant expiration = clock.instant().plusSeconds(durationSeconds); + public String createSTSTokenString(CreateSTSTokenParams params) throws IOException { + final Instant creationTime = params.getClock().instant(); + final Instant expiration = creationTime.plusSeconds(params.getDurationSeconds()); - // Get the current secret key for encryption - final ManagedSecretKey currentSecretKey = secretKeyClient.getCurrentSecretKey(); - final byte[] encryptionKey = currentSecretKey.getSecretKey().getEncoded(); + // Get the current secret key once for encryption, secretKeyId, and signing. + final ManagedSecretKey secretKey = secretKeyClient.getCurrentSecretKey(); + final byte[] encryptionKey = secretKey.getSecretKey().getEncoded(); // Note - the encryptionKey will NOT be encoded in the token. When generateToken() is called, it eventually calls // the write() method in STSTokenIdentifier which calls toProtoBuf(), and the encryptionKey is not // serialized there. - final STSTokenIdentifier identifier = new STSTokenIdentifier( - tempAccessKeyId, originalAccessKeyId, roleArn, expiration, secretAccessKey, sessionPolicy, encryptionKey); + final STSTokenIdentifier identifier = new STSTokenIdentifier(STSTokenIdentifier.Params.newBuilder() + .setTempAccessKeyId(params.getTempAccessKeyId()) + .setOriginalAccessKeyId(params.getOriginalAccessKeyId()) + .setRoleArn(params.getRoleArn()) + .setCreationTime(creationTime) + .setExpiry(expiration) + .setSecretAccessKey(params.getSecretAccessKey()) + .setSessionPolicy(params.getSessionPolicy()) + .setAssumedRoleId(params.getAssumedRoleId()) + .setAssumedRoleUserArn(params.getAssumedRoleUserArn()) + .setEncryptionKey(encryptionKey) + .build()); + identifier.setSecretKeyId(secretKey.getId()); - final Token token = generateToken(identifier); + final Token token = generateToken(identifier, secretKey); return token.encodeToUrlString(); } + + /** + * Parameters for {@link #createSTSTokenString(CreateSTSTokenParams)}. + */ + public static final class CreateSTSTokenParams { + private final String tempAccessKeyId; + private final String originalAccessKeyId; + private final String roleArn; + private final int durationSeconds; + private final String secretAccessKey; + private final String sessionPolicy; + private final String assumedRoleId; + private final String assumedRoleUserArn; + private final Clock clock; + + private CreateSTSTokenParams(Builder builder) { + this.tempAccessKeyId = builder.tempAccessKeyId; + this.originalAccessKeyId = builder.originalAccessKeyId; + this.roleArn = builder.roleArn; + this.durationSeconds = builder.durationSeconds; + this.secretAccessKey = builder.secretAccessKey; + this.sessionPolicy = builder.sessionPolicy; + this.assumedRoleId = builder.assumedRoleId; + this.assumedRoleUserArn = builder.assumedRoleUserArn; + this.clock = builder.clock; + } + + public static Builder newBuilder() { + return new Builder(); + } + + public String getTempAccessKeyId() { + return tempAccessKeyId; + } + + public String getOriginalAccessKeyId() { + return originalAccessKeyId; + } + + public String getRoleArn() { + return roleArn; + } + + public int getDurationSeconds() { + return durationSeconds; + } + + public String getSecretAccessKey() { + return secretAccessKey; + } + + public String getSessionPolicy() { + return sessionPolicy; + } + + public String getAssumedRoleId() { + return assumedRoleId; + } + + public String getAssumedRoleUserArn() { + return assumedRoleUserArn; + } + + public Clock getClock() { + return clock; + } + + /** + * Builder for {@link CreateSTSTokenParams}. + */ + public static final class Builder { + private String tempAccessKeyId; + private String originalAccessKeyId; + private String roleArn; + private int durationSeconds; + private String secretAccessKey; + private String sessionPolicy; + private String assumedRoleId; + private String assumedRoleUserArn; + private Clock clock; + + public Builder setTempAccessKeyId(String value) { + this.tempAccessKeyId = value; + return this; + } + + public Builder setOriginalAccessKeyId(String value) { + this.originalAccessKeyId = value; + return this; + } + + public Builder setRoleArn(String value) { + this.roleArn = value; + return this; + } + + public Builder setDurationSeconds(int value) { + this.durationSeconds = value; + return this; + } + + public Builder setSecretAccessKey(String value) { + this.secretAccessKey = value; + return this; + } + + public Builder setSessionPolicy(String value) { + this.sessionPolicy = value; + return this; + } + + public Builder setAssumedRoleId(String value) { + this.assumedRoleId = value; + return this; + } + + public Builder setAssumedRoleUserArn(String value) { + this.assumedRoleUserArn = value; + return this; + } + + public Builder setClock(Clock value) { + this.clock = value; + return this; + } + + public CreateSTSTokenParams build() { + return new CreateSTSTokenParams(this); + } + } + } } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java index d0d11c5ba94f..c284b460dd68 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java @@ -1535,9 +1535,9 @@ public void testS3RevokedStsTokenTablePutAndGet() throws Exception { assertNotNull(omMetadataManager.getS3RevokedStsTokenTable(), "s3RevokedStsTokenTable should be initialized"); final MockClock clock = MockClock.newInstance(); - final String sessionToken1 = "test-session-token-1"; + final String originalAccessKeyId1 = "orig-1"; final long insertionTime1 = clock.millis(); - final String sessionToken2 = "test-session-token-2"; + final String originalAccessKeyId2 = "orig-2"; final long insertionTime2 = insertionTime1 + 1234L; // This table is configured as FULL_CACHE in OmMetadataManagerImpl. @@ -1546,25 +1546,25 @@ public void testS3RevokedStsTokenTablePutAndGet() throws Exception { final TypedTable revokedTable = (TypedTable) omMetadataManager.getS3RevokedStsTokenTable(); - revokedTable.put(sessionToken1, insertionTime1); - revokedTable.put(sessionToken2, insertionTime2); + revokedTable.put(originalAccessKeyId1, insertionTime1); + revokedTable.put(originalAccessKeyId2, insertionTime2); // Verify the values are persisted in RocksDB. - assertEquals(insertionTime1, revokedTable.getSkipCache(sessionToken1)); - assertEquals(insertionTime2, revokedTable.getSkipCache(sessionToken2)); + assertEquals(insertionTime1, revokedTable.getSkipCache(originalAccessKeyId1)); + assertEquals(insertionTime2, revokedTable.getSkipCache(originalAccessKeyId2)); // Update cache to make get/getIfExist reflect the write for FULL_CACHE tables. - revokedTable.addCacheEntry(sessionToken1, insertionTime1, 1L); - revokedTable.addCacheEntry(sessionToken2, insertionTime2, 1L); + revokedTable.addCacheEntry(originalAccessKeyId1, insertionTime1, 1L); + revokedTable.addCacheEntry(originalAccessKeyId2, insertionTime2, 1L); // Verify get and getIfExist return the stored value - assertEquals(insertionTime1, revokedTable.get(sessionToken1)); - assertEquals(insertionTime1, revokedTable.getIfExist(sessionToken1)); - assertEquals(insertionTime2, revokedTable.get(sessionToken2)); - assertEquals(insertionTime2, revokedTable.getIfExist(sessionToken2)); + assertEquals(insertionTime1, revokedTable.get(originalAccessKeyId1)); + assertEquals(insertionTime1, revokedTable.getIfExist(originalAccessKeyId1)); + assertEquals(insertionTime2, revokedTable.get(originalAccessKeyId2)); + assertEquals(insertionTime2, revokedTable.getIfExist(originalAccessKeyId2)); - // Invalid sessionToken should return null for getIfExist - assertNull(revokedTable.getIfExist("INVALID_SESSION_TOKEN")); + // Invalid originalAccessKeyId should return null for getIfExist. + assertNull(revokedTable.getIfExist("INVALID_ORIGINAL_ACCESS_KEY_ID")); } @Test diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/security/TestS3RevokeSTSTokenRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/security/TestS3RevokeSTSTokenRequest.java index 0f3d2519b30c..1b6caed9cb40 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/security/TestS3RevokeSTSTokenRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/security/TestS3RevokeSTSTokenRequest.java @@ -19,7 +19,9 @@ import static org.apache.hadoop.security.authentication.util.KerberosName.DEFAULT_MECHANISM; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.eq; import static org.mockito.Mockito.mock; @@ -29,16 +31,16 @@ import java.io.IOException; import java.util.Optional; import java.util.UUID; -import org.apache.hadoop.hdds.security.symmetric.SecretKeyClient; import org.apache.hadoop.hdds.utils.db.Table; import org.apache.hadoop.hdds.utils.db.cache.CacheKey; -import org.apache.hadoop.hdds.utils.db.cache.CacheValue; import org.apache.hadoop.ipc_.ExternalCall; import org.apache.hadoop.ipc_.Server; +import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.audit.AuditLogger; import org.apache.hadoop.ozone.om.OMMetadataManager; import org.apache.hadoop.ozone.om.OMMultiTenantManager; import org.apache.hadoop.ozone.om.OzoneManager; +import org.apache.hadoop.ozone.om.S3SecretManager; import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.execution.flowcontrol.ExecutionContext; import org.apache.hadoop.ozone.om.request.OMClientRequest; @@ -46,11 +48,8 @@ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type; -import org.apache.hadoop.ozone.security.STSTokenSecretManager; -import org.apache.hadoop.ozone.security.SecretKeyTestClient; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.authentication.util.KerberosName; -import org.apache.ozone.test.MockClock; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -60,22 +59,22 @@ */ public class TestS3RevokeSTSTokenRequest { - private static final MockClock CLOCK = MockClock.newInstance(); + private static final String TEST_KERBEROS_RULES = + "RULE:[2:$1@$0](.*@EXAMPLE.COM)s/@.*//\n" + "RULE:[1:$1@$0](.*@EXAMPLE.COM)s/@.*//\n" + "DEFAULT"; - private STSTokenSecretManager stsTokenSecretManager; - private SecretKeyClient secretKeyClient; private OMMultiTenantManager omMultiTenantManager; + private String kerberosMechanismBeforeTest; + private String kerberosRulesBeforeTest; @BeforeEach public void setUp() throws Exception { + kerberosMechanismBeforeTest = KerberosName.getRuleMechanism(); + kerberosRulesBeforeTest = KerberosName.getRules(); + KerberosName.setRuleMechanism(DEFAULT_MECHANISM); // Initialize KerberosName rules so that UGI short names derived from // principals like "alice@EXAMPLE.COM" are computed correctly. - KerberosName.setRuleMechanism(DEFAULT_MECHANISM); - KerberosName.setRules( - "RULE:[2:$1@$0](.*@EXAMPLE.COM)s/@.*//\n" + "RULE:[1:$1@$0](.*@EXAMPLE.COM)s/@.*//\n" + "DEFAULT"); + KerberosName.setRules(TEST_KERBEROS_RULES); - secretKeyClient = new SecretKeyTestClient(); - stsTokenSecretManager = new STSTokenSecretManager(secretKeyClient); // Multi-tenant manager mock used for tests that exercise the S3 multi-tenancy permission branch. omMultiTenantManager = mock(OMMultiTenantManager.class); } @@ -83,15 +82,15 @@ public void setUp() throws Exception { @AfterEach public void tearDown() { Server.getCurCall().remove(); + KerberosName.setRuleMechanism(kerberosMechanismBeforeTest); + KerberosName.setRules(kerberosRulesBeforeTest); } @Test public void testPreExecuteFailsForNonOwnerOfOriginalAccessKey() throws Exception { - // Verify that preExecute enforces permissions based on the original access key id encoded in the STS token + // Verify that preExecute enforces permissions based on the request's original access key ID // and rejects revocation attempts from non-owners. - final String tempAccessKeyId = "ASIA12345678"; final String originalAccessKeyId = "original-access-key-id"; - final String sessionToken = createSessionToken(tempAccessKeyId, originalAccessKeyId); // An RPC call running another Kerberos identity should NOT be allowed to revoke the token whose original // access key id is different. @@ -100,24 +99,10 @@ public void testPreExecuteFailsForNonOwnerOfOriginalAccessKey() throws Exception OMException ex; try (OzoneManager ozoneManager = mock(OzoneManager.class)) { - when(ozoneManager.isS3MultiTenancyEnabled()).thenReturn(false); - when(ozoneManager.isS3Admin(any(UserGroupInformation.class))) - .thenReturn(false); - when(ozoneManager.getSecretKeyClient()).thenReturn(secretKeyClient); - - final OzoneManagerProtocolProtos.RevokeSTSTokenRequest revokeRequest = - OzoneManagerProtocolProtos.RevokeSTSTokenRequest.newBuilder() - .setSessionToken(sessionToken) - .build(); - - final OMRequest omRequest = OMRequest.newBuilder() - .setClientId(UUID.randomUUID().toString()) - .setCmdType(Type.RevokeSTSToken) - .setRevokeSTSTokenRequest(revokeRequest) - .build(); - - final OMClientRequest omClientRequest = new S3RevokeSTSTokenRequest(omRequest); + configureOzoneManagerForPreExecute(ozoneManager, originalAccessKeyId, true); + when(ozoneManager.isS3Admin(any(UserGroupInformation.class))).thenReturn(false); + final OMClientRequest omClientRequest = new S3RevokeSTSTokenRequest(buildRevokeOmRequest(originalAccessKeyId)); ex = assertThrows(OMException.class, () -> omClientRequest.preExecute(ozoneManager)); } assertEquals(OMException.ResultCodes.USER_MISMATCH, ex.getResult()); @@ -125,36 +110,25 @@ public void testPreExecuteFailsForNonOwnerOfOriginalAccessKey() throws Exception @Test public void testPreExecuteSucceedsForOriginalAccessKeyOwner() throws Exception { - // Verify that preExecute allows the owner of the original access key id (as encoded in the STS token) + // Verify that preExecute allows the owner of the original access key ID from the revoke request // to revoke the temporary credentials. - final String tempAccessKeyId = "ASIA4567891230"; final String originalAccessKeyId = "original-access-key-id"; - final String sessionToken = createSessionToken(tempAccessKeyId, originalAccessKeyId); // Simulate RPC call running as originalAccessKeyId final UserGroupInformation originalUgi = UserGroupInformation.createRemoteUser(originalAccessKeyId); Server.getCurCall().set(new StubCall(originalUgi)); final OzoneManager ozoneManager = mock(OzoneManager.class); - when(ozoneManager.isS3MultiTenancyEnabled()).thenReturn(false); - when(ozoneManager.isS3Admin(any(UserGroupInformation.class))) - .thenReturn(false); - when(ozoneManager.getSecretKeyClient()).thenReturn(secretKeyClient); - - final OzoneManagerProtocolProtos.RevokeSTSTokenRequest revokeRequest = - OzoneManagerProtocolProtos.RevokeSTSTokenRequest.newBuilder() - .setSessionToken(sessionToken) - .build(); - - final OMRequest omRequest = OMRequest.newBuilder() - .setClientId(UUID.randomUUID().toString()) - .setCmdType(Type.RevokeSTSToken) - .setRevokeSTSTokenRequest(revokeRequest) - .build(); + configureOzoneManagerForPreExecute(ozoneManager, originalAccessKeyId, true); + when(ozoneManager.isS3Admin(any(UserGroupInformation.class))).thenReturn(false); - final OMClientRequest omClientRequest = new S3RevokeSTSTokenRequest(omRequest); + final OMClientRequest omClientRequest = new S3RevokeSTSTokenRequest(buildRevokeOmRequest(originalAccessKeyId)); final OMRequest result = omClientRequest.preExecute(ozoneManager); + assertEquals(Type.RevokeSTSToken, result.getCmdType()); + assertTrue(result.getRevokeSTSTokenRequest().hasRevocationTimeMillis()); + assertEquals(originalAccessKeyId, result.getRevokeSTSTokenRequest().getOriginalAccessKeyId()); + assertTrue(result.getRevokeSTSTokenRequest().getRevocationTimeMillis() > 0L); } @Test @@ -163,40 +137,23 @@ public void testPreExecuteSucceedsForTenantAccessIdOwner() throws Exception { // the tenant access ID owner is allowed to revoke the temporary credentials. final String tenantId = "finance"; final String originalAccessKeyId = "alice@EXAMPLE.COM"; - final String tempAccessKeyId = "ASIA123456789"; - final String sessionToken = createSessionToken(tempAccessKeyId, originalAccessKeyId); // Caller short name "alice" should match the owner username returned from the multi-tenant manager. final UserGroupInformation callerUgi = UserGroupInformation.createRemoteUser(originalAccessKeyId); Server.getCurCall().set(new StubCall(callerUgi)); final OzoneManager ozoneManager = mock(OzoneManager.class); + configureOzoneManagerForPreExecute(ozoneManager, originalAccessKeyId, true); when(ozoneManager.isS3MultiTenancyEnabled()).thenReturn(true); when(ozoneManager.getMultiTenantManager()).thenReturn(omMultiTenantManager); - when(ozoneManager.getSecretKeyClient()).thenReturn(secretKeyClient); // Original access key id is assigned to a tenant and owned by "alice". - when(omMultiTenantManager.getTenantForAccessID(originalAccessKeyId)) - .thenReturn(Optional.of(tenantId)); - when(omMultiTenantManager.getUserNameGivenAccessId(originalAccessKeyId)) - .thenReturn("alice"); + when(omMultiTenantManager.getTenantForAccessID(originalAccessKeyId)).thenReturn(Optional.of(tenantId)); + when(omMultiTenantManager.getUserNameGivenAccessId(originalAccessKeyId)).thenReturn("alice"); // Not a tenant admin; ownership should be sufficient. - when(omMultiTenantManager.isTenantAdmin(callerUgi, tenantId, false)) - .thenReturn(false); - - final OzoneManagerProtocolProtos.RevokeSTSTokenRequest revokeRequest = - OzoneManagerProtocolProtos.RevokeSTSTokenRequest.newBuilder() - .setSessionToken(sessionToken) - .build(); - - final OMRequest omRequest = OMRequest.newBuilder() - .setClientId(UUID.randomUUID().toString()) - .setCmdType(Type.RevokeSTSToken) - .setRevokeSTSTokenRequest(revokeRequest) - .build(); - - final OMClientRequest omClientRequest = new S3RevokeSTSTokenRequest(omRequest); + when(omMultiTenantManager.isTenantAdmin(callerUgi, tenantId, false)).thenReturn(false); + final OMClientRequest omClientRequest = new S3RevokeSTSTokenRequest(buildRevokeOmRequest(originalAccessKeyId)); final OMRequest result = omClientRequest.preExecute(ozoneManager); assertEquals(Type.RevokeSTSToken, result.getCmdType()); } @@ -207,40 +164,23 @@ public void testPreExecuteSucceedsForTenantAdmin() throws Exception { // tenant admin (who is not the owner) is allowed to revoke the temporary credentials. final String tenantId = "finance"; final String originalAccessKeyId = "alice@EXAMPLE.COM"; - final String tempAccessKeyId = "ASIA4567890123"; - final String sessionToken = createSessionToken(tempAccessKeyId, originalAccessKeyId); // Caller short name "bob" does not own the access ID but will be configured as tenant admin. final UserGroupInformation callerUgi = UserGroupInformation.createRemoteUser("bob@EXAMPLE.COM"); Server.getCurCall().set(new StubCall(callerUgi)); final OzoneManager ozoneManager = mock(OzoneManager.class); + configureOzoneManagerForPreExecute(ozoneManager, originalAccessKeyId, true); when(ozoneManager.isS3MultiTenancyEnabled()).thenReturn(true); when(ozoneManager.getMultiTenantManager()).thenReturn(omMultiTenantManager); - when(ozoneManager.getSecretKeyClient()).thenReturn(secretKeyClient); // Original access key id is assigned to a tenant and owned by "alice". - when(omMultiTenantManager.getTenantForAccessID(originalAccessKeyId)) - .thenReturn(Optional.of(tenantId)); - when(omMultiTenantManager.getUserNameGivenAccessId(originalAccessKeyId)) - .thenReturn("alice"); + when(omMultiTenantManager.getTenantForAccessID(originalAccessKeyId)).thenReturn(Optional.of(tenantId)); + when(omMultiTenantManager.getUserNameGivenAccessId(originalAccessKeyId)).thenReturn("alice"); // Caller is configured as tenant admin so the check should pass. - when(omMultiTenantManager.isTenantAdmin(callerUgi, tenantId, false)) - .thenReturn(true); - - final OzoneManagerProtocolProtos.RevokeSTSTokenRequest revokeRequest = - OzoneManagerProtocolProtos.RevokeSTSTokenRequest.newBuilder() - .setSessionToken(sessionToken) - .build(); - - final OMRequest omRequest = OMRequest.newBuilder() - .setClientId(UUID.randomUUID().toString()) - .setCmdType(Type.RevokeSTSToken) - .setRevokeSTSTokenRequest(revokeRequest) - .build(); - - final OMClientRequest omClientRequest = new S3RevokeSTSTokenRequest(omRequest); + when(omMultiTenantManager.isTenantAdmin(callerUgi, tenantId, false)).thenReturn(true); + final OMClientRequest omClientRequest = new S3RevokeSTSTokenRequest(buildRevokeOmRequest(originalAccessKeyId)); final OMRequest result = omClientRequest.preExecute(ozoneManager); assertEquals(Type.RevokeSTSToken, result.getCmdType()); } @@ -251,8 +191,6 @@ public void testPreExecuteFailsForNonOwnerNonAdminInTenant() throws Exception { // non-owner, non-admin caller is rejected. final String tenantId = "finance"; final String originalAccessKeyId = "alice@EXAMPLE.COM"; - final String tempAccessKeyId = "ASIA123456789"; - final String sessionToken = createSessionToken(tempAccessKeyId, originalAccessKeyId); // Caller short name "carol" does not own the access ID and is not // configured as tenant admin. @@ -261,42 +199,65 @@ public void testPreExecuteFailsForNonOwnerNonAdminInTenant() throws Exception { final OMException ex; try (OzoneManager ozoneManager = mock(OzoneManager.class)) { + configureOzoneManagerForPreExecute(ozoneManager, originalAccessKeyId, true); when(ozoneManager.isS3MultiTenancyEnabled()).thenReturn(true); when(ozoneManager.getMultiTenantManager()).thenReturn(omMultiTenantManager); - when(ozoneManager.getSecretKeyClient()).thenReturn(secretKeyClient); - // Original access key id is assigned to a tenant and owned by "alice". - when(omMultiTenantManager.getTenantForAccessID(originalAccessKeyId)) - .thenReturn(Optional.of(tenantId)); - when(omMultiTenantManager.getUserNameGivenAccessId(originalAccessKeyId)) - .thenReturn("alice"); + when(omMultiTenantManager.getTenantForAccessID(originalAccessKeyId)).thenReturn(Optional.of(tenantId)); + when(omMultiTenantManager.getUserNameGivenAccessId(originalAccessKeyId)).thenReturn("alice"); // Caller is not a tenant admin. - when(omMultiTenantManager.isTenantAdmin(callerUgi, tenantId, false)) - .thenReturn(false); + when(omMultiTenantManager.isTenantAdmin(callerUgi, tenantId, false)).thenReturn(false); + + final OMClientRequest omClientRequest = new S3RevokeSTSTokenRequest(buildRevokeOmRequest(originalAccessKeyId)); + ex = assertThrows(OMException.class, () -> omClientRequest.preExecute(ozoneManager)); + } + assertEquals(OMException.ResultCodes.USER_MISMATCH, ex.getResult()); + } - final OzoneManagerProtocolProtos.RevokeSTSTokenRequest revokeRequest = - OzoneManagerProtocolProtos.RevokeSTSTokenRequest.newBuilder() - .setSessionToken(sessionToken) - .build(); + @Test + public void testPreExecuteRejectsUnknownOriginalAccessKeyId() throws Exception { + // Reject revocation when originalAccessKeyId has no S3 secret in RocksDB. + final String originalAccessKeyId = "unknown-access-key-id"; + final UserGroupInformation callerUgi = UserGroupInformation.createRemoteUser(originalAccessKeyId); + Server.getCurCall().set(new StubCall(callerUgi)); - final OMRequest omRequest = OMRequest.newBuilder() - .setClientId(UUID.randomUUID().toString()) - .setCmdType(Type.RevokeSTSToken) - .setRevokeSTSTokenRequest(revokeRequest) - .build(); + try (OzoneManager ozoneManager = mock(OzoneManager.class)) { + final S3SecretManager s3SecretManager = configureOzoneManagerForPreExecute( + ozoneManager, originalAccessKeyId, false); + final OMClientRequest omClientRequest = new S3RevokeSTSTokenRequest(buildRevokeOmRequest(originalAccessKeyId)); + final OMException ex = assertThrows(OMException.class, () -> omClientRequest.preExecute(ozoneManager)); + assertEquals(OMException.ResultCodes.ACCESS_ID_NOT_FOUND, ex.getResult()); + assertTrue(ex.getMessage().contains("does not exist")); + assertTrue(ex.getMessage().contains(originalAccessKeyId)); + verify(s3SecretManager).hasS3Secret(originalAccessKeyId); + } + } - final OMClientRequest omClientRequest = new S3RevokeSTSTokenRequest(omRequest); + @Test + public void testPreExecuteRejectsUnknownOriginalAccessKeyIdForS3Admin() throws Exception { + // S3 admins may revoke other principals' tokens, but not for unknown access key IDs. + final String originalAccessKeyId = "unknown-access-key-id"; + final UserGroupInformation adminUgi = UserGroupInformation.createRemoteUser("om-admin"); + Server.getCurCall().set(new StubCall(adminUgi)); - ex = assertThrows(OMException.class, () -> omClientRequest.preExecute(ozoneManager)); + try (OzoneManager ozoneManager = mock(OzoneManager.class)) { + final S3SecretManager s3SecretManager = configureOzoneManagerForPreExecute( + ozoneManager, originalAccessKeyId, false); + when(ozoneManager.isS3Admin(adminUgi)).thenReturn(true); + + final OMClientRequest omClientRequest = new S3RevokeSTSTokenRequest(buildRevokeOmRequest(originalAccessKeyId)); + final OMException ex = assertThrows(OMException.class, () -> omClientRequest.preExecute(ozoneManager)); + assertEquals(OMException.ResultCodes.ACCESS_ID_NOT_FOUND, ex.getResult()); + assertTrue(ex.getMessage().contains("does not exist")); + assertTrue(ex.getMessage().contains(originalAccessKeyId)); + verify(s3SecretManager).hasS3Secret(originalAccessKeyId); } - assertEquals(OMException.ResultCodes.USER_MISMATCH, ex.getResult()); } @Test - public void testValidateAndUpdateCacheUpdatesCacheImmediately() throws Exception { - final String tempAccessKeyId = "ASIA4567891230"; + public void testValidateAndUpdateCacheUpdatesCacheImmediately() { final String originalAccessKeyId = "original-access-key-id"; - final String sessionToken = createSessionToken(tempAccessKeyId, originalAccessKeyId); + final long revocationTimeMillis = 1_700_000_000_000L; final OzoneManager ozoneManager = mock(OzoneManager.class); final OMMetadataManager omMetadataManager = mock(OMMetadataManager.class); @@ -311,7 +272,8 @@ public void testValidateAndUpdateCacheUpdatesCacheImmediately() throws Exception final OzoneManagerProtocolProtos.RevokeSTSTokenRequest revokeRequest = OzoneManagerProtocolProtos.RevokeSTSTokenRequest.newBuilder() - .setSessionToken(sessionToken) + .setOriginalAccessKeyId(originalAccessKeyId) + .setRevocationTimeMillis(revocationTimeMillis) .build(); final OMRequest omRequest = OMRequest.newBuilder() @@ -324,12 +286,91 @@ public void testValidateAndUpdateCacheUpdatesCacheImmediately() throws Exception final OMClientResponse omClientResponse = s3RevokeSTSTokenRequest.validateAndUpdateCache(ozoneManager, context); assertEquals(OzoneManagerProtocolProtos.Status.OK, omClientResponse.getOMResponse().getStatus()); - verify(s3RevokedStsTokenTable).addCacheEntry(eq(new CacheKey<>(sessionToken)), any(CacheValue.class)); + verify(s3RevokedStsTokenTable).addCacheEntry( + eq(new CacheKey<>(originalAccessKeyId)), any()); + assertNotNull(s3RevokeSTSTokenRequest.getAuditBuilder().getAuditMap()); + assertEquals( + originalAccessKeyId, s3RevokeSTSTokenRequest.getAuditBuilder().getAuditMap().get( + OzoneConsts.S3_REVOKESTSTOKEN_USER)); + } + + @Test + public void testValidateAndUpdateCacheRejectsMissingRevocationTimeMillis() { + final String originalAccessKeyId = "original-access-key-id"; + + final OzoneManager ozoneManager = mock(OzoneManager.class); + final OMMetadataManager omMetadataManager = mock(OMMetadataManager.class); + @SuppressWarnings("unchecked") + final Table s3RevokedStsTokenTable = mock(Table.class); + final ExecutionContext context = mock(ExecutionContext.class); + + when(ozoneManager.getMetadataManager()).thenReturn(omMetadataManager); + when(omMetadataManager.getS3RevokedStsTokenTable()).thenReturn(s3RevokedStsTokenTable); + + final OzoneManagerProtocolProtos.RevokeSTSTokenRequest revokeRequest = + OzoneManagerProtocolProtos.RevokeSTSTokenRequest.newBuilder() + .setOriginalAccessKeyId(originalAccessKeyId) + .build(); + + final OMRequest omRequest = OMRequest.newBuilder() + .setClientId(UUID.randomUUID().toString()) + .setCmdType(Type.RevokeSTSToken) + .setRevokeSTSTokenRequest(revokeRequest) + .build(); + + final S3RevokeSTSTokenRequest s3RevokeSTSTokenRequest = new S3RevokeSTSTokenRequest(omRequest); + final OMClientResponse omClientResponse = + s3RevokeSTSTokenRequest.validateAndUpdateCache(ozoneManager, context); + assertEquals(OzoneManagerProtocolProtos.Status.INTERNAL_ERROR, omClientResponse.getOMResponse().getStatus()); + } + + @Test + public void testPreExecuteRejectsClientSuppliedRevocationTimeMillis() throws Exception { + final String originalAccessKeyId = "original-access-key-id"; + final UserGroupInformation callerUgi = UserGroupInformation.createRemoteUser(originalAccessKeyId); + Server.getCurCall().set(new StubCall(callerUgi)); + + final OzoneManagerProtocolProtos.RevokeSTSTokenRequest revokeRequest = + OzoneManagerProtocolProtos.RevokeSTSTokenRequest.newBuilder() + .setOriginalAccessKeyId(originalAccessKeyId) + .setRevocationTimeMillis(1_700_000_000_000L) + .build(); + final OMRequest omRequest = OMRequest.newBuilder() + .setClientId(UUID.randomUUID().toString()) + .setCmdType(Type.RevokeSTSToken) + .setRevokeSTSTokenRequest(revokeRequest) + .build(); + + try (OzoneManager ozoneManager = mock(OzoneManager.class)) { + configureOzoneManagerForPreExecute(ozoneManager, originalAccessKeyId, true); + final OMClientRequest omClientRequest = new S3RevokeSTSTokenRequest(omRequest); + final OMException ex = assertThrows(OMException.class, () -> omClientRequest.preExecute(ozoneManager)); + assertEquals(OMException.ResultCodes.INVALID_REQUEST, ex.getResult()); + } + } + + private static OMRequest buildRevokeOmRequest(String originalAccessKeyId) { + final OzoneManagerProtocolProtos.RevokeSTSTokenRequest revokeRequest = + OzoneManagerProtocolProtos.RevokeSTSTokenRequest.newBuilder() + .setOriginalAccessKeyId(originalAccessKeyId) + .build(); + + return OMRequest.newBuilder() + .setClientId(UUID.randomUUID().toString()) + .setCmdType(Type.RevokeSTSToken) + .setRevokeSTSTokenRequest(revokeRequest) + .build(); + } + + private static S3SecretManager configureOzoneManagerForPreExecute(OzoneManager ozoneManager, + String originalAccessKeyId, boolean hasSecret) throws IOException { + when(ozoneManager.isS3MultiTenancyEnabled()).thenReturn(false); + final S3SecretManager s3SecretManager = mock(S3SecretManager.class); + when(ozoneManager.getS3SecretManager()).thenReturn(s3SecretManager); + when(s3SecretManager.hasS3Secret(originalAccessKeyId)).thenReturn(hasSecret); + return s3SecretManager; } - /** - * Stub used to inject a remote user into the ProtobufRpcEngine.Server.getRemoteUser() thread-local. - */ private static final class StubCall extends ExternalCall { private final UserGroupInformation ugi; @@ -343,10 +384,4 @@ public UserGroupInformation getRemoteUser() { return ugi; } } - - private String createSessionToken(String tempAccessKeyId, String originalAccessKeyId) throws IOException { - return stsTokenSecretManager.createSTSTokenString( - tempAccessKeyId, originalAccessKeyId, "arn:aws:iam::123456789012:role/test-role", 3600, - "test-secret-access-key", "test-session-policy", CLOCK); - } } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestRevokedSTSTokenCleanupService.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestRevokedSTSTokenCleanupService.java index d7cf3630b955..2b734cea2456 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestRevokedSTSTokenCleanupService.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestRevokedSTSTokenCleanupService.java @@ -75,13 +75,13 @@ public void setUp() { @Test public void submitsCleanupRequestForOnlyExpiredTokens() throws Exception { - // If there are two revoked entries, one expired and one not expired, only the expired session token should be - // submitted for cleanup. + // If there are two revoked entries, one expired and one not expired, only the expired + // originalAccessKeyId should be submitted for cleanup. final long nowMillis = testClock.millis(); final long expiredCreationTimeMillis = nowMillis - TimeUnit.HOURS.toMillis(13); // older than 12h threshold final long validCreationTimeMillis = nowMillis - TimeUnit.HOURS.toMillis(1); - revokedStsTokenTable.put("session-token-a", expiredCreationTimeMillis); - revokedStsTokenTable.put("session-token-b", validCreationTimeMillis); + revokedStsTokenTable.put("original-access-key-a", expiredCreationTimeMillis); + revokedStsTokenTable.put("original-access-key-b", validCreationTimeMillis); final AtomicReference capturedRequest = new AtomicReference<>(); @@ -100,7 +100,7 @@ public void submitsCleanupRequestForOnlyExpiredTokens() throws Exception { final DeleteRevokedSTSTokensRequest deleteRevokedSTSTokensRequest = omRequest.getDeleteRevokedSTSTokensRequest(); - assertThat(deleteRevokedSTSTokensRequest.getSessionTokenList()).containsExactly("session-token-a"); + assertThat(deleteRevokedSTSTokensRequest.getOriginalAccessKeyIdList()).containsExactly("original-access-key-a"); } } @@ -109,8 +109,8 @@ public void doesNotSubmitRequestWhenThereAreNoExpiredTokens() throws Exception { // If only non-expired entries exist in the revoked sts token table, no cleanup request should be submitted and // no metrics should be updated. final long nowMillis = testClock.millis(); - revokedStsTokenTable.put("session-token-c", nowMillis - TimeUnit.HOURS.toMillis(1)); - revokedStsTokenTable.put("session-token-d", nowMillis - TimeUnit.HOURS.toMillis(2)); + revokedStsTokenTable.put("original-access-key-c", nowMillis - TimeUnit.HOURS.toMillis(1)); + revokedStsTokenTable.put("original-access-key-d", nowMillis - TimeUnit.HOURS.toMillis(2)); final AtomicReference capturedRequest = new AtomicReference<>(); @@ -149,8 +149,8 @@ public void doesNotUpdateMetricsOnRatisSubmissionServiceExceptionFailure() throw // If there are expired tokens in the table but the OM request submission to clean up the entries fails with a // service exception, the metrics should not be updated final long nowMillis = testClock.millis(); - revokedStsTokenTable.put("session-token-e", nowMillis - TimeUnit.HOURS.toMillis(13)); - revokedStsTokenTable.put("session-token-f", nowMillis - TimeUnit.HOURS.toMillis(14)); + revokedStsTokenTable.put("original-access-key-e", nowMillis - TimeUnit.HOURS.toMillis(13)); + revokedStsTokenTable.put("original-access-key-f", nowMillis - TimeUnit.HOURS.toMillis(14)); final AtomicInteger submitAttempts = new AtomicInteger(0); @@ -172,7 +172,7 @@ public void doesNotUpdateMetricsOnNonSuccessfulResponse() throws Exception { // If there is an expired token in the table but the OM request submission to clean up the entries gets a // non-successful response, the metrics should not be updated final long nowMillis = testClock.millis(); - revokedStsTokenTable.put("session-token-f", nowMillis - TimeUnit.HOURS.toMillis(20)); + revokedStsTokenTable.put("original-access-key-f", nowMillis - TimeUnit.HOURS.toMillis(20)); try (MockedStatic ozoneManagerRatisUtilsMock = mockStatic(OzoneManagerRatisUtils.class)) { // Return a non-successful response @@ -190,9 +190,9 @@ public void doesNotUpdateMetricsOnNonSuccessfulResponse() throws Exception { public void handlesAllExpiredTokens() throws Exception { // If all the tokens in the table are expired on a particular run, ensure the metrics are updated appropriately final long nowMillis = testClock.millis(); - revokedStsTokenTable.put("session-token-g", nowMillis - TimeUnit.HOURS.toMillis(13)); - revokedStsTokenTable.put("session-token-h", nowMillis - TimeUnit.HOURS.toMillis(14)); - revokedStsTokenTable.put("session-token-i", nowMillis - TimeUnit.HOURS.toMillis(15)); + revokedStsTokenTable.put("original-access-key-g", nowMillis - TimeUnit.HOURS.toMillis(13)); + revokedStsTokenTable.put("original-access-key-h", nowMillis - TimeUnit.HOURS.toMillis(14)); + revokedStsTokenTable.put("original-access-key-i", nowMillis - TimeUnit.HOURS.toMillis(15)); final AtomicReference capturedRequest = new AtomicReference<>(); @@ -211,8 +211,8 @@ public void handlesAllExpiredTokens() throws Exception { final DeleteRevokedSTSTokensRequest deleteRevokedSTSTokensRequest = omRequest.getDeleteRevokedSTSTokensRequest(); - assertThat(deleteRevokedSTSTokensRequest.getSessionTokenList()) - .containsExactlyInAnyOrder("session-token-g", "session-token-h", "session-token-i"); + assertThat(deleteRevokedSTSTokensRequest.getOriginalAccessKeyIdList()) + .containsExactlyInAnyOrder("original-access-key-g", "original-access-key-h", "original-access-key-i"); } } @@ -221,9 +221,9 @@ public void submitsMultipleRequestsWhenBatchSizeIsExceeded() throws Exception { // If the tokens exceed the configured batch size, multiple requests should be submitted final long nowMillis = testClock.millis(); - // Create 10 expired tokens + // Create 10 expired originalAccessKeyIds for (int i = 0; i < 10; i++) { - revokedStsTokenTable.put("session-token-" + i, nowMillis - TimeUnit.HOURS.toMillis(13)); + revokedStsTokenTable.put(String.format("AKIA%07d", i), nowMillis - TimeUnit.HOURS.toMillis(13)); } // Set a very small ratisByteLimit (100 bytes) to force batching. A single token request will be small, but 10 @@ -245,7 +245,7 @@ public void submitsMultipleRequestsWhenBatchSizeIsExceeded() throws Exception { // Verify all tokens were included across the requests final int totalTokens = capturedRequests.stream() - .mapToInt(r -> r.getDeleteRevokedSTSTokensRequest().getSessionTokenList().size()) + .mapToInt(r -> r.getDeleteRevokedSTSTokensRequest().getOriginalAccessKeyIdList().size()) .sum(); assertThat(totalTokens).isEqualTo(10); assertThat(revokedSTSTokenCleanupService.getSubmittedDeletedEntryCount()).isEqualTo(10); @@ -254,7 +254,7 @@ public void submitsMultipleRequestsWhenBatchSizeIsExceeded() throws Exception { @Test public void testSingleOversizedExpiredTokenAndItIsTheOnlyExpiredToken() throws Exception { - // One sessionToken is larger than the ratisByteLimit, and it is the only expired token + // One originalAccessKeyId is larger than the ratisByteLimit, and it is the only expired entry final long nowMillis = testClock.millis(); // Serialized size for largeToken is 102 > 90 (the effective ratisByteLimit) . final String largeToken = new String(new char[100]).replace('\0', 'a'); @@ -279,10 +279,10 @@ public void testSingleOversizedExpiredTokenAndItIsTheOnlyExpiredToken() throws E @Test public void testSingleOversizedExpiredTokenAndThereAreMultipleExpiredTokens() throws Exception { - // One sessionToken is larger than the ratisByteLimit, and it is not the only expired token + // One originalAccessKeyId is larger than the ratisByteLimit, and it is not the only expired entry final long nowMillis = testClock.millis(); - final String smallToken = "session-token-j"; - final String largeToken = "session-token-k-" + new String(new char[90]).replace('\0', 'a'); // > 90 bytes + final String smallToken = "AKIASMALL01"; + final String largeToken = "AKIALARGE-" + new String(new char[90]).replace('\0', 'a'); // > 90 bytes revokedStsTokenTable.put(smallToken, nowMillis - TimeUnit.HOURS.toMillis(13)); revokedStsTokenTable.put(largeToken, nowMillis - TimeUnit.HOURS.toMillis(13)); @@ -308,9 +308,9 @@ public void testExpiredAndNonExpiredTokensWithSmallRatisByteLimit() throws Excep // Expired and non-expired entries with ratisByteLimit of 100 final long nowMillis = testClock.millis(); - revokedStsTokenTable.put("session-token-l", nowMillis - TimeUnit.HOURS.toMillis(13)); - revokedStsTokenTable.put("session-token-m", nowMillis - TimeUnit.HOURS.toMillis(1)); // Should be skipped - revokedStsTokenTable.put("session-token-n", nowMillis - TimeUnit.HOURS.toMillis(13)); + revokedStsTokenTable.put("original-access-key-l", nowMillis - TimeUnit.HOURS.toMillis(13)); + revokedStsTokenTable.put("original-access-key-m", nowMillis - TimeUnit.HOURS.toMillis(1)); // Should be skipped + revokedStsTokenTable.put("original-access-key-n", nowMillis - TimeUnit.HOURS.toMillis(13)); ozoneConfiguration.setStorageSize( OMConfigKeys.OZONE_OM_RATIS_LOG_APPENDER_QUEUE_BYTE_LIMIT, 100, StorageUnit.BYTES); @@ -323,10 +323,11 @@ public void testExpiredAndNonExpiredTokensWithSmallRatisByteLimit() throws Excep final RevokedSTSTokenCleanupService revokedSTSTokenCleanupService = createAndRunCleanupService(); assertThat(revokedSTSTokenCleanupService.getRunCount()).isEqualTo(1); - // session-token-l and session-token-n fit in one batch. session-token-m is ignored because it is not expired. + // original-access-key-l and original-access-key-n fit in one batch. + // original-access-key-m is ignored because it is not expired. assertThat(capturedRequests).hasSize(1); - assertThat(capturedRequests.get(0).getDeleteRevokedSTSTokensRequest().getSessionTokenList()) - .containsExactly("session-token-l", "session-token-n"); + assertThat(capturedRequests.get(0).getDeleteRevokedSTSTokensRequest().getOriginalAccessKeyIdList()) + .containsExactly("original-access-key-l", "original-access-key-n"); assertThat(revokedSTSTokenCleanupService.getSubmittedDeletedEntryCount()).isEqualTo(2); } } @@ -359,12 +360,12 @@ public void testExpiredTokenMatchesRatisByteLimitExactly() throws Exception { public void testCallIdCountIncreasesAcrossBatches() throws Exception { // Force small batch of 40 bytes (which should trigger multiple calls to OzoneManagerRatisUtils.submitRequest) // and ensure the callIdCount increases across each batch - // session-token-1 and session-token-2 are in first batch, and session-token-3 is in second batch. + // AKIA0000001 and AKIA0000002 are in first batch, and AKIA0000003 is in second batch. final long nowMillis = testClock.millis(); - revokedStsTokenTable.put("session-token-1", nowMillis - TimeUnit.HOURS.toMillis(13)); - revokedStsTokenTable.put("session-token-2", nowMillis - TimeUnit.HOURS.toMillis(13)); - revokedStsTokenTable.put("session-token-3", nowMillis - TimeUnit.HOURS.toMillis(13)); + revokedStsTokenTable.put("AKIA0000001", nowMillis - TimeUnit.HOURS.toMillis(13)); + revokedStsTokenTable.put("AKIA0000002", nowMillis - TimeUnit.HOURS.toMillis(13)); + revokedStsTokenTable.put("AKIA0000003", nowMillis - TimeUnit.HOURS.toMillis(13)); ozoneConfiguration.setStorageSize(OMConfigKeys.OZONE_OM_RATIS_LOG_APPENDER_QUEUE_BYTE_LIMIT, 40, StorageUnit.BYTES); diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/TestS3SecurityUtil.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/TestS3SecurityUtil.java index 99c358b929d2..814f79b1e84b 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/TestS3SecurityUtil.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/TestS3SecurityUtil.java @@ -66,12 +66,10 @@ public class TestS3SecurityUtil { } @Test - public void testValidateS3CredentialFailsWhenTokenRevoked() throws Exception { - // If the revoked STS token table contains an entry for the session token, the request should be rejected with - // REVOKED_TOKEN + public void testValidateS3CredentialFailsWhenTokenCreatedBeforeRevocationCutoff() throws Exception { validateS3CredentialHelper( new TestConfig() - .setTokenRevoked(true) + .setRevocationCutoffOffsetMs(1) .setExpectedResult(REVOKED_TOKEN) .setExpectedMessage("STS token has been revoked")); } @@ -162,6 +160,22 @@ public void testValidateS3CredentialFailsWhenRequestAccessIdEmpty() throws Excep .setExpectedMessage("STS token validation failed - accessKeyId is invalid for session token")); } + @Test + public void testValidateS3CredentialSuccessWhenTokenCreatedAfterRevocationCutoff() throws Exception { + validateS3CredentialHelper( + new TestConfig() + .setRevocationCutoffOffsetMs(-1) + .setExpectedResult(null)); + } + + @Test + public void testValidateS3CredentialSuccessWhenTokenCreatedAtRevocationCutoff() throws Exception { + validateS3CredentialHelper( + new TestConfig() + .setRevocationCutoffOffsetMs(0) + .setExpectedResult(null)); + } + private void validateS3CredentialHelper(TestConfig config) throws Exception { try (OzoneManager ozoneManager = mock(OzoneManager.class)) { when(ozoneManager.isSecurityEnabled()).thenReturn(true); @@ -188,12 +202,13 @@ private void validateS3CredentialHelper(TestConfig config) throws Exception { } final String sessionToken = "session-token"; - if (config.isTokenRevoked && config.revokedSTSTokenTable != null) { - final long insertionTimeMillis = CLOCK.millis(); - config.revokedSTSTokenTable.put(sessionToken, insertionTimeMillis); - } - final STSTokenIdentifier stsTokenIdentifier = createSTSTokenIdentifier(); + final String originalAccessKeyId = stsTokenIdentifier.getOriginalAccessKeyId(); + if (config.revocationCutoffOffsetMs != null && config.revokedSTSTokenTable != null) { + final long revocationTimeMillis = stsTokenIdentifier.getCreationTime().toEpochMilli() + + config.revocationCutoffOffsetMs; + config.revokedSTSTokenTable.put(originalAccessKeyId, revocationTimeMillis); + } try (MockedStatic stsSecurityUtilMock = mockStatic(STSSecurityUtil.class, CALLS_REAL_METHODS); MockedStatic awsV4AuthValidatorMock = mockStatic( @@ -229,10 +244,16 @@ private void validateS3CredentialHelper(TestConfig config) throws Exception { } private STSTokenIdentifier createSTSTokenIdentifier() { - return new STSTokenIdentifier( - TEMP_ACCESS_KEY_ID, "original-access-key-id", "arn:aws:iam::123456789012:role/test-role", - CLOCK.instant().plusSeconds(3600), "secret-access-key", "session-policy", - ENCRYPTION_KEY); + return new STSTokenIdentifier(STSTokenIdentifier.Params.newBuilder() + .setTempAccessKeyId(TEMP_ACCESS_KEY_ID) + .setOriginalAccessKeyId("original-access-key-id") + .setRoleArn("arn:aws:iam::123456789012:role/test-role") + .setCreationTime(CLOCK.instant()) + .setExpiry(CLOCK.instant().plusSeconds(3600)) + .setSecretAccessKey("secret-access-key") + .setSessionPolicy("session-policy") + .setEncryptionKey(ENCRYPTION_KEY) + .build()); } private static OMRequest createRequestWithSessionToken(String accessId, boolean includeAccessId) { @@ -258,7 +279,7 @@ private static OMRequest createRequestWithSessionToken(String accessId, boolean private static final class TestConfig { private OMMetadataManager metadataManager = mock(OMMetadataManager.class); private Table revokedSTSTokenTable = new InMemoryTestTable<>(); - private boolean isTokenRevoked = false; + private Long revocationCutoffOffsetMs = null; private boolean isOriginalAccessKeyIdRevoked = false; private boolean shouldOriginalAccessKeyIdCheckThrowError = false; private String requestAccessId = TEMP_ACCESS_KEY_ID; @@ -277,9 +298,8 @@ TestConfig setRevokedSTSTokenTable(Table table) { return this; } - @SuppressWarnings("SameParameterValue") - TestConfig setTokenRevoked(boolean isRevoked) { - this.isTokenRevoked = isRevoked; + TestConfig setRevocationCutoffOffsetMs(long offsetMs) { + this.revocationCutoffOffsetMs = offsetMs; return this; } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/TestSTSSecurityUtil.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/TestSTSSecurityUtil.java index d2033deabec1..eb9ac4ecde5e 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/TestSTSSecurityUtil.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/TestSTSSecurityUtil.java @@ -17,6 +17,7 @@ package org.apache.hadoop.ozone.security; +import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.INVALID_TOKEN; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.TOKEN_EXPIRED; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -53,6 +54,9 @@ public class TestSTSSecurityUtil { private static final String ROLE_ARN = "arn:aws:iam::123456789012:role/test-role"; private static final String SECRET_ACCESS_KEY = "test-secret-access-key"; private static final String SESSION_POLICY = "test-session-policy"; + private static final String ASSUMED_ROLE_ID = "AROATEST123456789:testsess"; + private static final String ASSUMED_ROLE_USER_ARN = + "arn:aws:sts::123456789012:assumed-role/test-role/testsess"; private static final int DURATION_SECONDS = 3600; private static final byte[] ENCRYPTION_KEY = new byte[5]; @@ -85,8 +89,7 @@ public void testConstructValidateAndDecryptSTSTokenInvalidProtobuf() throws IOEx @Test public void testConstructValidateAndDecryptSTSTokenSuccess() throws IOException { // Create a valid token - final String tokenString = tokenSecretManager.createSTSTokenString( - TEMP_ACCESS_KEY, ORIGINAL_ACCESS_KEY, ROLE_ARN, DURATION_SECONDS, SECRET_ACCESS_KEY, SESSION_POLICY, clock); + final String tokenString = createStsTokenString(); // Validate and decrypt the token final STSTokenIdentifier result = STSSecurityUtil.constructValidateAndDecryptSTSToken( @@ -98,6 +101,7 @@ public void testConstructValidateAndDecryptSTSTokenSuccess() throws IOException assertThat(result.getRoleArn()).isEqualTo(ROLE_ARN); assertThat(result.getSecretAccessKey()).isEqualTo(SECRET_ACCESS_KEY); assertThat(result.getSessionPolicy()).isEqualTo(SESSION_POLICY); + assertThat(result.getCreationTime()).isEqualTo(clock.instant()); assertThat(result.isExpired(clock.instant())).isFalse(); final long expirationEpochMillis = result.getExpiry().toEpochMilli(); assertThat(expirationEpochMillis).isEqualTo(clock.millis() + (DURATION_SECONDS * 1000)); @@ -106,8 +110,7 @@ public void testConstructValidateAndDecryptSTSTokenSuccess() throws IOException @Test public void testConstructValidateAndDecryptSTSTokenSuccessWithNullSessionPolicy() throws Exception { // Create a valid token with null session policy - final String tokenString = tokenSecretManager.createSTSTokenString( - TEMP_ACCESS_KEY, ORIGINAL_ACCESS_KEY, ROLE_ARN, DURATION_SECONDS, SECRET_ACCESS_KEY, null, clock); + final String tokenString = createStsTokenString(DURATION_SECONDS, null); // Validate and decrypt the token final STSTokenIdentifier result = STSSecurityUtil.constructValidateAndDecryptSTSToken( @@ -126,11 +129,20 @@ public void testConstructValidateAndDecryptSTSTokenInvalidFormat() { .hasMessageContaining("Invalid STS token format: Failed to decode STS token string"); } + @Test + public void testConstructValidateAndDecryptSTSTokenRuntimeDecodeFailure() { + assertThatThrownBy(() -> + STSSecurityUtil.constructValidateAndDecryptSTSToken("not-a-valid-token", secretKeyClient, clock)) + .isInstanceOf(OMException.class) + .satisfies(exception -> assertThat(((OMException) exception).getResult()).isEqualTo(INVALID_TOKEN)) + .hasMessageContaining("Invalid STS token format: Failed to decode STS token string") + .hasMessageContaining("NegativeArraySizeException"); + } + @Test public void testConstructValidateAndDecryptSTSTokenInvalidKind() throws Exception { // Create a valid identifier to use as base - final String validTokenString = tokenSecretManager.createSTSTokenString( - TEMP_ACCESS_KEY, ORIGINAL_ACCESS_KEY, ROLE_ARN, DURATION_SECONDS, SECRET_ACCESS_KEY, SESSION_POLICY, clock); + final String validTokenString = createStsTokenString(); final Token validToken = new Token<>(); validToken.decodeFromUrlString(validTokenString); @@ -152,8 +164,7 @@ public void testConstructValidateAndDecryptSTSTokenInvalidKind() throws Exceptio @Test public void testConstructValidateAndDecryptSTSTokenInvalidService() throws Exception { // Create a token with incorrect service - final String validTokenString = tokenSecretManager.createSTSTokenString( - TEMP_ACCESS_KEY, ORIGINAL_ACCESS_KEY, ROLE_ARN, DURATION_SECONDS, SECRET_ACCESS_KEY, SESSION_POLICY, clock); + final String validTokenString = createStsTokenString(); final Token validToken = new Token<>(); validToken.decodeFromUrlString(validTokenString); @@ -173,8 +184,7 @@ public void testConstructValidateAndDecryptSTSTokenInvalidService() throws Excep @Test public void testConstructValidateAndDecryptSTSTokenExpired() throws Exception { // Create a token that expires immediately (durationSeconds of 0) - final String tokenString = tokenSecretManager.createSTSTokenString( - TEMP_ACCESS_KEY, ORIGINAL_ACCESS_KEY, ROLE_ARN, 0, SECRET_ACCESS_KEY, SESSION_POLICY, clock); + final String tokenString = createStsTokenString(0, SESSION_POLICY); // Fast-forward time to ensure token is expired clock.fastForward(100); @@ -190,8 +200,7 @@ public void testConstructValidateAndDecryptSTSTokenExpired() throws Exception { @Test public void testConstructValidateAndDecryptSTSTokenSecretKeyNotFound() throws Exception { // Create a valid token string - final String validTokenString = tokenSecretManager.createSTSTokenString( - TEMP_ACCESS_KEY, ORIGINAL_ACCESS_KEY, ROLE_ARN, DURATION_SECONDS, SECRET_ACCESS_KEY, SESSION_POLICY, clock); + final String validTokenString = createStsTokenString(); // Create a mock secret key client that returns null for the key final SecretKeyClient mockKeyClient = mock(SecretKeyClient.class); @@ -209,8 +218,7 @@ public void testConstructValidateAndDecryptSTSTokenSecretKeyNotFound() throws Ex @Test public void testConstructValidateAndDecryptSTSTokenInvalidSecretKeyId() throws Exception { // Create a valid identifier to use as base - final String validTokenString = tokenSecretManager.createSTSTokenString( - TEMP_ACCESS_KEY, ORIGINAL_ACCESS_KEY, ROLE_ARN, DURATION_SECONDS, SECRET_ACCESS_KEY, SESSION_POLICY, clock); + final String validTokenString = createStsTokenString(); final Token validToken = new Token<>(); validToken.decodeFromUrlString(validTokenString); @@ -235,8 +243,7 @@ public void testConstructValidateAndDecryptSTSTokenInvalidSecretKeyId() throws E @Test public void testConstructValidateAndDecryptSTSTokenExpiredSecretKey() throws Exception { // Create a valid token string - final String validTokenString = tokenSecretManager.createSTSTokenString( - TEMP_ACCESS_KEY, ORIGINAL_ACCESS_KEY, ROLE_ARN, DURATION_SECONDS, SECRET_ACCESS_KEY, SESSION_POLICY, clock); + final String validTokenString = createStsTokenString(); // Create a mock secret key that is expired final ManagedSecretKey expiredSecretKey = mock(ManagedSecretKey.class); @@ -258,8 +265,7 @@ public void testConstructValidateAndDecryptSTSTokenExpiredSecretKey() throws Exc @Test public void testConstructValidateAndDecryptSTSTokenSecretKeyRetrievalException() throws Exception { // Create a valid token string - final String validTokenString = tokenSecretManager.createSTSTokenString( - TEMP_ACCESS_KEY, ORIGINAL_ACCESS_KEY, ROLE_ARN, DURATION_SECONDS, SECRET_ACCESS_KEY, SESSION_POLICY, clock); + final String validTokenString = createStsTokenString(); // Create a mock secret key client that throws an exception final SecretKeyClient mockKeyClient = mock(SecretKeyClient.class); @@ -277,8 +283,7 @@ public void testConstructValidateAndDecryptSTSTokenSecretKeyRetrievalException() @Test public void testConstructValidateAndDecryptSTSTokenInvalidSignature() throws Exception { // Create a valid token string - final String validTokenString = tokenSecretManager.createSTSTokenString( - TEMP_ACCESS_KEY, ORIGINAL_ACCESS_KEY, ROLE_ARN, DURATION_SECONDS, SECRET_ACCESS_KEY, SESSION_POLICY, clock); + final String validTokenString = createStsTokenString(); final Token validToken = new Token<>(); validToken.decodeFromUrlString(validTokenString); @@ -303,19 +308,18 @@ public void testConstructValidateAndDecryptSTSTokenEmptyString() { assertThatThrownBy(() -> STSSecurityUtil.constructValidateAndDecryptSTSToken("", secretKeyClient, clock)) .isInstanceOf(OMException.class) - .hasMessage("Invalid STS token format: Failed to decode STS token string: java.io.EOFException"); + .hasMessage( + "Invalid STS token format: Failed to decode STS token string: java.io.EOFException"); } @Test public void testConstructValidateAndDecryptMultipleTokens() throws Exception { // Create multiple tokens and validate them all - final String token1 = tokenSecretManager.createSTSTokenString( - "temp-key-1", "orig-key-1", "role-arn-1", DURATION_SECONDS, - "secret-key-1", "policy-1", clock); + final String token1 = createStsTokenString(DURATION_SECONDS, "secret-key-1", "policy-1", + "temp-key-1", "orig-key-1", "role-arn-1"); - final String token2 = tokenSecretManager.createSTSTokenString( - "temp-key-2", "orig-key-2", "role-arn-2", DURATION_SECONDS, - "secret-key-2", "policy-2", clock); + final String token2 = createStsTokenString(DURATION_SECONDS, "secret-key-2", "policy-2", + "temp-key-2", "orig-key-2", "role-arn-2"); final STSTokenIdentifier result1 = STSSecurityUtil.constructValidateAndDecryptSTSToken( token1, secretKeyClient, clock); @@ -330,8 +334,7 @@ public void testConstructValidateAndDecryptMultipleTokens() throws Exception { @Test public void testEnsureEssentialFieldsArePresentInTokenMissingExpiry() { - final STSTokenIdentifier tokenIdentifier = new STSTokenIdentifier( - TEMP_ACCESS_KEY, ORIGINAL_ACCESS_KEY, ROLE_ARN, null, SECRET_ACCESS_KEY, SESSION_POLICY, ENCRYPTION_KEY); + final STSTokenIdentifier tokenIdentifier = new STSTokenIdentifier(paramsBuilder().setExpiry(null).build()); assertThatThrownBy(() -> STSSecurityUtil.ensureEssentialFieldsArePresentInToken(tokenIdentifier)) .isInstanceOf(SecretManager.InvalidToken.class) @@ -340,8 +343,7 @@ public void testEnsureEssentialFieldsArePresentInTokenMissingExpiry() { @Test public void testEnsureEssentialFieldsArePresentInTokenMissingTempAccessKeyId() { - final STSTokenIdentifier tokenIdentifier = new STSTokenIdentifier( - null, ORIGINAL_ACCESS_KEY, ROLE_ARN, clock.instant(), SECRET_ACCESS_KEY, SESSION_POLICY, ENCRYPTION_KEY); + final STSTokenIdentifier tokenIdentifier = new STSTokenIdentifier(paramsBuilder().setTempAccessKeyId(null).build()); assertThatThrownBy(() -> STSSecurityUtil.ensureEssentialFieldsArePresentInToken(tokenIdentifier)) .isInstanceOf(SecretManager.InvalidToken.class) @@ -350,8 +352,7 @@ public void testEnsureEssentialFieldsArePresentInTokenMissingTempAccessKeyId() { @Test public void testEnsureEssentialFieldsArePresentInTokenMissingRoleArn() { - final STSTokenIdentifier tokenIdentifier = new STSTokenIdentifier( - TEMP_ACCESS_KEY, ORIGINAL_ACCESS_KEY, null, clock.instant(), SECRET_ACCESS_KEY, SESSION_POLICY, ENCRYPTION_KEY); + final STSTokenIdentifier tokenIdentifier = new STSTokenIdentifier(paramsBuilder().setRoleArn(null).build()); assertThatThrownBy(() -> STSSecurityUtil.ensureEssentialFieldsArePresentInToken(tokenIdentifier)) .isInstanceOf(SecretManager.InvalidToken.class) @@ -361,7 +362,7 @@ public void testEnsureEssentialFieldsArePresentInTokenMissingRoleArn() { @Test public void testEnsureEssentialFieldsArePresentInTokenMissingOriginalAccessKeyId() { final STSTokenIdentifier tokenIdentifier = new STSTokenIdentifier( - TEMP_ACCESS_KEY, null, ROLE_ARN, clock.instant(), SECRET_ACCESS_KEY, SESSION_POLICY, ENCRYPTION_KEY); + paramsBuilder().setOriginalAccessKeyId(null).build()); assertThatThrownBy(() -> STSSecurityUtil.ensureEssentialFieldsArePresentInToken(tokenIdentifier)) .isInstanceOf(SecretManager.InvalidToken.class) @@ -370,18 +371,25 @@ public void testEnsureEssentialFieldsArePresentInTokenMissingOriginalAccessKeyId @Test public void testEnsureEssentialFieldsArePresentInTokenMissingSecretAccessKey() { - final STSTokenIdentifier tokenIdentifier = new STSTokenIdentifier( - TEMP_ACCESS_KEY, ORIGINAL_ACCESS_KEY, ROLE_ARN, clock.instant(), null, SESSION_POLICY, ENCRYPTION_KEY); + final STSTokenIdentifier tokenIdentifier = new STSTokenIdentifier(paramsBuilder().setSecretAccessKey(null).build()); assertThatThrownBy(() -> STSSecurityUtil.ensureEssentialFieldsArePresentInToken(tokenIdentifier)) .isInstanceOf(SecretManager.InvalidToken.class) .hasMessage("Invalid STS token - secretAccessKey is null/empty"); } + @Test + public void testEnsureEssentialFieldsArePresentInTokenMissingCreationTime() { + final STSTokenIdentifier tokenIdentifier = new STSTokenIdentifier(paramsBuilder().setCreationTime(null).build()); + + assertThatThrownBy(() -> STSSecurityUtil.ensureEssentialFieldsArePresentInToken(tokenIdentifier)) + .isInstanceOf(SecretManager.InvalidToken.class) + .hasMessage("Invalid STS token - creationTime is null"); + } + @Test public void testEnsureResolvedStsFieldsInvariantsSuccess() throws Exception { - final String tokenString = tokenSecretManager.createSTSTokenString( - TEMP_ACCESS_KEY, ORIGINAL_ACCESS_KEY, ROLE_ARN, DURATION_SECONDS, SECRET_ACCESS_KEY, SESSION_POLICY, clock); + final String tokenString = createStsTokenString(); final S3Authentication s3Auth = S3Authentication.newBuilder() .setSessionToken(tokenString) @@ -420,9 +428,7 @@ public void testEnsureResolvedStsFieldsInvariantsMissingSessionToken() { @Test public void testEnsureResolvedStsFieldsInvariantsMissingResolvedFields() throws Exception { - final String tokenString = tokenSecretManager.createSTSTokenString( - TEMP_ACCESS_KEY, ORIGINAL_ACCESS_KEY, ROLE_ARN, DURATION_SECONDS, - SECRET_ACCESS_KEY, SESSION_POLICY, clock); + final String tokenString = createStsTokenString(); final S3Authentication s3Auth = S3Authentication.newBuilder() .setSessionToken(tokenString) @@ -449,4 +455,42 @@ public void testEnsureResolvedStsFieldsInvariantsNoS3Auth() throws Exception { // Should not throw STSSecurityUtil.ensureResolvedStsFieldsInvariants(request); } + + private String createStsTokenString() throws IOException { + return createStsTokenString(DURATION_SECONDS, SECRET_ACCESS_KEY, SESSION_POLICY, + TEMP_ACCESS_KEY, ORIGINAL_ACCESS_KEY, ROLE_ARN); + } + + private String createStsTokenString(int durationSeconds, String sessionPolicy) + throws IOException { + return createStsTokenString(durationSeconds, TestSTSSecurityUtil.SECRET_ACCESS_KEY, sessionPolicy, + TEMP_ACCESS_KEY, ORIGINAL_ACCESS_KEY, ROLE_ARN); + } + + private String createStsTokenString(int durationSeconds, String secretAccessKey, String sessionPolicy, + String tempAccessKey, String originalAccessKey, String roleArn) throws IOException { + return tokenSecretManager.createSTSTokenString(STSTokenSecretManager.CreateSTSTokenParams.newBuilder() + .setTempAccessKeyId(tempAccessKey) + .setOriginalAccessKeyId(originalAccessKey) + .setRoleArn(roleArn) + .setDurationSeconds(durationSeconds) + .setSecretAccessKey(secretAccessKey) + .setSessionPolicy(sessionPolicy) + .setAssumedRoleId(ASSUMED_ROLE_ID) + .setAssumedRoleUserArn(ASSUMED_ROLE_USER_ARN) + .setClock(clock) + .build()); + } + + private STSTokenIdentifier.Params.Builder paramsBuilder() { + return STSTokenIdentifier.Params.newBuilder() + .setTempAccessKeyId(TEMP_ACCESS_KEY) + .setOriginalAccessKeyId(ORIGINAL_ACCESS_KEY) + .setRoleArn(ROLE_ARN) + .setCreationTime(clock.instant()) + .setExpiry(clock.instant().plusSeconds(DURATION_SECONDS)) + .setSecretAccessKey(SECRET_ACCESS_KEY) + .setSessionPolicy(SESSION_POLICY) + .setEncryptionKey(ENCRYPTION_KEY); + } } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/TestSTSTokenEncryption.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/TestSTSTokenEncryption.java index 1eb880f9dd03..870e99c3da18 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/TestSTSTokenEncryption.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/TestSTSTokenEncryption.java @@ -76,12 +76,21 @@ public void testSTSTokenIdentifierEncryption() throws Exception { final String roleArn = "arn:aws:iam::123456789012:role/TestRole"; final String secretAccessKey = "mySecretAccessKey123456"; // Use millisecond precision to match serialization format - final Instant expiry = Instant.ofEpochMilli(Instant.now().plusSeconds(3600).toEpochMilli()); + final Instant creationTime = Instant.ofEpochMilli(1_700_000_000_000L); + final Instant expiry = creationTime.plusSeconds(3600); final String sessionPolicy = "test-session-policy"; - + // Create token identifier with encryption - final STSTokenIdentifier tokenId = new STSTokenIdentifier( - tempAccessKeyId, originalAccessKeyId, roleArn, expiry, secretAccessKey, sessionPolicy, keyBytes); + final STSTokenIdentifier tokenId = new STSTokenIdentifier(STSTokenIdentifier.Params.newBuilder() + .setTempAccessKeyId(tempAccessKeyId) + .setOriginalAccessKeyId(originalAccessKeyId) + .setRoleArn(roleArn) + .setCreationTime(creationTime) + .setExpiry(expiry) + .setSecretAccessKey(secretAccessKey) + .setSessionPolicy(sessionPolicy) + .setEncryptionKey(keyBytes) + .build()); tokenId.setSecretKeyId(UUID.randomUUID()); // Convert to protobuf @@ -100,6 +109,7 @@ public void testSTSTokenIdentifierEncryption() throws Exception { assertEquals(roleArn, decodedTokenId.getRoleArn()); assertEquals(secretAccessKey, decodedTokenId.getSecretAccessKey()); assertEquals(expiry, decodedTokenId.getExpiry()); + assertEquals(creationTime, decodedTokenId.getCreationTime()); } @Test diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/TestSTSTokenIdentifier.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/TestSTSTokenIdentifier.java index 09a786faaea3..ba26747b90ab 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/TestSTSTokenIdentifier.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/TestSTSTokenIdentifier.java @@ -38,6 +38,7 @@ public class TestSTSTokenIdentifier { private static final byte[] ENCRYPTION_KEY = new byte[5]; + private static final Instant CREATION_TIME = Instant.ofEpochMilli(1_700_000_000_000L); { ThreadLocalRandom.current().nextBytes(ENCRYPTION_KEY); @@ -45,9 +46,14 @@ public class TestSTSTokenIdentifier { @Test public void testKindAndService() { - final STSTokenIdentifier stsTokenIdentifier = new STSTokenIdentifier( - "tempAccessKeyId", "originalAccessKeyId", "roleArn", - Instant.now().plusSeconds(3600), "secretAccessKey", "sessionPolicy", ENCRYPTION_KEY); + final STSTokenIdentifier stsTokenIdentifier = new STSTokenIdentifier(paramsBuilder() + .setTempAccessKeyId("tempAccessKeyId") + .setOriginalAccessKeyId("originalAccessKeyId") + .setRoleArn("roleArn") + .setExpiry(Instant.now().plusSeconds(3600)) + .setSecretAccessKey("secretAccessKey") + .setSessionPolicy("sessionPolicy") + .build()); assertEquals("STSToken", stsTokenIdentifier.getKind().toString()); assertEquals("STS", stsTokenIdentifier.getService()); @@ -59,9 +65,16 @@ public void testProtoBufRoundTrip() throws IOException { // so use a millisecond-precision Instant to avoid nanos-only differences across // platforms/JDKs during round-trips. final Instant expiry = Instant.now().plusSeconds(7200).truncatedTo(ChronoUnit.MILLIS); - final STSTokenIdentifier originalTokenIdentifier = new STSTokenIdentifier( - "tempAccess", "origAccess", "arn:aws:iam::123456789012:role/RoleY", - expiry, "secretKey", "sessionPolicy", ENCRYPTION_KEY); + final STSTokenIdentifier originalTokenIdentifier = new STSTokenIdentifier(paramsBuilder() + .setTempAccessKeyId("tempAccess") + .setOriginalAccessKeyId("origAccess") + .setRoleArn("arn:aws:iam::123456789012:role/RoleY") + .setExpiry(expiry) + .setSecretAccessKey("secretKey") + .setSessionPolicy("sessionPolicy") + .setAssumedRoleId("AROATEST123456789:testsess") + .setAssumedRoleUserArn("arn:aws:sts::123456789012:assumed-role/RoleY/testsess") + .build()); final UUID secretKeyId = UUID.randomUUID(); originalTokenIdentifier.setSecretKeyId(secretKeyId); @@ -69,10 +82,14 @@ public void testProtoBufRoundTrip() throws IOException { assertThat(proto.getType()).isEqualTo(OMTokenProto.Type.S3_STS_TOKEN); assertThat(proto.getOwner()).isEqualTo("tempAccess"); assertThat(proto.getMaxDate()).isEqualTo(expiry.toEpochMilli()); + assertThat(proto.getIssueDate()).isEqualTo(CREATION_TIME.toEpochMilli()); assertThat(proto.getOriginalAccessKeyId()).isEqualTo("origAccess"); assertThat(proto.getRoleArn()).isEqualTo("arn:aws:iam::123456789012:role/RoleY"); assertThat(proto.getSecretAccessKey()).isNotEqualTo("secretKey"); // must be encrypted assertThat(proto.getSessionPolicy()).isEqualTo("sessionPolicy"); + assertThat(proto.getAssumedRoleId()).isEqualTo("AROATEST123456789:testsess"); + assertThat(proto.getAssumedRoleUserArn()) + .isEqualTo("arn:aws:sts::123456789012:assumed-role/RoleY/testsess"); assertThat(proto.getSecretKeyId()).isEqualTo(secretKeyId.toString()); final STSTokenIdentifier parsedTokenIdentifier = new STSTokenIdentifier(); @@ -81,11 +98,15 @@ public void testProtoBufRoundTrip() throws IOException { assertThat(parsedTokenIdentifier.getOwnerId()).isEqualTo("tempAccess"); assertThat(parsedTokenIdentifier.getExpiry()).isEqualTo(expiry); + assertThat(parsedTokenIdentifier.getCreationTime()).isEqualTo(CREATION_TIME); assertThat(parsedTokenIdentifier.getOriginalAccessKeyId()).isEqualTo("origAccess"); assertThat(parsedTokenIdentifier.getRoleArn()).isEqualTo("arn:aws:iam::123456789012:role/RoleY"); assertThat(parsedTokenIdentifier.getSecretAccessKey()).isEqualTo("secretKey"); assertThat(parsedTokenIdentifier.getSecretKeyId()).isEqualTo(secretKeyId); assertThat(parsedTokenIdentifier.getSessionPolicy()).isEqualTo("sessionPolicy"); + assertThat(parsedTokenIdentifier.getAssumedRoleId()).isEqualTo("AROATEST123456789:testsess"); + assertThat(parsedTokenIdentifier.getAssumedRoleUserArn()) + .isEqualTo("arn:aws:sts::123456789012:assumed-role/RoleY/testsess"); assertThat(parsedTokenIdentifier).isEqualTo(originalTokenIdentifier); assertThat(parsedTokenIdentifier.hashCode()).isEqualTo(originalTokenIdentifier.hashCode()); } @@ -99,9 +120,14 @@ public void testFromProtoBufInvalidSecretKeyId() { .setSecretKeyId("not-a-uuid") .build(); - final STSTokenIdentifier stsTokenIdentifier = new STSTokenIdentifier( - "tempAccessKeyId", "originalAccessKeyId", "roleArn", Instant.now(), - "secretAccessKey", "sessionPolicy", ENCRYPTION_KEY); + final STSTokenIdentifier stsTokenIdentifier = new STSTokenIdentifier(paramsBuilder() + .setTempAccessKeyId("tempAccessKeyId") + .setOriginalAccessKeyId("originalAccessKeyId") + .setRoleArn("roleArn") + .setExpiry(Instant.now()) + .setSecretAccessKey("secretAccessKey") + .setSessionPolicy("sessionPolicy") + .build()); final IOException ex = assertThrows(IOException.class, () -> stsTokenIdentifier.fromProtoBuf(invalid)); assertThat(ex.getMessage()).isEqualTo("Invalid secretKeyId format in STS token: not-a-uuid"); @@ -110,9 +136,13 @@ public void testFromProtoBufInvalidSecretKeyId() { @Test public void testProtobufRoundTripWithNullSessionPolicy() throws IOException { final Instant expiry = Instant.now().plusSeconds(7200); - final STSTokenIdentifier stsTokenIdentifier = new STSTokenIdentifier( - "tempAccess", "origAccess", "arn:aws:iam::123456789012:role/RoleX", - expiry, "secretKey", null, ENCRYPTION_KEY); + final STSTokenIdentifier stsTokenIdentifier = new STSTokenIdentifier(paramsBuilder() + .setTempAccessKeyId("tempAccess") + .setOriginalAccessKeyId("origAccess") + .setRoleArn("arn:aws:iam::123456789012:role/RoleX") + .setExpiry(expiry) + .setSecretAccessKey("secretKey") + .build()); final UUID secretKeyId = UUID.randomUUID(); stsTokenIdentifier.setSecretKeyId(secretKeyId); @@ -129,9 +159,14 @@ public void testProtobufRoundTripWithNullSessionPolicy() throws IOException { @Test public void testProtobufRoundTripWithEmptySessionPolicy() throws IOException { final Instant expiry = Instant.now().plusSeconds(4000); - final STSTokenIdentifier stsTokenIdentifier = new STSTokenIdentifier( - "tempAccess", "origAccess", "arn:aws:iam::123456789012:role/RoleZ", - expiry, "secretKey", "", ENCRYPTION_KEY); + final STSTokenIdentifier stsTokenIdentifier = new STSTokenIdentifier(paramsBuilder() + .setTempAccessKeyId("tempAccess") + .setOriginalAccessKeyId("origAccess") + .setRoleArn("arn:aws:iam::123456789012:role/RoleZ") + .setExpiry(expiry) + .setSecretAccessKey("secretKey") + .setSessionPolicy("") + .build()); final UUID secretKeyId = UUID.randomUUID(); stsTokenIdentifier.setSecretKeyId(secretKeyId); @@ -153,9 +188,14 @@ public void testFromProtoBufInvalidTokenType() { .setMaxDate(Instant.now().toEpochMilli()) .build(); - final STSTokenIdentifier stsTokenIdentifier = new STSTokenIdentifier( - "tempAccessKeyId", "origAccessKeyId", "roleArn", Instant.now(), - "secretAccessKey", "sessionPolicy", ENCRYPTION_KEY); + final STSTokenIdentifier stsTokenIdentifier = new STSTokenIdentifier(paramsBuilder() + .setTempAccessKeyId("tempAccessKeyId") + .setOriginalAccessKeyId("origAccessKeyId") + .setRoleArn("roleArn") + .setExpiry(Instant.now()) + .setSecretAccessKey("secretAccessKey") + .setSessionPolicy("sessionPolicy") + .build()); final IllegalArgumentException ex = assertThrows( IllegalArgumentException.class, () -> stsTokenIdentifier.fromProtoBuf(invalidType)); @@ -169,9 +209,16 @@ public void testWriteToAndReadFromByteArray() throws Exception { // compared to the original object, which is compared using equals(). final Instant expiry = Instant.now().plusSeconds(1000).truncatedTo(ChronoUnit.MILLIS); - final STSTokenIdentifier originalTokenIdentifier = new STSTokenIdentifier( - "tempAccessKeyId", "originalAccessKeyId", "roleArn", expiry, - "secretAccessKey", "sessionPolicy", ENCRYPTION_KEY); + final STSTokenIdentifier originalTokenIdentifier = new STSTokenIdentifier(paramsBuilder() + .setTempAccessKeyId("tempAccessKeyId") + .setOriginalAccessKeyId("originalAccessKeyId") + .setRoleArn("roleArn") + .setExpiry(expiry) + .setSecretAccessKey("secretAccessKey") + .setSessionPolicy("sessionPolicy") + .setAssumedRoleId("AROATEST123456789:testsess") + .setAssumedRoleUserArn("arn:aws:sts::123456789012:assumed-role/test-role/testsess") + .build()); originalTokenIdentifier.setSecretKeyId(UUID.randomUUID()); final ByteArrayOutputStream baos = new ByteArrayOutputStream(); @@ -196,9 +243,14 @@ public void testWriteToAndReadFromByteArrayWithDifferentSecretKeyIds() throws Ex } final Instant expiry = Instant.now().plusSeconds(1500); - final STSTokenIdentifier originalTokenIdentifier = new STSTokenIdentifier( - "tempAccessKeyId", "originalAccessKeyId", "roleArn", expiry, - "secretAccessKey", "sessionPolicy", ENCRYPTION_KEY); + final STSTokenIdentifier originalTokenIdentifier = new STSTokenIdentifier(paramsBuilder() + .setTempAccessKeyId("tempAccessKeyId") + .setOriginalAccessKeyId("originalAccessKeyId") + .setRoleArn("roleArn") + .setExpiry(expiry) + .setSecretAccessKey("secretAccessKey") + .setSessionPolicy("sessionPolicy") + .build()); originalTokenIdentifier.setSecretKeyId(uuid1); final ByteArrayOutputStream baos1 = new ByteArrayOutputStream(); @@ -206,9 +258,14 @@ public void testWriteToAndReadFromByteArrayWithDifferentSecretKeyIds() throws Ex originalTokenIdentifier.write(out); } - final STSTokenIdentifier anotherTokenIdentifier = new STSTokenIdentifier( - "tempAccessKeyId", "originalAccessKeyId", "roleArn", expiry, - "secretAccessKey", "sessionPolicy", ENCRYPTION_KEY); + final STSTokenIdentifier anotherTokenIdentifier = new STSTokenIdentifier(paramsBuilder() + .setTempAccessKeyId("tempAccessKeyId") + .setOriginalAccessKeyId("originalAccessKeyId") + .setRoleArn("roleArn") + .setExpiry(expiry) + .setSecretAccessKey("secretAccessKey") + .setSessionPolicy("sessionPolicy") + .build()); anotherTokenIdentifier.setSecretKeyId(uuid2); final ByteArrayOutputStream baos2 = new ByteArrayOutputStream(); @@ -236,9 +293,14 @@ public void testWriteToAndReadFromByteArrayWithSameSecretKeyIds() throws Excepti final UUID uuid = UUID.randomUUID(); final Instant expiry = Instant.now().plusSeconds(1700); - final STSTokenIdentifier originalTokenIdentifier = new STSTokenIdentifier( - "tempAccessKeyId", "originalAccessKeyId", "roleArn", expiry, - "secretAccessKey", "sessionPolicy", ENCRYPTION_KEY); + final STSTokenIdentifier originalTokenIdentifier = new STSTokenIdentifier(paramsBuilder() + .setTempAccessKeyId("tempAccessKeyId") + .setOriginalAccessKeyId("originalAccessKeyId") + .setRoleArn("roleArn") + .setExpiry(expiry) + .setSecretAccessKey("secretAccessKey") + .setSessionPolicy("sessionPolicy") + .build()); originalTokenIdentifier.setSecretKeyId(uuid); final ByteArrayOutputStream baos1 = new ByteArrayOutputStream(); @@ -246,9 +308,14 @@ public void testWriteToAndReadFromByteArrayWithSameSecretKeyIds() throws Excepti originalTokenIdentifier.write(out); } - final STSTokenIdentifier anotherTokenIdentifier = new STSTokenIdentifier( - "tempAccessKeyId", "originalAccessKeyId", "roleArn", expiry, - "secretAccessKey", "sessionPolicy", ENCRYPTION_KEY); + final STSTokenIdentifier anotherTokenIdentifier = new STSTokenIdentifier(paramsBuilder() + .setTempAccessKeyId("tempAccessKeyId") + .setOriginalAccessKeyId("originalAccessKeyId") + .setRoleArn("roleArn") + .setExpiry(expiry) + .setSecretAccessKey("secretAccessKey") + .setSessionPolicy("sessionPolicy") + .build()); anotherTokenIdentifier.setSecretKeyId(uuid); final ByteArrayOutputStream baos2 = new ByteArrayOutputStream(); @@ -279,13 +346,20 @@ public void testGettersReturnCorrectValues() { final String secretAccessKey = "mySecretKey"; final String sessionPolicy = "myPolicy"; - final STSTokenIdentifier stsTokenIdentifier = new STSTokenIdentifier( - tempAccessKeyId, originalAccessKeyId, roleArn, expiry, secretAccessKey, sessionPolicy, ENCRYPTION_KEY); + final STSTokenIdentifier stsTokenIdentifier = new STSTokenIdentifier(paramsBuilder() + .setTempAccessKeyId(tempAccessKeyId) + .setOriginalAccessKeyId(originalAccessKeyId) + .setRoleArn(roleArn) + .setExpiry(expiry) + .setSecretAccessKey(secretAccessKey) + .setSessionPolicy(sessionPolicy) + .build()); assertThat(stsTokenIdentifier.getOwnerId()).isEqualTo(tempAccessKeyId); assertThat(stsTokenIdentifier.getTempAccessKeyId()).isEqualTo(tempAccessKeyId); assertThat(stsTokenIdentifier.getOriginalAccessKeyId()).isEqualTo(originalAccessKeyId); assertThat(stsTokenIdentifier.getRoleArn()).isEqualTo(roleArn); + assertThat(stsTokenIdentifier.getCreationTime()).isEqualTo(CREATION_TIME); assertThat(stsTokenIdentifier.getExpiry()).isEqualTo(expiry); assertThat(stsTokenIdentifier.getSecretAccessKey()).isEqualTo(secretAccessKey); assertThat(stsTokenIdentifier.getSessionPolicy()).isEqualTo(sessionPolicy); @@ -296,14 +370,24 @@ public void testEqualsAndHashCode() { final Instant expiry = Instant.now().plusSeconds(3600); final UUID uuid = UUID.randomUUID(); - final STSTokenIdentifier stsTokenIdentifier = new STSTokenIdentifier( - "tempAccessKeyId", "originalAccessKeyId", "roleArn", expiry, - "secretAccessKey", "sessionPolicy", ENCRYPTION_KEY); + final STSTokenIdentifier stsTokenIdentifier = new STSTokenIdentifier(paramsBuilder() + .setTempAccessKeyId("tempAccessKeyId") + .setOriginalAccessKeyId("originalAccessKeyId") + .setRoleArn("roleArn") + .setExpiry(expiry) + .setSecretAccessKey("secretAccessKey") + .setSessionPolicy("sessionPolicy") + .build()); stsTokenIdentifier.setSecretKeyId(uuid); - final STSTokenIdentifier stsTokenIdentifier2 = new STSTokenIdentifier( - "tempAccessKeyId", "originalAccessKeyId", "roleArn", expiry, - "secretAccessKey", "sessionPolicy", ENCRYPTION_KEY); + final STSTokenIdentifier stsTokenIdentifier2 = new STSTokenIdentifier(paramsBuilder() + .setTempAccessKeyId("tempAccessKeyId") + .setOriginalAccessKeyId("originalAccessKeyId") + .setRoleArn("roleArn") + .setExpiry(expiry) + .setSecretAccessKey("secretAccessKey") + .setSessionPolicy("sessionPolicy") + .build()); stsTokenIdentifier2.setSecretKeyId(uuid); assertThat(stsTokenIdentifier).isEqualTo(stsTokenIdentifier2); @@ -314,13 +398,23 @@ public void testEqualsAndHashCode() { public void testNotEqualsWhenTempAccessKeyIdDiffers() { final Instant expiry = Instant.now().plusSeconds(3600); - final STSTokenIdentifier stsTokenIdentifier = new STSTokenIdentifier( - "tempAccessKeyId1", "originalAccessKeyId", "roleArn", - expiry, "secretAccessKey", "sessionPolicy", ENCRYPTION_KEY); - - final STSTokenIdentifier stsTokenIdentifier2 = new STSTokenIdentifier( - "tempAccessKeyId2", "originalAccessKeyId", "roleArn", expiry, - "secretAccessKey", "sessionPolicy", ENCRYPTION_KEY); + final STSTokenIdentifier stsTokenIdentifier = new STSTokenIdentifier(paramsBuilder() + .setTempAccessKeyId("tempAccessKeyId1") + .setOriginalAccessKeyId("originalAccessKeyId") + .setRoleArn("roleArn") + .setExpiry(expiry) + .setSecretAccessKey("secretAccessKey") + .setSessionPolicy("sessionPolicy") + .build()); + + final STSTokenIdentifier stsTokenIdentifier2 = new STSTokenIdentifier(paramsBuilder() + .setTempAccessKeyId("tempAccessKeyId2") + .setOriginalAccessKeyId("originalAccessKeyId") + .setRoleArn("roleArn") + .setExpiry(expiry) + .setSecretAccessKey("secretAccessKey") + .setSessionPolicy("sessionPolicy") + .build()); assertThat(stsTokenIdentifier).isNotEqualTo(stsTokenIdentifier2); } @@ -329,13 +423,23 @@ public void testNotEqualsWhenTempAccessKeyIdDiffers() { public void testNotEqualsWhenOriginalAccessKeyIdDiffers() { final Instant expiry = Instant.now().plusSeconds(3600); - final STSTokenIdentifier stsTokenIdentifier = new STSTokenIdentifier( - "tempAccessKeyId", "originalAccessKeyId1", "roleArn", expiry, - "secretAccessKey", "sessionPolicy", ENCRYPTION_KEY); - - final STSTokenIdentifier stsTokenIdentifier2 = new STSTokenIdentifier( - "tempAccessKeyId", "originalAccessKeyId2", "roleArn", expiry, - "secretAccessKey", "sessionPolicy", ENCRYPTION_KEY); + final STSTokenIdentifier stsTokenIdentifier = new STSTokenIdentifier(paramsBuilder() + .setTempAccessKeyId("tempAccessKeyId") + .setOriginalAccessKeyId("originalAccessKeyId1") + .setRoleArn("roleArn") + .setExpiry(expiry) + .setSecretAccessKey("secretAccessKey") + .setSessionPolicy("sessionPolicy") + .build()); + + final STSTokenIdentifier stsTokenIdentifier2 = new STSTokenIdentifier(paramsBuilder() + .setTempAccessKeyId("tempAccessKeyId") + .setOriginalAccessKeyId("originalAccessKeyId2") + .setRoleArn("roleArn") + .setExpiry(expiry) + .setSecretAccessKey("secretAccessKey") + .setSessionPolicy("sessionPolicy") + .build()); assertThat(stsTokenIdentifier).isNotEqualTo(stsTokenIdentifier2); } @@ -344,26 +448,46 @@ public void testNotEqualsWhenOriginalAccessKeyIdDiffers() { public void testNotEqualsWhenRoleArnDiffers() { final Instant expiry = Instant.now().plusSeconds(3600); - final STSTokenIdentifier stsTokenIdentifier = new STSTokenIdentifier( - "tempAccessKeyId", "originalAccessKeyId", "roleArn1", expiry, - "secretAccessKey", "sessionPolicy", ENCRYPTION_KEY); - - final STSTokenIdentifier stsTokenIdentifier2 = new STSTokenIdentifier( - "tempAccessKeyId", "originalAccessKeyId", "roleArn2", expiry, - "secretAccessKey", "sessionPolicy", ENCRYPTION_KEY); + final STSTokenIdentifier stsTokenIdentifier = new STSTokenIdentifier(paramsBuilder() + .setTempAccessKeyId("tempAccessKeyId") + .setOriginalAccessKeyId("originalAccessKeyId") + .setRoleArn("roleArn1") + .setExpiry(expiry) + .setSecretAccessKey("secretAccessKey") + .setSessionPolicy("sessionPolicy") + .build()); + + final STSTokenIdentifier stsTokenIdentifier2 = new STSTokenIdentifier(paramsBuilder() + .setTempAccessKeyId("tempAccessKeyId") + .setOriginalAccessKeyId("originalAccessKeyId") + .setRoleArn("roleArn2") + .setExpiry(expiry) + .setSecretAccessKey("secretAccessKey") + .setSessionPolicy("sessionPolicy") + .build()); assertThat(stsTokenIdentifier).isNotEqualTo(stsTokenIdentifier2); } @Test public void testNotEqualsWhenExpirationDiffers() { - final STSTokenIdentifier stsTokenIdentifier = new STSTokenIdentifier( - "tempAccessKeyId", "originalAccessKeyId", "roleArn", - Instant.now().plusSeconds(3600), "secretAccessKey", "sessionPolicy", ENCRYPTION_KEY); - - final STSTokenIdentifier stsTokenIdentifier2 = new STSTokenIdentifier( - "tempAccessKeyId", "originalAccessKeyId", "roleArn", - Instant.now().plusSeconds(7600), "secretAccessKey", "sessionPolicy", ENCRYPTION_KEY); + final STSTokenIdentifier stsTokenIdentifier = new STSTokenIdentifier(paramsBuilder() + .setTempAccessKeyId("tempAccessKeyId") + .setOriginalAccessKeyId("originalAccessKeyId") + .setRoleArn("roleArn") + .setExpiry(Instant.now().plusSeconds(3600)) + .setSecretAccessKey("secretAccessKey") + .setSessionPolicy("sessionPolicy") + .build()); + + final STSTokenIdentifier stsTokenIdentifier2 = new STSTokenIdentifier(paramsBuilder() + .setTempAccessKeyId("tempAccessKeyId") + .setOriginalAccessKeyId("originalAccessKeyId") + .setRoleArn("roleArn") + .setExpiry(Instant.now().plusSeconds(7600)) + .setSecretAccessKey("secretAccessKey") + .setSessionPolicy("sessionPolicy") + .build()); assertThat(stsTokenIdentifier).isNotEqualTo(stsTokenIdentifier2); } @@ -372,13 +496,23 @@ public void testNotEqualsWhenExpirationDiffers() { public void testNotEqualsWhenSecretAccessKeyDiffers() { final Instant expiry = Instant.now().plusSeconds(3600); - final STSTokenIdentifier stsTokenIdentifier = new STSTokenIdentifier( - "tempAccessKeyId", "originalAccessKeyId", "roleArn", expiry, - "secretAccessKey1", "sessionPolicy", ENCRYPTION_KEY); - - final STSTokenIdentifier stsTokenIdentifier2 = new STSTokenIdentifier( - "tempAccessKeyId", "originalAccessKeyId", "roleArn", expiry, - "secretAccessKey2", "sessionPolicy", ENCRYPTION_KEY); + final STSTokenIdentifier stsTokenIdentifier = new STSTokenIdentifier(paramsBuilder() + .setTempAccessKeyId("tempAccessKeyId") + .setOriginalAccessKeyId("originalAccessKeyId") + .setRoleArn("roleArn") + .setExpiry(expiry) + .setSecretAccessKey("secretAccessKey1") + .setSessionPolicy("sessionPolicy") + .build()); + + final STSTokenIdentifier stsTokenIdentifier2 = new STSTokenIdentifier(paramsBuilder() + .setTempAccessKeyId("tempAccessKeyId") + .setOriginalAccessKeyId("originalAccessKeyId") + .setRoleArn("roleArn") + .setExpiry(expiry) + .setSecretAccessKey("secretAccessKey2") + .setSessionPolicy("sessionPolicy") + .build()); assertThat(stsTokenIdentifier).isNotEqualTo(stsTokenIdentifier2); } @@ -387,13 +521,23 @@ public void testNotEqualsWhenSecretAccessKeyDiffers() { public void testNotEqualsWhenSessionPolicyDiffers() { final Instant expiry = Instant.now().plusSeconds(3600); - final STSTokenIdentifier stsTokenIdentifier = new STSTokenIdentifier( - "tempAccessKeyId", "originalAccessKeyId", "roleArn", expiry, - "secretAccessKey", "sessionPolicy1", ENCRYPTION_KEY); - - final STSTokenIdentifier stsTokenIdentifier2 = new STSTokenIdentifier( - "tempAccessKeyId", "originalAccessKeyId", "roleArn", expiry, - "secretAccessKey", "sessionPolicy2", ENCRYPTION_KEY); + final STSTokenIdentifier stsTokenIdentifier = new STSTokenIdentifier(paramsBuilder() + .setTempAccessKeyId("tempAccessKeyId") + .setOriginalAccessKeyId("originalAccessKeyId") + .setRoleArn("roleArn") + .setExpiry(expiry) + .setSecretAccessKey("secretAccessKey") + .setSessionPolicy("sessionPolicy1") + .build()); + + final STSTokenIdentifier stsTokenIdentifier2 = new STSTokenIdentifier(paramsBuilder() + .setTempAccessKeyId("tempAccessKeyId") + .setOriginalAccessKeyId("originalAccessKeyId") + .setRoleArn("roleArn") + .setExpiry(expiry) + .setSecretAccessKey("secretAccessKey") + .setSessionPolicy("sessionPolicy2") + .build()); assertThat(stsTokenIdentifier).isNotEqualTo(stsTokenIdentifier2); } @@ -403,14 +547,24 @@ public void testToString() { final Instant expiry = Instant.now().plusSeconds(3600); final UUID uuid = UUID.randomUUID(); - final STSTokenIdentifier stsTokenIdentifier = new STSTokenIdentifier( - "tempAccessKeyId", "originalAccessKeyId", "roleArn", expiry, - "secretAccessKey", "sessionPolicy", ENCRYPTION_KEY); + final STSTokenIdentifier stsTokenIdentifier = new STSTokenIdentifier(paramsBuilder() + .setTempAccessKeyId("tempAccessKeyId") + .setOriginalAccessKeyId("originalAccessKeyId") + .setRoleArn("roleArn") + .setExpiry(expiry) + .setSecretAccessKey("secretAccessKey") + .setSessionPolicy("sessionPolicy") + .setAssumedRoleId("AROATEST123456789:testsess") + .setAssumedRoleUserArn("arn:aws:sts::123456789012:assumed-role/test-role/testsess") + .build()); stsTokenIdentifier.setSecretKeyId(uuid); final String stsTokenIdentifierStr = stsTokenIdentifier.toString(); final String expectedString = "STSTokenIdentifier{" + "tempAccessKeyId='tempAccessKeyId'" + - ", originalAccessKeyId='originalAccessKeyId'" + ", roleArn='roleArn'" + ", expiry='" + expiry + + ", originalAccessKeyId='originalAccessKeyId'" + ", roleArn='roleArn'" + + ", assumedRoleId='AROATEST123456789:testsess'" + + ", assumedRoleUserArn='arn:aws:sts::123456789012:assumed-role/test-role/testsess'" + + ", creationTime='" + CREATION_TIME + "', expiry='" + expiry + "', secretKeyId='" + uuid + "', sessionPolicy='sessionPolicy'" + '}'; assertEquals(expectedString, stsTokenIdentifierStr); @@ -418,9 +572,14 @@ public void testToString() { @Test public void testNotEqualsWithNull() { - final STSTokenIdentifier stsTokenIdentifier = new STSTokenIdentifier( - "tempAccessKeyId", "originalAccessKeyId", "roleArn", Instant.now(), - "secretAccessKey", "sessionPolicy", ENCRYPTION_KEY); + final STSTokenIdentifier stsTokenIdentifier = new STSTokenIdentifier(paramsBuilder() + .setTempAccessKeyId("tempAccessKeyId") + .setOriginalAccessKeyId("originalAccessKeyId") + .setRoleArn("roleArn") + .setExpiry(Instant.now()) + .setSecretAccessKey("secretAccessKey") + .setSessionPolicy("sessionPolicy") + .build()); assertThat(stsTokenIdentifier).isNotEqualTo(null); } @@ -431,24 +590,39 @@ public void testEqualsWithDifferentEncryptionKeys() { final UUID uuid = UUID.randomUUID(); // Create first identifier with the default key - final STSTokenIdentifier stsTokenIdentifier = new STSTokenIdentifier( - "tempAccessKeyId", "originalAccessKeyId", "roleArn", expiry, - "secretAccessKey", "sessionPolicy", ENCRYPTION_KEY); + final STSTokenIdentifier stsTokenIdentifier = new STSTokenIdentifier(paramsBuilder() + .setTempAccessKeyId("tempAccessKeyId") + .setOriginalAccessKeyId("originalAccessKeyId") + .setRoleArn("roleArn") + .setExpiry(expiry) + .setSecretAccessKey("secretAccessKey") + .setSessionPolicy("sessionPolicy") + .build()); stsTokenIdentifier.setSecretKeyId(uuid); // Create second identifier with a different encryption key but otherwise same parameters byte[] differentKey = new byte[5]; new SecureRandom().nextBytes(differentKey); - final STSTokenIdentifier stsTokenIdentifier2 = new STSTokenIdentifier( - "tempAccessKeyId", "originalAccessKeyId", "roleArn", expiry, - "secretAccessKey", "sessionPolicy", differentKey); + final STSTokenIdentifier stsTokenIdentifier2 = new STSTokenIdentifier(paramsBuilder() + .setTempAccessKeyId("tempAccessKeyId") + .setOriginalAccessKeyId("originalAccessKeyId") + .setRoleArn("roleArn") + .setExpiry(expiry) + .setSecretAccessKey("secretAccessKey") + .setSessionPolicy("sessionPolicy") + .setEncryptionKey(differentKey) + .build()); stsTokenIdentifier2.setSecretKeyId(uuid); // They should still be equal because encryptionKey is transient/ignored for identity assertThat(stsTokenIdentifier).isEqualTo(stsTokenIdentifier2); assertThat(stsTokenIdentifier.hashCode()).isEqualTo(stsTokenIdentifier2.hashCode()); } -} - + private static STSTokenIdentifier.Params.Builder paramsBuilder() { + return STSTokenIdentifier.Params.newBuilder() + .setCreationTime(CREATION_TIME) + .setEncryptionKey(ENCRYPTION_KEY); + } +} diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/TestSTSTokenSecretManager.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/TestSTSTokenSecretManager.java index 800aeabe97c5..2f21e8203e0d 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/TestSTSTokenSecretManager.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/TestSTSTokenSecretManager.java @@ -26,12 +26,16 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; +import java.time.Duration; import java.time.Instant; import java.time.ZoneOffset; +import java.util.HashMap; +import java.util.Map; import java.util.UUID; import javax.crypto.SecretKey; import javax.crypto.spec.SecretKeySpec; import org.apache.hadoop.hdds.security.symmetric.ManagedSecretKey; +import org.apache.hadoop.hdds.security.symmetric.SecretKeyClient; import org.apache.hadoop.hdds.security.symmetric.SecretKeySignerClient; import org.apache.hadoop.io.Text; import org.apache.hadoop.security.token.Token; @@ -52,6 +56,8 @@ public class TestSTSTokenSecretManager { private static final String ROLE_ARN = "arn:aws:iam::123456789012:role/test-role"; private static final String SECRET_ACCESS_KEY = "test-secret-access-key"; private static final String SESSION_POLICY = "test-session-policy"; + private static final String ASSUMED_ROLE_ID = "AROATEST123456789:testsess"; + private static final String ASSUMED_ROLE_USER_ARN = "arn:aws:sts::123456789012:assumed-role/test-role/testsess"; private static final int DURATION_SECONDS = 3600; private static SecretKey sharedSecretKey; @@ -70,7 +76,7 @@ public void setUp() throws Exception { final UUID keyId = UUID.fromString("00000000-0000-0000-0000-000000000000"); when(mockSecretKey.getId()).thenReturn(keyId); when(mockSecretKey.getSecretKey()).thenReturn(sharedSecretKey); - when(mockSecretKey.sign(any(STSTokenIdentifier.class))) + when(mockSecretKey.sign(any(byte[].class))) .thenReturn("mock-signature".getBytes(StandardCharsets.UTF_8)); when(mockSecretKeyClient.getCurrentSecretKey()).thenReturn(mockSecretKey); @@ -80,8 +86,7 @@ public void setUp() throws Exception { @Test public void testCreateSTSTokenStringContainsCorrectFields() throws IOException { - final String tokenString = secretManager.createSTSTokenString( - TEMP_ACCESS_KEY, ORIGINAL_ACCESS_KEY, ROLE_ARN, DURATION_SECONDS, SECRET_ACCESS_KEY, SESSION_POLICY, clock); + final String tokenString = secretManager.createSTSTokenString(createStsTokenParamsBuilder().build()); // Decode the token final Token token = new Token<>(); @@ -98,6 +103,9 @@ public void testCreateSTSTokenStringContainsCorrectFields() throws IOException { assertEquals(ROLE_ARN, identifier.getRoleArn()); assertEquals(SECRET_ACCESS_KEY, identifier.getSecretAccessKey()); assertEquals(SESSION_POLICY, identifier.getSessionPolicy()); + assertEquals(ASSUMED_ROLE_ID, identifier.getAssumedRoleId()); + assertEquals(ASSUMED_ROLE_USER_ARN, identifier.getAssumedRoleUserArn()); + assertEquals(clock.instant(), identifier.getCreationTime()); assertNotNull(identifier.getSecretKeyId()); assertEquals(new Text("STSToken"), identifier.getKind()); assertEquals("STS", identifier.getService()); @@ -107,7 +115,7 @@ public void testCreateSTSTokenStringContainsCorrectFields() throws IOException { @Test public void testCreateSTSTokenStringWithNullSessionPolicy() throws IOException { final String tokenString = secretManager.createSTSTokenString( - TEMP_ACCESS_KEY, ORIGINAL_ACCESS_KEY, ROLE_ARN, DURATION_SECONDS, SECRET_ACCESS_KEY, null, clock); + createStsTokenParamsBuilder().setSessionPolicy(null).build()); // Decode the token final Token token = new Token<>(); @@ -118,4 +126,84 @@ public void testCreateSTSTokenStringWithNullSessionPolicy() throws IOException { identifier.readFromByteArray(token.getIdentifier()); assertTrue(identifier.getSessionPolicy().isEmpty()); } + + /** + * createSTSTokenString() must use a single getCurrentSecretKey() for encryption, secretKeyId, and signing. If a + * second fetch happened during signing, a key rotation between calls would encrypt with the old key but stamp the + * token with the new key id. + */ + @Test + public void testCreateSTSTokenStringValidatesWhenSecretKeyRotatesDuringCreation() throws Exception { + // ManagedSecretKey.isExpired() uses Instant.now(), not the test clock. + final Instant keyCreationTime = Instant.now(); + final ManagedSecretKey encryptionKey = createManagedSecretKey( + UUID.fromString("11111111-1111-1111-1111-111111111111"), + "encryption-key-material-012345678901".getBytes(StandardCharsets.US_ASCII), + keyCreationTime); + final ManagedSecretKey signingKey = createManagedSecretKey( + UUID.fromString("22222222-2222-2222-2222-222222222222"), + "signing-key-material-01234567890123".getBytes(StandardCharsets.US_ASCII), + keyCreationTime); + + final RotatingSecretKeyTestClient rotatingSecretKeyClient = new RotatingSecretKeyTestClient( + encryptionKey, signingKey); + final STSTokenSecretManager rotatingSecretManager = new STSTokenSecretManager(rotatingSecretKeyClient); + + final String tokenString = rotatingSecretManager.createSTSTokenString(createStsTokenParamsBuilder().build()); + + final STSTokenIdentifier result = STSSecurityUtil.constructValidateAndDecryptSTSToken( + tokenString, rotatingSecretKeyClient, clock); + assertEquals(SECRET_ACCESS_KEY, result.getSecretAccessKey()); + assertEquals(encryptionKey.getId(), result.getSecretKeyId()); + assertEquals(1, rotatingSecretKeyClient.getCurrentSecretKeyCallCount()); + } + + private STSTokenSecretManager.CreateSTSTokenParams.Builder createStsTokenParamsBuilder() { + return STSTokenSecretManager.CreateSTSTokenParams.newBuilder() + .setTempAccessKeyId(TEMP_ACCESS_KEY) + .setOriginalAccessKeyId(ORIGINAL_ACCESS_KEY) + .setRoleArn(ROLE_ARN) + .setDurationSeconds(DURATION_SECONDS) + .setSecretAccessKey(SECRET_ACCESS_KEY) + .setSessionPolicy(SESSION_POLICY) + .setAssumedRoleId(ASSUMED_ROLE_ID) + .setAssumedRoleUserArn(ASSUMED_ROLE_USER_ARN) + .setClock(clock); + } + + private static ManagedSecretKey createManagedSecretKey(UUID id, byte[] keyBytes, Instant creationTime) { + final SecretKey secretKey = new SecretKeySpec(keyBytes, "HmacSHA256"); + return new ManagedSecretKey(id, creationTime, creationTime.plus(Duration.ofHours(1)), secretKey); + } + + /** + * Returns different current keys on consecutive getCurrentSecretKey() calls to simulate rotation. + */ + private static final class RotatingSecretKeyTestClient implements SecretKeyClient { + private final ManagedSecretKey firstKey; + private final ManagedSecretKey secondKey; + private final Map keysById = new HashMap<>(); + private int getCurrentSecretKeyCallCount; + + private RotatingSecretKeyTestClient(ManagedSecretKey firstKey, ManagedSecretKey secondKey) { + this.firstKey = firstKey; + this.secondKey = secondKey; + keysById.put(firstKey.getId(), firstKey); + keysById.put(secondKey.getId(), secondKey); + } + + @Override + public synchronized ManagedSecretKey getCurrentSecretKey() { + return getCurrentSecretKeyCallCount++ == 0 ? firstKey : secondKey; + } + + @Override + public ManagedSecretKey getSecretKey(UUID id) { + return keysById.get(id); + } + + private int getCurrentSecretKeyCallCount() { + return getCurrentSecretKeyCallCount; + } + } } diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/audit/S3GAction.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/audit/S3GAction.java index abdb64e5ff29..f215cd6cb0ff 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/audit/S3GAction.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/audit/S3GAction.java @@ -66,6 +66,7 @@ public enum S3GAction implements AuditAction { // STS endpoint ASSUME_ROLE, + GET_CALLER_IDENTITY, GET_OBJECT_ATTRIBUTES; diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3GActionIamMapper.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3GActionIamMapper.java index 9953ebe2020b..223b057b5bb3 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3GActionIamMapper.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3GActionIamMapper.java @@ -85,6 +85,7 @@ private S3GActionIamMapper() { case GENERATE_SECRET: case REVOKE_SECRET: case ASSUME_ROLE: + case GET_CALLER_IDENTITY: default: return null; } diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3sts/S3AssumeRoleResponseXml.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3sts/S3AssumeRoleResponseXml.java index bd4be9a7eafb..6c8b73906a76 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3sts/S3AssumeRoleResponseXml.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3sts/S3AssumeRoleResponseXml.java @@ -33,7 +33,7 @@ public class S3AssumeRoleResponseXml { private AssumeRoleResult assumeRoleResult; @XmlElement(name = "ResponseMetadata") - private ResponseMetadata responseMetadata; + private S3STSResponseMetadata responseMetadata; public AssumeRoleResult getAssumeRoleResult() { return assumeRoleResult; @@ -43,11 +43,11 @@ public void setAssumeRoleResult(AssumeRoleResult assumeRoleResult) { this.assumeRoleResult = assumeRoleResult; } - public ResponseMetadata getResponseMetadata() { + public S3STSResponseMetadata getResponseMetadata() { return responseMetadata; } - public void setResponseMetadata(ResponseMetadata responseMetadata) { + public void setResponseMetadata(S3STSResponseMetadata responseMetadata) { this.responseMetadata = responseMetadata; } @@ -157,23 +157,6 @@ public void setArn(String arn) { this.arn = arn; } } - - /** - * ResponseMetadata element. - */ - @XmlAccessorType(XmlAccessType.FIELD) - public static class ResponseMetadata { - @XmlElement(name = "RequestId") - private String requestId; - - public String getRequestId() { - return requestId; - } - - public void setRequestId(String requestId) { - this.requestId = requestId; - } - } } diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3sts/S3GetCallerIdentityResponseXml.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3sts/S3GetCallerIdentityResponseXml.java new file mode 100644 index 000000000000..594ed12e55de --- /dev/null +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3sts/S3GetCallerIdentityResponseXml.java @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.s3sts; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlRootElement; + +/** + * JAXB model for AWS STS GetCallerIdentityResponse. + */ +@XmlAccessorType(XmlAccessType.FIELD) +@XmlRootElement(name = "GetCallerIdentityResponse", namespace = "https://sts.amazonaws.com/doc/2011-06-15/") +public class S3GetCallerIdentityResponseXml { + + @XmlElement(name = "GetCallerIdentityResult") + private GetCallerIdentityResult getCallerIdentityResult; + + @XmlElement(name = "ResponseMetadata") + private S3STSResponseMetadata responseMetadata; + + public GetCallerIdentityResult getGetCallerIdentityResult() { + return getCallerIdentityResult; + } + + public void setGetCallerIdentityResult(GetCallerIdentityResult getCallerIdentityResult) { + this.getCallerIdentityResult = getCallerIdentityResult; + } + + public S3STSResponseMetadata getResponseMetadata() { + return responseMetadata; + } + + public void setResponseMetadata(S3STSResponseMetadata responseMetadata) { + this.responseMetadata = responseMetadata; + } + + /** + * GetCallerIdentityResult element. + */ + @XmlAccessorType(XmlAccessType.FIELD) + public static class GetCallerIdentityResult { + @XmlElement(name = "Arn") + private String arn; + + @XmlElement(name = "UserId") + private String userId; + + @XmlElement(name = "Account") + private String account; + + public String getArn() { + return arn; + } + + public void setArn(String arn) { + this.arn = arn; + } + + public String getUserId() { + return userId; + } + + public void setUserId(String userId) { + this.userId = userId; + } + + public String getAccount() { + return account; + } + + public void setAccount(String account) { + this.account = account; + } + } +} diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3sts/S3STSEndpoint.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3sts/S3STSEndpoint.java index d6ed5339a448..2c6200d1e661 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3sts/S3STSEndpoint.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3sts/S3STSEndpoint.java @@ -60,6 +60,7 @@ import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.helpers.AssumeRoleResponseInfo; import org.apache.hadoop.ozone.om.helpers.AwsRoleArnValidator; +import org.apache.hadoop.ozone.om.helpers.CallerIdentityInfo; import org.apache.hadoop.ozone.om.helpers.S3STSUtils; import org.apache.hadoop.ozone.s3.RequestIdentifier; import org.apache.hadoop.ozone.s3.exception.OS3Exception; @@ -73,8 +74,8 @@ * This endpoint provides temporary security credentials compatible with * AWS STS API, exposed on port 9880 or 9881 at the root path ({@code /}). *

- * Currently supports only AssumeRole operation. Other STS operations will - * return appropriate error responses. + * Currently supports AssumeRole and GetCallerIdentity operations. Other STS + * operations will return appropriate error responses. * * @see AWS STS API Reference */ @@ -113,7 +114,8 @@ public class S3STSEndpoint extends S3STSEndpointBase { static { try { - JAXB_CONTEXT = JAXBContext.newInstance(S3AssumeRoleResponseXml.class); + JAXB_CONTEXT = JAXBContext.newInstance( + S3AssumeRoleResponseXml.class, S3GetCallerIdentityResponseXml.class, S3STSResponseMetadata.class); } catch (JAXBException e) { throw new RuntimeException("Failed to initialize JAXBContext: " + e, e); } @@ -193,11 +195,12 @@ private Response handleSTSRequest(Set paramNamesToValidate, String actio case ASSUME_ROLE_ACTION: return handleAssumeRole( paramNamesToValidate, roleArn, roleSessionName, durationSeconds, awsIamSessionPolicy, version, requestId); + case GET_CALLER_IDENTITY_ACTION: + return handleGetCallerIdentity(version, requestId); // These operations are not supported yet case GET_SESSION_TOKEN_ACTION: case ASSUME_ROLE_WITH_SAML_ACTION: case ASSUME_ROLE_WITH_WEB_IDENTITY_ACTION: - case GET_CALLER_IDENTITY_ACTION: case DECODE_AUTHORIZATION_MESSAGE_ACTION: case GET_ACCESS_KEY_INFO_ACTION: throw new OSTSException(STS_INVALID_ACTION_NOT_IMPLEMENTED) @@ -307,39 +310,76 @@ private Response handleAssumeRole(Set paramNamesToValidate, String roleA .header("Content-Type", "text/xml") .build(); } catch (IOException e) { - LOG.error("Error during AssumeRole processing", e); - + throw toStsProcessingException( + S3GAction.ASSUME_ROLE, auditParams, e, action, "User is not authorized to perform: sts:AssumeRole on " + + "resource: " + roleArn); + } catch (Exception e) { getAuditLogger().logWriteFailure(buildAuditMessageForFailure(S3GAction.ASSUME_ROLE, auditParams, e)); + throw e; + } + } - if (e instanceof OMException) { - final OMException omException = (OMException) e; - if (omException.getResult() == OMException.ResultCodes.ACCESS_DENIED || - omException.getResult() == OMException.ResultCodes.PERMISSION_DENIED || - omException.getResult() == OMException.ResultCodes.TOKEN_EXPIRED) { - throw new OSTSException(ACCESS_DENIED) - .withMessage("User is not authorized to perform: sts:AssumeRole on resource: " + roleArn); - } - if (omException.getResult() == OMException.ResultCodes.INVALID_TOKEN) { - throw new OSTSException(STS_INVALID_CLIENT_TOKEN_ID); - } - if (omException.getResult() == OMException.ResultCodes.NOT_SUPPORTED_OPERATION || - omException.getResult() == OMException.ResultCodes.FEATURE_NOT_ENABLED) { - throw new OSTSException(STS_UNSUPPORTED_OPERATION).withMessage(omException.getMessage()); - } - if (omException.getResult() == OMException.ResultCodes.INVALID_REQUEST) { - throw new OSTSException(STS_VALIDATION_ERROR).withMessage(omException.getMessage()); - } - if (omException.getResult() == OMException.ResultCodes.MALFORMED_POLICY_DOCUMENT) { - throw new OSTSException(STS_MALFORMED_POLICY_DOCUMENT).withMessage(omException.getMessage()); - } - } - throw new OSTSException(STS_INTERNAL_FAILURE, e).withType("Receiver"); + private Response handleGetCallerIdentity(String version, String requestId) throws OSTSException { + final String action = GET_CALLER_IDENTITY_ACTION; + final Map auditParams = getAuditParameters(); + auditParams.put("action", action); + auditParams.put("requestId", requestId); + + if (version == null || !version.equals(EXPECTED_VERSION)) { + final OSTSException exception = new OSTSException(STS_INVALID_ACTION) + .withMessage("Could not find operation " + action + " for version " + + (version == null ? "NO_VERSION_SPECIFIED. Expected version is: " + EXPECTED_VERSION : version)); + getAuditLogger().logWriteFailure(buildAuditMessageForFailure( + S3GAction.GET_CALLER_IDENTITY, auditParams, exception)); + throw exception; + } + + try { + final CallerIdentityInfo identityInfo = getClient().getObjectStore().getCallerIdentity(); + final String responseXml = generateGetCallerIdentityResponse(identityInfo, requestId); + getAuditLogger().logWriteSuccess(buildAuditMessageForSuccess(S3GAction.GET_CALLER_IDENTITY, auditParams)); + return Response.ok(responseXml) + .header("Content-Type", "text/xml") + .build(); + } catch (IOException e) { + throw toStsProcessingException( + S3GAction.GET_CALLER_IDENTITY, auditParams, e, action, "User is not authorized to perform: " + + "sts:GetCallerIdentity"); } catch (Exception e) { - getAuditLogger().logWriteFailure(buildAuditMessageForFailure(S3GAction.ASSUME_ROLE, auditParams, e)); + getAuditLogger().logWriteFailure(buildAuditMessageForFailure(S3GAction.GET_CALLER_IDENTITY, auditParams, e)); throw e; } } + private OSTSException toStsProcessingException(S3GAction auditAction, Map auditParams, IOException e, + String operationName, String accessDeniedMessage) { + LOG.error("Error during {} processing", operationName, e); + getAuditLogger().logWriteFailure(buildAuditMessageForFailure(auditAction, auditParams, e)); + + if (e instanceof OMException) { + final OMException omException = (OMException) e; + if (omException.getResult() == OMException.ResultCodes.ACCESS_DENIED || + omException.getResult() == OMException.ResultCodes.PERMISSION_DENIED || + omException.getResult() == OMException.ResultCodes.TOKEN_EXPIRED) { + return new OSTSException(ACCESS_DENIED).withMessage(accessDeniedMessage); + } + if (omException.getResult() == OMException.ResultCodes.INVALID_TOKEN) { + return new OSTSException(STS_INVALID_CLIENT_TOKEN_ID); + } + if (omException.getResult() == OMException.ResultCodes.NOT_SUPPORTED_OPERATION || + omException.getResult() == OMException.ResultCodes.FEATURE_NOT_ENABLED) { + return new OSTSException(STS_UNSUPPORTED_OPERATION).withMessage(omException.getMessage()); + } + if (omException.getResult() == OMException.ResultCodes.INVALID_REQUEST) { + return new OSTSException(STS_VALIDATION_ERROR).withMessage(omException.getMessage()); + } + if (omException.getResult() == OMException.ResultCodes.MALFORMED_POLICY_DOCUMENT) { + return new OSTSException(STS_MALFORMED_POLICY_DOCUMENT).withMessage(omException.getMessage()); + } + } + return new OSTSException(STS_INTERNAL_FAILURE, e).withType("Receiver"); + } + private AssumeRoleParamValidationResult validateAssumeRoleParameters(Set paramNamesToValidate) { if (paramNamesToValidate == null || paramNamesToValidate.isEmpty()) { return AssumeRoleParamValidationResult.empty(); @@ -450,7 +490,7 @@ private String generateAssumeRoleResponse(String assumedRoleUserArn, AssumeRoleR user.setArn(assumedRoleUserArn); result.setAssumedRoleUser(user); response.setAssumeRoleResult(result); - final S3AssumeRoleResponseXml.ResponseMetadata meta = new S3AssumeRoleResponseXml.ResponseMetadata(); + final S3STSResponseMetadata meta = new S3STSResponseMetadata(); meta.setRequestId(requestId); response.setResponseMetadata(meta); @@ -463,5 +503,29 @@ private String generateAssumeRoleResponse(String assumedRoleUserArn, AssumeRoleR throw new IOException("Failed to marshal AssumeRole response", e); } } + + private String generateGetCallerIdentityResponse(CallerIdentityInfo identityInfo, String requestId) + throws IOException { + try { + final S3GetCallerIdentityResponseXml response = new S3GetCallerIdentityResponseXml(); + final S3GetCallerIdentityResponseXml.GetCallerIdentityResult result = + new S3GetCallerIdentityResponseXml.GetCallerIdentityResult(); + result.setAccount(identityInfo.getAccount()); + result.setArn(identityInfo.getArn()); + result.setUserId(identityInfo.getUserId()); + response.setGetCallerIdentityResult(result); + final S3STSResponseMetadata meta = new S3STSResponseMetadata(); + meta.setRequestId(requestId); + response.setResponseMetadata(meta); + + final Marshaller marshaller = JAXB_CONTEXT.createMarshaller(); + marshaller.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, Boolean.TRUE); + final StringWriter stringWriter = new StringWriter(); + marshaller.marshal(response, stringWriter); + return stringWriter.toString(); + } catch (JAXBException e) { + throw new IOException("Failed to marshal GetCallerIdentity response", e); + } + } } diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3sts/S3STSResponseMetadata.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3sts/S3STSResponseMetadata.java new file mode 100644 index 000000000000..a43ec76b4433 --- /dev/null +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3sts/S3STSResponseMetadata.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.s3sts; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlType; + +/** + * JAXB model for AWS STS ResponseMetadata element shared across STS responses. + */ +@XmlAccessorType(XmlAccessType.FIELD) +@XmlType(name = "ResponseMetadata", namespace = "https://sts.amazonaws.com/doc/2011-06-15/") +public class S3STSResponseMetadata { + + @XmlElement(name = "RequestId") + private String requestId; + + public String getRequestId() { + return requestId; + } + + public void setRequestId(String requestId) { + this.requestId = requestId; + } +} diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java index b79984fa93e7..6fd318dc9061 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java @@ -39,6 +39,7 @@ import org.apache.hadoop.ozone.client.protocol.ClientProtocol; import org.apache.hadoop.ozone.client.protocol.ListStatusLightOptions; import org.apache.hadoop.ozone.om.helpers.AssumeRoleResponseInfo; +import org.apache.hadoop.ozone.om.helpers.CallerIdentityInfo; import org.apache.hadoop.ozone.om.helpers.DeleteTenantState; import org.apache.hadoop.ozone.om.helpers.ErrorInfo; import org.apache.hadoop.ozone.om.helpers.LeaseKeyInfo; @@ -899,7 +900,12 @@ public AssumeRoleResponseInfo assumeRole(String roleArn, String roleSessionName, } @Override - public void revokeSTSToken(String sessionToken) throws IOException { + public CallerIdentityInfo getCallerIdentity() throws IOException { + return null; + } + + @Override + public void revokeSTSToken(String originalAccessKeyId) throws IOException { } @Override diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/util/TestS3GActionIamMapper.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/util/TestS3GActionIamMapper.java index c7ae9e4e924c..82f0134cffec 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/util/TestS3GActionIamMapper.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/util/TestS3GActionIamMapper.java @@ -62,6 +62,7 @@ public void copyActionsReturnNull() { @Test public void nonIamActionsReturnNull() { assertNull(S3GActionIamMapper.toS3ActionString(S3GAction.ASSUME_ROLE)); + assertNull(S3GActionIamMapper.toS3ActionString(S3GAction.GET_CALLER_IDENTITY)); assertNull(S3GActionIamMapper.toS3ActionString(S3GAction.GENERATE_SECRET)); assertNull(S3GActionIamMapper.toS3ActionString(S3GAction.REVOKE_SECRET)); } diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3sts/TestS3STSEndpoint.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3sts/TestS3STSEndpoint.java index 36adf2359c4a..379bd27eb981 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3sts/TestS3STSEndpoint.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3sts/TestS3STSEndpoint.java @@ -51,6 +51,7 @@ import org.apache.hadoop.ozone.client.OzoneClientStub; import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.helpers.AssumeRoleResponseInfo; +import org.apache.hadoop.ozone.om.helpers.CallerIdentityInfo; import org.apache.hadoop.ozone.s3.OzoneConfigurationHolder; import org.apache.hadoop.ozone.s3.RequestIdentifier; import org.apache.hadoop.ozone.s3.exception.OSTSException; @@ -103,6 +104,11 @@ public void setup() throws Exception { "session-token", Instant.now().plusSeconds(3600).getEpochSecond(), "AROA1234567890123456:test-session")); + when(objectStore.getCallerIdentity()) + .thenReturn(new CallerIdentityInfo( + "123456789012", + "arn:aws:iam::123456789012:user/test-user", + "test-user")); when(clientStub.getObjectStore()).thenReturn(objectStore); endpoint = new S3STSEndpoint(); @@ -560,6 +566,79 @@ public void testStsWhenActionNotImplemented() throws Exception { "Operation GetSessionToken is not supported yet."); } + @Test + public void testStsGetCallerIdentitySuccessForGetMethod() throws Exception { + final Response response = endpoint.get("GetCallerIdentity", null, null, null, "2011-06-15", null); + + assertEquals(200, response.getStatus()); + verify(objectStore).getCallerIdentity(); + verify(auditLogger).logWriteSuccess(any(AuditMessage.class)); + verify(auditLogger, never()).logWriteFailure(any(AuditMessage.class)); + + final Document doc = parseXml((String) response.getEntity()); + assertEquals("GetCallerIdentityResponse", doc.getDocumentElement().getLocalName()); + assertEquals(STS_NS, doc.getDocumentElement().getNamespaceURI()); + assertEquals( + "123456789012", doc.getElementsByTagNameNS(STS_NS, "Account").item(0).getTextContent()); + assertEquals( + "arn:aws:iam::123456789012:user/test-user", doc.getElementsByTagNameNS(STS_NS, "Arn").item(0).getTextContent()); + assertEquals( + "test-user", doc.getElementsByTagNameNS(STS_NS, "UserId").item(0).getTextContent()); + } + + @Test + public void testStsGetCallerIdentityIgnoresExtraParameters() throws Exception { + final Response response = endpoint.get("GetCallerIdentity", ROLE_ARN, ROLE_SESSION_NAME, 3600, "2011-06-15", null); + + assertEquals(200, response.getStatus()); + verify(objectStore).getCallerIdentity(); + } + + @Test + public void testStsGetCallerIdentityIgnoresExtraParametersForPostMethod() throws Exception { + formParameters = new Form(); + formParameters.param("Action", "GetCallerIdentity"); + formParameters.param("Version", "2011-06-15"); + formParameters.param("RoleArn", ROLE_ARN); + formParameters.param("RoleSessionName", ROLE_SESSION_NAME); + formParameters.param("DurationSeconds", "3600"); + + final Response response = endpoint.post(formParameters); + + assertEquals(200, response.getStatus()); + verify(objectStore).getCallerIdentity(); + } + + @Test + public void testStsGetCallerIdentityRejectsMissingVersion() throws Exception { + final OSTSException ex = assertThrows( + OSTSException.class, () -> endpoint.get("GetCallerIdentity", null, null, null, null, null)); + + assertEquals(400, ex.getHttpCode()); + verify(auditLogger).logWriteFailure(any(AuditMessage.class)); + verify(objectStore, never()).getCallerIdentity(); + + ex.setRequestId(REQUEST_ID); + assertStsErrorXml( + ex.toXml(), AWS_FAULT_NS, "Sender", "InvalidAction", + "Could not find operation GetCallerIdentity for version NO_VERSION_SPECIFIED"); + } + + @Test + public void testStsGetCallerIdentityRejectsInvalidVersion() throws Exception { + final OSTSException ex = assertThrows( + OSTSException.class, () -> endpoint.get("GetCallerIdentity", null, null, null, "2020-01-01", null)); + + assertEquals(400, ex.getHttpCode()); + verify(auditLogger).logWriteFailure(any(AuditMessage.class)); + verify(objectStore, never()).getCallerIdentity(); + + ex.setRequestId(REQUEST_ID); + assertStsErrorXml( + ex.toXml(), AWS_FAULT_NS, "Sender", "InvalidAction", + "Could not find operation GetCallerIdentity for version 2020-01-01"); + } + @Test public void testStsMissingRoleSessionName() throws Exception { final OSTSException ex = assertThrows(OSTSException.class, () ->