HDDS-16110. [STS] Update key in sts revocation table

hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java

@@ -314,6 +314,8 @@ public final class OzoneConsts {
   public static final String S3_SETSECRET_USER = "S3SetSecretUser";
   public static final String S3_REVOKESECRET_USER = "S3RevokeSecretUser";
   public static final String S3_REVOKESTSTOKEN_USER = "S3RevokeSTSTokenUser";
+  public static final String S3_STS_ORIGINAL_ACCESS_KEY_ID = "originalAccessKeyId";
+  public static final String S3_STS_TEMP_ACCESS_KEY_ID = "tempAccessKeyId";
   public static final String RENAMED_KEYS_MAP = "renamedKeysMap";
   public static final String UNRENAMED_KEYS_MAP = "unRenamedKeysMap";
   public static final String MULTIPART_UPLOAD_PART_NUMBER = "partNumber";

hadoop-hdds/common/src/main/resources/ozone-default.xml

@@ -5254,9 +5254,10 @@
     <value>3h</value>
     <tag>OZONE, OM, PERFORMANCE, SECURITY</tag>
     <description>
-      A background job that periodically checks revoked STS token entries and
-      deletes ones that have existed for 12 hours. This entry controls the interval of this
-      cleanup check. Unit could be defined with postfix (ns,ms,s,m,h,d).
+      A background service that periodically scans the s3RevokedStsTokenTable and deletes
+      revocation entries whose cutoff is older than the maximum STS token lifetime (12 hours).
+      This property controls how often the cleanup service runs. Unit could be defined with
+      postfix (ns,ms,s,m,h,d).
     </description>
   </property>
   <property>

hadoop-hdds/docs/content/design/ozone-sts.md

@@ -139,17 +139,24 @@ was included with the AssumeRole request, the String return value will also incl
 would further limit the scope of the permissions, resources and actions granted by the role in Ranger, such that the temporary 
 credential will have the permissions and actions comprising the intersection of the role permissions and actions and the sessionPolicy permissions and actions.
 - HMAC-SHA256 signature - used to ensure the sessionToken was created by Ozone and was not altered since it was created.
+- creation time of the token (via `OMTokenProto#issueDate`, exposed as `STSTokenIdentifier#getCreationTime()`)
 - expiration time of the token (via `ShortLivedTokenIdentifier#getExpiry()`)
 - UUID of the OzoneManager secret key used to sign the sessionToken and encrypt the secretAccessKey (via `ShortLivedTokenIdentifier#getSecretKeyId()`)
 
 ## 3.5 STS Token Revocation
 
 In the rare event temporary credentials need to be revoked (ex. for security reasons), a table in the OzoneManager RocksDB will be created
-to store revoked tokens, and a command-line utility will be created to add tokens to the table.  A background cleaner service
-will be created to run every 3 hours to delete revoked tokens that have been in the table for more than 12 hours.  The
-input parameter for the command-line utility will be the sessionToken - this value is returned in plain text as a result 
-of the AssumeRole call (mentioned above).  In this way, specific STS tokens can be revoked as opposed to all tokens.  Furthermore, 
-AWS doesn't have a standard API to revoke tokens therefore we are creating our own system.
+to store revocation cutoffs per originalAccessKeyId, and a command-line utility will be created to add entries to the table. 
+A background cleaner service will be created to run every 3 hours to delete revocation entries whose cutoff is more than 12 hours old.
+
+The command-line utility accepts only `originalAccessKeyId`. The OM stores revocations by keying the table on
+`originalAccessKeyId` and storing the revocation cutoff time in milliseconds as the value. When the command is issued,
+all STS tokens created by that `originalAccessKeyId` whose signed `creationTime` is strictly before the cutoff are
+revoked. Tokens created at or after the cutoff remain valid.
+
+Before writing a revocation entry, the OM verifies that `originalAccessKeyId` corresponds to a real Kerberos identity by
+checking that an S3 secret exists for it. This prevents bogus entries from filling the table. Non-admins may only
+revoke their own `originalAccessKeyId`; S3 and tenant admins may revoke other principals.
 
 Additionally, if the Kerberos identity of the user that created the STS token is revoked via the `ozone s3 revokesecret`
 command, then all the existing and unexpired STS tokens that user created will be revoked.
@@ -221,7 +228,8 @@ created in Ranger as per the Prerequisites above.
 originalAccessKeyId in the session token and perform the following checks:
   - Ensure that if the accessKeyId starts with "ASIA", that a sessionToken was included in the `x-amz-security-token` header
   - Ensure the sessionToken is not expired
-  - Ensure the sessionToken is not revoked via a `keyMayExist` check in OzoneManager RocksDB
+  - Ensure the STS credentials are not revoked by looking up the revocation cutoff for the token's originalAccessKeyId
+    and comparing it against the token's signed creationTime
   - Validate the HMAC-SHA256 signature in the sessionToken
   - Decrypt the secretAccessKey from the sessionToken and validate the AWS signature
   - Authorize the call with either RangerOzoneAuthorizer or OzoneNativeAuthorizer

hadoop-ozone/cli-shell/src/main/java/org/apache/hadoop/ozone/shell/s3/RevokeSTSTokenHandler.java

@@ -29,18 +29,18 @@
 /**
  * Executes revocation of STS tokens.
  *
- * <p>This command marks the specified STS token as revoked by adding it to the OM's revoked STS token table.
- * Subsequent S3 requests using the same session token will be rejected once the revocation
- * state has propagated.</p>
+ * <p>This command records a revocation cutoff for the given original access key ID in the OM's
+ * revoked STS token table. Subsequent S3 requests using STS tokens created before that cutoff
+ * will be rejected once the revocation state has propagated.</p>
  */
 @Command(name = "revokeststoken",
-    description = "Revoke S3 STS token for the given session token")
+    description = "Revoke S3 STS tokens for the given original access key ID")
 public class RevokeSTSTokenHandler extends S3Handler {
 
-  @Option(names = "-t",
+  @Option(names = {"-o", "--original-access-key-id"},
       required = true,
-      description = "STS session token")
-  private String sessionToken;
+      description = "Original long-lived access key ID whose STS tokens should be revoked")
+  private String originalAccessKeyId;
 
   @Option(names = "-y",
       description = "Continue without interactive user confirmation")
@@ -56,8 +56,8 @@ protected void execute(OzoneClient client, OzoneAddress address)
       throws IOException {
 
     if (!yes) {
-      out().print("Enter 'y' to confirm STS token revocation for sessionToken '" +
-          sessionToken + "': ");
+      out().print(
+          "Enter 'y' to confirm STS token revocation for originalAccessKeyId '" + originalAccessKeyId + "': ");
       out().flush();
       final Scanner scanner = new Scanner(new InputStreamReader(System.in, StandardCharsets.UTF_8));
       final String confirmation = scanner.next().trim().toLowerCase();
@@ -67,7 +67,7 @@ protected void execute(OzoneClient client, OzoneAddress address)
       }
     }
 
-    client.getObjectStore().revokeSTSToken(sessionToken);
-    out().println("STS token revoked for sessionToken '" + sessionToken + "'.");
+    client.getObjectStore().revokeSTSToken(originalAccessKeyId);
+    out().println("STS tokens revoked for originalAccessKeyId '" + originalAccessKeyId + "'.");
   }
 }

hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java

@@ -813,12 +813,12 @@ public AssumeRoleResponseInfo assumeRole(String roleArn, String roleSessionName,
   }
 
   /**
-   * Revokes an STS token.
-   * @param sessionToken            The STS sessionToken
+   * Revokes STS tokens for the given original access key ID.
+   * @param originalAccessKeyId     The original long-lived access key ID whose STS tokens to revoke
    * @throws IOException            if an error occurs while revoking the STS token
    */
-  public void revokeSTSToken(String sessionToken) throws IOException  {
-    proxy.revokeSTSToken(sessionToken);
+  public void revokeSTSToken(String originalAccessKeyId) throws IOException  {
+    proxy.revokeSTSToken(originalAccessKeyId);
   }
 
   /**

hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java

@@ -142,8 +142,9 @@ OzoneVolume getVolumeDetails(String volumeName)
       throws IOException;
 
   /**
-   * @return Raw GetS3VolumeContextResponse.
-   * S3Auth won't be updated with actual userPrincipal by this call.
+   * @return S3 volume context from OM.
+   * When thread-local {@link S3Auth} is set, implementations update it with OM-returned
+   * {@code userPrincipal} and, when present, validated STS {@code originalAccessKeyId}.
    * @throws IOException
    */
   S3VolumeContext getS3VolumeContext() throws IOException;
@@ -1648,11 +1649,11 @@ AssumeRoleResponseInfo assumeRole(String roleArn, String roleSessionName, int du
       String awsIamSessionPolicy, String requestId) throws IOException;
 
   /**
-   * Revokes an STS token.
-   * @param sessionToken            The STS sessionToken
+   * Revokes STS tokens for the given original access key ID.
+   * @param originalAccessKeyId     The original long-lived access key ID whose STS tokens to revoke
    * @throws IOException            if an error occurs while revoking the STS token
    */
-  void revokeSTSToken(String sessionToken) throws IOException;
+  void revokeSTSToken(String originalAccessKeyId) throws IOException;
 
   /**
    * Gets the lifecycle configuration information.

hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java

@@ -211,6 +211,8 @@ public class RpcClient implements ClientProtocol {
   private final XceiverClientFactory xceiverClientManager;
   private final UserGroupInformation ugi;
   private UserGroupInformation s3gUgi;
+  // Cached per thread for the current S3 Gateway request - cleared with thread-local S3Auth.
+  private final ThreadLocal<S3VolumeContext> cachedS3VolumeContext = new ThreadLocal<>();
   private final ClientId clientId = ClientId.randomId();
   private final boolean unsafeByteBufferConversion;
   private Text dtService;
@@ -505,12 +507,31 @@ public OzoneVolume getVolumeDetails(String volumeName)
 
   @Override
   public S3VolumeContext getS3VolumeContext() throws IOException {
-    S3VolumeContext resp = ozoneManagerClient.getS3VolumeContext();
-    String userPrincipal = resp.getUserPrincipal();
-    updateS3Principal(userPrincipal);
+    final S3VolumeContext cached = cachedS3VolumeContext.get();
+    if (cached != null) {
+      return cached;
+    }
+    final S3VolumeContext resp = ozoneManagerClient.getS3VolumeContext();
+    updateS3Principal(resp.getUserPrincipal());
+    updateValidatedStsOriginalAccessKeyId(resp.getStsOriginalAccessKeyId());
+    cachedS3VolumeContext.set(resp);
     return resp;
   }
 
+  private void updateValidatedStsOriginalAccessKeyId(String stsOriginalAccessKeyId) {
+    final S3Auth s3Auth = this.getThreadLocalS3Auth();
+    if (s3Auth != null && StringUtils.isNotEmpty(stsOriginalAccessKeyId)) {
+      LOG.debug("Updating S3Auth.validatedStsOriginalAccessKeyId to {}", stsOriginalAccessKeyId);
+      s3Auth.setValidatedStsOriginalAccessKeyId(stsOriginalAccessKeyId);
+      this.setThreadLocalS3Auth(s3Auth);
+    }
+  }
+
+  private void updateS3Context(KeyInfoWithVolumeContext keyInfoWithS3Context) {
+    keyInfoWithS3Context.getUserPrincipal().ifPresent(this::updateS3Principal);
+    keyInfoWithS3Context.getStsOriginalAccessKeyId().ifPresent(this::updateValidatedStsOriginalAccessKeyId);
+  }
+
   private void updateS3Principal(String userPrincipal) {
     S3Auth s3Auth = this.getThreadLocalS3Auth();
     // Update user principal if needed to be used for KMS client
@@ -1979,7 +2000,7 @@ private OmKeyInfo getS3KeyInfo(
         .build();
     KeyInfoWithVolumeContext keyInfoWithS3Context =
         ozoneManagerClient.getKeyInfo(keyArgs, true);
-    keyInfoWithS3Context.getUserPrincipal().ifPresent(this::updateS3Principal);
+    updateS3Context(keyInfoWithS3Context);
     return keyInfoWithS3Context.getKeyInfo();
   }
 
@@ -2004,7 +2025,7 @@ private OmKeyInfo getS3PartKeyInfo(
         .build();
     KeyInfoWithVolumeContext keyInfoWithS3Context =
         ozoneManagerClient.getKeyInfo(keyArgs, true);
-    keyInfoWithS3Context.getUserPrincipal().ifPresent(this::updateS3Principal);
+    updateS3Context(keyInfoWithS3Context);
     return keyInfoWithS3Context.getKeyInfo();
   }
 
@@ -2896,6 +2917,7 @@ public OzoneKey headS3Object(String bucketName, String keyName)
   @Override
   public void setThreadLocalS3Auth(
       S3Auth ozoneSharedSecretAuth) {
+    cachedS3VolumeContext.remove();
     ozoneManagerClient.setThreadLocalS3Auth(ozoneSharedSecretAuth);
     this.s3gUgi = UserGroupInformation.createRemoteUser(getThreadLocalS3Auth().getUserPrincipal());
   }
@@ -2908,6 +2930,7 @@ public S3Auth getThreadLocalS3Auth() {
   @Override
   public void clearThreadLocalS3Auth() {
     ozoneManagerClient.clearThreadLocalS3Auth();
+    cachedS3VolumeContext.remove();
   }
 
   @Override
@@ -3022,8 +3045,8 @@ public AssumeRoleResponseInfo assumeRole(String roleArn, String roleSessionName,
   }
 
   @Override
-  public void revokeSTSToken(String sessionToken) throws IOException {
-    ozoneManagerClient.revokeSTSToken(sessionToken);
+  public void revokeSTSToken(String originalAccessKeyId) throws IOException {
+    ozoneManagerClient.revokeSTSToken(originalAccessKeyId);
   }
 
   @Override

hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/rpc/TestRpcClient.java

@@ -21,20 +21,30 @@
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
 import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertSame;
 import static org.junit.jupiter.api.Assertions.assertThrows;
 
 import java.io.IOException;
 import java.util.LinkedList;
 import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
 import org.apache.hadoop.hdds.conf.OzoneConfiguration;
 import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
 import org.apache.hadoop.hdds.scm.XceiverClientFactory;
 import org.apache.hadoop.ozone.OzoneManagerVersion;
 import org.apache.hadoop.ozone.client.MockOmTransport;
 import org.apache.hadoop.ozone.client.MockXceiverClientFactory;
+import org.apache.hadoop.ozone.om.helpers.S3VolumeContext;
 import org.apache.hadoop.ozone.om.helpers.ServiceInfo;
 import org.apache.hadoop.ozone.om.helpers.ServiceInfoEx;
+import org.apache.hadoop.ozone.om.protocol.S3Auth;
 import org.apache.hadoop.ozone.om.protocolPB.OmTransport;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.GetS3VolumeContextResponse;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Status;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.VolumeInfo;
 import org.apache.ozone.test.GenericTestUtils;
 import org.apache.ozone.test.GenericTestUtils.LogCapturer;
 import org.junit.jupiter.api.Test;
@@ -228,6 +238,64 @@ public void testFutureVersionShouldNotBeAnExpectedVersion() {
         () -> validateOmVersion(OzoneManagerVersion.FUTURE_VERSION, null));
   }
 
+  @Test
+  public void testGetS3VolumeContextCachesResponseWithinSameS3Auth() throws IOException {
+    final CountingS3VolumeContextTransport transport = new CountingS3VolumeContextTransport();
+    final RpcClient rpcClient = createRpcClient(transport);
+    try {
+      final S3Auth s3Auth = new S3Auth("sign", "sig", "ASIAEXAMPLE", "ASIAEXAMPLE");
+      rpcClient.setThreadLocalS3Auth(s3Auth);
+
+      final S3VolumeContext first = rpcClient.getS3VolumeContext();
+      final S3VolumeContext second = rpcClient.getS3VolumeContext();
+
+      assertEquals(1, transport.getS3VolumeContextCallCount());
+      assertSame(first, second);
+      assertEquals("AKIAORIGINAL123", s3Auth.getValidatedStsOriginalAccessKeyId());
+      assertEquals("alice", s3Auth.getUserPrincipal());
+    } finally {
+      rpcClient.close();
+    }
+  }
+
+  @Test
+  public void testClearThreadLocalS3AuthClearsS3VolumeContextCache() throws IOException {
+    final CountingS3VolumeContextTransport transport = new CountingS3VolumeContextTransport();
+    final RpcClient rpcClient = createRpcClient(transport);
+    try {
+      rpcClient.setThreadLocalS3Auth(new S3Auth("sign", "sig", "ASIAEXAMPLE", "ASIAEXAMPLE"));
+      rpcClient.getS3VolumeContext();
+      rpcClient.getS3VolumeContext();
+      assertEquals(1, transport.getS3VolumeContextCallCount());
+
+      rpcClient.clearThreadLocalS3Auth();
+      rpcClient.setThreadLocalS3Auth(new S3Auth("sign", "sig", "ASIAEXAMPLE", "ASIAEXAMPLE"));
+      rpcClient.getS3VolumeContext();
+
+      assertEquals(2, transport.getS3VolumeContextCallCount());
+    } finally {
+      rpcClient.close();
+    }
+  }
+
+  @Test
+  public void testSetThreadLocalS3AuthClearsS3VolumeContextCache() throws IOException {
+    final CountingS3VolumeContextTransport transport = new CountingS3VolumeContextTransport();
+    final RpcClient rpcClient = createRpcClient(transport);
+    try {
+      rpcClient.setThreadLocalS3Auth(new S3Auth("sign", "sig", "ASIAEXAMPLE", "ASIAEXAMPLE"));
+      rpcClient.getS3VolumeContext();
+      assertEquals(1, transport.getS3VolumeContextCallCount());
+
+      rpcClient.setThreadLocalS3Auth(new S3Auth("sign2", "sig2", "ASIAEXAMPLE2", "ASIAEXAMPLE2"));
+      rpcClient.getS3VolumeContext();
+
+      assertEquals(2, transport.getS3VolumeContextCallCount());
+    } finally {
+      rpcClient.close();
+    }
+  }
+
   @Test
   public void testCloseTwiceDoesNotWarn() throws IOException {
     RpcClient rpcClient = createRpcClient();
@@ -250,11 +318,15 @@ public void testCloseTwiceDoesNotWarn() throws IOException {
   }
 
   private static RpcClient createRpcClient() throws IOException {
+    return createRpcClient(new MockOmTransport());
+  }
+
+  private static RpcClient createRpcClient(MockOmTransport transport) throws IOException {
     OzoneConfiguration config = new OzoneConfiguration();
     return new RpcClient(config, null) {
       @Override
       protected OmTransport createOmTransport(String omServiceId) {
-        return new MockOmTransport();
+        return transport;
       }
 
       @Override
@@ -264,4 +336,37 @@ protected XceiverClientFactory createXceiverClientFactory(
       }
     };
   }
+
+  private static final class CountingS3VolumeContextTransport extends MockOmTransport {
+    private final AtomicInteger getS3VolumeContextCallCount = new AtomicInteger();
+
+    @Override
+    public OMResponse submitRequest(OMRequest payload) throws IOException {
+      if (payload.getCmdType() == Type.GetS3VolumeContext) {
+        getS3VolumeContextCallCount.incrementAndGet();
+        final VolumeInfo volumeInfo = VolumeInfo.newBuilder()
+            .setVolume("s3v")
+            .setAdminName("admin")
+            .setOwnerName("owner")
+            .build();
+        final GetS3VolumeContextResponse getS3VolumeContextResponse =
+            GetS3VolumeContextResponse.newBuilder()
+                .setVolumeInfo(volumeInfo)
+                .setUserPrincipal("alice")
+                .setStsOriginalAccessKeyId("AKIAORIGINAL123")
+                .build();
+        return OMResponse.newBuilder()
+            .setCmdType(payload.getCmdType())
+            .setSuccess(true)
+            .setStatus(Status.OK)
+            .setGetS3VolumeContextResponse(getS3VolumeContextResponse)
+            .build();
+      }
+      return super.submitRequest(payload);
+    }
+
+    private int getS3VolumeContextCallCount() {
+      return getS3VolumeContextCallCount.get();
+    }
+  }
 }