Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view

Large diffs are not rendered by default.

Large diffs are not rendered by default.

Original file line number Diff line number Diff line change
Expand Up @@ -137,7 +137,7 @@
import org.apache.hadoop.hdds.scm.container.ReplicationManagerReport;
import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
import org.apache.hadoop.hdds.scm.protocol.StorageContainerLocationProtocol;
import org.apache.hadoop.hdds.scm.protocol.StorageContainerLocationInternalInterface;
import org.apache.hadoop.hdds.scm.proxy.SCMContainerLocationFailoverProxyProvider;
import org.apache.hadoop.hdds.tracing.TracingUtil;
import org.apache.hadoop.io_.retry.RetryProxy;
Expand All @@ -154,12 +154,12 @@

/**
* This class is the client-side translator to translate the requests made on
* the {@link StorageContainerLocationProtocol} interface to the RPC server
* the {@link StorageContainerLocationInternalInterface} interface to the RPC server
* implementing {@link StorageContainerLocationProtocolPB}.
*/
@InterfaceAudience.Private
public final class StorageContainerLocationProtocolClientSideTranslatorPB
implements StorageContainerLocationProtocol, ProtocolTranslator, Closeable {
implements StorageContainerLocationInternalInterface, ProtocolTranslator, Closeable {

/**
* RpcController is not used and hence is set to null.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@
import org.apache.hadoop.hdds.scm.ScmInfo;
import org.apache.hadoop.hdds.scm.ha.SCMHAUtils;
import org.apache.hadoop.hdds.scm.protocol.ScmBlockLocationProtocol;
import org.apache.hadoop.hdds.scm.protocol.StorageContainerLocationProtocol;
import org.apache.hadoop.hdds.scm.protocol.StorageContainerLocationInternalInterface;
import org.apache.hadoop.hdds.scm.protocolPB.ScmBlockLocationProtocolClientSideTranslatorPB;
import org.apache.hadoop.hdds.scm.protocolPB.StorageContainerLocationProtocolClientSideTranslatorPB;
import org.apache.hadoop.hdds.scm.protocolPB.StorageContainerLocationProtocolClientSideTranslatorPB.ScmNodeTarget;
Expand Down Expand Up @@ -139,38 +139,38 @@ public static ScmBlockLocationProtocol getScmBlockClient(
conf);
}

public static StorageContainerLocationProtocol getScmContainerClient(
public static StorageContainerLocationInternalInterface getScmContainerClient(
ConfigurationSource conf) {
SCMContainerLocationFailoverProxyProvider proxyProvider =
new SCMContainerLocationFailoverProxyProvider(conf, null);
StorageContainerLocationProtocol scmContainerClient =
StorageContainerLocationInternalInterface scmContainerClient =
TracingUtil.createProxy(
new StorageContainerLocationProtocolClientSideTranslatorPB(
proxyProvider), StorageContainerLocationProtocol.class, conf);
proxyProvider), StorageContainerLocationInternalInterface.class, conf);
return scmContainerClient;
}

@VisibleForTesting
public static StorageContainerLocationProtocol getScmContainerClient(
public static StorageContainerLocationInternalInterface getScmContainerClient(
ConfigurationSource conf, UserGroupInformation userGroupInformation) {
SCMContainerLocationFailoverProxyProvider proxyProvider =
new SCMContainerLocationFailoverProxyProvider(conf,
userGroupInformation);
StorageContainerLocationProtocol scmContainerClient =
StorageContainerLocationInternalInterface scmContainerClient =
TracingUtil.createProxy(
new StorageContainerLocationProtocolClientSideTranslatorPB(
proxyProvider), StorageContainerLocationProtocol.class, conf);
proxyProvider), StorageContainerLocationInternalInterface.class, conf);
return scmContainerClient;
}

public static StorageContainerLocationProtocol getScmContainerClientForNode(
public static StorageContainerLocationInternalInterface getScmContainerClientForNode(
ConfigurationSource conf, ScmNodeTarget targetScmNode) {
SCMContainerLocationFailoverProxyProvider proxyProvider =
new SCMContainerLocationFailoverProxyProvider(conf, null);
StorageContainerLocationProtocol scmContainerClient =
StorageContainerLocationInternalInterface scmContainerClient =
TracingUtil.createProxy(
new StorageContainerLocationProtocolClientSideTranslatorPB(
proxyProvider, targetScmNode), StorageContainerLocationProtocol.class, conf);
proxyProvider, targetScmNode), StorageContainerLocationInternalInterface.class, conf);
return scmContainerClient;
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,13 @@
package org.apache.hadoop.hdds.scm.protocolPB;

import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertTrue;

import java.lang.reflect.Modifier;
import org.apache.hadoop.hdds.scm.ScmConfig;
import org.apache.hadoop.hdds.scm.protocol.StorageContainerLocationInternalInterface;
import org.apache.hadoop.hdds.scm.protocol.StorageContainerLocationProtocol;
import org.apache.hadoop.ipc_.RPC;
import org.apache.hadoop.security.KerberosInfo;
Expand All @@ -44,4 +48,11 @@ public void testProtocolMetadata() {
assertEquals(ScmConfig.ConfigStrings.HDDS_SCM_KERBEROS_PRINCIPAL_KEY,
kerberosInfo.serverPrincipal());
}

@Test
public void testProtocolIdentityIsSeparateFromInternalInterface() {
assertFalse(StorageContainerLocationProtocol.class.isInterface());
assertTrue(Modifier.isFinal(StorageContainerLocationProtocol.class.getModifiers()));
assertTrue(StorageContainerLocationInternalInterface.class.isInterface());
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,9 @@
import static org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.Type.GetPipeline;
import static org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.Type.ListContainer;
import static org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.Type.ListPipelines;
import static org.apache.hadoop.hdds.scm.protocol.StorageContainerLocationProtocol.ADMIN_COMMAND_TYPE;
import static org.apache.hadoop.hdds.scm.protocol.StorageContainerLocationProtocol.FOLLOWER_READABLE_COMMAND_TYPES;
import static org.apache.hadoop.hdds.scm.protocol.StorageContainerLocationInternalInterface.ADMIN_COMMAND_TYPE;
import static org.apache.hadoop.hdds.scm.protocol.StorageContainerLocationInternalInterface
.FOLLOWER_READABLE_COMMAND_TYPES;

import com.google.protobuf.RpcController;
import com.google.protobuf.ServiceException;
Expand Down Expand Up @@ -160,7 +161,7 @@
/**
* This class is the server-side translator that forwards requests received on
* {@link StorageContainerLocationProtocolPB} to the
* {@link StorageContainerLocationProtocol} server implementation.
* {@link StorageContainerLocationInternalInterface} server implementation.
*/
@InterfaceAudience.Private
@SuppressWarnings({"method"})
Expand All @@ -181,7 +182,7 @@ public final class StorageContainerLocationProtocolServerSideTranslatorPB
+ " Please upgrade the client to a version that supports Erasure"
+ " Coded data, and retry!";

private final StorageContainerLocationProtocol impl;
private final StorageContainerLocationInternalInterface impl;
private final StorageContainerManager scm;
private static final String ROLE_TYPE = "SCM";

Expand All @@ -192,12 +193,12 @@ public final class StorageContainerLocationProtocolServerSideTranslatorPB
/**
* Creates a new StorageContainerLocationProtocolServerSideTranslatorPB.
*
* @param impl {@link StorageContainerLocationProtocol} server
* @param impl {@link StorageContainerLocationInternalInterface} server
* implementation
* @param protocolMetrics
*/
public StorageContainerLocationProtocolServerSideTranslatorPB(
StorageContainerLocationProtocol impl,
StorageContainerLocationInternalInterface impl,
StorageContainerManager scm,
ProtocolMessageMetrics<StorageContainerLocationProtocolProtos.Type> protocolMetrics)
throws IOException {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,7 @@
import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
import org.apache.hadoop.hdds.scm.pipeline.PipelineManager;
import org.apache.hadoop.hdds.scm.pipeline.PipelineNotFoundException;
import org.apache.hadoop.hdds.scm.protocol.StorageContainerLocationProtocol;
import org.apache.hadoop.hdds.scm.protocol.StorageContainerLocationInternalInterface;
import org.apache.hadoop.hdds.scm.protocol.StorageContainerLocationProtocolServerSideTranslatorPB;
import org.apache.hadoop.hdds.scm.protocolPB.StorageContainerLocationProtocolPB;
import org.apache.hadoop.hdds.security.SecurityConfig;
Expand Down Expand Up @@ -132,7 +132,7 @@
* The RPC server that listens to requests from clients.
*/
public class SCMClientProtocolServer implements
StorageContainerLocationProtocol, Auditor {
StorageContainerLocationInternalInterface, Auditor {
private static final Logger LOG =
LoggerFactory.getLogger(SCMClientProtocolServer.class);
private static final AuditLogger AUDIT =
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@
import org.apache.hadoop.hdds.scm.container.ReplicationManagerReport;
import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
import org.apache.hadoop.hdds.scm.protocol.StorageContainerLocationProtocol;
import org.apache.hadoop.hdds.scm.protocol.StorageContainerLocationInternalInterface;
import org.apache.hadoop.hdds.scm.protocolPB.StorageContainerLocationProtocolClientSideTranslatorPB.ScmNodeTarget;
import org.apache.hadoop.hdds.scm.storage.ContainerProtocolCalls;
import org.apache.hadoop.hdds.security.x509.certificate.client.CACertificateProvider;
Expand All @@ -70,7 +70,7 @@ public class ContainerOperationClient implements ScmClient {

private static final Logger LOG =
LoggerFactory.getLogger(ContainerOperationClient.class);
private final StorageContainerLocationProtocol
private final StorageContainerLocationInternalInterface
storageContainerLocationClient;
private final SecretKeyProtocolScm secretKeyClient;
private final boolean containerTokenEnabled;
Expand Down Expand Up @@ -119,12 +119,12 @@ private XceiverClientManager newXCeiverClientManager(ConfigurationSource conf)
return manager;
}

public static StorageContainerLocationProtocol newContainerRpcClient(
public static StorageContainerLocationInternalInterface newContainerRpcClient(
ConfigurationSource configSource) {
return HAUtils.getScmContainerClient(configSource);
}

public static StorageContainerLocationProtocol newContainerRpcClientForNode(
public static StorageContainerLocationInternalInterface newContainerRpcClientForNode(
ConfigurationSource configSource, ScmNodeTarget targetScmNode) {
return HAUtils.getScmContainerClientForNode(configSource, targetScmNode);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.conf.TimeDurationUtil;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
import org.apache.hadoop.hdds.scm.protocol.StorageContainerLocationProtocol;
import org.apache.hadoop.hdds.scm.protocol.StorageContainerLocationInternalInterface;
import org.apache.hadoop.hdds.tracing.TracingUtil;
import org.apache.hadoop.hdds.utils.HAUtils;
import org.apache.hadoop.ipc_.ProtobufRpcEngine;
Expand Down Expand Up @@ -444,14 +444,14 @@ public OzoneManagerProtocolClientSideTranslatorPB createOmClient(
return new OzoneManagerProtocolClientSideTranslatorPB(transport, clientId);
}

public StorageContainerLocationProtocol createStorageContainerLocationClient(
public StorageContainerLocationInternalInterface createStorageContainerLocationClient(
OzoneConfiguration ozoneConf) throws IOException {
return HAUtils.getScmContainerClient(ozoneConf);
}

@SuppressWarnings("java:S3864") // Stream.peek (for debug)
public static Pipeline findPipelineForTest(String pipelineId,
StorageContainerLocationProtocol client, Logger log) throws IOException {
StorageContainerLocationInternalInterface client, Logger log) throws IOException {
Stream<Pipeline> pipelines = client.listPipelines().stream();
Pipeline pipeline;
if (log.isDebugEnabled()) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@
import org.apache.hadoop.hdds.scm.XceiverClientFactory;
import org.apache.hadoop.hdds.scm.XceiverClientSpi;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
import org.apache.hadoop.hdds.scm.protocol.StorageContainerLocationProtocol;
import org.apache.hadoop.hdds.scm.protocol.StorageContainerLocationInternalInterface;
import org.apache.hadoop.ozone.OzoneSecurityUtil;
import org.apache.hadoop.ozone.common.Checksum;
import org.kohsuke.MetaInfServices;
Expand Down Expand Up @@ -96,7 +96,7 @@ public Void call() throws Exception {
"datanode-block-putter is not supported in secure environment");
}

try (StorageContainerLocationProtocol scmLocationClient =
try (StorageContainerLocationInternalInterface scmLocationClient =
createStorageContainerLocationClient(ozoneConf)) {
Pipeline pipeline =
findPipelineForTest(pipelineId, scmLocationClient, LOG);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@
import org.apache.hadoop.hdds.scm.XceiverClientReply;
import org.apache.hadoop.hdds.scm.XceiverClientSpi;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
import org.apache.hadoop.hdds.scm.protocol.StorageContainerLocationProtocol;
import org.apache.hadoop.hdds.scm.protocol.StorageContainerLocationInternalInterface;
import org.apache.hadoop.ozone.OzoneSecurityUtil;
import org.apache.hadoop.ozone.common.Checksum;
import org.apache.ratis.thirdparty.com.google.protobuf.ByteString;
Expand Down Expand Up @@ -115,7 +115,7 @@ public Void call() throws Exception {

Set<Pipeline> pipelines;

try (StorageContainerLocationProtocol scmLocationClient =
try (StorageContainerLocationInternalInterface scmLocationClient =
createStorageContainerLocationClient(ozoneConf);
XceiverClientFactory xceiverClientManager =
new XceiverClientCreator(ozoneConf)) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@
import org.apache.hadoop.hdds.scm.XceiverClientFactory;
import org.apache.hadoop.hdds.scm.XceiverClientSpi;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
import org.apache.hadoop.hdds.scm.protocol.StorageContainerLocationProtocol;
import org.apache.hadoop.hdds.scm.protocol.StorageContainerLocationInternalInterface;
import org.apache.hadoop.ozone.OzoneSecurityUtil;
import org.apache.hadoop.ozone.common.Checksum;
import org.apache.hadoop.ozone.common.ChecksumData;
Expand Down Expand Up @@ -88,7 +88,7 @@ public Void call() throws Exception {
);
}

try (StorageContainerLocationProtocol scmClient =
try (StorageContainerLocationInternalInterface scmClient =
createStorageContainerLocationClient(ozoneConf)) {
Pipeline pipeline = findPipelineForTest(pipelineId, scmClient, LOG);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@
import org.apache.hadoop.hdds.scm.container.balancer.IllegalContainerBalancerStateException;
import org.apache.hadoop.hdds.scm.container.balancer.InvalidContainerBalancerConfigurationException;
import org.apache.hadoop.hdds.scm.protocol.ScmBlockLocationProtocol;
import org.apache.hadoop.hdds.scm.protocol.StorageContainerLocationProtocol;
import org.apache.hadoop.hdds.scm.protocol.StorageContainerLocationInternalInterface;
import org.apache.hadoop.hdds.scm.protocolPB.ScmBlockLocationProtocolClientSideTranslatorPB;
import org.apache.hadoop.hdds.scm.protocolPB.StorageContainerLocationProtocolClientSideTranslatorPB;
import org.apache.hadoop.hdds.scm.proxy.SCMBlockLocationFailoverProxyProvider;
Expand Down Expand Up @@ -131,10 +131,10 @@ public void testFailover() throws Exception {
Level.DEBUG);
logCapture = LogCapturer.captureLogs(SCMContainerLocationFailoverProxyProvider.class);
proxyProvider.changeCurrentProxy(scm.getSCMNodeId());
StorageContainerLocationProtocol scmContainerClient =
StorageContainerLocationInternalInterface scmContainerClient =
TracingUtil.createProxy(
new StorageContainerLocationProtocolClientSideTranslatorPB(
proxyProvider), StorageContainerLocationProtocol.class, conf);
proxyProvider), StorageContainerLocationInternalInterface.class, conf);

scmContainerClient.allocateContainer(HddsProtos.ReplicationType.RATIS,
HddsProtos.ReplicationFactor.ONE, "ozone");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@
import org.apache.hadoop.hdds.scm.container.ContainerInfo;
import org.apache.hadoop.hdds.scm.node.states.NodeNotFoundException;
import org.apache.hadoop.hdds.scm.pipeline.PipelineManager;
import org.apache.hadoop.hdds.scm.protocol.StorageContainerLocationProtocol;
import org.apache.hadoop.hdds.scm.protocol.StorageContainerLocationInternalInterface;
import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
import org.apache.hadoop.hdds.scm.server.upgrade.FinalizationCheckpoint;
import org.apache.hadoop.ozone.HddsDatanodeService;
Expand All @@ -67,7 +67,7 @@ public final class HddsUpgradeTestUtils {
private HddsUpgradeTestUtils() { }

public static void waitForFinalizationFromClient(
StorageContainerLocationProtocol scmClient, String clientID)
StorageContainerLocationInternalInterface scmClient, String clientID)
throws Exception {
LambdaTestUtils.await(60_000, 1_000, () -> {
UpgradeFinalization.Status status = scmClient
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@
import java.util.concurrent.TimeUnit;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.scm.ScmConfig;
import org.apache.hadoop.hdds.scm.protocol.StorageContainerLocationProtocol;
import org.apache.hadoop.hdds.scm.protocol.StorageContainerLocationInternalInterface;
import org.apache.hadoop.hdds.scm.server.SCMConfigurator;
import org.apache.hadoop.hdds.scm.server.SCMStorageConfig;
import org.apache.hadoop.ozone.HddsDatanodeService;
Expand Down Expand Up @@ -66,7 +66,7 @@ public class TestDNDataDistributionFinalization {
private static final Logger LOG =
LoggerFactory.getLogger(TestDNDataDistributionFinalization.class);

private StorageContainerLocationProtocol scmClient;
private StorageContainerLocationInternalInterface scmClient;
private MiniOzoneHAClusterImpl cluster;

private static final int NUM_DATANODES = 3;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@
import org.apache.hadoop.hdds.scm.block.SCMDeletedBlockTransactionStatusManager;
import org.apache.hadoop.hdds.scm.ha.SCMHADBTransactionBuffer;
import org.apache.hadoop.hdds.scm.metadata.DBTransactionBuffer;
import org.apache.hadoop.hdds.scm.protocol.StorageContainerLocationProtocol;
import org.apache.hadoop.hdds.scm.protocol.StorageContainerLocationInternalInterface;
import org.apache.hadoop.hdds.scm.server.SCMConfigurator;
import org.apache.hadoop.hdds.scm.server.SCMStorageConfig;
import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
Expand Down Expand Up @@ -98,7 +98,7 @@ public class TestScmDataDistributionFinalization {
private static final Logger LOG =
LoggerFactory.getLogger(TestScmDataDistributionFinalization.class);

private StorageContainerLocationProtocol scmClient;
private StorageContainerLocationInternalInterface scmClient;
private MiniOzoneHAClusterImpl cluster;
private static final int NUM_DATANODES = 3;
private static final int NUM_SCMS = 3;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline;
import org.apache.hadoop.hdds.scm.protocol.StorageContainerLocationProtocol;
import org.apache.hadoop.hdds.scm.protocol.StorageContainerLocationInternalInterface;
import org.apache.hadoop.hdds.scm.server.SCMConfigurator;
import org.apache.hadoop.hdds.scm.server.SCMStorageConfig;
import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
Expand Down Expand Up @@ -73,7 +73,7 @@ public class TestScmHAFinalization {
"org.apache.hadoop.hdds.upgrade" +
".TestScmHAFinalization#injectionPointsToTest";

private StorageContainerLocationProtocol scmClient;
private StorageContainerLocationInternalInterface scmClient;
private MiniOzoneHAClusterImpl cluster;
private static final int NUM_DATANODES = 3;
private static final int NUM_SCMS = 3;
Expand Down
Loading