From 1080cfd31be63e9d7c37cc9461bbd5a396831ebe Mon Sep 17 00:00:00 2001 From: rjgoyln Date: Tue, 25 Aug 2026 19:09:06 +0800 Subject: [PATCH] HDDS-16206. Start MiniOzoneCluster Datanodes finalized MiniOzoneCluster wrote a datanode.id file before each Datanode started, so that a Datanode given a synthetic hostname could report an IP address it has no way to resolve. A datanode.id file with no VERSION file beside it means "upgraded from an installation predating the upgrade framework", so every Datanode came up at metadata layout version 0 and reached the current version only once SCM ordered it to finalize. On master SCM finalizes those Datanodes before they can satisfy safemode, which hides the problem. With ZDU, pre-finalized Datanodes are valid for all existing operations, so a test that expects finalized Datanodes can be handed one that has not finalized yet and fails intermittently. --- .../hadoop/ozone/TestMiniOzoneCluster.java | 25 +++++++++++++++++++ .../hadoop/ozone/MiniOzoneClusterImpl.java | 24 +++++++++++++++--- 2 files changed, 45 insertions(+), 4 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestMiniOzoneCluster.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestMiniOzoneCluster.java index a227242d3027..746da53c76ca 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestMiniOzoneCluster.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestMiniOzoneCluster.java @@ -19,6 +19,7 @@ import static org.apache.hadoop.hdds.protocol.DatanodeDetails.Port; import static org.apache.hadoop.hdds.protocol.MockDatanodeDetails.randomDatanodeDetails; +import static org.apache.hadoop.hdds.upgrade.HDDSLayoutVersionManager.maxLayoutVersion; import static org.apache.hadoop.ozone.OzoneConfigKeys.HDDS_CONTAINER_RATIS_IPC_RANDOM_PORT; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -41,6 +42,7 @@ import org.apache.hadoop.hdds.scm.pipeline.Pipeline; import org.apache.hadoop.hdds.scm.pipeline.PipelineID; import org.apache.hadoop.hdds.scm.server.StorageContainerManager; +import org.apache.hadoop.ozone.container.common.DatanodeLayoutStorage; import org.apache.hadoop.ozone.container.common.SCMTestUtils; import org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine; import org.apache.hadoop.ozone.container.common.statemachine.EndpointStateMachine; @@ -49,6 +51,8 @@ import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; /** * Test cases for mini ozone cluster. @@ -103,6 +107,27 @@ public void testStartMultipleDatanodes() throws Exception { } } + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testDatanodesStartWithLatestLayoutVersion(boolean withCustomHostnames) + throws Exception { + MiniOzoneCluster.Builder builder = + MiniOzoneCluster.newBuilder(new OzoneConfiguration(conf)) + .setNumDatanodes(1) + .setStartDataNodes(false); + if (withCustomHostnames) { + builder.setHosts(new String[] {"host0.test"}); + } + cluster = builder.build(); + + OzoneConfiguration dnConf = cluster.getHddsDatanodes().get(0).getConf(); + assertEquals(maxLayoutVersion(), + new DatanodeLayoutStorage(dnConf).getLayoutVersion(), + "Datanode must start at the latest metadata layout version. A " + + "datanode.id file written before the datanode starts makes it " + + "come up pre-finalized, as if upgraded from an older install."); + } + @Test void testContainerRandomPort(@TempDir File tempDir) throws IOException { OzoneConfiguration ozoneConf = SCMTestUtils.getConf(tempDir); diff --git a/hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java b/hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java index e13719dde6e5..c385c80ae46f 100644 --- a/hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java +++ b/hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java @@ -20,6 +20,7 @@ import static java.util.Collections.singletonList; import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState.HEALTHY; import static org.apache.hadoop.hdds.server.http.BaseHttpServer.SERVER_DIR; +import static org.apache.hadoop.hdds.upgrade.HDDSLayoutVersionManager.maxLayoutVersion; import static org.apache.hadoop.ozone.OzoneConsts.OZONE_RATIS_SNAPSHOT_DIR; import static org.apache.ozone.test.GenericTestUtils.PortAllocator.getFreePort; import static org.apache.ozone.test.GenericTestUtils.PortAllocator.localhostWithFreePort; @@ -77,6 +78,7 @@ import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.client.OzoneClientFactory; import org.apache.hadoop.ozone.common.Storage.StorageState; +import org.apache.hadoop.ozone.container.common.DatanodeLayoutStorage; import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils; import org.apache.hadoop.ozone.container.common.utils.ContainerCache; import org.apache.hadoop.ozone.container.common.utils.DatanodeStoreCache; @@ -774,10 +776,9 @@ protected List createHddsDatanodes() OzoneConfiguration dnConf = dnFactory.apply(conf); if (hosts != null) { dnConf.set(HddsConfigKeys.HDDS_DATANODE_HOST_NAME_KEY, hosts[i]); + initializeDatanodeIdentity(dnConf, hosts[i]); } - // Bypass InetAddress.getName() resolution for custom hostnames by starting DN via YAML. - confDatanodeViaYaml(dnConf); HddsDatanodeService datanode = new HddsDatanodeService(NO_ARGS); dnConf.setStrings(ScmConfigKeys.OZONE_SCM_NAMES, conf.getStrings(ScmConfigKeys.OZONE_SCM_NAMES)); datanode.setConfiguration(dnConf); @@ -787,13 +788,28 @@ protected List createHddsDatanodes() return hddsDatanodes; } - private void confDatanodeViaYaml(OzoneConfiguration dnConf) throws IOException { + /** + * Seeds the datanode.id file for a Datanode with a synthetic hostname: such a hostname + * cannot be resolved, so the Datanode cannot discover its own IP address, which is a + * required field of the registration request. The layout version is stamped as well, + * since a datanode.id file without a VERSION file marks the Datanode as an installation + * predating the upgrade framework, which would make it start pre-finalized. + */ + private void initializeDatanodeIdentity(OzoneConfiguration dnConf, String hostName) + throws IOException { DatanodeDetails datanodeDetails = DatanodeDetails.newBuilder() .setID(DatanodeID.randomID()) - .setHostName(dnConf.get(HddsConfigKeys.HDDS_DATANODE_HOST_NAME_KEY)) + .setHostName(hostName) .setIpAddress("127.0.0.1") .build(); datanodeDetails.setNetworkName(datanodeDetails.getUuidString()); + + DatanodeLayoutStorage layoutStorage = new DatanodeLayoutStorage(dnConf, + datanodeDetails.getUuidString(), maxLayoutVersion()); + if (layoutStorage.getState() != StorageState.INITIALIZED) { + layoutStorage.initialize(); + } + String dnFilePath = HddsServerUtil.getDatanodeIdFilePath(dnConf); ContainerUtils.writeDatanodeDetailsTo(datanodeDetails, new File(dnFilePath), dnConf); }