diff --git a/deploy/nuvolaris-permissions/nuvolaris-wsku-roles.yaml b/deploy/nuvolaris-permissions/nuvolaris-wsku-roles.yaml index 101cff05..bdcf8bbd 100644 --- a/deploy/nuvolaris-permissions/nuvolaris-wsku-roles.yaml +++ b/deploy/nuvolaris-permissions/nuvolaris-wsku-roles.yaml @@ -33,6 +33,11 @@ rules: - apiGroups: [""] resources: ["configmaps"] verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] +# allow the admin api to read registry credentials used by BuildKit +- apiGroups: [""] + resources: ["secrets"] + resourceNames: ["registry-pull-secret", "registry-pull-secret-int"] + verbs: ["get"] # assign the possibility to operate on jobs (admin api) - apiGroups: ["batch"] resources: ["jobs"] diff --git a/deploy/nuvolaris-permissions/whisk-crd.yaml b/deploy/nuvolaris-permissions/whisk-crd.yaml index 5cc235d5..1bf8ea73 100644 --- a/deploy/nuvolaris-permissions/whisk-crd.yaml +++ b/deploy/nuvolaris-permissions/whisk-crd.yaml @@ -829,6 +829,9 @@ spec: hostname: description: used to configure the repo hostname (if set to auto and mode=internal it will be img.) type: string + pull-hostname: + description: node-reachable registry hostname used in action image references + type: string ingress: description: configuration option for global REGISTRY ingresses exposure, will be taken into account only if deployment=interna type: object @@ -1003,4 +1006,4 @@ spec: type: string priority: 0 jsonPath: .status.whisk_create.seaweedfs - description: Seaweedfs \ No newline at end of file + description: Seaweedfs diff --git a/deploy/registry/02-registry-sts.yaml b/deploy/registry/02-registry-sts.yaml index 83f579bd..a08cd965 100644 --- a/deploy/registry/02-registry-sts.yaml +++ b/deploy/registry/02-registry-sts.yaml @@ -43,6 +43,8 @@ spec: ports: - containerPort: 5000 env: + - name: OTEL_TRACES_EXPORTER + value: "none" - name: REGISTRY_AUTH value: "htpasswd" - name: REGISTRY_AUTH_HTPASSWD_REALM @@ -63,4 +65,4 @@ spec: claimName: registry-pvc - name: auth-volume secret: - secretName: registry-auth-secret \ No newline at end of file + secretName: registry-auth-secret diff --git a/nuvolaris/registry_deploy.py b/nuvolaris/registry_deploy.py index e9c13fe4..13398b73 100644 --- a/nuvolaris/registry_deploy.py +++ b/nuvolaris/registry_deploy.py @@ -35,6 +35,7 @@ def create_external_registry(data, owner=None): logging.info("setting up an external registry") + assign_registry_endpoints(data) #set the registry secret registrySecret = SecretHtpasswordData(data['registryUsername'],data['registryPassword']) @@ -44,10 +45,16 @@ def create_external_registry(data, owner=None): os.remove(path_to_template_yaml) #set the registry pull secret - registryPullSecret = ImagePullSecretData(data['registryUsername'],data['registryPassword'],data['repoHostname']) + registryPullSecret = ImagePullSecretData(data['registryUsername'],data['registryPassword'],data['repoPullHostname']) registryPullSecret.with_secret_name("registry-pull-secret") path_to_template_yaml = registryPullSecret.render_template("nuvolaris") + res += kube.kubectl("apply", "-f",path_to_template_yaml) + os.remove(path_to_template_yaml) + + registryPushSecret = ImagePullSecretData(data['registryUsername'],data['registryPassword'],data['repoPushHostname']) + registryPushSecret.with_secret_name("registry-pull-secret-int") + path_to_template_yaml = registryPushSecret.render_template("nuvolaris") res += kube.kubectl("apply", "-f",path_to_template_yaml) os.remove(path_to_template_yaml) _annotate_registry_metadata(data) @@ -56,6 +63,7 @@ def create_external_registry(data, owner=None): def create_internal_registry(data, owner=None): logging.info("setting up an internal registry") + assign_registry_endpoints(data) tplp = ["pvc-attach.yaml"] if(data['affinity'] or data['tolerations']): @@ -69,13 +77,7 @@ def create_internal_registry(data, owner=None): kust += registrySecret.generateHtPasswordPatch() #path the registry pull secret - repoInternalHost = data['repoHostname'] - if kube.detect_kind(): - # when repo is kind, the registry pull secret will point - # to the node port exposed on the node - repoInternalHost = "127.0.0.1:32000" - - registryPullSecret = ImagePullSecretData(data['registryUsername'],data['registryPassword'],repoInternalHost) + registryPullSecret = ImagePullSecretData(data['registryUsername'],data['registryPassword'],data['repoPullHostname']) registryPullSecret.with_secret_name("registry-pull-secret") kust += registryPullSecret.generatePullSecretPatch() @@ -87,6 +89,17 @@ def create_internal_registry(data, owner=None): cfg.put("state.registry.spec", spec) res = kube.apply(spec) + # BuildKit pushes from inside the cluster, while action containers are + # pulled by the node runtime. Keep separate credentials because the Docker + # config auth key must match the endpoint used by each client. + registryPushSecret = ImagePullSecretData( + data['registryUsername'], data['registryPassword'], data['repoPushHostname'] + ) + registryPushSecret.with_secret_name("registry-pull-secret-int") + path_to_template_yaml = registryPushSecret.render_template("nuvolaris") + res += kube.kubectl("apply", "-f", path_to_template_yaml) + os.remove(path_to_template_yaml) + wait_for_registry_ready() _annotate_registry_metadata(data) @@ -110,8 +123,10 @@ def _annotate_registry_metadata(data): annotate nuvolaris configmap with entries for registry connectivity REGISTRY_ENDPOINT, REGISTRY_USERNAME, RESIGTRY_PASSWORD """ try: - openwhisk.annotate(f"registry_host={data['repoHostname']}") - openwhisk.annotate(f"registry_internal_host={data['repoSvcHostname']}") + openwhisk.annotate(f"registry_host={data['repoPullHostname']}") + openwhisk.annotate(f"registry_internal_host={data['repoPushHostname']}") + openwhisk.annotate(f"registry_pull_host={data['repoPullHostname']}") + openwhisk.annotate(f"registry_push_host={data['repoPushHostname']}") openwhisk.annotate(f"registry_username={data['registryUsername']}") openwhisk.annotate(f"registry_password={data['registryPassword']}") @@ -142,6 +157,53 @@ def assign_registry_hostname(data): logging.info(f"assigned registry hostname {data['repoHostname']}") +def _without_scheme(hostname): + return str(hostname or "").replace("https://", "").replace("http://", "").rstrip("/") + + +def _k3s_node_registry_host(): + try: + addresses = kube.kubectl( + "get", "nodes", namespace=None, + jsonpath="{.items[0].status.addresses[?(@.type == 'InternalIP')].address}", + ) + if isinstance(addresses, list) and addresses: + return f"{addresses[0]}:32000" + if isinstance(addresses, str) and addresses: + return f"{addresses}:32000" + except Exception as exc: + logging.warning(f"cannot determine K3s registry node address: {exc}") + return "" + + +def assign_registry_endpoints(data): + """Resolve pod-side push and node-side pull registry endpoints.""" + assign_registry_hostname(data) + mode = data.get("mode", "internal") + runtime = cfg.get('nuvolaris.kube') + if mode == "external": + endpoint = _without_scheme(data['repoHostname']) + data['repoPushHostname'] = endpoint + data['repoPullHostname'] = endpoint + return + + data['repoPushHostname'] = _without_scheme(data['repoSvcHostname']) + configured_pull = data.get('repoPullHostname') + if configured_pull and configured_pull != "auto": + data['repoPullHostname'] = _without_scheme(configured_pull) + elif data.get('ingressEnabled'): + data['repoPullHostname'] = _without_scheme(data['repoHostname']) + elif runtime == "kind" or kube.detect_kind(): + data['repoPullHostname'] = "127.0.0.1:32000" + elif runtime == "k3s": + data['repoPullHostname'] = _k3s_node_registry_host() + else: + data['repoPullHostname'] = _without_scheme(data['repoHostname']) + + if not data['repoPullHostname']: + raise ValueError("registry pull hostname is not configured") + + def wait_for_registry_ready(): # dynamically detect registry pod and wait for readiness util.wait_for_pod_ready("{.items[?(@.metadata.labels.name == 'registry')].metadata.name}") diff --git a/nuvolaris/util.py b/nuvolaris/util.py index b6168516..321e50d8 100644 --- a/nuvolaris/util.py +++ b/nuvolaris/util.py @@ -891,6 +891,7 @@ def get_registry_config_data(): "size": cfg.get("registry.volume-size", "REGISTRY_VOLUME_SIZE", 20), "storageClass": cfg.get("nuvolaris.storageclass"), "repoHostname": cfg.get('registry.hostname') or "auto", + "repoPullHostname": cfg.get('registry.pull-hostname') or "auto", "ingressEnabled": cfg.get('registry.ingress.enabled') or False, "registryUsername": cfg.get('registry.auth.username') or "openserverless", "registryPassword": cfg.get('registry.auth.password') or "4pwdregistry", diff --git a/tests/registry_config_test.py b/tests/registry_config_test.py new file mode 100644 index 00000000..73afa6ed --- /dev/null +++ b/tests/registry_config_test.py @@ -0,0 +1,76 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import unittest +from unittest.mock import patch + +import nuvolaris.registry_deploy as registry_deploy + + +def registry_data(pull="auto"): + return { + "mode": "internal", + "repoHostname": "auto", + "repoPullHostname": pull, + "repoSvcHostname": "nuvolaris-registry-svc:5000", + "ingressEnabled": False, + } + + +class RegistryEndpointTest(unittest.TestCase): + @patch.object(registry_deploy, "assign_registry_hostname") + @patch.object(registry_deploy.kube, "detect_kind", return_value=True) + @patch.object(registry_deploy.cfg, "get", return_value="kind") + def test_kind_uses_service_for_push_and_node_port_for_pull( + self, _cfg_get, _detect_kind, _assign_hostname + ): + data = registry_data() + + registry_deploy.assign_registry_endpoints(data) + + self.assertEqual("nuvolaris-registry-svc:5000", data["repoPushHostname"]) + self.assertEqual("127.0.0.1:32000", data["repoPullHostname"]) + + @patch.object(registry_deploy, "assign_registry_hostname") + @patch.object(registry_deploy.kube, "detect_kind", return_value=False) + @patch.object(registry_deploy.cfg, "get", return_value="k3s") + def test_k3s_preserves_configured_node_reachable_host( + self, _cfg_get, _detect_kind, _assign_hostname + ): + data = registry_data("192.0.2.10:32000") + + registry_deploy.assign_registry_endpoints(data) + + self.assertEqual("nuvolaris-registry-svc:5000", data["repoPushHostname"]) + self.assertEqual("192.0.2.10:32000", data["repoPullHostname"]) + + @patch.object(registry_deploy, "assign_registry_hostname") + @patch.object(registry_deploy.kube, "detect_kind", return_value=False) + @patch.object(registry_deploy.cfg, "get", return_value="k3s") + @patch.object(registry_deploy, "_k3s_node_registry_host", return_value="192.0.2.20:32000") + def test_k3s_can_derive_pull_host_from_node_address( + self, _node_host, _cfg_get, _detect_kind, _assign_hostname + ): + data = registry_data() + + registry_deploy.assign_registry_endpoints(data) + + self.assertEqual("192.0.2.20:32000", data["repoPullHostname"]) + + +if __name__ == "__main__": + unittest.main()