Separate registry push and pull endpoints

deploy/nuvolaris-permissions/nuvolaris-wsku-roles.yaml

@@ -33,6 +33,11 @@ rules:
 - apiGroups: [""]
   resources: ["configmaps"]
   verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]  
+# allow the admin api to read registry credentials used by BuildKit
+- apiGroups: [""]
+  resources: ["secrets"]
+  resourceNames: ["registry-pull-secret", "registry-pull-secret-int"]
+  verbs: ["get"]
 # assign the possibility to operate on jobs (admin api)
 - apiGroups: ["batch"]
   resources: ["jobs"]

deploy/nuvolaris-permissions/whisk-crd.yaml

@@ -829,6 +829,9 @@ spec:
                     hostname:
                       description: used to configure the repo hostname (if set to auto and mode=internal it will be img.<apihost>)
                       type: string
+                    pull-hostname:
+                      description: node-reachable registry hostname used in action image references
+                      type: string
                     ingress:
                       description: configuration option for global REGISTRY ingresses exposure, will be taken into account only if deployment=interna
                       type: object
@@ -1003,4 +1006,4 @@ spec:
           type: string
           priority: 0
           jsonPath: .status.whisk_create.seaweedfs
-          description: Seaweedfs                                                                                                                        
+          description: Seaweedfs

deploy/registry/02-registry-sts.yaml

@@ -43,6 +43,8 @@ spec:
           ports:
             - containerPort: 5000
           env:
+            - name: OTEL_TRACES_EXPORTER
+              value: "none"
             - name: REGISTRY_AUTH
               value: "htpasswd"
             - name: REGISTRY_AUTH_HTPASSWD_REALM
@@ -63,4 +65,4 @@ spec:
             claimName: registry-pvc
         - name: auth-volume
           secret:
-            secretName: registry-auth-secret      
+            secretName: registry-auth-secret

nuvolaris/registry_deploy.py

@@ -35,6 +35,7 @@
 
 def create_external_registry(data, owner=None):
     logging.info("setting up an external registry")
+    assign_registry_endpoints(data)
 
     #set the registry secret
     registrySecret = SecretHtpasswordData(data['registryUsername'],data['registryPassword'])
@@ -44,10 +45,16 @@ def create_external_registry(data, owner=None):
     os.remove(path_to_template_yaml)
 
     #set the registry pull secret
-    registryPullSecret = ImagePullSecretData(data['registryUsername'],data['registryPassword'],data['repoHostname'])
+    registryPullSecret = ImagePullSecretData(data['registryUsername'],data['registryPassword'],data['repoPullHostname'])
     registryPullSecret.with_secret_name("registry-pull-secret")
     path_to_template_yaml = registryPullSecret.render_template("nuvolaris")
 
+    res += kube.kubectl("apply", "-f",path_to_template_yaml)
+    os.remove(path_to_template_yaml)
+
+    registryPushSecret = ImagePullSecretData(data['registryUsername'],data['registryPassword'],data['repoPushHostname'])
+    registryPushSecret.with_secret_name("registry-pull-secret-int")
+    path_to_template_yaml = registryPushSecret.render_template("nuvolaris")
     res += kube.kubectl("apply", "-f",path_to_template_yaml)
     os.remove(path_to_template_yaml)
     _annotate_registry_metadata(data)
@@ -56,6 +63,7 @@ def create_external_registry(data, owner=None):
 
 def create_internal_registry(data, owner=None):
     logging.info("setting up an internal registry")
+    assign_registry_endpoints(data)
     tplp = ["pvc-attach.yaml"]
 
     if(data['affinity'] or data['tolerations']):
@@ -69,13 +77,7 @@ def create_internal_registry(data, owner=None):
     kust += registrySecret.generateHtPasswordPatch()
 
     #path the registry pull secret
-    repoInternalHost = data['repoHostname']
-    if kube.detect_kind():
-        # when repo is kind, the registry pull secret will point
-        # to the node port exposed on the node
-        repoInternalHost = "127.0.0.1:32000"
-
-    registryPullSecret = ImagePullSecretData(data['registryUsername'],data['registryPassword'],repoInternalHost)
+    registryPullSecret = ImagePullSecretData(data['registryUsername'],data['registryPassword'],data['repoPullHostname'])
     registryPullSecret.with_secret_name("registry-pull-secret")    
     kust += registryPullSecret.generatePullSecretPatch()
 
@@ -87,6 +89,17 @@ def create_internal_registry(data, owner=None):
         cfg.put("state.registry.spec", spec)
     res = kube.apply(spec)
 
+    # BuildKit pushes from inside the cluster, while action containers are
+    # pulled by the node runtime. Keep separate credentials because the Docker
+    # config auth key must match the endpoint used by each client.
+    registryPushSecret = ImagePullSecretData(
+        data['registryUsername'], data['registryPassword'], data['repoPushHostname']
+    )
+    registryPushSecret.with_secret_name("registry-pull-secret-int")
+    path_to_template_yaml = registryPushSecret.render_template("nuvolaris")
+    res += kube.kubectl("apply", "-f", path_to_template_yaml)
+    os.remove(path_to_template_yaml)
+
     wait_for_registry_ready()
     _annotate_registry_metadata(data)
 
@@ -110,8 +123,10 @@ def _annotate_registry_metadata(data):
     annotate nuvolaris configmap with entries for registry connectivity REGISTRY_ENDPOINT, REGISTRY_USERNAME, RESIGTRY_PASSWORD    
     """ 
     try:
-        openwhisk.annotate(f"registry_host={data['repoHostname']}")
-        openwhisk.annotate(f"registry_internal_host={data['repoSvcHostname']}")
+        openwhisk.annotate(f"registry_host={data['repoPullHostname']}")
+        openwhisk.annotate(f"registry_internal_host={data['repoPushHostname']}")
+        openwhisk.annotate(f"registry_pull_host={data['repoPullHostname']}")
+        openwhisk.annotate(f"registry_push_host={data['repoPushHostname']}")
         openwhisk.annotate(f"registry_username={data['registryUsername']}")
         openwhisk.annotate(f"registry_password={data['registryPassword']}")
 
@@ -142,6 +157,53 @@ def assign_registry_hostname(data):
     logging.info(f"assigned registry hostname {data['repoHostname']}")
 
 
+def _without_scheme(hostname):
+    return str(hostname or "").replace("https://", "").replace("http://", "").rstrip("/")
+
+
+def _k3s_node_registry_host():
+    try:
+        addresses = kube.kubectl(
+            "get", "nodes", namespace=None,
+            jsonpath="{.items[0].status.addresses[?(@.type == 'InternalIP')].address}",
+        )
+        if isinstance(addresses, list) and addresses:
+            return f"{addresses[0]}:32000"
+        if isinstance(addresses, str) and addresses:
+            return f"{addresses}:32000"
+    except Exception as exc:
+        logging.warning(f"cannot determine K3s registry node address: {exc}")
+    return ""
+
+
+def assign_registry_endpoints(data):
+    """Resolve pod-side push and node-side pull registry endpoints."""
+    assign_registry_hostname(data)
+    mode = data.get("mode", "internal")
+    runtime = cfg.get('nuvolaris.kube')
+    if mode == "external":
+        endpoint = _without_scheme(data['repoHostname'])
+        data['repoPushHostname'] = endpoint
+        data['repoPullHostname'] = endpoint
+        return
+
+    data['repoPushHostname'] = _without_scheme(data['repoSvcHostname'])
+    configured_pull = data.get('repoPullHostname')
+    if configured_pull and configured_pull != "auto":
+        data['repoPullHostname'] = _without_scheme(configured_pull)
+    elif data.get('ingressEnabled'):
+        data['repoPullHostname'] = _without_scheme(data['repoHostname'])
+    elif runtime == "kind" or kube.detect_kind():
+        data['repoPullHostname'] = "127.0.0.1:32000"
+    elif runtime == "k3s":
+        data['repoPullHostname'] = _k3s_node_registry_host()
+    else:
+        data['repoPullHostname'] = _without_scheme(data['repoHostname'])
+
+    if not data['repoPullHostname']:
+        raise ValueError("registry pull hostname is not configured")
+
+
 def wait_for_registry_ready():
     # dynamically detect registry pod and wait for readiness
     util.wait_for_pod_ready("{.items[?(@.metadata.labels.name == 'registry')].metadata.name}")

nuvolaris/util.py

@@ -891,6 +891,7 @@ def get_registry_config_data():
         "size": cfg.get("registry.volume-size", "REGISTRY_VOLUME_SIZE", 20),
         "storageClass": cfg.get("nuvolaris.storageclass"),
         "repoHostname": cfg.get('registry.hostname') or "auto",
+        "repoPullHostname": cfg.get('registry.pull-hostname') or "auto",
         "ingressEnabled": cfg.get('registry.ingress.enabled') or False,
         "registryUsername": cfg.get('registry.auth.username') or "openserverless",
         "registryPassword": cfg.get('registry.auth.password') or "4pwdregistry",

tests/registry_config_test.py

@@ -0,0 +1,76 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import unittest
+from unittest.mock import patch
+
+import nuvolaris.registry_deploy as registry_deploy
+
+
+def registry_data(pull="auto"):
+    return {
+        "mode": "internal",
+        "repoHostname": "auto",
+        "repoPullHostname": pull,
+        "repoSvcHostname": "nuvolaris-registry-svc:5000",
+        "ingressEnabled": False,
+    }
+
+
+class RegistryEndpointTest(unittest.TestCase):
+    @patch.object(registry_deploy, "assign_registry_hostname")
+    @patch.object(registry_deploy.kube, "detect_kind", return_value=True)
+    @patch.object(registry_deploy.cfg, "get", return_value="kind")
+    def test_kind_uses_service_for_push_and_node_port_for_pull(
+        self, _cfg_get, _detect_kind, _assign_hostname
+    ):
+        data = registry_data()
+
+        registry_deploy.assign_registry_endpoints(data)
+
+        self.assertEqual("nuvolaris-registry-svc:5000", data["repoPushHostname"])
+        self.assertEqual("127.0.0.1:32000", data["repoPullHostname"])
+
+    @patch.object(registry_deploy, "assign_registry_hostname")
+    @patch.object(registry_deploy.kube, "detect_kind", return_value=False)
+    @patch.object(registry_deploy.cfg, "get", return_value="k3s")
+    def test_k3s_preserves_configured_node_reachable_host(
+        self, _cfg_get, _detect_kind, _assign_hostname
+    ):
+        data = registry_data("192.0.2.10:32000")
+
+        registry_deploy.assign_registry_endpoints(data)
+
+        self.assertEqual("nuvolaris-registry-svc:5000", data["repoPushHostname"])
+        self.assertEqual("192.0.2.10:32000", data["repoPullHostname"])
+
+    @patch.object(registry_deploy, "assign_registry_hostname")
+    @patch.object(registry_deploy.kube, "detect_kind", return_value=False)
+    @patch.object(registry_deploy.cfg, "get", return_value="k3s")
+    @patch.object(registry_deploy, "_k3s_node_registry_host", return_value="192.0.2.20:32000")
+    def test_k3s_can_derive_pull_host_from_node_address(
+        self, _node_host, _cfg_get, _detect_kind, _assign_hostname
+    ):
+        data = registry_data()
+
+        registry_deploy.assign_registry_endpoints(data)
+
+        self.assertEqual("192.0.2.20:32000", data["repoPullHostname"])
+
+
+if __name__ == "__main__":
+    unittest.main()