From 45142ecc55014a1d2c6ed45d553b5b99ce9e55a4 Mon Sep 17 00:00:00 2001 From: tonic Date: Thu, 18 Jun 2026 19:47:30 +0800 Subject: [PATCH 1/4] feat(server): redirect blob GETs to presigned object-store URLs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit v2GetBlob proxies every blob byte through the epoch process: it reads from the object store and io.Copy's the body back to the client, paying TLS on both sides for multi-GiB VM disk/memory blobs. A single stream tops out at ~20 MB/s even with 3 CPUs, so a 10.5 GB snapshot pull takes ~9 min, and concurrent transfers saturate the pod. When EPOCH_BLOB_REDIRECT is set, v2GetBlob now responds with a 307 to a presigned object-store URL (minio-go PresignedGetObject), so blob bytes flow client<->storage directly and epoch leaves the data path entirely. GCS honors the SigV4-signed URL and serves Range requests natively. Measured on a same-region GCE VM pulling the 5.7 GB memory-ranges blob: path single-stream 10.5 GB pull proxy (1 CPU, before) 3.4 MB/s ~51 min proxy (3 CPU) 20 MB/s ~9 min presigned direct-to-GCS 275 MB/s ~38 s ~13x over the proxy with zero client changes — any redirect-following OCI client (registryclient/vk-cocoon, oras, crane, docker) benefits transparently, and CopyBlobExact still verifies the digest end to end. Intra-blob Range parallelism (measured 1.28 GB/s at 8 connections) would need a client-side downloader and is out of scope here. The flag defaults off. On any server-side failure (existence check or presign) the handler falls back to streaming. Note there is no fallback once the 307 is sent: every client hitting /v2/blobs must have egress to the object-store host when the flag is enabled. --- objectstore/client.go | 10 +++++++ registry/registry.go | 6 ++++ server/registry_v2_blobs.go | 47 ++++++++++++++++++++++++++++++++ server/registry_v2_blobs_test.go | 25 +++++++++++++++++ server/server.go | 30 +++++++++++++------- 5 files changed, 108 insertions(+), 10 deletions(-) create mode 100644 server/registry_v2_blobs_test.go diff --git a/objectstore/client.go b/objectstore/client.go index f789844..3db8db4 100644 --- a/objectstore/client.go +++ b/objectstore/client.go @@ -58,6 +58,16 @@ func (c *Client) Put(ctx context.Context, key string, body io.Reader, size int64 return nil } +// PresignGet returns a time-limited URL that lets a client GET the object +// directly from the backing store, bypassing this process. +func (c *Client) PresignGet(ctx context.Context, key string, ttl time.Duration) (string, error) { + u, err := c.client.PresignedGetObject(ctx, c.cfg.Bucket, c.fullKey(key), ttl, nil) + if err != nil { + return "", fmt.Errorf("presign get %s: %w", key, err) + } + return u.String(), nil +} + // Get returns a streaming reader and size for the given key. func (c *Client) Get(ctx context.Context, key string) (io.ReadCloser, int64, error) { obj, err := c.client.GetObject(ctx, c.cfg.Bucket, c.fullKey(key), minio.GetObjectOptions{}) diff --git a/registry/registry.go b/registry/registry.go index 5dc7799..4c6cb1b 100644 --- a/registry/registry.go +++ b/registry/registry.go @@ -74,6 +74,12 @@ func (r *Registry) StreamBlob(ctx context.Context, digest string) (io.ReadCloser return r.client.Get(ctx, blobKey(digest)) } +// PresignBlobGet returns a time-limited URL for fetching a blob directly from +// the object store, bypassing this process. +func (r *Registry) PresignBlobGet(ctx context.Context, digest string, ttl time.Duration) (string, error) { + return r.client.PresignGet(ctx, blobKey(digest), ttl) +} + // BlobSize returns the size of a blob in bytes. func (r *Registry) BlobSize(ctx context.Context, digest string) (int64, error) { return r.client.Head(ctx, blobKey(digest)) diff --git a/server/registry_v2_blobs.go b/server/registry_v2_blobs.go index 34463d4..7f27088 100644 --- a/server/registry_v2_blobs.go +++ b/server/registry_v2_blobs.go @@ -4,13 +4,35 @@ import ( "io" "net/http" "strconv" + "time" + + "github.com/projecteru2/core/log" "github.com/cocoonstack/epoch/manifest" ) +const defaultBlobRedirectTTL = time.Hour + +// resolveBlobRedirectTTL parses EPOCH_BLOB_REDIRECT_TTL, falling back to the +// default for empty, unparseable, or non-positive values. +func resolveBlobRedirectTTL(raw string) time.Duration { + if raw == "" { + return defaultBlobRedirectTTL + } + d, err := time.ParseDuration(raw) + if err != nil || d <= 0 { + return defaultBlobRedirectTTL + } + return d +} + func (s *Server) v2GetBlob(w http.ResponseWriter, r *http.Request) { dgst := stripSHA256Prefix(urlVar(r, "digest")) + if s.blobRedirect && s.redirectBlob(w, r, dgst) { + return + } + body, size, err := s.reg.StreamBlob(r.Context(), dgst) if err != nil { if isNotFound(err) { @@ -31,6 +53,31 @@ func (s *Server) v2GetBlob(w http.ResponseWriter, r *http.Request) { _, _ = io.Copy(w, body) } +// redirectBlob points the client at a presigned object-store URL so blob bytes +// flow directly from storage instead of being proxied through this process. +// Returns false without writing a response when the blob can't be redirected, +// so v2GetBlob falls back to streaming. +func (s *Server) redirectBlob(w http.ResponseWriter, r *http.Request, dgst string) bool { + logger := log.WithFunc("server.redirectBlob") + exists, err := s.reg.BlobExists(r.Context(), dgst) + if err != nil { + logger.Warnf(r.Context(), "blob exists check for %s failed, falling back to proxy: %v", dgst, err) + return false + } + if !exists { + v2Error(w, http.StatusNotFound, "BLOB_UNKNOWN", "blob not found") + return true + } + url, err := s.reg.PresignBlobGet(r.Context(), dgst, s.blobRedirectTTL) + if err != nil { + logger.Warnf(r.Context(), "presign blob %s failed, falling back to proxy: %v", dgst, err) + return false + } + w.Header().Set("Docker-Content-Digest", "sha256:"+dgst) + http.Redirect(w, r, url, http.StatusTemporaryRedirect) + return true +} + func (s *Server) v2HeadBlob(w http.ResponseWriter, r *http.Request) { dgst := stripSHA256Prefix(urlVar(r, "digest")) diff --git a/server/registry_v2_blobs_test.go b/server/registry_v2_blobs_test.go new file mode 100644 index 0000000..0a31c0e --- /dev/null +++ b/server/registry_v2_blobs_test.go @@ -0,0 +1,25 @@ +package server + +import ( + "testing" + "time" +) + +func TestResolveBlobRedirectTTL(t *testing.T) { + cases := []struct { + raw string + want time.Duration + }{ + {"", defaultBlobRedirectTTL}, + {"30m", 30 * time.Minute}, + {"2h", 2 * time.Hour}, + {"garbage", defaultBlobRedirectTTL}, + {"0", defaultBlobRedirectTTL}, + {"-5m", defaultBlobRedirectTTL}, + } + for _, c := range cases { + if got := resolveBlobRedirectTTL(c.raw); got != c.want { + t.Errorf("resolveBlobRedirectTTL(%q) = %s, want %s", c.raw, got, c.want) + } + } +} diff --git a/server/server.go b/server/server.go index c734e63..50108e3 100644 --- a/server/server.go +++ b/server/server.go @@ -9,6 +9,7 @@ import ( "net" "net/http" "os" + "strconv" "time" commonhttpx "github.com/cocoonstack/cocoon-common/httpx" @@ -29,9 +30,11 @@ var _ http.ResponseWriter = (*responseWriter)(nil) // Server is the Epoch HTTP server providing OCI Distribution and control plane APIs. type Server struct { - addr string // config - registryToken string // config — Bearer token for /v2/ (empty = no token required) - sso *SSOConfig // config — nil = UI auth disabled + addr string // config + registryToken string // config — Bearer token for /v2/ (empty = no token required) + sso *SSOConfig // config — nil = UI auth disabled + blobRedirect bool // config — redirect blob GETs to presigned object-store URLs + blobRedirectTTL time.Duration reg *registry.Registry // resources store *store.Store // resources @@ -54,14 +57,21 @@ func New(ctx context.Context, reg *registry.Registry, st *store.Store, addr stri if regToken != "" { logger.Info(ctx, "registry token auth enabled") } + blobRedirect, _ := strconv.ParseBool(os.Getenv("EPOCH_BLOB_REDIRECT")) + blobRedirectTTL := resolveBlobRedirectTTL(os.Getenv("EPOCH_BLOB_REDIRECT_TTL")) + if blobRedirect { + logger.Infof(ctx, "blob redirect enabled, ttl=%s", blobRedirectTTL) + } s := &Server{ - addr: addr, - registryToken: regToken, - sso: sso, - reg: reg, - store: st, - router: mux.NewRouter(), - uploads: newUploadSessions(resolveUploadDir(ctx)), + addr: addr, + registryToken: regToken, + sso: sso, + blobRedirect: blobRedirect, + blobRedirectTTL: blobRedirectTTL, + reg: reg, + store: st, + router: mux.NewRouter(), + uploads: newUploadSessions(resolveUploadDir(ctx)), } s.setupRoutes(ctx) return s From 13b78789b990ef1ee025c616c058a33ebb034c61 Mon Sep 17 00:00:00 2001 From: tonic Date: Thu, 18 Jun 2026 19:47:30 +0800 Subject: [PATCH 2/4] chore(deploy): enable blob redirect and right-size epoch-server Turn on EPOCH_BLOB_REDIRECT so pulls bypass the proxy. Bump resources from 1 CPU / 512Mi to 3 CPU / 4Gi: pushes still stream through epoch (this PR only redirects GETs) and the redirect fallback path also proxies, so the pod still needs headroom for multi-GiB transfers. --- epoch-server.yaml | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/epoch-server.yaml b/epoch-server.yaml index 9c1b374..51fcdd0 100644 --- a/epoch-server.yaml +++ b/epoch-server.yaml @@ -68,6 +68,11 @@ spec: # (tmpfs in most clusters) and OOMs on multi-GiB pushes. - name: EPOCH_UPLOAD_DIR value: /var/cache/epoch/uploads + # Serve blob GETs as redirects to presigned object-store URLs so + # multi-GiB bytes flow client<->GCS directly instead of being + # proxied (and TLS-re-encrypted) through this pod. + - name: EPOCH_BLOB_REDIRECT + value: "true" # Optional: absolute base URL clients reach the server at. Used # to anchor the OCI WWW-Authenticate realm + /v2/token. Required # when fronting epoch with a proxy that does NOT set @@ -131,11 +136,11 @@ spec: periodSeconds: 10 resources: requests: - cpu: 100m - memory: 128Mi - limits: cpu: "1" - memory: 512Mi + memory: 1Gi + limits: + cpu: "3" + memory: 4Gi volumeMounts: - name: upload-spool mountPath: /var/cache/epoch/uploads From 88558c46f9f97006587fd7a97f75cc9de5d3d3ca Mon Sep 17 00:00:00 2001 From: CMGS Date: Mon, 29 Jun 2026 00:12:55 +0800 Subject: [PATCH 3/4] chore(deps): bump cocoon-common to v0.2.2 (merged #3) Pin the merged cocoon-common main (cross-node migration API surface, PR #3) via pseudo-version until v0.2.2 is tagged. --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 954e34d..ba90602 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.25.6 require ( github.com/DATA-DOG/go-sqlmock v1.5.2 - github.com/cocoonstack/cocoon-common v0.2.1 + github.com/cocoonstack/cocoon-common v0.2.2-0.20260628160944-6e06987c3211 github.com/go-sql-driver/mysql v1.9.3 github.com/google/uuid v1.6.0 github.com/gorilla/mux v1.8.1 diff --git a/go.sum b/go.sum index 5b65b65..7d6af2d 100644 --- a/go.sum +++ b/go.sum @@ -31,8 +31,8 @@ github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZe github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs= github.com/cockroachdb/redact v1.1.3 h1:AKZds10rFSIj7qADf0g46UixK8NNLwWTNdCIGS5wfSQ= github.com/cockroachdb/redact v1.1.3/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= -github.com/cocoonstack/cocoon-common v0.2.1 h1:phf3UehIzTMxT/lIu8+RVAtaheH8KH3dBlA+uVYbp44= -github.com/cocoonstack/cocoon-common v0.2.1/go.mod h1:xIXbJ83vngQ2mrLC6q0Tw7h21M9BYBBqqYTcHaUrm1Y= +github.com/cocoonstack/cocoon-common v0.2.2-0.20260628160944-6e06987c3211 h1:1PpD7GG3juMZ9Do08bAQ3dq+qmba4zxJTak7psj90SA= +github.com/cocoonstack/cocoon-common v0.2.2-0.20260628160944-6e06987c3211/go.mod h1:xIXbJ83vngQ2mrLC6q0Tw7h21M9BYBBqqYTcHaUrm1Y= github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0/go.mod h1:4Zcjuz89kmFXt9morQgcfYZAYZ5n8WHjt81YYWIwtTM= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= From 4ebffbdaa583127f2efe49630c002c9d0ae73511 Mon Sep 17 00:00:00 2001 From: CMGS Date: Mon, 29 Jun 2026 00:21:35 +0800 Subject: [PATCH 4/4] fix(server): /code review hardening of blob redirect - clamp EPOCH_BLOB_REDIRECT_TTL to [1s, 7d]: an out-of-range value fails every presign and silently degrades to proxying, so fall back to the default below 1s (the minio presign floor) and clamp above 7d - read the redirect env via commonk8s.EnvBool / EnvDuration, dropping the hand-rolled strconv/ParseDuration to match the package convention - mark the blobRedirectTTL config field; note why redirectBlob HEADs before presigning; document the EPOCH_BLOB_REDIRECT_TTL knob --- epoch-server.yaml | 8 +++++--- objectstore/client.go | 3 +-- registry/registry.go | 3 +-- server/registry_v2_blobs.go | 35 ++++++++++++++++---------------- server/registry_v2_blobs_test.go | 22 ++++++++++---------- server/server.go | 16 +++++++-------- 6 files changed, 43 insertions(+), 44 deletions(-) diff --git a/epoch-server.yaml b/epoch-server.yaml index 51fcdd0..27986ed 100644 --- a/epoch-server.yaml +++ b/epoch-server.yaml @@ -68,11 +68,13 @@ spec: # (tmpfs in most clusters) and OOMs on multi-GiB pushes. - name: EPOCH_UPLOAD_DIR value: /var/cache/epoch/uploads - # Serve blob GETs as redirects to presigned object-store URLs so - # multi-GiB bytes flow client<->GCS directly instead of being - # proxied (and TLS-re-encrypted) through this pod. + # 307-redirect blob GETs to presigned URLs so multi-GiB bytes + # bypass this pod. - name: EPOCH_BLOB_REDIRECT value: "true" + # Optional redirect URL lifetime, Go duration (default 1h, max 7d). + # - name: EPOCH_BLOB_REDIRECT_TTL + # value: "1h" # Optional: absolute base URL clients reach the server at. Used # to anchor the OCI WWW-Authenticate realm + /v2/token. Required # when fronting epoch with a proxy that does NOT set diff --git a/objectstore/client.go b/objectstore/client.go index 3db8db4..db6013b 100644 --- a/objectstore/client.go +++ b/objectstore/client.go @@ -58,8 +58,7 @@ func (c *Client) Put(ctx context.Context, key string, body io.Reader, size int64 return nil } -// PresignGet returns a time-limited URL that lets a client GET the object -// directly from the backing store, bypassing this process. +// PresignGet returns a time-limited URL for a direct GET, bypassing this process. func (c *Client) PresignGet(ctx context.Context, key string, ttl time.Duration) (string, error) { u, err := c.client.PresignedGetObject(ctx, c.cfg.Bucket, c.fullKey(key), ttl, nil) if err != nil { diff --git a/registry/registry.go b/registry/registry.go index 4c6cb1b..68687ad 100644 --- a/registry/registry.go +++ b/registry/registry.go @@ -74,8 +74,7 @@ func (r *Registry) StreamBlob(ctx context.Context, digest string) (io.ReadCloser return r.client.Get(ctx, blobKey(digest)) } -// PresignBlobGet returns a time-limited URL for fetching a blob directly from -// the object store, bypassing this process. +// PresignBlobGet returns a time-limited URL for a direct blob GET. func (r *Registry) PresignBlobGet(ctx context.Context, digest string, ttl time.Duration) (string, error) { return r.client.PresignGet(ctx, blobKey(digest), ttl) } diff --git a/server/registry_v2_blobs.go b/server/registry_v2_blobs.go index 7f27088..6c1d2ec 100644 --- a/server/registry_v2_blobs.go +++ b/server/registry_v2_blobs.go @@ -11,20 +11,11 @@ import ( "github.com/cocoonstack/epoch/manifest" ) -const defaultBlobRedirectTTL = time.Hour - -// resolveBlobRedirectTTL parses EPOCH_BLOB_REDIRECT_TTL, falling back to the -// default for empty, unparseable, or non-positive values. -func resolveBlobRedirectTTL(raw string) time.Duration { - if raw == "" { - return defaultBlobRedirectTTL - } - d, err := time.ParseDuration(raw) - if err != nil || d <= 0 { - return defaultBlobRedirectTTL - } - return d -} +const ( + defaultBlobRedirectTTL = time.Hour + // maxBlobRedirectTTL is the presign expiry cap; over it every presign fails. + maxBlobRedirectTTL = 7 * 24 * time.Hour +) func (s *Server) v2GetBlob(w http.ResponseWriter, r *http.Request) { dgst := stripSHA256Prefix(urlVar(r, "digest")) @@ -53,12 +44,12 @@ func (s *Server) v2GetBlob(w http.ResponseWriter, r *http.Request) { _, _ = io.Copy(w, body) } -// redirectBlob points the client at a presigned object-store URL so blob bytes -// flow directly from storage instead of being proxied through this process. -// Returns false without writing a response when the blob can't be redirected, -// so v2GetBlob falls back to streaming. +// redirectBlob 307s the client to a presigned URL. Returns false without +// writing a response when it can't, so v2GetBlob falls back to streaming. func (s *Server) redirectBlob(w http.ResponseWriter, r *http.Request, dgst string) bool { logger := log.WithFunc("server.redirectBlob") + // presign succeeds even for a missing object, so HEAD first to return an + // OCI BLOB_UNKNOWN rather than 307 to a backend 404. exists, err := s.reg.BlobExists(r.Context(), dgst) if err != nil { logger.Warnf(r.Context(), "blob exists check for %s failed, falling back to proxy: %v", dgst, err) @@ -106,3 +97,11 @@ func (s *Server) v2PutBlob(w http.ResponseWriter, r *http.Request) { } s.persistMonolithicUpload(w, r, name, "sha256:"+dgst) } + +// clampBlobRedirectTTL keeps a TTL in the presign-valid range: sub-1s → default, over-7d → cap. +func clampBlobRedirectTTL(d time.Duration) time.Duration { + if d < time.Second { + return defaultBlobRedirectTTL + } + return min(d, maxBlobRedirectTTL) +} diff --git a/server/registry_v2_blobs_test.go b/server/registry_v2_blobs_test.go index 0a31c0e..54421c6 100644 --- a/server/registry_v2_blobs_test.go +++ b/server/registry_v2_blobs_test.go @@ -5,21 +5,21 @@ import ( "time" ) -func TestResolveBlobRedirectTTL(t *testing.T) { +func TestClampBlobRedirectTTL(t *testing.T) { cases := []struct { - raw string + in time.Duration want time.Duration }{ - {"", defaultBlobRedirectTTL}, - {"30m", 30 * time.Minute}, - {"2h", 2 * time.Hour}, - {"garbage", defaultBlobRedirectTTL}, - {"0", defaultBlobRedirectTTL}, - {"-5m", defaultBlobRedirectTTL}, + {30 * time.Minute, 30 * time.Minute}, + {2 * time.Hour, 2 * time.Hour}, + {0, defaultBlobRedirectTTL}, + {-5 * time.Minute, defaultBlobRedirectTTL}, + {500 * time.Millisecond, defaultBlobRedirectTTL}, // under the 1s presign floor + {200 * time.Hour, maxBlobRedirectTTL}, // over the 7-day presign cap } - for _, c := range cases { - if got := resolveBlobRedirectTTL(c.raw); got != c.want { - t.Errorf("resolveBlobRedirectTTL(%q) = %s, want %s", c.raw, got, c.want) + for _, tc := range cases { + if got := clampBlobRedirectTTL(tc.in); got != tc.want { + t.Errorf("clampBlobRedirectTTL(%s) = %s, want %s", tc.in, got, tc.want) } } } diff --git a/server/server.go b/server/server.go index 50108e3..73f89ec 100644 --- a/server/server.go +++ b/server/server.go @@ -9,10 +9,10 @@ import ( "net" "net/http" "os" - "strconv" "time" commonhttpx "github.com/cocoonstack/cocoon-common/httpx" + commonk8s "github.com/cocoonstack/cocoon-common/k8s" "github.com/gorilla/mux" "github.com/projecteru2/core/log" @@ -30,11 +30,11 @@ var _ http.ResponseWriter = (*responseWriter)(nil) // Server is the Epoch HTTP server providing OCI Distribution and control plane APIs. type Server struct { - addr string // config - registryToken string // config — Bearer token for /v2/ (empty = no token required) - sso *SSOConfig // config — nil = UI auth disabled - blobRedirect bool // config — redirect blob GETs to presigned object-store URLs - blobRedirectTTL time.Duration + addr string // config + registryToken string // config — Bearer token for /v2/ (empty = no token required) + sso *SSOConfig // config — nil = UI auth disabled + blobRedirect bool // config — redirect blob GETs to presigned object-store URLs + blobRedirectTTL time.Duration // config — presigned redirect URL lifetime reg *registry.Registry // resources store *store.Store // resources @@ -57,8 +57,8 @@ func New(ctx context.Context, reg *registry.Registry, st *store.Store, addr stri if regToken != "" { logger.Info(ctx, "registry token auth enabled") } - blobRedirect, _ := strconv.ParseBool(os.Getenv("EPOCH_BLOB_REDIRECT")) - blobRedirectTTL := resolveBlobRedirectTTL(os.Getenv("EPOCH_BLOB_REDIRECT_TTL")) + blobRedirect := commonk8s.EnvBool("EPOCH_BLOB_REDIRECT", false) + blobRedirectTTL := clampBlobRedirectTTL(commonk8s.EnvDuration("EPOCH_BLOB_REDIRECT_TTL", defaultBlobRedirectTTL)) if blobRedirect { logger.Infof(ctx, "blob redirect enabled, ttl=%s", blobRedirectTTL) }