From 30bc09bfa726149953c884e02864a0d78b3a66ab Mon Sep 17 00:00:00 2001 From: Luis Padron Date: Tue, 21 Jul 2026 15:55:19 -0400 Subject: [PATCH 1/2] feat(git): per-repo partial-clone snapshot filters Some repos' snapshot artifacts are dominated by history no checkout ever touches. A new snapshot-filters config maps host/org/repo to a git partial-clone filter (e.g. blob:none) applied to the workstation snapshot clone via --no-local --filter. Filtered snapshots keep full commit/tree history but only HEAD blobs; clients lazily fetch older blobs through cachew. Mirror and LFS snapshots are unchanged. Co-authored-by: Claude Code Ai-assisted: true --- cachew.hcl | 4 ++ internal/strategy/git/git.go | 2 + internal/strategy/git/snapshot.go | 70 ++++++++++++++++++--- internal/strategy/git/snapshot_test.go | 86 ++++++++++++++++++++++++++ 4 files changed, 152 insertions(+), 10 deletions(-) diff --git a/cachew.hcl b/cachew.hcl index 79feccfb..38ce2e4a 100644 --- a/cachew.hcl +++ b/cachew.hcl @@ -39,6 +39,10 @@ strategy git { #bundle-interval = "24h" snapshot-interval = "1h" repack-interval = "1h" + # Serve partial-clone snapshots for repos whose history dwarfs their checkout. + #snapshot-filters = { + # "github.com/org/repo": "blob:none", + #} } strategy host "https://ghcr.io" { diff --git a/internal/strategy/git/git.go b/internal/strategy/git/git.go index c0c329fb..923fc18e 100644 --- a/internal/strategy/git/git.go +++ b/internal/strategy/git/git.go @@ -50,6 +50,8 @@ type Config struct { RepackInterval time.Duration `hcl:"repack-interval,optional" help:"How often to run full repack. 0 disables." default:"0"` ZstdThreads int `hcl:"zstd-threads,optional" help:"Threads for zstd compression/decompression. 0 = all CPU cores; useful for short-lived CLI invocations but risky on a long-running server where multiple snapshot/restore operations can run concurrently." default:"4"` BundleCacheTTL time.Duration `hcl:"bundle-cache-ttl,optional" help:"TTL of cached server-side git bundles." default:"2h"` + + SnapshotFilters map[string]string `hcl:"snapshot-filters,optional" help:"Per-repository git partial-clone filter applied to workstation snapshots, keyed by host/org/repo (e.g. {\"github.com/org/repo\": \"blob:none\"}). Filtered snapshots contain full history metadata but only the blobs needed for the HEAD checkout; clients lazily fetch historical blobs through cachew on demand."` } type Strategy struct { diff --git a/internal/strategy/git/snapshot.go b/internal/strategy/git/snapshot.go index 9805b078..dc9b19f7 100644 --- a/internal/strategy/git/snapshot.go +++ b/internal/strategy/git/snapshot.go @@ -58,16 +58,36 @@ func lfsSnapshotCacheKey(upstreamURL string) cache.Key { } // cloneForSnapshot clones the mirror into destDir under repo's read lock, -// then fixes the remote URL to point through cachew (or upstream). -func (s *Strategy) cloneForSnapshot(ctx context.Context, repo *gitclone.Repository, destDir string) error { +// then fixes the remote URL to point through cachew (or upstream). A non-empty +// filter produces a partial clone (e.g. blob:none) that carries full history +// metadata but only the blobs needed for the HEAD checkout. +func (s *Strategy) cloneForSnapshot(ctx context.Context, repo *gitclone.Repository, destDir, filter string) error { if err := repo.WithReadLock(func() error { + args := []string{"clone"} + if filter != "" { + // Local (hardlink) clones copy the entire object store and silently + // ignore --filter; --no-local forces the transport path so the filter + // applies and objects unreachable from cloned refs (e.g. refs/pull + // history in the mirror) are left behind. + args = append(args, "--no-local", "--filter="+filter) + } + args = append(args, repo.Path(), destDir) // #nosec G204 - repo.Path() and destDir are controlled by us - cmd := exec.CommandContext(ctx, "git", "clone", repo.Path(), destDir) + cmd := exec.CommandContext(ctx, "git", args...) cmd.Env = append(os.Environ(), "GIT_LFS_SKIP_SMUDGE=1") if output, err := cmd.CombinedOutput(); err != nil { return errors.Wrapf(err, "git clone for snapshot: %s", string(output)) } + if filter != "" { + // git silently falls back to a full clone when the source repo does + // not advertise the filter capability; fail loudly rather than cache + // a full-size artifact for a repo configured to be filtered. + if err := verifyPartialClone(ctx, destDir); err != nil { + return errors.WithStack(err) + } + } + // git clone from a local path sets remote.origin.url to that path; restore // it to the upstream URL. Clients use insteadOf to route through cachew, so // embedding the cachew URL here would couple snapshots to a specific instance. @@ -83,7 +103,30 @@ func (s *Strategy) cloneForSnapshot(ctx context.Context, repo *gitclone.Reposito return nil } -func (s *Strategy) withSnapshotClone(ctx context.Context, repo *gitclone.Repository, suffix string, fn func(workDir string) error) error { +// snapshotFilterFor reports the partial-clone filter configured for the repo, +// or empty when the snapshot should be a full clone. +func (s *Strategy) snapshotFilterFor(upstreamURL string) string { + if len(s.config.SnapshotFilters) == 0 { + return "" + } + repoPath, err := gitclone.RepoPathFromURL(upstreamURL) + if err != nil { + return "" + } + return s.config.SnapshotFilters[repoPath] +} + +// verifyPartialClone reports an error when repoDir is not a partial clone. +func verifyPartialClone(ctx context.Context, repoDir string) error { + cmd := exec.CommandContext(ctx, "git", "-C", repoDir, "config", "--get", "remote.origin.promisor") // #nosec G204 + output, err := cmd.Output() + if err != nil || strings.TrimSpace(string(output)) != "true" { + return errors.New("snapshot filter was ignored by the mirror (uploadpack.allowFilter unset?)") + } + return nil +} + +func (s *Strategy) withSnapshotClone(ctx context.Context, repo *gitclone.Repository, suffix, filter string, fn func(workDir string) error) error { logger := logging.FromContext(ctx) mirrorRoot := s.cloneManager.Config().MirrorRoot workDir, err := snapshotDirForURL(mirrorRoot, repo.UpstreamURL()) @@ -100,7 +143,7 @@ func (s *Strategy) withSnapshotClone(ctx context.Context, repo *gitclone.Reposit return errors.Wrap(err, "create snapshot work dir parent") } - if err := s.cloneForSnapshot(ctx, repo, workDir); err != nil { + if err := s.cloneForSnapshot(ctx, repo, workDir, filter); err != nil { _ = os.RemoveAll(workDir) return err } @@ -134,7 +177,11 @@ func (s *Strategy) generateAndUploadSnapshot(ctx context.Context, repo *gitclone logger := logging.FromContext(ctx) start := time.Now() - logger.InfoContext(ctx, "Snapshot generation started", "upstream", upstream) + filter := s.snapshotFilterFor(upstream) + if filter != "" { + span.SetAttributes(attribute.String("cachew.snapshot.filter", filter)) + } + logger.InfoContext(ctx, "Snapshot generation started", "upstream", upstream, "filter", filter) mu := s.snapshotMutexFor(upstream) mu.Lock() @@ -146,7 +193,7 @@ func (s *Strategy) generateAndUploadSnapshot(ctx context.Context, repo *gitclone logger.InfoContext(ctx, "Snapshot unchanged, skipping generation", "upstream", upstream, "commit", head) return "", nil } - if err := s.withSnapshotClone(ctx, repo, "base", func(workDir string) error { + if err := s.withSnapshotClone(ctx, repo, "base", filter, func(workDir string) error { // Capture the snapshot's HEAD so we can later build a delta bundle between // the cached snapshot and the current mirror state. headSHA, err := revParse(ctx, workDir, "HEAD") @@ -958,7 +1005,7 @@ func (s *Strategy) streamSnapshotDirect(w http.ResponseWriter, r *http.Request, defer func() { _ = os.RemoveAll(snapshotDir) }() repoDir := filepath.Join(snapshotDir, "repo") - if err := s.cloneForSnapshot(ctx, repo, repoDir); err != nil { + if err := s.cloneForSnapshot(ctx, repo, repoDir, s.snapshotFilterFor(repo.UpstreamURL())); err != nil { return errors.Wrap(err, "clone for snapshot streaming") } @@ -999,7 +1046,7 @@ func (s *Strategy) prepareSnapshotSpool(ctx context.Context, repo *gitclone.Repo } repoDir = filepath.Join(snapshotDir, "repo") - if err := s.cloneForSnapshot(ctx, repo, repoDir); err != nil { + if err := s.cloneForSnapshot(ctx, repo, repoDir, s.snapshotFilterFor(upstreamURL)); err != nil { spool.MarkError(err) s.snapshotSpools.Delete(upstreamURL) _ = os.RemoveAll(snapshotDir) @@ -1217,7 +1264,10 @@ func (s *Strategy) generateAndUploadLFSSnapshot(ctx context.Context, repo *gitcl excludePatterns := []string{"*.lock"} cloneStart := time.Now() cloneRecorded := false - if err := s.withSnapshotClone(ctx, repo, "lfs", func(workDir string) error { + // The LFS clone stays unfiltered: after the remote is reset to upstream, a + // partial clone's lazy fetches would hit GitHub directly without the + // credential injection that repo.GitCommand provides. + if err := s.withSnapshotClone(ctx, repo, "lfs", "", func(workDir string) error { s.metrics.recordLFSPhase(ctx, upstream, "clone", "success", time.Since(cloneStart)) cloneRecorded = true diff --git a/internal/strategy/git/snapshot_test.go b/internal/strategy/git/snapshot_test.go index 0f19f7e2..02d3f906 100644 --- a/internal/strategy/git/snapshot_test.go +++ b/internal/strategy/git/snapshot_test.go @@ -318,6 +318,92 @@ func TestSnapshotGenerationViaLocalClone(t *testing.T) { assert.True(t, os.IsNotExist(err)) } +func TestSnapshotGenerationWithFilter(t *testing.T) { + if _, err := exec.LookPath("git"); err != nil { + t.Skip("git not found in PATH") + } + + _, ctx := logging.Configure(context.Background(), logging.Config{}) + tmpDir := t.TempDir() + mirrorRoot := filepath.Join(tmpDir, "mirrors") + upstreamURL := "https://github.com/org/repo" + + mirrorPath := filepath.Join(mirrorRoot, "github.com", "org", "repo") + historicalBlob := createTestMirrorRepoWithHistory(t, mirrorPath) + + memCache, err := cache.NewMemory(ctx, cache.MemoryConfig{MaxTTL: time.Hour}) + assert.NoError(t, err) + mux := newTestMux() + + cm := gitclone.NewManagerProvider(ctx, gitclone.Config{MirrorRoot: mirrorRoot}, nil) + s, err := git.New(ctx, git.Config{SnapshotFilters: map[string]string{"github.com/org/repo": "blob:none"}}, newTestScheduler(ctx, t), memCache, mux, cm, func() (*githubapp.TokenManager, error) { return nil, nil }) //nolint:nilnil + assert.NoError(t, err) + + manager, err := cm() + assert.NoError(t, err) + repo, err := manager.GetOrCreate(ctx, upstreamURL) + assert.NoError(t, err) + assert.Equal(t, gitclone.StateReady, repo.State()) + + waitForReady(t, s) + err = s.GenerateAndUploadSnapshot(ctx, repo) + assert.NoError(t, err) + + cacheKey := cache.NewKey(upstreamURL + ".snapshot") + restoreDir := filepath.Join(tmpDir, "restored") + err = snapshot.Restore(ctx, memCache, cacheKey, restoreDir, 0) + assert.NoError(t, err) + + data, err := os.ReadFile(filepath.Join(restoreDir, "data.txt")) + assert.NoError(t, err) + assert.Equal(t, "current\n", string(data)) + + cmd := exec.Command("git", "-C", restoreDir, "config", "--get", "remote.origin.partialclonefilter") + output, err := cmd.CombinedOutput() + assert.NoError(t, err, string(output)) + assert.Equal(t, "blob:none\n", string(output)) + + cmd = exec.Command("git", "-C", restoreDir, "cat-file", "--batch-all-objects", "--batch-check=%(objectname)") + output, err = cmd.CombinedOutput() + assert.NoError(t, err, string(output)) + assert.False(t, strings.Contains(string(output), historicalBlob)) + + cmd = exec.Command("git", "-C", restoreDir, "remote", "get-url", "origin") + output, err = cmd.CombinedOutput() + assert.NoError(t, err, string(output)) + assert.Equal(t, upstreamURL+"\n", string(output)) +} + +// createTestMirrorRepoWithHistory creates a mirror whose history contains a +// superseded version of data.txt, returning that historical blob's SHA. +func createTestMirrorRepoWithHistory(t *testing.T, mirrorPath string) string { + t.Helper() + tmpWork := t.TempDir() + + run := func(args ...string) string { + cmd := exec.Command("git", args...) + output, err := cmd.CombinedOutput() + assert.NoError(t, err, string(output)) + return strings.TrimSpace(string(output)) + } + + run("init", tmpWork) + run("-C", tmpWork, "config", "user.email", "test@test.com") + run("-C", tmpWork, "config", "user.name", "Test") + assert.NoError(t, os.WriteFile(filepath.Join(tmpWork, "data.txt"), []byte("historical\n"), 0o644)) + run("-C", tmpWork, "add", ".") + run("-C", tmpWork, "commit", "-m", "one") + historicalBlob := run("-C", tmpWork, "rev-parse", "HEAD:data.txt") + assert.NoError(t, os.WriteFile(filepath.Join(tmpWork, "data.txt"), []byte("current\n"), 0o644)) + run("-C", tmpWork, "commit", "-am", "two") + run("clone", "--mirror", tmpWork, mirrorPath) + // configureMirror sets this on managed mirrors; a premade test mirror lacks + // it, and without it git silently ignores --filter. + run("-C", mirrorPath, "config", "uploadpack.allowfilter", "true") + run("-C", mirrorPath, "cat-file", "-e", historicalBlob) + return historicalBlob +} + func TestSnapshotGenerationIncludesTrackedLockFiles(t *testing.T) { if _, err := exec.LookPath("git"); err != nil { t.Skip("git not found in PATH") From 64dbbc05d9c7033676cbe4c7fe9cb056f0aa6213 Mon Sep 17 00:00:00 2001 From: Luis Padron Date: Tue, 21 Jul 2026 16:42:20 -0400 Subject: [PATCH 2/2] fix(git): detect ignored snapshot filters from clone output git writes remote.origin.promisor and partialclonefilter even when the server does not support filtering and the clone silently falls back to a full clone, so the promisor config is no proof the filter applied. Detect the fallback from git's warning in the clone output instead, with LC_ALL=C pinned so the message is not localized. Co-authored-by: Claude Code Ai-assisted: true --- internal/strategy/git/snapshot.go | 34 ++++++++----------- internal/strategy/git/snapshot_test.go | 45 ++++++++++++++++++++++++++ 2 files changed, 59 insertions(+), 20 deletions(-) diff --git a/internal/strategy/git/snapshot.go b/internal/strategy/git/snapshot.go index dc9b19f7..b514f8df 100644 --- a/internal/strategy/git/snapshot.go +++ b/internal/strategy/git/snapshot.go @@ -74,18 +74,22 @@ func (s *Strategy) cloneForSnapshot(ctx context.Context, repo *gitclone.Reposito args = append(args, repo.Path(), destDir) // #nosec G204 - repo.Path() and destDir are controlled by us cmd := exec.CommandContext(ctx, "git", args...) - cmd.Env = append(os.Environ(), "GIT_LFS_SKIP_SMUDGE=1") - if output, err := cmd.CombinedOutput(); err != nil { + // LC_ALL=C pins git's messages to English so the filter fallback + // warning below is detectable regardless of the host locale. + cmd.Env = append(os.Environ(), "GIT_LFS_SKIP_SMUDGE=1", "LC_ALL=C") + output, err := cmd.CombinedOutput() + if err != nil { return errors.Wrapf(err, "git clone for snapshot: %s", string(output)) } - if filter != "" { - // git silently falls back to a full clone when the source repo does - // not advertise the filter capability; fail loudly rather than cache - // a full-size artifact for a repo configured to be filtered. - if err := verifyPartialClone(ctx, destDir); err != nil { - return errors.WithStack(err) - } + // When the source repo does not advertise the filter capability, git + // falls back to a full clone with only a warning — while still writing + // remote.origin.promisor and partialclonefilter as if the filter had + // applied, so the clone's config cannot be trusted as proof. The + // warning is the reliable signal; detect it and fail loudly rather + // than cache a full-size artifact for a repo configured to be filtered. + if filter != "" && strings.Contains(string(output), "filtering not recognized by server") { + return errors.Errorf("snapshot filter %q was ignored by the mirror (uploadpack.allowFilter unset?)", filter) } // git clone from a local path sets remote.origin.url to that path; restore @@ -93,7 +97,7 @@ func (s *Strategy) cloneForSnapshot(ctx context.Context, repo *gitclone.Reposito // embedding the cachew URL here would couple snapshots to a specific instance. // #nosec G204 - upstreamURL is derived from controlled inputs cmd = exec.CommandContext(ctx, "git", "-C", destDir, "remote", "set-url", "origin", repo.UpstreamURL()) - if output, err := cmd.CombinedOutput(); err != nil { + if output, err = cmd.CombinedOutput(); err != nil { return errors.Wrapf(err, "fix snapshot remote URL: %s", string(output)) } return nil @@ -116,16 +120,6 @@ func (s *Strategy) snapshotFilterFor(upstreamURL string) string { return s.config.SnapshotFilters[repoPath] } -// verifyPartialClone reports an error when repoDir is not a partial clone. -func verifyPartialClone(ctx context.Context, repoDir string) error { - cmd := exec.CommandContext(ctx, "git", "-C", repoDir, "config", "--get", "remote.origin.promisor") // #nosec G204 - output, err := cmd.Output() - if err != nil || strings.TrimSpace(string(output)) != "true" { - return errors.New("snapshot filter was ignored by the mirror (uploadpack.allowFilter unset?)") - } - return nil -} - func (s *Strategy) withSnapshotClone(ctx context.Context, repo *gitclone.Repository, suffix, filter string, fn func(workDir string) error) error { logger := logging.FromContext(ctx) mirrorRoot := s.cloneManager.Config().MirrorRoot diff --git a/internal/strategy/git/snapshot_test.go b/internal/strategy/git/snapshot_test.go index 02d3f906..739f86aa 100644 --- a/internal/strategy/git/snapshot_test.go +++ b/internal/strategy/git/snapshot_test.go @@ -374,6 +374,51 @@ func TestSnapshotGenerationWithFilter(t *testing.T) { assert.Equal(t, upstreamURL+"\n", string(output)) } +func TestSnapshotGenerationWithFilterFailsWhenMirrorIgnoresFilter(t *testing.T) { + if _, err := exec.LookPath("git"); err != nil { + t.Skip("git not found in PATH") + } + + _, ctx := logging.Configure(context.Background(), logging.Config{}) + tmpDir := t.TempDir() + mirrorRoot := filepath.Join(tmpDir, "mirrors") + upstreamURL := "https://github.com/org/repo" + + mirrorPath := filepath.Join(mirrorRoot, "github.com", "org", "repo") + createTestMirrorRepoWithHistory(t, mirrorPath) + + memCache, err := cache.NewMemory(ctx, cache.MemoryConfig{MaxTTL: time.Hour}) + assert.NoError(t, err) + mux := newTestMux() + + cm := gitclone.NewManagerProvider(ctx, gitclone.Config{MirrorRoot: mirrorRoot}, nil) + s, err := git.New(ctx, git.Config{SnapshotFilters: map[string]string{"github.com/org/repo": "blob:none"}}, newTestScheduler(ctx, t), memCache, mux, cm, func() (*githubapp.TokenManager, error) { return nil, nil }) //nolint:nilnil + assert.NoError(t, err) + + manager, err := cm() + assert.NoError(t, err) + repo, err := manager.GetOrCreate(ctx, upstreamURL) + assert.NoError(t, err) + assert.Equal(t, gitclone.StateReady, repo.State()) + + waitForReady(t, s) + + // A mirror missing uploadpack.allowFilter makes git silently fall back to + // a full clone; generation must fail rather than cache the full artifact. + // Unset only after waitForReady: startup warming reruns configureMirror, + // which would re-enable the capability. + cmd := exec.Command("git", "-C", mirrorPath, "config", "--unset", "uploadpack.allowfilter") + output, err := cmd.CombinedOutput() + assert.NoError(t, err, string(output)) + + err = s.GenerateAndUploadSnapshot(ctx, repo) + assert.Error(t, err) + assert.Contains(t, err.Error(), "ignored by the mirror") + + _, err = memCache.Stat(ctx, cache.NewKey(upstreamURL+".snapshot")) + assert.IsError(t, err, os.ErrNotExist) +} + // createTestMirrorRepoWithHistory creates a mirror whose history contains a // superseded version of data.txt, returning that historical blob's SHA. func createTestMirrorRepoWithHistory(t *testing.T, mirrorPath string) string {