feat(git): per-repo partial-clone snapshot filters

cachew.hcl

@@ -39,6 +39,10 @@ strategy git {
   #bundle-interval = "24h"
   snapshot-interval = "1h"
   repack-interval = "1h"
+  # Serve partial-clone snapshots for repos whose history dwarfs their checkout.
+  #snapshot-filters = {
+  #  "github.com/org/repo": "blob:none",
+  #}
 }
 
 strategy host "https://ghcr.io" {

internal/strategy/git/git.go

@@ -50,6 +50,8 @@ type Config struct {
 	RepackInterval         time.Duration `hcl:"repack-interval,optional" help:"How often to run full repack. 0 disables." default:"0"`
 	ZstdThreads            int           `hcl:"zstd-threads,optional" help:"Threads for zstd compression/decompression. 0 = all CPU cores; useful for short-lived CLI invocations but risky on a long-running server where multiple snapshot/restore operations can run concurrently." default:"4"`
 	BundleCacheTTL         time.Duration `hcl:"bundle-cache-ttl,optional" help:"TTL of cached server-side git bundles." default:"2h"`
+
+	SnapshotFilters map[string]string `hcl:"snapshot-filters,optional" help:"Per-repository git partial-clone filter applied to workstation snapshots, keyed by host/org/repo (e.g. {\"github.com/org/repo\": \"blob:none\"}). Filtered snapshots contain full history metadata but only the blobs needed for the HEAD checkout; clients lazily fetch historical blobs through cachew on demand."`
 }
 
 type Strategy struct {

internal/strategy/git/snapshot.go

@@ -58,22 +58,46 @@ func lfsSnapshotCacheKey(upstreamURL string) cache.Key {
 }
 
 // cloneForSnapshot clones the mirror into destDir under repo's read lock,
-// then fixes the remote URL to point through cachew (or upstream).
-func (s *Strategy) cloneForSnapshot(ctx context.Context, repo *gitclone.Repository, destDir string) error {
+// then fixes the remote URL to point through cachew (or upstream). A non-empty
+// filter produces a partial clone (e.g. blob:none) that carries full history
+// metadata but only the blobs needed for the HEAD checkout.
+func (s *Strategy) cloneForSnapshot(ctx context.Context, repo *gitclone.Repository, destDir, filter string) error {
 	if err := repo.WithReadLock(func() error {
+		args := []string{"clone"}
+		if filter != "" {
+			// Local (hardlink) clones copy the entire object store and silently
+			// ignore --filter; --no-local forces the transport path so the filter
+			// applies and objects unreachable from cloned refs (e.g. refs/pull
+			// history in the mirror) are left behind.
+			args = append(args, "--no-local", "--filter="+filter)
+		}
+		args = append(args, repo.Path(), destDir)
 		// #nosec G204 - repo.Path() and destDir are controlled by us
-		cmd := exec.CommandContext(ctx, "git", "clone", repo.Path(), destDir)
-		cmd.Env = append(os.Environ(), "GIT_LFS_SKIP_SMUDGE=1")
-		if output, err := cmd.CombinedOutput(); err != nil {
+		cmd := exec.CommandContext(ctx, "git", args...)
+		// LC_ALL=C pins git's messages to English so the filter fallback
+		// warning below is detectable regardless of the host locale.
+		cmd.Env = append(os.Environ(), "GIT_LFS_SKIP_SMUDGE=1", "LC_ALL=C")
+		output, err := cmd.CombinedOutput()
+		if err != nil {
 			return errors.Wrapf(err, "git clone for snapshot: %s", string(output))
 		}
 
+		// When the source repo does not advertise the filter capability, git
+		// falls back to a full clone with only a warning — while still writing
+		// remote.origin.promisor and partialclonefilter as if the filter had
+		// applied, so the clone's config cannot be trusted as proof. The
+		// warning is the reliable signal; detect it and fail loudly rather
+		// than cache a full-size artifact for a repo configured to be filtered.
+		if filter != "" && strings.Contains(string(output), "filtering not recognized by server") {
+			return errors.Errorf("snapshot filter %q was ignored by the mirror (uploadpack.allowFilter unset?)", filter)
+		}
+
 		// git clone from a local path sets remote.origin.url to that path; restore
 		// it to the upstream URL. Clients use insteadOf to route through cachew, so
 		// embedding the cachew URL here would couple snapshots to a specific instance.
 		// #nosec G204 - upstreamURL is derived from controlled inputs
 		cmd = exec.CommandContext(ctx, "git", "-C", destDir, "remote", "set-url", "origin", repo.UpstreamURL())
-		if output, err := cmd.CombinedOutput(); err != nil {
+		if output, err = cmd.CombinedOutput(); err != nil {
 			return errors.Wrapf(err, "fix snapshot remote URL: %s", string(output))
 		}
 		return nil
@@ -83,7 +107,20 @@ func (s *Strategy) cloneForSnapshot(ctx context.Context, repo *gitclone.Reposito
 	return nil
 }
 
-func (s *Strategy) withSnapshotClone(ctx context.Context, repo *gitclone.Repository, suffix string, fn func(workDir string) error) error {
+// snapshotFilterFor reports the partial-clone filter configured for the repo,
+// or empty when the snapshot should be a full clone.
+func (s *Strategy) snapshotFilterFor(upstreamURL string) string {
+	if len(s.config.SnapshotFilters) == 0 {
+		return ""
+	}
+	repoPath, err := gitclone.RepoPathFromURL(upstreamURL)
+	if err != nil {
+		return ""
+	}
+	return s.config.SnapshotFilters[repoPath]
+}
+
+func (s *Strategy) withSnapshotClone(ctx context.Context, repo *gitclone.Repository, suffix, filter string, fn func(workDir string) error) error {
 	logger := logging.FromContext(ctx)
 	mirrorRoot := s.cloneManager.Config().MirrorRoot
 	workDir, err := snapshotDirForURL(mirrorRoot, repo.UpstreamURL())
@@ -100,7 +137,7 @@ func (s *Strategy) withSnapshotClone(ctx context.Context, repo *gitclone.Reposit
 		return errors.Wrap(err, "create snapshot work dir parent")
 	}
 
-	if err := s.cloneForSnapshot(ctx, repo, workDir); err != nil {
+	if err := s.cloneForSnapshot(ctx, repo, workDir, filter); err != nil {
 		_ = os.RemoveAll(workDir)
 		return err
 	}
@@ -134,7 +171,11 @@ func (s *Strategy) generateAndUploadSnapshot(ctx context.Context, repo *gitclone
 	logger := logging.FromContext(ctx)
 	start := time.Now()
 
-	logger.InfoContext(ctx, "Snapshot generation started", "upstream", upstream)
+	filter := s.snapshotFilterFor(upstream)
+	if filter != "" {
+		span.SetAttributes(attribute.String("cachew.snapshot.filter", filter))
+	}
+	logger.InfoContext(ctx, "Snapshot generation started", "upstream", upstream, "filter", filter)
 
 	mu := s.snapshotMutexFor(upstream)
 	mu.Lock()
@@ -146,7 +187,7 @@ func (s *Strategy) generateAndUploadSnapshot(ctx context.Context, repo *gitclone
 		logger.InfoContext(ctx, "Snapshot unchanged, skipping generation", "upstream", upstream, "commit", head)
 		return "", nil
 	}
-	if err := s.withSnapshotClone(ctx, repo, "base", func(workDir string) error {
+	if err := s.withSnapshotClone(ctx, repo, "base", filter, func(workDir string) error {
 		// Capture the snapshot's HEAD so we can later build a delta bundle between
 		// the cached snapshot and the current mirror state.
 		headSHA, err := revParse(ctx, workDir, "HEAD")
@@ -958,7 +999,7 @@ func (s *Strategy) streamSnapshotDirect(w http.ResponseWriter, r *http.Request,
 	defer func() { _ = os.RemoveAll(snapshotDir) }()
 
 	repoDir := filepath.Join(snapshotDir, "repo")
-	if err := s.cloneForSnapshot(ctx, repo, repoDir); err != nil {
+	if err := s.cloneForSnapshot(ctx, repo, repoDir, s.snapshotFilterFor(repo.UpstreamURL())); err != nil {
 		return errors.Wrap(err, "clone for snapshot streaming")
 	}
 
@@ -999,7 +1040,7 @@ func (s *Strategy) prepareSnapshotSpool(ctx context.Context, repo *gitclone.Repo
 	}
 
 	repoDir = filepath.Join(snapshotDir, "repo")
-	if err := s.cloneForSnapshot(ctx, repo, repoDir); err != nil {
+	if err := s.cloneForSnapshot(ctx, repo, repoDir, s.snapshotFilterFor(upstreamURL)); err != nil {
 		spool.MarkError(err)
 		s.snapshotSpools.Delete(upstreamURL)
 		_ = os.RemoveAll(snapshotDir)
@@ -1217,7 +1258,10 @@ func (s *Strategy) generateAndUploadLFSSnapshot(ctx context.Context, repo *gitcl
 	excludePatterns := []string{"*.lock"}
 	cloneStart := time.Now()
 	cloneRecorded := false
-	if err := s.withSnapshotClone(ctx, repo, "lfs", func(workDir string) error {
+	// The LFS clone stays unfiltered: after the remote is reset to upstream, a
+	// partial clone's lazy fetches would hit GitHub directly without the
+	// credential injection that repo.GitCommand provides.
+	if err := s.withSnapshotClone(ctx, repo, "lfs", "", func(workDir string) error {
 		s.metrics.recordLFSPhase(ctx, upstream, "clone", "success", time.Since(cloneStart))
 		cloneRecorded = true
 

internal/strategy/git/snapshot_test.go

@@ -318,6 +318,137 @@ func TestSnapshotGenerationViaLocalClone(t *testing.T) {
 	assert.True(t, os.IsNotExist(err))
 }
 
+func TestSnapshotGenerationWithFilter(t *testing.T) {
+	if _, err := exec.LookPath("git"); err != nil {
+		t.Skip("git not found in PATH")
+	}
+
+	_, ctx := logging.Configure(context.Background(), logging.Config{})
+	tmpDir := t.TempDir()
+	mirrorRoot := filepath.Join(tmpDir, "mirrors")
+	upstreamURL := "https://github.com/org/repo"
+
+	mirrorPath := filepath.Join(mirrorRoot, "github.com", "org", "repo")
+	historicalBlob := createTestMirrorRepoWithHistory(t, mirrorPath)
+
+	memCache, err := cache.NewMemory(ctx, cache.MemoryConfig{MaxTTL: time.Hour})
+	assert.NoError(t, err)
+	mux := newTestMux()
+
+	cm := gitclone.NewManagerProvider(ctx, gitclone.Config{MirrorRoot: mirrorRoot}, nil)
+	s, err := git.New(ctx, git.Config{SnapshotFilters: map[string]string{"github.com/org/repo": "blob:none"}}, newTestScheduler(ctx, t), memCache, mux, cm, func() (*githubapp.TokenManager, error) { return nil, nil }) //nolint:nilnil
+	assert.NoError(t, err)
+
+	manager, err := cm()
+	assert.NoError(t, err)
+	repo, err := manager.GetOrCreate(ctx, upstreamURL)
+	assert.NoError(t, err)
+	assert.Equal(t, gitclone.StateReady, repo.State())
+
+	waitForReady(t, s)
+	err = s.GenerateAndUploadSnapshot(ctx, repo)
+	assert.NoError(t, err)
+
+	cacheKey := cache.NewKey(upstreamURL + ".snapshot")
+	restoreDir := filepath.Join(tmpDir, "restored")
+	err = snapshot.Restore(ctx, memCache, cacheKey, restoreDir, 0)
+	assert.NoError(t, err)
+
+	data, err := os.ReadFile(filepath.Join(restoreDir, "data.txt"))
+	assert.NoError(t, err)
+	assert.Equal(t, "current\n", string(data))
+
+	cmd := exec.Command("git", "-C", restoreDir, "config", "--get", "remote.origin.partialclonefilter")
+	output, err := cmd.CombinedOutput()
+	assert.NoError(t, err, string(output))
+	assert.Equal(t, "blob:none\n", string(output))
+
+	cmd = exec.Command("git", "-C", restoreDir, "cat-file", "--batch-all-objects", "--batch-check=%(objectname)")
+	output, err = cmd.CombinedOutput()
+	assert.NoError(t, err, string(output))
+	assert.False(t, strings.Contains(string(output), historicalBlob))
+
+	cmd = exec.Command("git", "-C", restoreDir, "remote", "get-url", "origin")
+	output, err = cmd.CombinedOutput()
+	assert.NoError(t, err, string(output))
+	assert.Equal(t, upstreamURL+"\n", string(output))
+}
+
+func TestSnapshotGenerationWithFilterFailsWhenMirrorIgnoresFilter(t *testing.T) {
+	if _, err := exec.LookPath("git"); err != nil {
+		t.Skip("git not found in PATH")
+	}
+
+	_, ctx := logging.Configure(context.Background(), logging.Config{})
+	tmpDir := t.TempDir()
+	mirrorRoot := filepath.Join(tmpDir, "mirrors")
+	upstreamURL := "https://github.com/org/repo"
+
+	mirrorPath := filepath.Join(mirrorRoot, "github.com", "org", "repo")
+	createTestMirrorRepoWithHistory(t, mirrorPath)
+
+	memCache, err := cache.NewMemory(ctx, cache.MemoryConfig{MaxTTL: time.Hour})
+	assert.NoError(t, err)
+	mux := newTestMux()
+
+	cm := gitclone.NewManagerProvider(ctx, gitclone.Config{MirrorRoot: mirrorRoot}, nil)
+	s, err := git.New(ctx, git.Config{SnapshotFilters: map[string]string{"github.com/org/repo": "blob:none"}}, newTestScheduler(ctx, t), memCache, mux, cm, func() (*githubapp.TokenManager, error) { return nil, nil }) //nolint:nilnil
+	assert.NoError(t, err)
+
+	manager, err := cm()
+	assert.NoError(t, err)
+	repo, err := manager.GetOrCreate(ctx, upstreamURL)
+	assert.NoError(t, err)
+	assert.Equal(t, gitclone.StateReady, repo.State())
+
+	waitForReady(t, s)
+
+	// A mirror missing uploadpack.allowFilter makes git silently fall back to
+	// a full clone; generation must fail rather than cache the full artifact.
+	// Unset only after waitForReady: startup warming reruns configureMirror,
+	// which would re-enable the capability.
+	cmd := exec.Command("git", "-C", mirrorPath, "config", "--unset", "uploadpack.allowfilter")
+	output, err := cmd.CombinedOutput()
+	assert.NoError(t, err, string(output))
+
+	err = s.GenerateAndUploadSnapshot(ctx, repo)
+	assert.Error(t, err)
+	assert.Contains(t, err.Error(), "ignored by the mirror")
+
+	_, err = memCache.Stat(ctx, cache.NewKey(upstreamURL+".snapshot"))
+	assert.IsError(t, err, os.ErrNotExist)
+}
+
+// createTestMirrorRepoWithHistory creates a mirror whose history contains a
+// superseded version of data.txt, returning that historical blob's SHA.
+func createTestMirrorRepoWithHistory(t *testing.T, mirrorPath string) string {
+	t.Helper()
+	tmpWork := t.TempDir()
+
+	run := func(args ...string) string {
+		cmd := exec.Command("git", args...)
+		output, err := cmd.CombinedOutput()
+		assert.NoError(t, err, string(output))
+		return strings.TrimSpace(string(output))
+	}
+
+	run("init", tmpWork)
+	run("-C", tmpWork, "config", "user.email", "test@test.com")
+	run("-C", tmpWork, "config", "user.name", "Test")
+	assert.NoError(t, os.WriteFile(filepath.Join(tmpWork, "data.txt"), []byte("historical\n"), 0o644))
+	run("-C", tmpWork, "add", ".")
+	run("-C", tmpWork, "commit", "-m", "one")
+	historicalBlob := run("-C", tmpWork, "rev-parse", "HEAD:data.txt")
+	assert.NoError(t, os.WriteFile(filepath.Join(tmpWork, "data.txt"), []byte("current\n"), 0o644))
+	run("-C", tmpWork, "commit", "-am", "two")
+	run("clone", "--mirror", tmpWork, mirrorPath)
+	// configureMirror sets this on managed mirrors; a premade test mirror lacks
+	// it, and without it git silently ignores --filter.
+	run("-C", mirrorPath, "config", "uploadpack.allowfilter", "true")
+	run("-C", mirrorPath, "cat-file", "-e", historicalBlob)
+	return historicalBlob
+}
+
 func TestSnapshotGenerationIncludesTrackedLockFiles(t *testing.T) {
 	if _, err := exec.LookPath("git"); err != nil {
 		t.Skip("git not found in PATH")