package gitrepo import ( "container/list" "errors" "log" "os" "path/filepath" "sync" "github.com/go-git/go-billy/v5/osfs" "github.com/go-git/go-git/v5" "github.com/go-git/go-git/v5/plumbing/cache" "github.com/go-git/go-git/v5/plumbing/storer" "github.com/go-git/go-git/v5/storage/filesystem" "gitweb/internal/store" ) const ( // defaultCacheBudget bounds the total bytes of materialized repos kept on // disk; least-recently-used repos are evicted past this. defaultCacheBudget = 512 << 20 // 512 MiB // objectCacheSize is the single shared go-git decompressed-object LRU, // replacing the 96 MiB-per-repo default. Objects are keyed by content // hash, so one cache is safely shared across all repos and pushes. objectCacheSize = 32 * cache.MiByte ) // CacheOptions configures a Cache. type CacheOptions struct { // Dir is the cache directory. Required. ALL of its children are deleted // when the cache is created (crash leftovers) and it is managed // exclusively by the cache thereafter. Dir string // BudgetBytes is the target total size of cached repos on disk; 0 means // the 512 MiB default. It is a soft target: entries still referenced by // in-flight requests survive eviction until released. BudgetBytes int64 // MaxRepoBytes caps a single repo's materialized size; 0 means the // 256 MiB default. MaxRepoBytes int64 } // Cache materializes upspin-stored repos into per-entry directories under a // cache root and caches the opened repositories, keyed by upspin repo path. // An entry is reused while the repo's store Sequence is unchanged; any write // to the repo bumps that sequence and forces re-materialization on the next // Get. Total size is kept near BudgetBytes by LRU eviction. type Cache struct { st store.Store dir string budget int64 maxRepo int64 objects cache.Object // shared across all storers (content-hash keyed) mu sync.Mutex m map[string]*entry // repoPath -> live entry lru *list.List // front = most recently used; Value = *entry total int64 // Σ size over live entries } // entry is a materialized, opened repo. Lifecycle: LIVE (in m/lru, size // counted in total) → RETIRED (dropped from m/lru on staleness, eviction, or // Invalidate; dir still on disk for in-flight holders) → GONE (dir removed, // once retired and the last reference is released). type entry struct { key string repo *git.Repository seq int64 // store Sequence at materialization time dir string // per-entry directory under Cache.dir size int64 // Σ manifest sizes refs int // in-flight holders (Get callers who have not released) retired bool elem *list.Element // nil once retired } // NewCache returns a repo cache backed by st, materializing into opts.Dir. // Any existing children of opts.Dir are deleted. func NewCache(st store.Store, opts CacheOptions) (*Cache, error) { if opts.Dir == "" { return nil, errors.New("gitrepo: CacheOptions.Dir is required") } if err := os.MkdirAll(opts.Dir, 0o700); err != nil { return nil, err } // Sweep leftovers from a previous run. Children only — the root may be a // mountpoint. children, err := os.ReadDir(opts.Dir) if err != nil { return nil, err } for _, ch := range children { if err := os.RemoveAll(filepath.Join(opts.Dir, ch.Name())); err != nil { return nil, err } } budget := opts.BudgetBytes if budget == 0 { budget = defaultCacheBudget } maxRepo := opts.MaxRepoBytes if maxRepo == 0 { maxRepo = defaultMaxRepoBytes } return &Cache{ st: st, dir: opts.Dir, budget: budget, maxRepo: maxRepo, objects: cache.NewObjectLRU(objectCacheSize), m: make(map[string]*entry), lru: list.New(), }, nil } // Get returns an opened repository for repoPath, materializing it if absent or // stale, together with a release func the caller must invoke once it is done // with the repository (defer it). The repo's on-disk backing survives eviction // and invalidation until every holder has released. The returned repository is // read-only and safe for concurrent use across requests. release is idempotent // and nil when err is non-nil. func (c *Cache) Get(repoPath string) (*git.Repository, func(), error) { seq, err := c.st.Seq(repoPath) if err != nil { return nil, nil, err } c.mu.Lock() if e, ok := c.m[repoPath]; ok { if e.seq == seq { e.refs++ c.lru.MoveToFront(e.elem) c.mu.Unlock() return e.repo, c.releaseFunc(e), nil } // Stale: retire it and re-materialize below. dir := c.retire(e) c.mu.Unlock() removeDir(dir) } else { c.mu.Unlock() } // Materialize outside the lock (network/IO heavy). ne, err := c.open(repoPath, seq) if err != nil { return nil, nil, err } c.mu.Lock() // Another request may have populated it meanwhile; prefer the existing // entry if it is current. Ours was never published and has no holders, so // its directory can be deleted outright. var dirs []string if e, ok := c.m[repoPath]; ok { if e.seq == seq { e.refs++ c.lru.MoveToFront(e.elem) c.mu.Unlock() removeDir(ne.dir) return e.repo, c.releaseFunc(e), nil } if d := c.retire(e); d != "" { dirs = append(dirs, d) } } ne.refs = 1 ne.elem = c.lru.PushFront(ne) c.m[repoPath] = ne c.total += ne.size dirs = append(dirs, c.evict()...) c.mu.Unlock() for _, d := range dirs { removeDir(d) } return ne.repo, c.releaseFunc(ne), nil } // GetStorer returns the go-git storer for repoPath plus a release func, with // the same lifecycle as Get. The git smart-HTTP paths need the storer (not the // higher-level *git.Repository); upload-pack in particular streams from it for // the whole response, so release must not be called before the response body // is fully written. The returned storer is read-only in practice; push uses a // separate private storer (see push.go). func (c *Cache) GetStorer(repoPath string) (storer.Storer, func(), error) { repo, release, err := c.Get(repoPath) if err != nil { return nil, nil, err } return repo.Storer, release, nil } // Invalidate retires any cached entry for repoPath, forcing the next Get to // re-materialize. Called after a successful push so readers see the new refs. // The entry's directory survives until in-flight holders release. func (c *Cache) Invalidate(repoPath string) { c.mu.Lock() var dir string if e, ok := c.m[repoPath]; ok { dir = c.retire(e) } c.mu.Unlock() removeDir(dir) } // open materializes repoPath into a fresh directory under c.dir and opens it // (no locking). On error the partial directory is removed. func (c *Cache) open(repoPath string, seq int64) (e *entry, err error) { dir, err := os.MkdirTemp(c.dir, "repo-") if err != nil { return nil, err } defer func() { if err != nil { removeDir(dir) } }() fs := osfs.New(dir) man, err := materialize(c.st, repoPath, fs, c.maxRepo) if err != nil { return nil, err } sto := filesystem.NewStorage(fs, c.objects) repo, err := git.Open(sto, nil) // nil worktree => bare if err != nil { return nil, err } var size int64 for _, fi := range man { size += fi.Size } return &entry{key: repoPath, repo: repo, seq: seq, dir: dir, size: size}, nil } // releaseFunc returns the idempotent release closure handed to Get callers. func (c *Cache) releaseFunc(e *entry) func() { var once sync.Once return func() { once.Do(func() { c.mu.Lock() e.refs-- var dir string if e.retired && e.refs == 0 { dir = e.dir } c.mu.Unlock() removeDir(dir) }) } } // retire drops e from the map and LRU and uncounts its size (caller holds mu). // It returns e's directory if no holder remains — the caller must delete it // after unlocking — or "" if deletion is deferred to the last release. func (c *Cache) retire(e *entry) string { if e.retired { return "" } e.retired = true c.lru.Remove(e.elem) e.elem = nil delete(c.m, e.key) c.total -= e.size if e.refs == 0 { return e.dir } return "" } // evict retires least-recently-used entries until the live total fits the // budget (caller holds mu), returning directories the caller must delete after // unlocking. At least one entry is always kept, so a single repo larger than // the whole budget remains servable — the budget is a target, not a hard cap. func (c *Cache) evict() (dirs []string) { for c.total > c.budget && c.lru.Len() > 1 { e := c.lru.Back().Value.(*entry) if d := c.retire(e); d != "" { dirs = append(dirs, d) } } return dirs } // removeDir deletes a retired entry's directory. A failure leaks disk space // until the next startup sweep, so it is logged rather than ignored. func removeDir(dir string) { if dir == "" { return } if err := os.RemoveAll(dir); err != nil { log.Printf("gitrepo: removing cached repo dir: %v", err) } }