git@cosmocr.at/gitweb.git · summary refs
Materialize repos onto disk with a refcounted byte-budget cache
commit 44d91419df1a0bc40b09f76a8b643fcfe5653188
Patrick Gallagher <rpg137@gmail.com> · 2026-07-06 22:10 -0400
parent 2e0a533d
The whole design held every materialized repo in RAM: up to 8 repos ×
256 MiB in the read cache, a 96 MiB go-git object LRU per repo, plus a
full private copy per push — a ~3 GiB worst case on a 1 GB host, where
the first ~150 MiB repo would OOM-kill the service during its own
materialization.

Materialization now writes to an osfs directory under a cache root
(-cache_dir, wiped at startup), so repo bytes live in the kernel page
cache — reclaimable under pressure — instead of the Go heap.

The cache becomes a byte-budget LRU (-cache_budget_mb, default 512)
with refcounted entries: Get/GetStorer return a release func, and an
evicted or invalidated entry's directory is deleted only after the last
holder releases. upload-pack streams from the storer for the entire
HTTP response, so its handler holds the reference until the response
body is fully written. A single entry may exceed the budget (it is a
target, not a hard cap) so one large repo stays servable.

Push materializes its private copy into a push-* dir under the same
root (always removed, success or error) and is bounded by a global
2-slot semaphore on top of the per-repo locks. stableStatFS is deleted:
it existed only because memfs Stat returns time.Now(), which trapped
go-git's packed-refs mtime-retry loop; real files have real mtimes.
walkManifest also skips dotgit ._packed-refs temp files so a leaked
scratch file can never be written back to the store.

One shared 32 MiB object cache (content-hash keyed, mutex-guarded)
replaces the per-storer 96 MiB default across reads and pushes.
internal/gitrepo/cache.go
package gitrepo
import (
"container/list"
+ "errors"
+ "log"
+ "os"
+ "path/filepath"
"sync"
- "github.com/go-git/go-billy/v5"
+ "github.com/go-git/go-billy/v5/osfs"
"github.com/go-git/go-git/v5"
"github.com/go-git/go-git/v5/plumbing/cache"
"github.com/go-git/go-git/v5/plumbing/storer"
"github.com/go-git/go-git/v5/storage/filesystem"
"gitweb/internal/store"
)
-// defaultMaxRepos bounds how many materialized repos are kept in memory at
-// once; least-recently-used repos are evicted past this.
-const defaultMaxRepos = 8
+const (
+ // defaultCacheBudget bounds the total bytes of materialized repos kept on
+ // disk; least-recently-used repos are evicted past this.
+ defaultCacheBudget = 512 << 20 // 512 MiB
-// Repo is a materialized, opened bare repository.
-type Repo struct {
- Repo *git.Repository
- fs billy.Filesystem
- seq int64 // store Sequence at materialization time
+ // objectCacheSize is the single shared go-git decompressed-object LRU,
+ // replacing the 96 MiB-per-repo default. Objects are keyed by content
+ // hash, so one cache is safely shared across all repos and pushes.
+ objectCacheSize = 32 * cache.MiByte
+)
+
+// CacheOptions configures a Cache.
+type CacheOptions struct {
+ // Dir is the cache directory. Required. ALL of its children are deleted
+ // when the cache is created (crash leftovers) and it is managed
+ // exclusively by the cache thereafter.
+ Dir string
+ // BudgetBytes is the target total size of cached repos on disk; 0 means
+ // the 512 MiB default. It is a soft target: entries still referenced by
+ // in-flight requests survive eviction until released.
+ BudgetBytes int64
+ // MaxRepoBytes caps a single repo's materialized size; 0 means the
+ // 256 MiB default.
+ MaxRepoBytes int64
}
-// Cache materializes and caches opened repositories, keyed by upspin repo path.
-// An entry is reused while the repo's store Sequence is unchanged; any write to
-// the repo bumps that sequence and forces re-materialization on the next Get.
+// Cache materializes upspin-stored repos into per-entry directories under a
+// cache root and caches the opened repositories, keyed by upspin repo path.
+// An entry is reused while the repo's store Sequence is unchanged; any write
+// to the repo bumps that sequence and forces re-materialization on the next
+// Get. Total size is kept near BudgetBytes by LRU eviction.
type Cache struct {
- st store.Store
- maxRepos int
+ st store.Store
+ dir string
+ budget int64
+ maxRepo int64
+ objects cache.Object // shared across all storers (content-hash keyed)
- mu sync.Mutex
- m map[string]*list.Element // repoPath -> element in lru
- lru *list.List // front = most recently used; Value = *entry
+ mu sync.Mutex
+ m map[string]*entry // repoPath -> live entry
+ lru *list.List // front = most recently used; Value = *entry
+ total int64 // Σ size over live entries
}
+// entry is a materialized, opened repo. Lifecycle: LIVE (in m/lru, size
+// counted in total) → RETIRED (dropped from m/lru on staleness, eviction, or
+// Invalidate; dir still on disk for in-flight holders) → GONE (dir removed,
+// once retired and the last reference is released).
type entry struct {
key string
- repo *Repo
+ repo *git.Repository
+ seq int64 // store Sequence at materialization time
+ dir string // per-entry directory under Cache.dir
+ size int64 // Σ manifest sizes
+
+ refs int // in-flight holders (Get callers who have not released)
+ retired bool
+ elem *list.Element // nil once retired
}
-// NewCache returns a repo cache backed by st.
-func NewCache(st store.Store) *Cache {
+// NewCache returns a repo cache backed by st, materializing into opts.Dir.
+// Any existing children of opts.Dir are deleted.
+func NewCache(st store.Store, opts CacheOptions) (*Cache, error) {
+ if opts.Dir == "" {
+ return nil, errors.New("gitrepo: CacheOptions.Dir is required")
+ }
+ if err := os.MkdirAll(opts.Dir, 0o700); err != nil {
+ return nil, err
+ }
+ // Sweep leftovers from a previous run. Children only — the root may be a
+ // mountpoint.
+ children, err := os.ReadDir(opts.Dir)
+ if err != nil {
+ return nil, err
+ }
+ for _, ch := range children {
+ if err := os.RemoveAll(filepath.Join(opts.Dir, ch.Name())); err != nil {
+ return nil, err
+ }
+ }
+ budget := opts.BudgetBytes
+ if budget == 0 {
+ budget = defaultCacheBudget
+ }
+ maxRepo := opts.MaxRepoBytes
+ if maxRepo == 0 {
+ maxRepo = defaultMaxRepoBytes
+ }
return &Cache{
- st: st,
- maxRepos: defaultMaxRepos,
- m: make(map[string]*list.Element),
- lru: list.New(),
- }
+ st: st,
+ dir: opts.Dir,
+ budget: budget,
+ maxRepo: maxRepo,
+ objects: cache.NewObjectLRU(objectCacheSize),
+ m: make(map[string]*entry),
+ lru: list.New(),
+ }, nil
}
// Get returns an opened repository for repoPath, materializing it if absent or
-// stale. The returned *git.Repository is read-only and safe for concurrent use
-// across requests (go-git reads do not mutate the storer).
-func (c *Cache) Get(repoPath string) (*git.Repository, error) {
+// stale, together with a release func the caller must invoke once it is done
+// with the repository (defer it). The repo's on-disk backing survives eviction
+// and invalidation until every holder has released. The returned repository is
+// read-only and safe for concurrent use across requests. release is idempotent
+// and nil when err is non-nil.
+func (c *Cache) Get(repoPath string) (*git.Repository, func(), error) {
seq, err := c.st.Seq(repoPath)
if err != nil {
- return nil, err
+ return nil, nil, err
}
c.mu.Lock()
- if el, ok := c.m[repoPath]; ok {
- if e := el.Value.(*entry); e.repo.seq == seq {
- c.lru.MoveToFront(el)
+ if e, ok := c.m[repoPath]; ok {
+ if e.seq == seq {
+ e.refs++
+ c.lru.MoveToFront(e.elem)
c.mu.Unlock()
- return e.repo.Repo, nil
+ return e.repo, c.releaseFunc(e), nil
}
- // Stale: drop it and re-materialize below.
- c.removeElem(el)
+ // Stale: retire it and re-materialize below.
+ dir := c.retire(e)
+ c.mu.Unlock()
+ removeDir(dir)
+ } else {
+ c.mu.Unlock()
}
- c.mu.Unlock()
// Materialize outside the lock (network/IO heavy).
- r, err := open(c.st, repoPath, seq)
+ ne, err := c.open(repoPath, seq)
if err != nil {
- return nil, err
+ return nil, nil, err
}
c.mu.Lock()
- defer c.mu.Unlock()
- // Another request may have populated it meanwhile; prefer the existing one
- // if it is current.
- if el, ok := c.m[repoPath]; ok {
- if e := el.Value.(*entry); e.repo.seq == seq {
- c.lru.MoveToFront(el)
- return e.repo.Repo, nil
+ // Another request may have populated it meanwhile; prefer the existing
+ // entry if it is current. Ours was never published and has no holders, so
+ // its directory can be deleted outright.
+ var dirs []string
+ if e, ok := c.m[repoPath]; ok {
+ if e.seq == seq {
+ e.refs++
+ c.lru.MoveToFront(e.elem)
+ c.mu.Unlock()
+ removeDir(ne.dir)
+ return e.repo, c.releaseFunc(e), nil
+ }
+ if d := c.retire(e); d != "" {
+ dirs = append(dirs, d)
}
- c.removeElem(el)
}
- el := c.lru.PushFront(&entry{key: repoPath, repo: r})
- c.m[repoPath] = el
- c.evict()
- return r.Repo, nil
+ ne.refs = 1
+ ne.elem = c.lru.PushFront(ne)
+ c.m[repoPath] = ne
+ c.total += ne.size
+ dirs = append(dirs, c.evict()...)
+ c.mu.Unlock()
+ for _, d := range dirs {
+ removeDir(d)
+ }
+ return ne.repo, c.releaseFunc(ne), nil
}
-// GetStorer returns the go-git storer for repoPath, materializing/caching the
-// repo as Get does. The git smart-HTTP upload-pack path needs the storer (not
-// the higher-level *git.Repository). The returned storer is read-only in
-// practice; push uses a separate private storer (see push.go).
-func (c *Cache) GetStorer(repoPath string) (storer.Storer, error) {
- repo, err := c.Get(repoPath)
+// GetStorer returns the go-git storer for repoPath plus a release func, with
+// the same lifecycle as Get. The git smart-HTTP paths need the storer (not the
+// higher-level *git.Repository); upload-pack in particular streams from it for
+// the whole response, so release must not be called before the response body
+// is fully written. The returned storer is read-only in practice; push uses a
+// separate private storer (see push.go).
+func (c *Cache) GetStorer(repoPath string) (storer.Storer, func(), error) {
+ repo, release, err := c.Get(repoPath)
if err != nil {
- return nil, err
+ return nil, nil, err
}
- return repo.Storer, nil
+ return repo.Storer, release, nil
}
-// Invalidate drops any cached entry for repoPath, forcing the next Get to
+// Invalidate retires any cached entry for repoPath, forcing the next Get to
// re-materialize. Called after a successful push so readers see the new refs.
+// The entry's directory survives until in-flight holders release.
func (c *Cache) Invalidate(repoPath string) {
c.mu.Lock()
- defer c.mu.Unlock()
- if el, ok := c.m[repoPath]; ok {
- c.removeElem(el)
+ var dir string
+ if e, ok := c.m[repoPath]; ok {
+ dir = c.retire(e)
}
+ c.mu.Unlock()
+ removeDir(dir)
}
-// open materializes and opens a repo (no locking).
-func open(st store.Store, repoPath string, seq int64) (*Repo, error) {
- fs, _, err := materialize(st, repoPath)
+// open materializes repoPath into a fresh directory under c.dir and opens it
+// (no locking). On error the partial directory is removed.
+func (c *Cache) open(repoPath string, seq int64) (e *entry, err error) {
+ dir, err := os.MkdirTemp(c.dir, "repo-")
if err != nil {
return nil, err
}
- storer := filesystem.NewStorage(fs, cache.NewObjectLRUDefault())
- repo, err := git.Open(storer, nil) // nil worktree => bare
+ defer func() {
+ if err != nil {
+ removeDir(dir)
+ }
+ }()
+ fs := osfs.New(dir)
+ man, err := materialize(c.st, repoPath, fs, c.maxRepo)
if err != nil {
return nil, err
}
- return &Repo{Repo: repo, fs: fs, seq: seq}, nil
+ sto := filesystem.NewStorage(fs, c.objects)
+ repo, err := git.Open(sto, nil) // nil worktree => bare
+ if err != nil {
+ return nil, err
+ }
+ var size int64
+ for _, fi := range man {
+ size += fi.Size
+ }
+ return &entry{key: repoPath, repo: repo, seq: seq, dir: dir, size: size}, nil
+}
+
+// releaseFunc returns the idempotent release closure handed to Get callers.
+func (c *Cache) releaseFunc(e *entry) func() {
+ var once sync.Once
+ return func() {
+ once.Do(func() {
+ c.mu.Lock()
+ e.refs--
+ var dir string
+ if e.retired && e.refs == 0 {
+ dir = e.dir
+ }
+ c.mu.Unlock()
+ removeDir(dir)
+ })
+ }
+}
+
+// retire drops e from the map and LRU and uncounts its size (caller holds mu).
+// It returns e's directory if no holder remains — the caller must delete it
+// after unlocking — or "" if deletion is deferred to the last release.
+func (c *Cache) retire(e *entry) string {
+ if e.retired {
+ return ""
+ }
+ e.retired = true
+ c.lru.Remove(e.elem)
+ e.elem = nil
+ delete(c.m, e.key)
+ c.total -= e.size
+ if e.refs == 0 {
+ return e.dir
+ }
+ return ""
}
-// evict drops least-recently-used entries past maxRepos (caller holds mu).
-func (c *Cache) evict() {
- for c.lru.Len() > c.maxRepos {
- c.removeElem(c.lru.Back())
+// evict retires least-recently-used entries until the live total fits the
+// budget (caller holds mu), returning directories the caller must delete after
+// unlocking. At least one entry is always kept, so a single repo larger than
+// the whole budget remains servable — the budget is a target, not a hard cap.
+func (c *Cache) evict() (dirs []string) {
+ for c.total > c.budget && c.lru.Len() > 1 {
+ e := c.lru.Back().Value.(*entry)
+ if d := c.retire(e); d != "" {
+ dirs = append(dirs, d)
+ }
}
+ return dirs
}
-func (c *Cache) removeElem(el *list.Element) {
- if el == nil {
+// removeDir deletes a retired entry's directory. A failure leaks disk space
+// until the next startup sweep, so it is logged rather than ignored.
+func removeDir(dir string) {
+ if dir == "" {
return
}
- c.lru.Remove(el)
- delete(c.m, el.Value.(*entry).key)
+ if err := os.RemoveAll(dir); err != nil {
+ log.Printf("gitrepo: removing cached repo dir: %v", err)
+ }
}
added internal/gitrepo/cache_test.go
+package gitrepo
+
+import (
+ "os"
+ "path/filepath"
+ "sync"
+ "testing"
+ "time"
+
+ "gitweb/internal/store"
+)
+
+// dirCount returns the number of children under the cache root.
+func dirCount(t *testing.T, root string) int {
+ t.Helper()
+ children, err := os.ReadDir(root)
+ if err != nil {
+ t.Fatal(err)
+ }
+ return len(children)
+}
+
+// twoRepos builds two distinct bare repos in one LocalStore and returns the
+// store and their repo paths.
+func twoRepos(t *testing.T) (st store.Store, pathA, pathB string) {
+ t.Helper()
+ root := t.TempDir()
+ pathA = makeBareRepo(t, root, "u@d.com", "a.git")
+ pathB = makeBareRepo(t, root, "u@d.com", "b.git")
+ return store.NewLocalStore(root), pathA, pathB
+}
+
+// repoBytes returns the materialized size of repoPath (sum of manifest sizes).
+func repoBytes(t *testing.T, st store.Store, repoPath string) int64 {
+ t.Helper()
+ c := newTestCache(t, st, CacheOptions{})
+ _, release, err := c.Get(repoPath)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer release()
+ c.mu.Lock()
+ defer c.mu.Unlock()
+ return c.total
+}
+
+func TestCacheByteEviction(t *testing.T) {
+ st, pathA, pathB := twoRepos(t)
+ sizeA := repoBytes(t, st, pathA)
+
+ // Budget fits one repo but not two: getting B must evict A.
+ dir := t.TempDir()
+ c := newTestCache(t, st, CacheOptions{Dir: dir, BudgetBytes: sizeA + sizeA/2})
+
+ rA, releaseA, err := c.Get(pathA)
+ if err != nil {
+ t.Fatal(err)
+ }
+ releaseA()
+ if got := dirCount(t, dir); got != 1 {
+ t.Fatalf("after Get A: %d dirs, want 1", got)
+ }
+
+ _, releaseB, err := c.Get(pathB)
+ if err != nil {
+ t.Fatal(err)
+ }
+ releaseB()
+ if got := dirCount(t, dir); got != 1 {
+ t.Fatalf("after Get B evicted A: %d dirs, want 1", got)
+ }
+
+ // A re-materializes on demand as a fresh entry.
+ rA2, releaseA2, err := c.Get(pathA)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer releaseA2()
+ if rA2 == rA {
+ t.Fatal("expected evicted repo to re-materialize (different pointer)")
+ }
+}
+
+func TestCacheHoldAcrossEviction(t *testing.T) {
+ st, pathA, pathB := twoRepos(t)
+ sizeA := repoBytes(t, st, pathA)
+
+ dir := t.TempDir()
+ c := newTestCache(t, st, CacheOptions{Dir: dir, BudgetBytes: sizeA + sizeA/2})
+
+ rA, releaseA, err := c.Get(pathA)
+ if err != nil {
+ t.Fatal(err)
+ }
+ // A is still held when B evicts it: its dir must survive.
+ _, releaseB, err := c.Get(pathB)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer releaseB()
+ if got := dirCount(t, dir); got != 2 {
+ t.Fatalf("evicted-but-held entry deleted early: %d dirs, want 2", got)
+ }
+ // The held repo still reads from its (retired) backing dir.
+ if _, err := rA.Head(); err != nil {
+ t.Fatalf("held repo unreadable after eviction: %v", err)
+ }
+ releaseA()
+ if got := dirCount(t, dir); got != 1 {
+ t.Fatalf("after release: %d dirs, want 1", got)
+ }
+}
+
+func TestCacheInvalidateDuringHold(t *testing.T) {
+ root := t.TempDir()
+ repoPath := makeBareRepo(t, root, "u@d.com", "demo.git")
+ st := store.NewLocalStore(root)
+
+ dir := t.TempDir()
+ c := newTestCache(t, st, CacheOptions{Dir: dir})
+
+ r1, release1, err := c.Get(repoPath)
+ if err != nil {
+ t.Fatal(err)
+ }
+ c.Invalidate(repoPath)
+ if got := dirCount(t, dir); got != 1 {
+ t.Fatalf("invalidated-but-held entry deleted early: %d dirs, want 1", got)
+ }
+ if _, err := r1.Head(); err != nil {
+ t.Fatalf("held repo unreadable after Invalidate: %v", err)
+ }
+
+ // A new Get re-materializes into a fresh dir.
+ r2, release2, err := c.Get(repoPath)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer release2()
+ if r2 == r1 {
+ t.Fatal("expected fresh repo after Invalidate")
+ }
+ if got := dirCount(t, dir); got != 2 {
+ t.Fatalf("old + new dirs: %d, want 2", got)
+ }
+
+ release1()
+ if got := dirCount(t, dir); got != 1 {
+ t.Fatalf("old dir survives its last release: %d dirs, want 1", got)
+ }
+}
+
+func TestCacheConcurrentGet(t *testing.T) {
+ root := t.TempDir()
+ repoPath := makeBareRepo(t, root, "u@d.com", "demo.git")
+ st := store.NewLocalStore(root)
+
+ dir := t.TempDir()
+ c := newTestCache(t, st, CacheOptions{Dir: dir})
+
+ const n = 8
+ var wg sync.WaitGroup
+ errs := make(chan error, n)
+ for range n {
+ wg.Add(1)
+ go func() {
+ defer wg.Done()
+ repo, release, err := c.Get(repoPath)
+ if err != nil {
+ errs <- err
+ return
+ }
+ defer release()
+ if _, err := repo.Head(); err != nil {
+ errs <- err
+ }
+ }()
+ }
+ wg.Wait()
+ close(errs)
+ for err := range errs {
+ t.Fatal(err)
+ }
+
+ // Race losers cleaned up after themselves: one dir remains.
+ if got := dirCount(t, dir); got != 1 {
+ t.Fatalf("%d dirs after concurrent Gets, want 1", got)
+ }
+ // And it is a usable cache hit.
+ _, release, err := c.Get(repoPath)
+ if err != nil {
+ t.Fatal(err)
+ }
+ release()
+}
+
+func TestCacheStaleReleaseDeletesOldDir(t *testing.T) {
+ root := t.TempDir()
+ repoPath := makeBareRepo(t, root, "u@d.com", "demo.git")
+ st := store.NewLocalStore(root)
+
+ dir := t.TempDir()
+ c := newTestCache(t, st, CacheOptions{Dir: dir})
+
+ _, release1, err := c.Get(repoPath)
+ if err != nil {
+ t.Fatal(err)
+ }
+ release1()
+
+ // Bump the repo's mtime so LocalStore.Seq changes; the stale entry (now
+ // unreferenced) is retired and deleted during the next Get.
+ later := time.Now().Add(2 * time.Second)
+ if err := os.Chtimes(filepath.Join(root, repoPath, "HEAD"), later, later); err != nil {
+ t.Fatal(err)
+ }
+ _, release2, err := c.Get(repoPath)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer release2()
+ if got := dirCount(t, dir); got != 1 {
+ t.Fatalf("stale dir not deleted: %d dirs, want 1", got)
+ }
+}
+
+func TestCacheSingleRepoOverBudget(t *testing.T) {
+ root := t.TempDir()
+ repoPath := makeBareRepo(t, root, "u@d.com", "demo.git")
+ st := store.NewLocalStore(root)
+
+ // Budget of 1 byte: the sole entry must still be kept and servable.
+ dir := t.TempDir()
+ c := newTestCache(t, st, CacheOptions{Dir: dir, BudgetBytes: 1})
+
+ r1, release1, err := c.Get(repoPath)
+ if err != nil {
+ t.Fatal(err)
+ }
+ release1()
+ if got := dirCount(t, dir); got != 1 {
+ t.Fatalf("sole over-budget entry evicted: %d dirs, want 1", got)
+ }
+ r2, release2, err := c.Get(repoPath)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer release2()
+ if r1 != r2 {
+ t.Fatal("expected the over-budget entry to be reused")
+ }
+}
internal/gitrepo/materialize.go
// Package gitrepo bridges upspin-stored bare git repositories to go-git. A bare
-// repo (HEAD, config, objects/, refs/) is copied out of upspin into an
-// in-memory billy filesystem and opened with go-git — for read-only browsing
-// and clone, and (via a private copy) for push write-back.
+// repo (HEAD, config, objects/, refs/) is copied out of upspin into a billy
+// filesystem and opened with go-git — for read-only browsing and clone, and
+// (via a private copy) for push write-back.
package gitrepo
import (
"crypto/sha256"
+ "errors"
"fmt"
- "io"
"path"
"strings"
"github.com/go-git/go-billy/v5"
- "github.com/go-git/go-billy/v5/memfs"
"gitweb/internal/store"
)
// FileInfo records a materialized file's size and content hash, used by the
// push write-back to detect which files changed.
type FileInfo struct {
Size int64
Sum [32]byte
}
// Manifest maps a repo-relative file path (as stored in the billy filesystem,
// using forward slashes and no leading slash) to its FileInfo.
type Manifest map[string]FileInfo
-// maxRepoBytes caps the total size materialized into memory. The whole bare
-// repo (objects, packfiles) is held in RAM, so this bounds a single request's
-// footprint; very large repos are rejected rather than OOMing the process.
-const maxRepoBytes = 256 << 20 // 256 MiB
+// defaultMaxRepoBytes caps the total size materialized per repo. It bounds a
+// single request's footprint; very large repos are rejected rather than
+// exhausting the cache filesystem.
+const defaultMaxRepoBytes = 256 << 20 // 256 MiB
-// errRepoTooLarge is returned by materialize when a repo exceeds maxRepoBytes.
-var errRepoTooLarge = fmt.Errorf("repository too large to browse (limit %d bytes)", maxRepoBytes)
+// ErrRepoTooLarge is returned (wrapped) by materialize when a repo exceeds the
+// materialization size cap.
+var ErrRepoTooLarge = errors.New("repository too large")
// materialize copies the bare-repo subtree rooted at repoPath out of the store
-// into a fresh in-memory billy filesystem, with the repo's internal files
-// (HEAD, config, objects/, refs/) at the filesystem root — the layout go-git's
-// filesystem storage expects. It mirrors the recursive Glob/recurse/copy walk
-// of upspin's `upspin tar` command (cmd/upspin/tar.go doArchive), but goes
-// through the store.Store interface so it also works against LocalStore in dev.
-func materialize(st store.Store, repoPath string) (billy.Filesystem, Manifest, error) {
- fs := memfs.New()
+// into fs, with the repo's internal files (HEAD, config, objects/, refs/) at
+// the filesystem root — the layout go-git's filesystem storage expects. fs
+// must be empty; total copied bytes are capped at maxBytes. It mirrors the
+// recursive Glob/recurse/copy walk of upspin's `upspin tar` command
+// (cmd/upspin/tar.go doArchive), but goes through the store.Store interface so
+// it also works against LocalStore in dev.
+func materialize(st store.Store, repoPath string, fs billy.Filesystem, maxBytes int64) (Manifest, error) {
man := Manifest{}
var total int64
- if err := copyTree(st, strings.TrimRight(repoPath, "/"), "", fs, man, &total); err != nil {
- return nil, nil, err
+ if err := copyTree(st, strings.TrimRight(repoPath, "/"), "", fs, man, &total, maxBytes); err != nil {
+ return nil, err
}
// Sanity-check the result really is a bare repo at the root.
if _, err := fs.Stat("HEAD"); err != nil {
- return nil, nil, fmt.Errorf("materialize %s: no HEAD at repo root: %w", repoPath, err)
+ return nil, fmt.Errorf("materialize %s: no HEAD at repo root: %w", repoPath, err)
}
if fi, err := fs.Stat("objects"); err != nil || !fi.IsDir() {
- return nil, nil, fmt.Errorf("materialize %s: no objects/ dir at repo root", repoPath)
+ return nil, fmt.Errorf("materialize %s: no objects/ dir at repo root", repoPath)
}
- return fs, man, nil
+ return man, nil
}
// copyTree recursively copies upspin path `src` into billy path `dst`, recording
// each copied file into man.
-func copyTree(st store.Store, src, dst string, fs billy.Filesystem, man Manifest, total *int64) error {
+func copyTree(st store.Store, src, dst string, fs billy.Filesystem, man Manifest, total *int64, maxBytes int64) error {
entries, err := st.List(src)
if err != nil {
return err
}
for _, e := range entries {
srcChild := src + "/" + e.Name
dstChild := e.Name
if dst != "" {
dstChild = path.Join(dst, e.Name)
}
if e.IsDir {
if err := fs.MkdirAll(dstChild, 0o755); err != nil {
return err
}
- if err := copyTree(st, srcChild, dstChild, fs, man, total); err != nil {
+ if err := copyTree(st, srcChild, dstChild, fs, man, total, maxBytes); err != nil {
return err
}
continue
}
data, err := st.Get(srcChild)
if err != nil {
return err
}
*total += int64(len(data))
- if *total > maxRepoBytes {
- return errRepoTooLarge
+ if *total > maxBytes {
+ return fmt.Errorf("%w (limit %d bytes)", ErrRepoTooLarge, maxBytes)
}
f, err := fs.Create(dstChild)
if err != nil {
return err
}
- if _, err := io.Copy(f, strings.NewReader(string(data))); err != nil {
+ if _, err := f.Write(data); err != nil {
f.Close()
return err
}
if err := f.Close(); err != nil {
return err
}
man[dstChild] = FileInfo{Size: int64(len(data)), Sum: sha256.Sum256(data)}
}
return nil
}
internal/gitrepo/materialize_test.go
package gitrepo
import (
+ "errors"
"os"
"os/exec"
"path/filepath"
"testing"
"time"
+ "github.com/go-git/go-billy/v5/osfs"
"github.com/go-git/go-git/v5"
"github.com/go-git/go-git/v5/plumbing/object"
"gitweb/internal/store"
)
// makeBareRepo builds a real bare repo at <root>/<user>/<name> via the git CLI
// and returns the store-relative repo path "<user>/<name>". Skips the test if
// git is unavailable.
func makeBareRepo(t *testing.T, root, user, name string) string {
t.Helper()
if _, err := exec.LookPath("git"); err != nil {
t.Skip("git not available")
}
work := t.TempDir()
run := func(dir string, args ...string) {
t.Helper()
cmd := exec.Command("git", args...)
cmd.Dir = dir
cmd.Env = append(os.Environ(),
"GIT_AUTHOR_NAME=Test", "GIT_AUTHOR_EMAIL=test@example.com",
"GIT_COMMITTER_NAME=Test", "GIT_COMMITTER_EMAIL=test@example.com",
)
if out, err := cmd.CombinedOutput(); err != nil {
t.Fatalf("git %v: %v\n%s", args, err, out)
}
}
run(work, "init", "-b", "main")
if err := os.WriteFile(filepath.Join(work, "README.md"), []byte("# Demo\n\nhi\n"), 0o644); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(work, "main.go"), []byte("package main\n\nfunc main() {}\n"), 0o644); err != nil {
t.Fatal(err)
}
run(work, "add", ".")
run(work, "commit", "-m", "initial commit")
run(work, "checkout", "-b", "feature/x")
if err := os.WriteFile(filepath.Join(work, "main.go"), []byte("package main\n\nfunc main() { _ = 1 }\n"), 0o644); err != nil {
t.Fatal(err)
}
run(work, "commit", "-am", "tweak main")
run(work, "tag", "v1.0.0")
run(work, "checkout", "main")
bare := filepath.Join(root, user, name)
if err := os.MkdirAll(filepath.Dir(bare), 0o755); err != nil {
t.Fatal(err)
}
run(work, "clone", "--bare", work, bare)
return user + "/" + name
}
+// newTestCache builds a Cache over st rooted at a test tempdir.
+func newTestCache(t *testing.T, st store.Store, opts CacheOptions) *Cache {
+ t.Helper()
+ if opts.Dir == "" {
+ opts.Dir = t.TempDir()
+ }
+ c, err := NewCache(st, opts)
+ if err != nil {
+ t.Fatalf("NewCache: %v", err)
+ }
+ return c
+}
+
func TestMaterializeAndOpen(t *testing.T) {
root := t.TempDir()
repoPath := makeBareRepo(t, root, "u@d.com", "demo.git")
st := store.NewLocalStore(root)
- repo, err := open(st, repoPath, 0)
+ c := newTestCache(t, st, CacheOptions{})
+ repo, release, err := c.Get(repoPath)
if err != nil {
- t.Fatalf("open: %v", err)
+ t.Fatalf("Get: %v", err)
}
+ defer release()
- head, err := repo.Repo.Head()
+ head, err := repo.Head()
if err != nil {
t.Fatalf("Head: %v", err)
}
if head.Name().Short() != "main" {
t.Fatalf("head = %q, want main", head.Name().Short())
}
// Log walks.
- iter, err := repo.Repo.Log(&git.LogOptions{From: head.Hash()})
+ iter, err := repo.Log(&git.LogOptions{From: head.Hash()})
if err != nil {
t.Fatalf("Log: %v", err)
}
var n int
err = iter.ForEach(func(*object.Commit) error { n++; return nil })
if err != nil {
t.Fatalf("Log iter: %v", err)
}
if n == 0 {
t.Fatal("expected at least one commit")
}
// Tree walks: README.md present at HEAD.
- commit, err := repo.Repo.CommitObject(head.Hash())
+ commit, err := repo.CommitObject(head.Hash())
if err != nil {
t.Fatalf("CommitObject: %v", err)
}
tree, err := commit.Tree()
if err != nil {
t.Fatalf("Tree: %v", err)
}
if _, err := tree.File("README.md"); err != nil {
t.Fatalf("README.md not in tree: %v", err)
}
// Branch + tag are visible.
- if _, err := repo.Repo.Reference("refs/heads/feature/x", false); err != nil {
+ if _, err := repo.Reference("refs/heads/feature/x", false); err != nil {
t.Fatalf("feature/x branch missing: %v", err)
}
- if _, err := repo.Repo.Tag("v1.0.0"); err != nil {
+ if _, err := repo.Tag("v1.0.0"); err != nil {
t.Fatalf("v1.0.0 tag missing: %v", err)
}
}
+func TestMaterializeToDisk(t *testing.T) {
+ root := t.TempDir()
+ repoPath := makeBareRepo(t, root, "u@d.com", "demo.git")
+ st := store.NewLocalStore(root)
+
+ dir := t.TempDir()
+ man, err := materialize(st, repoPath, osfs.New(dir), defaultMaxRepoBytes)
+ if err != nil {
+ t.Fatalf("materialize: %v", err)
+ }
+ // The bare-repo layout lands on the real filesystem.
+ if _, err := os.Stat(filepath.Join(dir, "HEAD")); err != nil {
+ t.Fatalf("HEAD not on disk: %v", err)
+ }
+ if fi, err := os.Stat(filepath.Join(dir, "objects")); err != nil || !fi.IsDir() {
+ t.Fatalf("objects/ not a dir on disk: %v", err)
+ }
+ // Manifest entries match what is on disk.
+ for name, info := range man {
+ fi, err := os.Stat(filepath.Join(dir, filepath.FromSlash(name)))
+ if err != nil {
+ t.Fatalf("manifest file %s missing on disk: %v", name, err)
+ }
+ if fi.Size() != info.Size {
+ t.Errorf("%s: size %d on disk, %d in manifest", name, fi.Size(), info.Size)
+ }
+ }
+}
+
+func TestMaterializeTooLarge(t *testing.T) {
+ root := t.TempDir()
+ repoPath := makeBareRepo(t, root, "u@d.com", "demo.git")
+ st := store.NewLocalStore(root)
+
+ _, err := materialize(st, repoPath, osfs.New(t.TempDir()), 10)
+ if !errors.Is(err, ErrRepoTooLarge) {
+ t.Fatalf("err = %v, want ErrRepoTooLarge", err)
+ }
+}
+
func TestCacheReusesAndInvalidates(t *testing.T) {
root := t.TempDir()
repoPath := makeBareRepo(t, root, "u@d.com", "demo.git")
st := store.NewLocalStore(root)
- c := NewCache(st)
+ c := newTestCache(t, st, CacheOptions{})
- r1, err := c.Get(repoPath)
+ r1, release1, err := c.Get(repoPath)
if err != nil {
t.Fatalf("Get: %v", err)
}
- r2, err := c.Get(repoPath)
+ defer release1()
+ r2, release2, err := c.Get(repoPath)
if err != nil {
t.Fatalf("Get: %v", err)
}
+ defer release2()
if r1 != r2 {
t.Fatal("expected cached repo to be reused (same pointer)")
}
// Bump the repo's mtime so LocalStore.Seq changes -> re-materialization.
later := time.Now().Add(2 * time.Second)
if err := os.Chtimes(filepath.Join(root, repoPath, "HEAD"), later, later); err != nil {
t.Fatal(err)
}
- r3, err := c.Get(repoPath)
+ r3, release3, err := c.Get(repoPath)
if err != nil {
t.Fatalf("Get: %v", err)
}
+ defer release3()
if r3 == r1 {
t.Fatal("expected re-materialization after edit (different pointer)")
}
}
internal/gitrepo/push.go
package gitrepo
import (
"context"
"crypto/sha256"
"errors"
"fmt"
"os"
"path"
"sort"
"strings"
"sync"
- "time"
"github.com/go-git/go-billy/v5"
+ "github.com/go-git/go-billy/v5/osfs"
"github.com/go-git/go-billy/v5/util"
- "github.com/go-git/go-git/v5/plumbing/cache"
"github.com/go-git/go-git/v5/plumbing/protocol/packp"
"github.com/go-git/go-git/v5/plumbing/storer"
"github.com/go-git/go-git/v5/storage/filesystem"
"gitweb/internal/githttp"
"gitweb/internal/store"
)
// ErrConcurrentModification is returned when a repo's store Sequence changed
// between materialization and write-back, meaning the tree was modified out of
// band during the push. The push is abandoned without writing anything.
var ErrConcurrentModification = errors.New("remote repository changed during push; please retry")
+// defaultPushConcurrency caps pushes in flight across all repos; each one
+// holds a full private copy of its repo on disk while it runs.
+const defaultPushConcurrency = 2
+
// Pusher applies a git receive-pack (push) to an upspin-stored repo. It
// materializes a private, writable copy of the repo, runs the push against it
// with go-git, and writes the changed files back to the store. Pushes to the
-// same repo are serialized.
+// same repo are serialized, and total pushes in flight are bounded.
type Pusher struct {
st store.WriteStore
cache *Cache
+ sem chan struct{} // bounds concurrent pushes
mu sync.Mutex
locks map[string]*sync.Mutex
}
// NewPusher returns a Pusher writing through st and invalidating c on success.
func NewPusher(st store.WriteStore, c *Cache) *Pusher {
- return &Pusher{st: st, cache: c, locks: map[string]*sync.Mutex{}}
+ return &Pusher{
+ st: st,
+ cache: c,
+ sem: make(chan struct{}, defaultPushConcurrency),
+ locks: map[string]*sync.Mutex{},
+ }
}
func (p *Pusher) lockFor(repoPath string) *sync.Mutex {
p.mu.Lock()
defer p.mu.Unlock()
l := p.locks[repoPath]
if l == nil {
l = &sync.Mutex{}
p.locks[repoPath] = l
}
return l
}
-// stableStatFS wraps a billy filesystem so Stat reports a constant ModTime.
-// memfs answers every ModTime call with time.Now(), which traps go-git's
-// packed-refs locking loop — open, stat, lock, stat, retry until the mtimes
-// match to rule out a concurrent change — in an infinite spin whenever it
-// rewrites packed-refs (e.g. deleting a ref). The materialized copy is private
-// to one push, so a constant ModTime loses nothing.
-type stableStatFS struct{ billy.Filesystem }
-
-func (s stableStatFS) Stat(name string) (os.FileInfo, error) {
- fi, err := s.Filesystem.Stat(name)
- if err != nil {
- return nil, err
- }
- return stableInfo{fi}, nil
-}
-
-// Chroot rewraps so the fix survives go-git chrooting into subdirectories.
-func (s stableStatFS) Chroot(path string) (billy.Filesystem, error) {
- fs, err := s.Filesystem.Chroot(path)
- if err != nil {
- return nil, err
- }
- return stableStatFS{fs}, nil
-}
-
-type stableInfo struct{ os.FileInfo }
-
-func (stableInfo) ModTime() time.Time { return time.Time{} }
-
// noPackfileWriter hides the storer's optional PackfileWriter capability from
// the receive-pack session. With it visible, go-git copies the client's
// packfile verbatim into objects/pack, and that path cannot resolve the
// REF_DELTA entries of a thin pack — which git sends on every incremental
// push, deltifying against objects that live in the repo, not in the pack —
// so such pushes fail with "reference delta not found". Hidden, go-git parses
// the pack against the storer, resolving thin deltas and storing the new
// objects loose.
type noPackfileWriter struct{ storer.Storer }
// ReceivePack runs the push described by req against repoPath and writes the
// result back to the store. The returned ReportStatus should be sent to the
// client even when err is non-nil (it carries per-ref status).
func (p *Pusher) ReceivePack(ctx context.Context, repoPath string, req *packp.ReferenceUpdateRequest) (*packp.ReportStatus, error) {
lock := p.lockFor(repoPath)
lock.Lock()
defer lock.Unlock()
+ // Bound pushes in flight (each holds a private on-disk repo copy). Taken
+ // after the per-repo lock: holders always release, so no cycle.
+ select {
+ case p.sem <- struct{}{}:
+ defer func() { <-p.sem }()
+ case <-ctx.Done():
+ return nil, ctx.Err()
+ }
+
seq0, err := p.st.Seq(repoPath)
if err != nil {
return nil, err
}
- // A private copy: never the shared read cache, which serves concurrent
- // readers and must not be mutated.
- fs, before, err := materialize(p.st, repoPath)
+ // A private on-disk copy: never the shared read cache, which serves
+ // concurrent readers and must not be mutated.
+ dir, err := os.MkdirTemp(p.cache.dir, "push-")
+ if err != nil {
+ return nil, err
+ }
+ defer removeDir(dir)
+ fs := osfs.New(dir)
+ before, err := materialize(p.st, repoPath, fs, p.cache.maxRepo)
if err != nil {
return nil, err
}
- sto := filesystem.NewStorage(stableStatFS{fs}, cache.NewObjectLRUDefault())
+ sto := filesystem.NewStorage(fs, p.cache.objects)
sess, err := githttp.NewReceivePackSession(noPackfileWriter{sto})
if err != nil {
return nil, err
}
defer sess.Close()
rs, rpErr := sess.ReceivePack(ctx, req)
if rpErr != nil {
// Unpack or ref update failed; the storer may be in a partial state, so
// do not write anything back. The report status still goes to the client.
return rs, rpErr
}
- after, err := walkManifest(fs)
+ after, err := walkManifest(fs, p.cache.maxRepo)
if err != nil {
return failReport(rs, err), err
}
// Guard against an out-of-band change to the tree since we materialized.
// The per-repo lock already excludes other pushes through this server.
seq1, err := p.st.Seq(repoPath)
if err != nil {
return failReport(rs, err), err
}
if seq1 != seq0 {
return failReport(rs, ErrConcurrentModification), ErrConcurrentModification
}
if err := p.writeBack(fs, before, after, repoPath); err != nil {
return failReport(rs, err), err
}
p.cache.Invalidate(repoPath)
return rs, nil
}
// failReport downgrades every "ok" per-ref status in rs to err's message. The
// session updated only the private copy; when persisting to the store fails,
// the client must see its refs rejected, not an "ok" report for updates that
// were never written. The message is flattened to one line so it survives
// pkt-line framing.
func failReport(rs *packp.ReportStatus, err error) *packp.ReportStatus {
if rs == nil {
return nil
}
msg := strings.Join(strings.Fields(err.Error()), " ")
for _, cs := range rs.CommandStatuses {
if cs.Error() == nil { // "ok"
cs.Status = msg
}
}
return rs
}
// writeBack applies the difference between the before and after manifests to the
// store: new/changed files are written objects-first then refs, and files that
// disappeared are deleted last. This ordering keeps a concurrent reader's view
// consistent — it may briefly see extra unreachable objects, never a ref that
// points at a missing object.
func (p *Pusher) writeBack(fs billy.Filesystem, before, after Manifest, repoPath string) error {
var objectPuts, refPuts, dels []string
for name, info := range after {
if old, ok := before[name]; ok && old.Sum == info.Sum {
continue // unchanged
}
if strings.HasPrefix(name, "objects/") {
objectPuts = append(objectPuts, name)
} else {
refPuts = append(refPuts, name)
}
}
for name := range before {
if _, ok := after[name]; !ok {
dels = append(dels, name)
}
}
// Deterministic order (aids testing and reproducibility).
sort.Strings(objectPuts)
sort.Strings(refPuts)
sort.Strings(dels)
put := func(name string) error {
data, err := util.ReadFile(fs, name)
if err != nil {
return err
}
return p.st.Put(path.Join(repoPath, name), data)
}
for _, name := range objectPuts {
if err := put(name); err != nil {
return fmt.Errorf("push write object %s: %w", name, err)
}
}
for _, name := range refPuts {
if err := put(name); err != nil {
return fmt.Errorf("push write ref %s: %w", name, err)
}
}
for _, name := range dels {
if err := p.st.Delete(path.Join(repoPath, name)); err != nil && err != store.ErrNotFound {
return fmt.Errorf("push delete %s: %w", name, err)
}
}
return nil
}
// walkManifest builds a manifest of every regular file currently in fs,
// enforcing the size cap and skipping go-git's dotgit temp files.
-func walkManifest(fs billy.Filesystem) (Manifest, error) {
+func walkManifest(fs billy.Filesystem, maxBytes int64) (Manifest, error) {
man := Manifest{}
var total int64
err := util.Walk(fs, "/", func(p string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if info.IsDir() {
return nil
}
rel := strings.TrimPrefix(p, "/")
- if strings.HasPrefix(path.Base(rel), "tmp_") {
+ base := path.Base(rel)
+ if strings.HasPrefix(base, "tmp_") || strings.HasPrefix(base, "._packed-refs") {
return nil // go-git dotgit scratch file, not part of the repo
}
total += info.Size()
- if total > maxRepoBytes {
- return errRepoTooLarge
+ if total > maxBytes {
+ return fmt.Errorf("%w (limit %d bytes)", ErrRepoTooLarge, maxBytes)
}
data, err := util.ReadFile(fs, rel)
if err != nil {
return err
}
man[rel] = FileInfo{Size: info.Size(), Sum: sha256.Sum256(data)}
return nil
})
if err != nil {
return nil, err
}
return man, nil
}
internal/gitrepo/push_test.go
package gitrepo
import (
+ "context"
"errors"
"strings"
"testing"
"github.com/go-git/go-billy/v5"
"github.com/go-git/go-billy/v5/memfs"
"github.com/go-git/go-billy/v5/util"
"github.com/go-git/go-git/v5/plumbing/protocol/packp"
"gitweb/internal/store"
)
// recordingStore records Put/Delete calls for assertions. Reads are unused by
// writeBack (it reads file bytes from the billy fs), so they are stubs.
type recordingStore struct {
puts []string
dels []string
}
func (r *recordingStore) Stat(string) (bool, error) { return false, store.ErrNotFound }
func (r *recordingStore) Get(string) ([]byte, error) { return nil, store.ErrNotFound }
func (r *recordingStore) List(string) ([]store.Entry, error) { return nil, store.ErrNotFound }
func (r *recordingStore) Seq(string) (int64, error) { return 0, nil }
func (r *recordingStore) Put(p string, _ []byte) error { r.puts = append(r.puts, p); return nil }
func (r *recordingStore) Delete(p string) error { r.dels = append(r.dels, p); return nil }
func writeFile(t *testing.T, fs billy.Filesystem, name, content string) {
t.Helper()
if err := util.WriteFile(fs, name, []byte(content), 0o644); err != nil {
t.Fatal(err)
}
}
func TestWriteBackDiffAndOrdering(t *testing.T) {
fs := memfs.New()
// Initial state.
writeFile(t, fs, "HEAD", "ref: refs/heads/main\n")
writeFile(t, fs, "objects/pack/pack-old.pack", "OLD")
writeFile(t, fs, "refs/heads/main", "aaaa\n")
writeFile(t, fs, "refs/heads/stale", "bbbb\n")
- before, err := walkManifest(fs)
+ before, err := walkManifest(fs, defaultMaxRepoBytes)
if err != nil {
t.Fatal(err)
}
// Simulate a push: new packfile, updated main ref, deleted stale ref.
writeFile(t, fs, "objects/pack/pack-new.pack", "NEW")
writeFile(t, fs, "refs/heads/main", "cccc\n")
if err := fs.Remove("refs/heads/stale"); err != nil {
t.Fatal(err)
}
- after, err := walkManifest(fs)
+ after, err := walkManifest(fs, defaultMaxRepoBytes)
if err != nil {
t.Fatal(err)
}
rec := &recordingStore{}
- p := NewPusher(rec, NewCache(rec))
+ p := NewPusher(rec, newTestCache(t, rec, CacheOptions{}))
if err := p.writeBack(fs, before, after, "u@d.com/r.git"); err != nil {
t.Fatalf("writeBack: %v", err)
}
// Expected: the new packfile and the changed ref are Put; the removed ref is
// Deleted. HEAD and the old packfile are unchanged and must not be rewritten.
wantPuts := map[string]bool{
"u@d.com/r.git/objects/pack/pack-new.pack": true,
"u@d.com/r.git/refs/heads/main": true,
}
if len(rec.puts) != len(wantPuts) {
t.Fatalf("puts = %v, want keys %v", rec.puts, wantPuts)
}
for _, p := range rec.puts {
if !wantPuts[p] {
t.Errorf("unexpected put %q", p)
}
}
// Object writes must precede ref writes.
if !strings.Contains(rec.puts[0], "/objects/") {
t.Errorf("expected object put first, got order %v", rec.puts)
}
// The stale ref must be deleted.
if len(rec.dels) != 1 || rec.dels[0] != "u@d.com/r.git/refs/heads/stale" {
t.Fatalf("dels = %v, want [u@d.com/r.git/refs/heads/stale]", rec.dels)
}
}
func TestFailReport(t *testing.T) {
rs := packp.NewReportStatus()
rs.UnpackStatus = "ok"
rs.CommandStatuses = []*packp.CommandStatus{
{ReferenceName: "refs/heads/main", Status: "ok"},
{ReferenceName: "refs/heads/other", Status: "already failed"},
}
failReport(rs, errors.New("store broke:\n\tdetail"))
if got := rs.CommandStatuses[0].Status; got != "store broke: detail" {
t.Errorf("ok status = %q, want flattened error", got)
}
// A status that already carries a failure is not overwritten.
if got := rs.CommandStatuses[1].Status; got != "already failed" {
t.Errorf("failed status overwritten: %q", got)
}
if failReport(nil, errors.New("x")) != nil {
t.Error("failReport(nil) != nil")
}
}
+// TestReceivePackCleansUpOnError: a failed materialization must not leave a
+// push-* directory behind in the cache root.
+func TestReceivePackCleansUpOnError(t *testing.T) {
+ rec := &recordingStore{} // every read fails with ErrNotFound
+ dir := t.TempDir()
+ p := NewPusher(rec, newTestCache(t, rec, CacheOptions{Dir: dir}))
+
+ _, err := p.ReceivePack(context.Background(), "u@d.com/r.git", packp.NewReferenceUpdateRequest())
+ if err == nil {
+ t.Fatal("expected error from failing store")
+ }
+ if got := dirCount(t, dir); got != 0 {
+ t.Fatalf("%d dirs left in cache root after failed push, want 0", got)
+ }
+}
+
+// TestReceivePackSemaphoreHonorsContext: with the push semaphore full, a
+// cancelled context aborts instead of blocking forever.
+func TestReceivePackSemaphoreHonorsContext(t *testing.T) {
+ rec := &recordingStore{}
+ p := NewPusher(rec, newTestCache(t, rec, CacheOptions{}))
+ for range cap(p.sem) {
+ p.sem <- struct{}{}
+ }
+ ctx, cancel := context.WithCancel(context.Background())
+ cancel()
+ _, err := p.ReceivePack(ctx, "u@d.com/r.git", packp.NewReferenceUpdateRequest())
+ if !errors.Is(err, context.Canceled) {
+ t.Fatalf("err = %v, want context.Canceled", err)
+ }
+}
+
func TestWalkManifestSkipsTmp(t *testing.T) {
fs := memfs.New()
writeFile(t, fs, "HEAD", "x")
writeFile(t, fs, "objects/tmp_pack_123", "scratch")
- man, err := walkManifest(fs)
+ writeFile(t, fs, "._packed-refs123", "scratch")
+ man, err := walkManifest(fs, defaultMaxRepoBytes)
if err != nil {
t.Fatal(err)
}
if _, ok := man["objects/tmp_pack_123"]; ok {
t.Error("walkManifest included a tmp_ scratch file")
+ }
+ if _, ok := man["._packed-refs123"]; ok {
+ t.Error("walkManifest included a ._packed-refs temp file")
}
if _, ok := man["HEAD"]; !ok {
t.Error("walkManifest missing HEAD")
}
}
internal/gitrepo/refs_test.go
package gitrepo
import (
"testing"
"gitweb/internal/store"
)
// TestResolveRefAmbiguity exercises the slash-in-ref disambiguation against a
// real repo that has a branch "feature/x" and a tag "v1.0.0".
func TestResolveRefAmbiguity(t *testing.T) {
root := t.TempDir()
repoPath := makeBareRepo(t, root, "u@d.com", "demo.git")
st := store.NewLocalStore(root)
- r, err := open(st, repoPath, 0)
+ c := newTestCache(t, st, CacheOptions{})
+ repo, release, err := c.Get(repoPath)
if err != nil {
- t.Fatalf("open: %v", err)
+ t.Fatalf("Get: %v", err)
}
- repo := r.Repo
+ defer release()
tests := []struct {
name string
in string
wantRef string
wantSubPath string
}{
{"empty -> HEAD", "", "HEAD", ""},
{"branch only", "main", "main", ""},
{"branch + path", "main/main.go", "main", "main.go"},
{"slashed branch only", "feature/x", "feature/x", ""},
{"slashed branch + path", "feature/x/main.go", "feature/x", "main.go"},
{"tag only", "v1.0.0", "v1.0.0", ""},
{"tag + path", "v1.0.0/README.md", "v1.0.0", "README.md"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
h, ref, sub, err := ResolveRef(repo, tt.in)
if err != nil {
t.Fatalf("ResolveRef(%q): %v", tt.in, err)
}
if ref != tt.wantRef || sub != tt.wantSubPath {
t.Fatalf("ResolveRef(%q) = ref %q sub %q, want ref %q sub %q",
tt.in, ref, sub, tt.wantRef, tt.wantSubPath)
}
if h.IsZero() {
t.Fatalf("ResolveRef(%q): zero hash", tt.in)
}
if _, err := repo.CommitObject(h); err != nil {
t.Fatalf("ResolveRef(%q): hash %s is not a commit: %v", tt.in, h, err)
}
})
}
// A full commit hash resolves as its own ref.
head, _ := repo.Head()
full := head.Hash().String()
h, ref, sub, err := ResolveRef(repo, full+"/main.go")
if err != nil {
t.Fatalf("ResolveRef(hash): %v", err)
}
if ref != full || sub != "main.go" || h != head.Hash() {
t.Fatalf("hash ref: got ref %q sub %q hash %s", ref, sub, h)
}
// An unknown ref errors.
if _, _, _, err := ResolveRef(repo, "nope/x.go"); err != ErrRefNotFound {
t.Fatalf("want ErrRefNotFound, got %v", err)
}
}
internal/server/clone_push_test.go
package server
import (
"crypto/sha256"
"encoding/hex"
"fmt"
"net/http"
"net/http/httptest"
"net/url"
"os"
"os/exec"
"path/filepath"
"strings"
"testing"
"gitweb/internal/store"
)
// gitEnv returns a hermetic environment for invoking git in tests: no prompts,
// no system/global config, no credential helpers, and an isolated HOME.
func gitEnv(t *testing.T) []string {
return append(os.Environ(),
"HOME="+t.TempDir(),
"GIT_TERMINAL_PROMPT=0",
"GIT_CONFIG_NOSYSTEM=1",
"GIT_ASKPASS=true",
"GIT_AUTHOR_NAME=T", "GIT_AUTHOR_EMAIL=t@e.com",
"GIT_COMMITTER_NAME=T", "GIT_COMMITTER_EMAIL=t@e.com",
)
}
func runGit(t *testing.T, env []string, dir string, args ...string) (string, error) {
t.Helper()
c := exec.Command("git", args...)
c.Dir = dir
c.Env = env
out, err := c.CombinedOutput()
return string(out), err
}
func mustGit(t *testing.T, env []string, dir string, args ...string) string {
t.Helper()
out, err := runGit(t, env, dir, args...)
if err != nil {
t.Fatalf("git %s: %v\n%s", strings.Join(args, " "), err, out)
}
return out
}
// makeBareRepo builds a bare repo at ns/<repoPath> with one commit, and writes
// an Access file with the given content beside it.
func makeBareRepo(t *testing.T, env []string, ns, repoPath, access string) {
t.Helper()
work := t.TempDir()
mustGit(t, env, work, "init", "-q", "-b", "main")
if err := os.WriteFile(filepath.Join(work, "README.md"), []byte("# hello\n"), 0o644); err != nil {
t.Fatal(err)
}
mustGit(t, env, work, "add", "-A")
mustGit(t, env, work, "commit", "-qm", "initial")
dst := filepath.Join(ns, filepath.FromSlash(repoPath))
if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil {
t.Fatal(err)
}
mustGit(t, env, work, "clone", "-q", "--bare", work, dst)
if err := os.WriteFile(filepath.Join(dst, "Access"), []byte(access), 0o644); err != nil {
t.Fatal(err)
}
}
// writeTokens writes a gitweb tokens file granting the given plaintext tokens to
// user (by storing their SHA-256 hashes).
func writeTokens(t *testing.T, ns, user string, tokens ...string) {
t.Helper()
dir := filepath.Join(ns, user, "gitweb")
if err := os.MkdirAll(dir, 0o755); err != nil {
t.Fatal(err)
}
var b strings.Builder
for _, tok := range tokens {
h := sha256.Sum256([]byte(tok))
fmt.Fprintf(&b, "%s token\n", hex.EncodeToString(h[:]))
}
if err := os.WriteFile(filepath.Join(dir, "tokens"), []byte(b.String()), 0o644); err != nil {
t.Fatal(err)
}
}
// cloneURL builds an http clone URL, optionally embedding Basic-auth creds.
func cloneURL(base, repoPath, user, token string) string {
u, _ := url.Parse(base)
if user != "" {
u.User = url.UserPassword(user, token)
}
u.Path = "/" + repoPath
return u.String()
}
func TestCloneAndPush(t *testing.T) {
if _, err := exec.LookPath("git"); err != nil {
t.Skip("git not available")
}
env := gitEnv(t)
ns := t.TempDir()
const owner = "u@d.com"
const writer = "w@d.com"
const reader = "r@d.com"
makeBareRepo(t, env, ns, owner+"/public.git", "read,list: all\n*: "+owner+"\n")
makeBareRepo(t, env, ns, owner+"/private.git",
"*: "+owner+"\nread,list,write,create,delete: "+writer+"\nread,list: "+reader+"\n")
writeTokens(t, ns, writer, "writer-token")
writeTokens(t, ns, reader, "reader-token")
web := os.DirFS("../../web")
- srv, err := New(Config{Store: store.NewLocalStore(ns), Web: http.FS(web)})
+ cacheDir := t.TempDir()
+ srv, err := New(Config{Store: store.NewLocalStore(ns), Web: http.FS(web), CacheDir: cacheDir})
if err != nil {
t.Fatal(err)
}
ts := httptest.NewServer(srv.Handler())
defer ts.Close()
t.Run("anon clone public", func(t *testing.T) {
dst := filepath.Join(t.TempDir(), "pub")
if out, err := runGit(t, env, ".", "clone", "-q", cloneURL(ts.URL, owner+"/public.git", "", ""), dst); err != nil {
t.Fatalf("clone public failed: %v\n%s", err, out)
}
if _, err := os.Stat(filepath.Join(dst, "README.md")); err != nil {
t.Fatalf("cloned repo missing README: %v", err)
}
})
t.Run("anon clone private denied", func(t *testing.T) {
dst := filepath.Join(t.TempDir(), "priv")
if out, err := runGit(t, env, ".", "clone", "-q", cloneURL(ts.URL, owner+"/private.git", "", ""), dst); err == nil {
t.Fatalf("anonymous clone of private repo succeeded, want failure\n%s", out)
}
})
t.Run("authed clone private", func(t *testing.T) {
dst := filepath.Join(t.TempDir(), "priv-ok")
u := cloneURL(ts.URL, owner+"/private.git", writer, "writer-token")
if out, err := runGit(t, env, ".", "clone", "-q", u, dst); err != nil {
t.Fatalf("authed clone failed: %v\n%s", err, out)
}
})
t.Run("wrong token denied", func(t *testing.T) {
dst := filepath.Join(t.TempDir(), "priv-bad")
u := cloneURL(ts.URL, owner+"/private.git", writer, "nope")
if out, err := runGit(t, env, ".", "clone", "-q", u, dst); err == nil {
t.Fatalf("clone with wrong token succeeded, want failure\n%s", out)
}
})
t.Run("push as writer", func(t *testing.T) {
dst := filepath.Join(t.TempDir(), "push")
u := cloneURL(ts.URL, owner+"/private.git", writer, "writer-token")
mustGit(t, env, ".", "clone", "-q", u, dst)
if err := os.WriteFile(filepath.Join(dst, "new.txt"), []byte("added\n"), 0o644); err != nil {
t.Fatal(err)
}
mustGit(t, env, dst, "add", "-A")
mustGit(t, env, dst, "commit", "-qm", "add new.txt")
if out, err := runGit(t, env, dst, "push", "-q", "origin", "main"); err != nil {
t.Fatalf("push failed: %v\n%s", err, out)
}
// A fresh clone must see the pushed commit.
fresh := filepath.Join(t.TempDir(), "fresh")
mustGit(t, env, ".", "clone", "-q", u, fresh)
if _, err := os.Stat(filepath.Join(fresh, "new.txt")); err != nil {
t.Fatalf("pushed file not visible after re-clone: %v", err)
}
})
t.Run("incremental push (thin pack)", func(t *testing.T) {
dst := filepath.Join(t.TempDir(), "thin")
u := cloneURL(ts.URL, owner+"/private.git", writer, "writer-token")
mustGit(t, env, ".", "clone", "-q", u, dst)
// A file large enough that git deltas the second version against the
// first. On the second push that base blob is not in the pack (the
// server already has it), so git sends a thin pack with a REF_DELTA —
// the case that used to fail with "reference delta not found".
lines := strings.Repeat("a line of text that compresses and deltas well\n", 200)
if err := os.WriteFile(filepath.Join(dst, "big.txt"), []byte(lines), 0o644); err != nil {
t.Fatal(err)
}
mustGit(t, env, dst, "add", "-A")
mustGit(t, env, dst, "commit", "-qm", "add big.txt")
mustGit(t, env, dst, "push", "-q", "origin", "main")
if err := os.WriteFile(filepath.Join(dst, "big.txt"), []byte(lines+"one changed line\n"), 0o644); err != nil {
t.Fatal(err)
}
mustGit(t, env, dst, "add", "-A")
mustGit(t, env, dst, "commit", "-qm", "append to big.txt")
if out, err := runGit(t, env, dst, "push", "origin", "main"); err != nil {
t.Fatalf("incremental push failed: %v\n%s", err, out)
}
fresh := filepath.Join(t.TempDir(), "fresh-thin")
mustGit(t, env, ".", "clone", "-q", u, fresh)
got, err := os.ReadFile(filepath.Join(fresh, "big.txt"))
if err != nil {
t.Fatal(err)
}
if string(got) != lines+"one changed line\n" {
t.Error("re-clone does not have the incrementally pushed content")
}
})
t.Run("push branch create and delete", func(t *testing.T) {
dst := filepath.Join(t.TempDir(), "branch")
u := cloneURL(ts.URL, owner+"/private.git", writer, "writer-token")
mustGit(t, env, ".", "clone", "-q", u, dst)
mustGit(t, env, dst, "push", "-q", "origin", "main:refs/heads/scratch")
// Deleting a ref sends no packfile at all; the server must not choke
// on the empty pack stream.
if out, err := runGit(t, env, dst, "push", "origin", ":refs/heads/scratch"); err != nil {
t.Fatalf("delete push failed: %v\n%s", err, out)
}
refs := mustGit(t, env, dst, "ls-remote", "origin")
if strings.Contains(refs, "scratch") {
t.Errorf("scratch branch still on remote:\n%s", refs)
+ }
+ })
+
+ t.Run("push copies cleaned up", func(t *testing.T) {
+ // Every push materializes a private push-* dir under the cache root and
+ // must remove it when done.
+ children, err := os.ReadDir(cacheDir)
+ if err != nil {
+ t.Fatal(err)
+ }
+ for _, ch := range children {
+ if strings.HasPrefix(ch.Name(), "push-") {
+ t.Errorf("leftover push dir %s in cache root", ch.Name())
+ }
}
})
t.Run("push as reader denied", func(t *testing.T) {
dst := filepath.Join(t.TempDir(), "push-ro")
u := cloneURL(ts.URL, owner+"/private.git", reader, "reader-token")
mustGit(t, env, ".", "clone", "-q", u, dst)
if err := os.WriteFile(filepath.Join(dst, "sneaky.txt"), []byte("x\n"), 0o644); err != nil {
t.Fatal(err)
}
mustGit(t, env, dst, "add", "-A")
mustGit(t, env, dst, "commit", "-qm", "sneak")
if out, err := runGit(t, env, dst, "push", "-q", "origin", "main"); err == nil {
t.Fatalf("reader push succeeded, want 403 failure\n%s", out)
}
})
}
internal/server/git.go
package server
import (
"bytes"
"errors"
"html/template"
"net/http"
"net/url"
"path"
"strconv"
"strings"
"github.com/go-git/go-git/v5"
"github.com/go-git/go-git/v5/plumbing"
"github.com/go-git/go-git/v5/plumbing/filemode"
"github.com/go-git/go-git/v5/plumbing/object"
"gitweb/internal/gitrepo"
"gitweb/internal/render"
)
var gitTmpl = template.Must(template.ParseFS(tmplFS, "templates/git.html"))
const logLimit = 100
// serveGit handles a request that Locate resolved to a bare repo. repoPath is
// the upspin repo root (e.g. "u@d.com/proj.git"); rest is the remaining URL
// after it (e.g. "blob/main/src/x.go"). base is "/"+repoPath.
func (s *Server) serveGit(w http.ResponseWriter, r *http.Request, repoPath, rest string) {
- repo, err := s.repos.Get(repoPath)
+ repo, release, err := s.repos.Get(repoPath)
if err != nil {
s.storeErr(w, err)
return
}
+ defer release()
base := "/" + repoPath
rest = strings.Trim(rest, "/")
verb, rem, _ := strings.Cut(rest, "/")
switch verb {
case "":
s.gitSummary(w, repo, repoPath, base)
case "refs":
s.gitRefs(w, repo, repoPath, base)
case "tree":
s.gitTree(w, repo, repoPath, base, rem)
case "blob":
s.gitBlob(w, repo, repoPath, base, rem, false, r.URL.Query())
case "raw":
s.gitBlob(w, repo, repoPath, base, rem, true, nil)
case "log":
s.gitLog(w, repo, repoPath, base, rem)
case "commit":
s.gitCommit(w, repo, repoPath, base, rem)
default:
s.notFound(w)
}
}
// --- view models ---
type gitNav struct {
RepoName string
Base string
Ref string
LogURL string
}
func newNav(repoName, base, ref string) gitNav {
n := gitNav{RepoName: repoName, Base: base, Ref: ref}
if ref != "" {
n.LogURL = base + "/log/" + escapePath(ref)
}
return n
}
type treeEntry struct {
Name string
IsDir bool
URL string
Size string
}
type crumb struct {
Name string
URL string // empty for the last crumb
}
type refLink struct {
Name string
Short string
URL string
LogURL string
}
// --- summary ---
func (s *Server) gitSummary(w http.ResponseWriter, repo *git.Repository, repoName, base string) {
hash, ref, _, err := gitrepo.ResolveRef(repo, "")
if err != nil {
s.notFound(w)
return
}
nav := newNav(repoName, base, ref)
tree, err := treeAt(repo, hash, "")
if err != nil {
s.serverErr(w, err)
return
}
entries := treeEntries(repo, tree, base, ref, "")
branches, tags, err := gitrepo.Refs(repo)
if err != nil {
s.serverErr(w, err)
return
}
var readme template.HTML
if name := findReadme(tree); name != "" {
if f, err := tree.File(name); err == nil {
if data, err := f.Contents(); err == nil {
if h, err := s.renderer.ToHTML([]byte(data)); err == nil {
readme = rebaseRepoHTML(template.HTML(h), base, ref, path.Dir(name))
}
}
}
}
s.renderGit(w, repoName, "summary", struct {
Nav gitNav
Branches []refLink
Tags []refLink
Entries []treeEntry
Readme template.HTML
}{
Nav: nav,
Branches: refLinks(branches, base, "tree"),
Tags: refLinks(tags, base, "tree"),
Entries: entries,
Readme: readme,
})
}
// --- tree ---
func (s *Server) gitTree(w http.ResponseWriter, repo *git.Repository, repoName, base, rem string) {
hash, ref, sub, err := gitrepo.ResolveRef(repo, rem)
if err != nil {
s.notFound(w)
return
}
tree, err := treeAt(repo, hash, sub)
if err != nil {
s.notFound(w)
return
}
nav := newNav(repoName, base, ref)
title := repoName
if sub != "" {
title = repoName + "/" + sub
}
s.renderGit(w, repoName, "tree", struct {
Nav gitNav
Title string
Crumbs []crumb
Entries []treeEntry
}{
Nav: nav,
Title: title,
Crumbs: buildCrumbs(base, ref, sub, true),
Entries: treeEntries(repo, tree, base, ref, sub),
})
}
// --- blob / raw ---
func (s *Server) gitBlob(w http.ResponseWriter, repo *git.Repository, repoName, base, rem string, raw bool, q url.Values) {
hash, ref, sub, err := gitrepo.ResolveRef(repo, rem)
if err != nil || sub == "" {
s.notFound(w)
return
}
if isAccessFile(path.Base(sub)) {
s.notFound(w) // never render an upspin ACL, blob or raw
return
}
tree, err := treeAt(repo, hash, "")
if err != nil {
s.notFound(w)
return
}
file, err := tree.File(sub)
if err != nil {
s.notFound(w)
return
}
data, err := file.Contents()
if err != nil {
s.serverErr(w, err)
return
}
content := []byte(data)
if raw {
ctype := http.DetectContentType(content)
w.Header().Set("Content-Type", ctype)
w.Write(content)
return
}
rawURL := base + "/raw/" + escapePath(ref) + "/" + escapePath(sub)
binary := render.IsBinary(content)
name := path.Base(sub)
force := q.Get("force") == "1"
selected := s.renderers.Select(name, content, q.Get("render"), force)
var body template.HTML
var choices []rendererChoice
var forceURL string
if !binary {
body, err = selected.Render(name, content)
if err != nil {
s.serverErr(w, err)
return
}
if selected.ID == "markdown" {
// Rewrite relative links/images so they resolve to the file's
// directory in the repo rather than the current URL.
body = rebaseRepoHTML(body, base, ref, path.Dir(sub))
}
blobURL := base + "/blob/" + escapePath(ref) + "/" + escapePath(sub)
choices, forceURL = renderControls(s.renderers, blobURL, name, content, selected.ID, force)
}
s.renderGit(w, repoName, "blob", blobView{
Nav: newNav(repoName, base, ref),
Crumbs: buildCrumbs(base, ref, sub, true),
Path: sub,
Size: humanSize(file.Size),
RawURL: rawURL,
Binary: binary,
Body: body,
Renderers: choices,
Force: force,
ForceURL: forceURL,
})
}
// blobView is the file-view model, including the renderer control bar.
type blobView struct {
Nav gitNav
Crumbs []crumb
Path string
Size string
RawURL string
Binary bool
Body template.HTML
Renderers []rendererChoice
Force bool
ForceURL string
}
// rendererChoice is one selectable renderer link in the control bar.
type rendererChoice struct {
Label string
URL string
Active bool
}
// renderControls builds the renderer switch links and the force-toggle link for
// a blob. With force on, every registered renderer is offered; otherwise only
// the eligible ones. The default renderer's link carries no ?render= so its URL
// stays clean.
func renderControls(reg *render.Registry, blobURL, name string, content []byte, selectedID string, force bool) ([]rendererChoice, string) {
mods := reg.Eligible(name, content)
if force {
mods = reg.All()
}
defID := reg.Default().ID
choices := make([]rendererChoice, 0, len(mods))
for _, m := range mods {
choices = append(choices, rendererChoice{
Label: m.Label,
URL: blobURL + blobQuery(m.ID, defID, force),
Active: m.ID == selectedID,
})
}
var forceURL string
if force {
// Turning force off: keep the selection only if it stays eligible.
keep := selectedID
if _, ok := find(reg.Eligible(name, content), keep); !ok {
keep = defID
}
forceURL = blobURL + blobQuery(keep, defID, false)
} else {
forceURL = blobURL + blobQuery(selectedID, defID, true)
}
return choices, forceURL
}
// blobQuery builds the query string for a blob link with the given renderer and
// force state. The default renderer is encoded as no ?render= param.
func blobQuery(id, defID string, force bool) string {
v := url.Values{}
if id != "" && id != defID {
v.Set("render", id)
}
if force {
v.Set("force", "1")
}
if len(v) == 0 {
return ""
}
return "?" + v.Encode()
}
func find(mods []render.Module, id string) (render.Module, bool) {
for _, m := range mods {
if m.ID == id {
return m, true
}
}
return render.Module{}, false
}
// --- log ---
func (s *Server) gitLog(w http.ResponseWriter, repo *git.Repository, repoName, base, rem string) {
hash, ref, _, err := gitrepo.ResolveRef(repo, rem)
if err != nil {
s.notFound(w)
return
}
iter, err := repo.Log(&git.LogOptions{From: hash})
if err != nil {
s.serverErr(w, err)
return
}
type commitRow struct {
Subject, Author, When, Short, URL string
}
var commits []commitRow
_ = iter.ForEach(func(c *object.Commit) error {
if len(commits) >= logLimit {
return errStopIter
}
commits = append(commits, commitRow{
Subject: subjectOf(c.Message),
Author: c.Author.Name,
When: c.Author.When.Format("2006-01-02"),
Short: c.Hash.String()[:8],
URL: base + "/commit/" + c.Hash.String(),
})
return nil
})
s.renderGit(w, repoName, "log", struct {
Nav gitNav
Commits interface{}
}{
Nav: newNav(repoName, base, ref),
Commits: commits,
})
}
// --- commit ---
func (s *Server) gitCommit(w http.ResponseWriter, repo *git.Repository, repoName, base, rem string) {
rem = strings.Trim(rem, "/")
h, err := repo.ResolveRevision(plumbing.Revision(rem))
if err != nil {
s.notFound(w)
return
}
c, err := repo.CommitObject(*h)
if err != nil {
s.notFound(w)
return
}
var fromTree *object.Tree
var parents []struct{ Short, URL string }
if c.NumParents() > 0 {
p, err := c.Parent(0)
if err == nil {
fromTree, _ = p.Tree()
parents = append(parents, struct{ Short, URL string }{
Short: p.Hash.String()[:8],
URL: base + "/commit/" + p.Hash.String(),
})
}
}
toTree, err := c.Tree()
if err != nil {
s.serverErr(w, err)
return
}
changes, err := object.DiffTree(fromTree, toTree)
if err != nil {
s.serverErr(w, err)
return
}
patch, err := changes.Patch()
if err != nil {
s.serverErr(w, err)
return
}
s.renderGit(w, repoName, "commit", struct {
Nav gitNav
Subject string
Body string
Hash string
Author string
When string
Parents []struct{ Short, URL string }
Diff template.HTML
}{
Nav: newNav(repoName, base, ""),
Subject: subjectOf(c.Message),
Body: bodyOf(c.Message),
Hash: c.Hash.String(),
Author: c.Author.Name + " <" + c.Author.Email + ">",
When: c.Author.When.Format("2006-01-02 15:04 -0700"),
Parents: parents,
Diff: render.Diff(patch),
})
}
// --- refs ---
func (s *Server) gitRefs(w http.ResponseWriter, repo *git.Repository, repoName, base string) {
branches, tags, err := gitrepo.Refs(repo)
if err != nil {
s.serverErr(w, err)
return
}
s.renderGit(w, repoName, "refs", struct {
Nav gitNav
Branches []refLink
Tags []refLink
}{
Nav: newNav(repoName, base, ""),
Branches: refLinks(branches, base, "tree"),
Tags: refLinks(tags, base, "tree"),
})
}
// --- helpers ---
// errStopIter is a sentinel to stop a commit ForEach early.
var errStopIter = errors.New("stop")
func (s *Server) renderGit(w http.ResponseWriter, repoName, name string, data interface{}) {
var body bytes.Buffer
if err := gitTmpl.ExecuteTemplate(&body, name, data); err != nil {
s.serverErr(w, err)
return
}
s.renderPage(w, repoName, template.HTML(body.String()))
}
// treeAt returns the tree at sub within the commit identified by hash.
func treeAt(repo *git.Repository, hash plumbing.Hash, sub string) (*object.Tree, error) {
c, err := repo.CommitObject(hash)
if err != nil {
return nil, err
}
tree, err := c.Tree()
if err != nil {
return nil, err
}
if sub == "" {
return tree, nil
}
return tree.Tree(sub)
}
func treeEntries(repo *git.Repository, tree *object.Tree, base, ref, sub string) []treeEntry {
out := make([]treeEntry, 0, len(tree.Entries))
for _, e := range tree.Entries {
if isAccessFile(e.Name) {
continue // don't list an upspin ACL committed into the tree
}
child := e.Name
if sub != "" {
child = sub + "/" + e.Name
}
isDir := e.Mode == filemode.Dir
te := treeEntry{Name: e.Name, IsDir: isDir}
if isDir {
te.URL = base + "/tree/" + escapePath(ref) + "/" + escapePath(child)
} else {
te.URL = base + "/blob/" + escapePath(ref) + "/" + escapePath(child)
if b, err := repo.BlobObject(e.Hash); err == nil {
te.Size = humanSize(b.Size)
}
}
out = append(out, te)
}
// Directories first, then by name.
sortTreeEntries(out)
return out
}
func sortTreeEntries(es []treeEntry) {
// Simple insertion sort keeps dirs first then name (lists are small).
for i := 1; i < len(es); i++ {
for j := i; j > 0 && less(es[j], es[j-1]); j-- {
es[j], es[j-1] = es[j-1], es[j]
}
}
}
func less(a, b treeEntry) bool {
if a.IsDir != b.IsDir {
return a.IsDir
}
return a.Name < b.Name
}
func buildCrumbs(base, ref, sub string, dirLinks bool) []crumb {
crumbs := []crumb{{Name: ref, URL: base + "/tree/" + escapePath(ref)}}
if sub == "" {
return crumbs
}
parts := strings.Split(sub, "/")
acc := ""
for i, p := range parts {
if acc == "" {
acc = p
} else {
acc = acc + "/" + p
}
c := crumb{Name: p}
last := i == len(parts)-1
if !last || dirLinks {
c.URL = base + "/tree/" + escapePath(ref) + "/" + escapePath(acc)
}
if last {
c.URL = "" // current location, not a link
}
crumbs = append(crumbs, c)
}
return crumbs
}
func refLinks(refs []gitrepo.RefName, base, verb string) []refLink {
out := make([]refLink, 0, len(refs))
for _, r := range refs {
out = append(out, refLink{
Name: r.Name,
Short: r.Hash.String()[:8],
URL: base + "/" + verb + "/" + escapePath(r.Name),
LogURL: base + "/log/" + escapePath(r.Name),
})
}
return out
}
func findReadme(tree *object.Tree) string {
for _, name := range []string{"README.md", "readme.md", "README.markdown", "README"} {
if _, err := tree.File(name); err == nil {
return name
}
}
return ""
}
// rebaseRepoHTML rewrites relative links/images in rendered repo markdown so
// they resolve to the source file's location in the repo (blob view for links,
// raw for images) at ref, given the markdown file's directory dir within the
// tree ("" or "." for the repo root).
func rebaseRepoHTML(h template.HTML, base, ref, dir string) template.HTML {
prefix := escapePath(ref) + "/"
if dir != "" && dir != "." {
prefix += escapePath(dir) + "/"
}
return render.Rebase(h, base+"/blob/"+prefix, base+"/raw/"+prefix)
}
// escapePath URL-escapes each slash-separated segment, preserving slashes.
func escapePath(p string) string {
parts := strings.Split(p, "/")
for i, s := range parts {
parts[i] = url.PathEscape(s)
}
return strings.Join(parts, "/")
}
func subjectOf(msg string) string {
if i := strings.IndexByte(msg, '\n'); i >= 0 {
return strings.TrimSpace(msg[:i])
}
return strings.TrimSpace(msg)
}
func bodyOf(msg string) string {
if i := strings.IndexByte(msg, '\n'); i >= 0 {
return strings.TrimSpace(msg[i+1:])
}
return ""
}
func humanSize(n int64) string {
const unit = 1024
if n < unit {
return strconv.FormatInt(n, 10) + " B"
}
div, exp := int64(unit), 0
for m := n / unit; m >= unit; m /= unit {
div *= unit
exp++
}
return strconv.FormatFloat(float64(n)/float64(div), 'f', 1, 64) + " " + string("KMGT"[exp]) + "iB"
}
internal/server/git_test.go
package server
import (
"io"
"net/http"
"net/http/httptest"
"os"
"os/exec"
"path/filepath"
"strings"
"testing"
"gitweb/internal/store"
)
// newTestServer builds a Server over a LocalStore namespace containing a bare
// repo at u@d.com/demo.git and a plain dir u@d.com/docs.
func newTestServer(t *testing.T) (*httptest.Server, func()) {
t.Helper()
if _, err := exec.LookPath("git"); err != nil {
t.Skip("git not available")
}
ns := t.TempDir()
if err := os.MkdirAll(filepath.Join(ns, "u@d.com", "docs"), 0o755); err != nil {
t.Fatal(err)
}
os.WriteFile(filepath.Join(ns, "u@d.com", "docs", "note.txt"), []byte("hi"), 0o644)
// Browsing is gated by Access files; make this whole namespace world-readable
// so the anonymous browse tests below succeed.
os.WriteFile(filepath.Join(ns, "u@d.com", "Access"), []byte("read,list: all\n*: u@d.com\n"), 0o644)
work := t.TempDir()
git := func(dir string, a ...string) {
t.Helper()
c := exec.Command("git", a...)
c.Dir = dir
c.Env = append(os.Environ(),
"GIT_AUTHOR_NAME=T", "GIT_AUTHOR_EMAIL=t@e.com",
"GIT_COMMITTER_NAME=T", "GIT_COMMITTER_EMAIL=t@e.com")
if out, err := c.CombinedOutput(); err != nil {
t.Fatalf("git %v: %v\n%s", a, err, out)
}
}
git(work, "init", "-q", "-b", "main")
os.WriteFile(filepath.Join(work, "README.md"), []byte("# Demo\n\nhello\n"), 0o644)
os.WriteFile(filepath.Join(work, "main.go"), []byte("package main\n\nfunc main() {}\n"), 0o644)
// An Access file committed into the repo tree, to prove it is hidden there too.
os.WriteFile(filepath.Join(work, "Access"), []byte("read,list: all\n*: secret@d.com\n"), 0o644)
git(work, "add", "-A")
git(work, "commit", "-qm", "initial")
git(work, "checkout", "-q", "-b", "feature/x")
os.WriteFile(filepath.Join(work, "main.go"), []byte("package main\n\nfunc main() { _ = 1 }\n"), 0o644)
git(work, "commit", "-qam", "tweak")
git(work, "tag", "v1.0.0")
git(work, "checkout", "-q", "main")
git(work, "clone", "-q", "--bare", work, filepath.Join(ns, "u@d.com", "demo.git"))
web := os.DirFS("../../web")
- srv, err := New(Config{Store: store.NewLocalStore(ns), Web: http.FS(web)})
+ srv, err := New(Config{Store: store.NewLocalStore(ns), Web: http.FS(web), CacheDir: t.TempDir()})
if err != nil {
t.Fatal(err)
}
ts := httptest.NewServer(srv.Handler())
return ts, ts.Close
}
// TestAccessFilesHidden verifies that upspin Access files are never listed or
// rendered — neither in the plain namespace browse nor in a git repo tree — so
// the set of users/groups with access is not exposed to browsers.
func TestAccessFilesHidden(t *testing.T) {
ts, done := newTestServer(t)
defer done()
get := func(p string) (int, string) {
resp, err := http.Get(ts.URL + p)
if err != nil {
t.Fatal(err)
}
defer resp.Body.Close()
b, _ := io.ReadAll(resp.Body)
return resp.StatusCode, string(b)
}
// Namespace listing must not link to the Access file...
if code, body := get("/u@d.com/"); code != 200 {
t.Fatalf("namespace browse code %d", code)
} else if strings.Contains(body, `href="Access"`) {
t.Error("namespace listing exposed the Access file")
}
// ...and its contents must not be servable directly.
if code, _ := get("/u@d.com/Access"); code != 404 {
t.Errorf("GET /u@d.com/Access = %d, want 404", code)
}
// Git tree listing must not link to a committed Access file...
if code, body := get("/u@d.com/demo.git/tree/main/"); code != 200 {
t.Fatalf("tree code %d", code)
} else if strings.Contains(body, "/blob/main/Access") {
t.Error("git tree listing exposed a committed Access file")
}
// ...and neither blob nor raw view will render it.
if code, body := get("/u@d.com/demo.git/blob/main/Access"); code != 404 {
t.Errorf("blob Access = %d, want 404", code)
} else if strings.Contains(body, "secret@d.com") {
t.Error("blob view leaked Access contents")
}
if code, _ := get("/u@d.com/demo.git/raw/main/Access"); code != 404 {
t.Errorf("raw Access = %d, want 404", code)
}
}
func TestGitVerbs(t *testing.T) {
ts, done := newTestServer(t)
defer done()
get := func(p string) (int, string) {
resp, err := http.Get(ts.URL + p)
if err != nil {
t.Fatal(err)
}
defer resp.Body.Close()
b, _ := io.ReadAll(resp.Body)
return resp.StatusCode, string(b)
}
cases := []struct {
path string
wantCode int
contains []string
}{
{"/u@d.com/docs/", 200, []string{"Index of", "note.txt"}},
{"/u@d.com/demo.git/", 200, []string{"feature/x", "v1.0.0", "README", "Demo"}},
{"/u@d.com/demo.git/refs", 200, []string{"Branches", "Tags", "feature/x", "v1.0.0"}},
{"/u@d.com/demo.git/tree/main/", 200, []string{"main.go", "README.md"}},
{"/u@d.com/demo.git/blob/main/main.go", 200, []string{"chroma", "func"}},
{"/u@d.com/demo.git/blob/feature/x/main.go", 200, []string{"chroma"}},
{"/u@d.com/demo.git/log/feature/x", 200, []string{"tweak", "/commit/"}},
{"/u@d.com/demo.git/bogus", 404, nil},
}
for _, c := range cases {
t.Run(c.path, func(t *testing.T) {
code, body := get(c.path)
if code != c.wantCode {
t.Fatalf("code = %d, want %d", code, c.wantCode)
}
for _, want := range c.contains {
if !strings.Contains(body, want) {
t.Fatalf("body missing %q", want)
}
}
})
}
// Renderer switch: default README.md is highlighted source, with a
// Markdown switch offered; ?render=markdown renders the markdown.
{
_, src := get("/u@d.com/demo.git/blob/main/README.md")
if !strings.Contains(src, "chroma") {
t.Fatal("README.md default view should be highlighted source")
}
if !strings.Contains(src, `href="/u@d.com/demo.git/blob/main/README.md?render=markdown"`) {
t.Fatal("README.md should offer a Markdown switch link")
}
_, md := get("/u@d.com/demo.git/blob/main/README.md?render=markdown")
if !strings.Contains(md, "<h1") {
t.Fatal("?render=markdown should render markdown (expected <h1>)")
}
}
// Non-markdown file offers only the default; force exposes & runs markdown.
{
_, src := get("/u@d.com/demo.git/blob/main/main.go")
if strings.Contains(src, "?render=markdown\"") {
t.Fatal("main.go should not offer markdown without force")
}
if !strings.Contains(src, "force=1") {
t.Fatal("main.go should offer a force toggle")
}
code, _ := get("/u@d.com/demo.git/blob/main/main.go?render=markdown&force=1")
if code != 200 {
t.Fatalf("forced markdown on main.go: code %d", code)
}
}
// CSP header is present on git pages. Scripts are restricted to same-origin
// with no 'unsafe-inline', so markdown-injected inline script cannot run
// (mermaid is served as a same-origin file).
resp, _ := http.Get(ts.URL + "/u@d.com/demo.git/")
resp.Body.Close()
csp := resp.Header.Get("Content-Security-Policy")
if !strings.Contains(csp, "script-src 'self'") || strings.Contains(csp, "script-src 'self' 'unsafe-inline'") {
t.Fatalf("missing/weak script CSP on git page: %q", csp)
}
// Commit page renders a diff.
_, logBody := get("/u@d.com/demo.git/log/feature/x")
i := strings.Index(logBody, "/u@d.com/demo.git/commit/")
if i < 0 {
t.Fatal("no commit link in log")
}
url := logBody[i : i+len("/u@d.com/demo.git/commit/")+40]
code, body := get(url)
if code != 200 || !strings.Contains(body, "diff-add") || !strings.Contains(body, "diff-del") {
t.Fatalf("commit page: code %d, diff classes present=%v", code,
strings.Contains(body, "diff-add"))
}
}
internal/server/githttp.go
package server
import (
"log"
"net/http"
"path"
"strings"
"gitweb/internal/githttp"
"gitweb/internal/gitrepo"
)
// gitKind identifies which git smart-HTTP endpoint a request hit.
type gitKind int
const (
gitInfoRefs gitKind = iota
gitUploadPack
gitReceivePack
)
// gitReq describes a matched git smart-HTTP request.
type gitReq struct {
kind gitKind
service githttp.Service // the effective service (upload-pack or receive-pack)
urlPath string // URL path with the git suffix stripped, e.g. "/u@d.com/x.git"
}
// gitEndpoint recognizes the three git smart-HTTP endpoints by method and path
// suffix. It returns ok=false for ordinary browse requests.
func gitEndpoint(r *http.Request) (gitReq, bool) {
p := r.URL.Path
switch {
case r.Method == http.MethodGet && strings.HasSuffix(p, "/info/refs"):
svc := r.URL.Query().Get("service")
switch githttp.Service(svc) {
case githttp.UploadPack, githttp.ReceivePack:
return gitReq{gitInfoRefs, githttp.Service(svc), strings.TrimSuffix(p, "/info/refs")}, true
}
// info/refs without a smart service param: fall through to browse (this
// also lets someone view a repo's literal info/refs file).
return gitReq{}, false
case r.Method == http.MethodPost && strings.HasSuffix(p, "/git-upload-pack"):
return gitReq{gitUploadPack, githttp.UploadPack, strings.TrimSuffix(p, "/git-upload-pack")}, true
case r.Method == http.MethodPost && strings.HasSuffix(p, "/git-receive-pack"):
return gitReq{gitReceivePack, githttp.ReceivePack, strings.TrimSuffix(p, "/git-receive-pack")}, true
}
return gitReq{}, false
}
// handleGitHTTP serves a git smart-HTTP request: authenticate, resolve the repo,
// authorize (read for clone/fetch, write for push), then run the protocol.
func (s *Server) handleGitHTTP(w http.ResponseWriter, r *http.Request, ep gitReq) {
user, clean, ok := parseUpspinPath(ep.urlPath)
if !ok {
s.notFound(w)
return
}
authUser, ok := s.authenticate(w, r)
if !ok {
return // 401 already written
}
repoPath, rest, isRepo, err := gitrepo.Locate(s.store, user, clean)
if err != nil {
s.storeErr(w, err)
return
}
// Endpoints live only at a repo root.
if !isRepo || rest != "" {
s.denyRead(w, authUser)
return
}
writing := ep.service == githttp.ReceivePack
if writing {
// Push always requires an authenticated, authorized writer.
if authUser == "" {
s.challenge(w)
return
}
can, err := s.authz.CanWrite(authUser, repoPath)
if err != nil {
s.serverErr(w, err)
return
}
if !can {
// The caller is known; distinguish "can't read" (hide it) from
// "can read but not write" (explicit 403).
readable, rerr := s.authz.CanRead(authUser, repoPath)
if rerr != nil {
s.serverErr(w, rerr)
return
}
if !readable {
s.notFound(w)
return
}
http.Error(w, "403 forbidden: no write access", http.StatusForbidden)
return
}
} else {
can, err := s.authz.CanRead(authUser, repoPath)
if err != nil {
s.serverErr(w, err)
return
}
if !can {
s.denyRead(w, authUser)
return
}
}
switch ep.kind {
case gitInfoRefs:
- storer, err := s.repos.GetStorer(repoPath)
+ storer, release, err := s.repos.GetStorer(repoPath)
if err != nil {
s.storeErr(w, err)
return
}
+ defer release()
if err := githttp.InfoRefs(w, storer, ep.service); err != nil {
log.Printf("gitweb: info/refs %s: %v", repoPath, err)
}
case gitUploadPack:
- storer, err := s.repos.GetStorer(repoPath)
+ // upload-pack streams the packfile from the storer for the whole
+ // response; the deferred release keeps the repo's on-disk backing
+ // alive until ServeUploadPack returns.
+ storer, release, err := s.repos.GetStorer(repoPath)
if err != nil {
s.storeErr(w, err)
return
}
+ defer release()
if err := githttp.ServeUploadPack(w, r, storer); err != nil {
log.Printf("gitweb: upload-pack %s: %v", repoPath, err)
}
case gitReceivePack:
req, err := githttp.DecodeReceivePack(r)
if err != nil {
http.Error(w, "400 bad request", http.StatusBadRequest)
return
}
rs, err := s.pusher.ReceivePack(r.Context(), repoPath, req)
if err != nil {
log.Printf("gitweb: receive-pack %s: %v", repoPath, err)
if rs == nil {
s.serverErr(w, err)
return
}
// rs carries per-ref failure detail; report it to the client.
}
if err := githttp.WriteReceivePackResult(w, rs); err != nil {
log.Printf("gitweb: receive-pack result %s: %v", repoPath, err)
}
}
}
// cleanRest cleans a URL remainder into an upspin path suffix beginning with
// "/". path.Clean collapses any ".." so it cannot escape the user's tree.
func cleanRest(rest string) string {
return path.Clean("/" + rest)
}
internal/server/mermaid_test.go
package server
import (
"io"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"testing"
"gitweb/internal/store"
)
// TestMermaidPageAndAssets checks that a markdown page with a mermaid block
// renders the diagram container, that the page wires the init script, and that
// the mermaid assets are served same-origin.
func TestMermaidPageAndAssets(t *testing.T) {
ns := t.TempDir()
dir := filepath.Join(ns, "u@d.com", "docs")
if err := os.MkdirAll(dir, 0o755); err != nil {
t.Fatal(err)
}
os.WriteFile(filepath.Join(ns, "u@d.com", "Access"), []byte("read,list: all\n*: u@d.com\n"), 0o644)
os.WriteFile(filepath.Join(dir, "diagram.md"),
[]byte("# Diagram\n\n```mermaid\ngraph TD\n A-->B\n```\n"), 0o644)
web := os.DirFS("../../web")
- srv, err := New(Config{Store: store.NewLocalStore(ns), Web: http.FS(web)})
+ srv, err := New(Config{Store: store.NewLocalStore(ns), Web: http.FS(web), CacheDir: t.TempDir()})
if err != nil {
t.Fatal(err)
}
ts := httptest.NewServer(srv.Handler())
defer ts.Close()
get := func(p string) (int, string) {
resp, err := http.Get(ts.URL + p)
if err != nil {
t.Fatal(err)
}
defer resp.Body.Close()
b, _ := io.ReadAll(resp.Body)
return resp.StatusCode, string(b)
}
code, body := get("/u@d.com/docs/diagram.md")
if code != 200 {
t.Fatalf("page status %d", code)
}
if !strings.Contains(body, `<pre class="mermaid">`) {
t.Errorf("rendered page missing mermaid container:\n%s", body)
}
if !strings.Contains(body, "/_assets/mermaid-init.js") {
t.Errorf("page did not include the mermaid init script")
}
if code, _ := get("/_assets/mermaid-init.js"); code != 200 {
t.Errorf("mermaid-init.js status %d, want 200", code)
}
code, bundle := get("/_assets/mermaid.min.js")
if code != 200 {
t.Fatalf("mermaid.min.js status %d, want 200", code)
}
if !strings.Contains(bundle, "mermaid") {
t.Errorf("served bundle does not look like mermaid")
}
}
internal/server/server.go
// Package server is the HTTP layer of the git browser. The upspin path is the
// URL path: GET /<user@domain>/<path...>. If the path resolves inside a bare
// git repository the request is handled by the git browser (summary, tree,
// blob, log, commit, refs); otherwise it falls back to plain upspin namespace
// browsing (directory listings and raw files) so users can drill down to repos.
package server
import (
"bytes"
"embed"
"errors"
"html/template"
"io"
"log"
"mime"
"net/http"
+ "os"
"path"
"strings"
"gitweb/internal/auth"
"gitweb/internal/gitrepo"
"gitweb/internal/render"
"gitweb/internal/store"
)
//go:embed templates/*.html
var tmplFS embed.FS
var pageTmpl = template.Must(template.ParseFS(tmplFS, "templates/page.html"))
// Config configures a Server.
type Config struct {
Store store.WriteStore // read for browse/clone; write for push
Web http.FileSystem // static assets (style.css, landing index.html)
Realm string // HTTP Basic auth realm; defaults to "gitweb"
// Vault, if non-nil, verifies vault-issued SSO bearer tokens (the passkey
// login path). When nil, only PATs are accepted.
Vault *auth.VaultVerifier
+ // CacheDir is where repos are materialized on disk. Its contents are
+ // deleted at startup. Empty means an ephemeral directory under the system
+ // temp dir (dev and tests).
+ CacheDir string
+ // CacheBudgetBytes and MaxRepoBytes bound the repo cache; 0 means the
+ // gitrepo defaults (512 MiB / 256 MiB).
+ CacheBudgetBytes int64
+ MaxRepoBytes int64
}
// Server serves upspin-stored git repositories over HTTP: a browse UI, plus the
// git smart-HTTP protocol for clone and push. Access is gated by upspin Access
// files; private repos require Basic-auth credentials (a vault SSO token or a
// personal access token).
type Server struct {
store store.Store
web http.FileSystem
renderer *render.Renderer
renderers *render.Registry
repos *gitrepo.Cache
authz *auth.Authorizer
pat *auth.PATVerifier
vault *auth.VaultVerifier // may be nil
pusher *gitrepo.Pusher
realm string
chroma []byte // generated highlighter stylesheet
landing []byte // /index.html, served at "/"
}
// New builds a Server.
func New(c Config) (*Server, error) {
css, err := render.CSS()
if err != nil {
return nil, err
}
rnd := render.New()
- repos := gitrepo.NewCache(c.Store)
+ cacheDir := c.CacheDir
+ if cacheDir == "" {
+ cacheDir, err = os.MkdirTemp("", "gitweb-cache-")
+ if err != nil {
+ return nil, err
+ }
+ }
+ repos, err := gitrepo.NewCache(c.Store, gitrepo.CacheOptions{
+ Dir: cacheDir,
+ BudgetBytes: c.CacheBudgetBytes,
+ MaxRepoBytes: c.MaxRepoBytes,
+ })
+ if err != nil {
+ return nil, err
+ }
realm := c.Realm
if realm == "" {
realm = "gitweb"
}
s := &Server{
store: c.Store,
web: c.Web,
renderer: rnd,
renderers: render.NewRegistry(rnd),
repos: repos,
authz: auth.NewAuthorizer(c.Store),
pat: auth.NewPATVerifier(c.Store),
vault: c.Vault,
pusher: gitrepo.NewPusher(c.Store, repos),
realm: realm,
chroma: css,
}
if f, err := c.Web.Open("index.html"); err == nil {
s.landing, _ = io.ReadAll(f)
f.Close()
}
return s, nil
}
// Handler returns the HTTP handler.
func (s *Server) Handler() http.Handler {
mux := http.NewServeMux()
mux.HandleFunc("GET /_assets/chroma.css", s.handleChromaCSS)
mux.Handle("GET /_assets/", http.StripPrefix("/_assets/", http.FileServer(s.web)))
// The catch-all accepts every method: git smart-HTTP needs POST, and
// ServeMux cannot match the "/info/refs", "/git-upload-pack",
// "/git-receive-pack" suffixes, so handleRoot dispatches by suffix itself.
mux.HandleFunc("/", s.handleRoot)
return securityHeaders(mux)
}
// handleRoot dispatches between the git smart-HTTP endpoints and the browse UI.
func (s *Server) handleRoot(w http.ResponseWriter, r *http.Request) {
if ep, ok := gitEndpoint(r); ok {
s.handleGitHTTP(w, r, ep)
return
}
if r.Method != http.MethodGet {
http.Error(w, "405 method not allowed", http.StatusMethodNotAllowed)
return
}
s.handlePath(w, r)
}
func (s *Server) handleChromaCSS(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/css; charset=utf-8")
w.Write(s.chroma)
}
// handlePath dispatches a request whose URL path is an upspin path.
func (s *Server) handlePath(w http.ResponseWriter, r *http.Request) {
raw := strings.TrimPrefix(r.URL.Path, "/")
if raw == "" {
s.serveLanding(w)
return
}
// First element is the upspin user; validate before building any path.
user, rest, _ := strings.Cut(raw, "/")
if !store.ValidUser(user) {
s.notFound(w)
return
}
// Clean the remainder; path.Clean collapses any ".." so it cannot escape
// the user's tree. clean always begins with "/".
clean := path.Clean("/" + rest)
upath := user + clean // e.g. "u@d.com/proj.git"; root is "u@d.com/"
// Identify the caller (anonymous if no/blank credentials) before deciding
// what they may see.
authUser, ok := s.authenticate(w, r)
if !ok {
return // 401 already written
}
// If the path lies inside a bare git repo, hand off to the git browser;
// otherwise fall back to plain upspin namespace browsing.
repoPath, gitRest, isRepo, err := gitrepo.Locate(s.store, user, clean)
if err != nil {
s.storeErr(w, err)
return
}
// Gate the browse UI by the same public/private rule as clone: anonymous
// users see only world-readable paths; others must be granted read.
resource := upath
if isRepo {
resource = repoPath
}
allowed, err := s.authz.CanRead(authUser, resource)
if err != nil {
s.serverErr(w, err)
return
}
if !allowed {
s.denyRead(w, authUser)
return
}
if isRepo {
s.serveGit(w, r, repoPath, gitRest)
return
}
isDir, err := s.store.Stat(upath)
if err != nil {
s.storeErr(w, err)
return
}
if isDir {
s.serveDir(w, r, user, clean, upath)
return
}
s.serveFile(w, upath, clean)
}
// serveDir renders a directory: README.md/index.md if present, else a listing.
func (s *Server) serveDir(w http.ResponseWriter, r *http.Request, user, clean, upath string) {
// A trailing slash is required so relative links resolve against the dir.
if !strings.HasSuffix(r.URL.Path, "/") {
http.Redirect(w, r, r.URL.Path+"/", http.StatusMovedPermanently)
return
}
entries, err := s.store.List(upath)
if err != nil {
s.storeErr(w, err)
return
}
// Prefer an index document if the directory has one.
for _, name := range []string{"index.md", "README.md", "readme.md"} {
if hasEntry(entries, name) {
s.serveFile(w, strings.TrimRight(upath, "/")+"/"+name, clean)
return
}
}
var body bytes.Buffer
body.WriteString(`<h1>Index of `)
body.WriteString(template.HTMLEscapeString(displayPath(user, clean)))
body.WriteString("</h1>\n<ul class=\"dir\">\n")
if clean != "/" {
body.WriteString(`<li><a href="../">../</a></li>` + "\n")
}
for _, e := range entries {
if isAccessFile(e.Name) {
continue // never expose upspin ACLs in a listing
}
name := e.Name
if e.IsDir {
name += "/"
}
esc := template.HTMLEscapeString(name)
body.WriteString(`<li><a href="` + esc + `">` + esc + "</a></li>\n")
}
body.WriteString("</ul>\n")
s.renderPage(w, displayPath(user, clean), template.HTML(body.String()))
}
// serveFile renders a markdown file, or serves any other file raw.
func (s *Server) serveFile(w http.ResponseWriter, upath, clean string) {
// Never serve an upspin Access file's contents (its user/group list),
// even when requested directly by path.
if isAccessFile(path.Base(upath)) {
s.notFound(w)
return
}
data, err := s.store.Get(upath)
if err != nil {
s.storeErr(w, err)
return
}
if isMarkdown(upath) {
html, err := s.renderer.ToHTML(data)
if err != nil {
s.serverErr(w, err)
return
}
s.renderPage(w, path.Base(clean), template.HTML(html))
return
}
// Raw asset (images referenced by markdown, etc.).
ctype := mime.TypeByExtension(path.Ext(upath))
if ctype == "" {
ctype = http.DetectContentType(data)
}
w.Header().Set("Content-Type", ctype)
w.Write(data)
}
func (s *Server) renderPage(w http.ResponseWriter, title string, body template.HTML) {
w.Header().Set("Content-Type", "text/html; charset=utf-8")
err := pageTmpl.Execute(w, struct {
Title string
Body template.HTML
}{Title: title, Body: body})
if err != nil {
log.Printf("gitweb: template: %v", err)
}
}
func (s *Server) serveLanding(w http.ResponseWriter) {
if s.landing == nil {
s.notFound(w)
return
}
w.Header().Set("Content-Type", "text/html; charset=utf-8")
w.Write(s.landing)
}
// --- errors ---
func (s *Server) storeErr(w http.ResponseWriter, err error) {
if errors.Is(err, store.ErrNotFound) {
s.notFound(w)
return
}
s.serverErr(w, err)
}
func (s *Server) notFound(w http.ResponseWriter) {
http.Error(w, "404 not found", http.StatusNotFound)
}
// denyRead responds to a caller who may not read the requested resource. An
// anonymous caller gets a Basic-auth challenge (so a git/browser client can
// retry with credentials); an already-authenticated caller gets a 404, which
// does not reveal whether the resource exists.
func (s *Server) denyRead(w http.ResponseWriter, authUser string) {
if authUser == "" {
s.challenge(w)
return
}
s.notFound(w)
}
// challenge writes a 401 with a Basic-auth challenge. The body is deliberately
// vague so anonymous callers cannot distinguish "private" from "absent".
func (s *Server) challenge(w http.ResponseWriter) {
w.Header().Set("WWW-Authenticate", `Basic realm="`+s.realm+`", charset="UTF-8"`)
http.Error(w, "not found, or authentication required", http.StatusUnauthorized)
}
func (s *Server) serverErr(w http.ResponseWriter, err error) {
log.Printf("gitweb: 500: %v", err)
http.Error(w, "500 internal server error", http.StatusInternalServerError)
}
// --- helpers ---
func isMarkdown(p string) bool {
switch strings.ToLower(path.Ext(p)) {
case ".md", ".markdown":
return true
}
return false
}
// isAccessFile reports whether name is an upspin Access file. gitweb never lists
// or renders these: their contents enumerate the upspin users and groups granted
// access, which must not be exposed to browsers.
func isAccessFile(name string) bool { return name == "Access" }
func hasEntry(entries []store.Entry, name string) bool {
for _, e := range entries {
if !e.IsDir && e.Name == name {
return true
}
}
return false
}
// displayPath renders the upspin path for a heading: "u@d.com/docs/".
func displayPath(user, clean string) string {
if clean == "/" {
return user + "/"
}
return user + clean
}
func securityHeaders(h http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("X-Content-Type-Options", "nosniff")
w.Header().Set("X-Frame-Options", "DENY")
w.Header().Set("Referrer-Policy", "no-referrer")
// script-src 'self': the only scripts are our own, same-origin mermaid
// files (mermaid-init.js and the vendored bundle). No 'unsafe-inline', so
// script injected via rendered markdown cannot execute. style-src allows
// 'unsafe-inline' because mermaid injects diagram styles at render time;
// this permits inline styles but not scripts. Markdown may embed external
// https images; everything else is same-origin.
w.Header().Set("Content-Security-Policy",
"default-src 'self'; script-src 'self'; style-src 'self' 'unsafe-inline'; "+
"img-src 'self' data: https:; base-uri 'none'; form-action 'none'; frame-ancestors 'none'")
h.ServeHTTP(w, r)
})
}
main.go
// Command gitweb serves git repositories stored in upspin over HTTPS: a browse
// UI plus the git smart-HTTP protocol for clone and push. It runs as a single
// upspin user (git@cosmocr.at); a repo owner publishes a path by granting that
// user access in an upspin Access file. The upspin path is the URL path, so a
// repo lives at https://git.cosmocr.at/<user@domain>/<path-to-repo>.
//
// Public vs private is derived from the repo's governing Access file: a repo
// readable by all is public (browse and clone anonymously); otherwise callers
// must authenticate with HTTP Basic auth, supplying either a vault-issued SSO
// token (see cmd/git-credential-cosmocr) or a personal access token. Push
// requires write access in the Access file. See README.md.
package main
import (
"crypto/tls"
"embed"
"flag"
"io/fs"
"log"
"net/http"
"os"
"sync"
"time"
"gitweb/internal/auth"
"gitweb/internal/server"
"gitweb/internal/store"
"upspin.io/config"
"upspin.io/rpc"
"upspin.io/transports"
"upspin.io/upspin"
// Packers used by the upspin client when reading packed files.
_ "upspin.io/pack/ee"
_ "upspin.io/pack/eeintegrity"
_ "upspin.io/pack/plain"
)
//go:embed all:web
var embeddedWeb embed.FS
func main() {
var (
addr = flag.String("addr", ":8446", "listen address")
certFile = flag.String("tls_cert", "", "TLS certificate (PEM)")
keyFile = flag.String("tls_key", "", "TLS key (PEM)")
upspinConf = flag.String("upspin_config", "/gitweb/config", "upspin config file for the gitweb user")
webDir = flag.String("web", "", "serve static assets from this dir (default: embedded)")
devDir = flag.String("dev", "", "if set, read files from this local dir instead of upspin (dev only)")
vaultIss = flag.String("vault_issuer", "vault@cosmocr.at", "upspin user that issues SSO tokens ('' disables passkey/vault auth)")
vaultAud = flag.String("vault_audience", "gitweb", "expected audience of accepted vault SSO tokens")
+ cacheDir = flag.String("cache_dir", "", "repo cache directory — ALL its contents are DELETED at startup (default: fresh dir under the system temp dir)")
+ cacheMB = flag.Int64("cache_budget_mb", 512, "target total size of the on-disk repo cache, in MiB")
+ maxRepoMB = flag.Int64("max_repo_mb", 256, "reject repos larger than this when materializing, in MiB")
)
flag.Parse()
var st store.WriteStore
var vault *auth.VaultVerifier
if *devDir != "" {
st = store.NewLocalStore(*devDir)
log.Printf("gitweb: DEV MODE — local file store at %s", *devDir)
} else {
cfg, err := config.FromFile(*upspinConf)
if err != nil {
log.Fatalf("gitweb: load upspin config: %v", err)
}
transports.Init(cfg)
st = store.NewUpspinStore(cfg)
if *vaultIss != "" {
// Verify vault SSO tokens by looking up the issuer's public key
// through the upspin keyserver.
lookup := rpc.PublicUserKeyService(cfg)
vault = auth.NewVaultVerifier(upspin.UserName(*vaultIss), *vaultAud,
auth.KeyLookup(lookup))
log.Printf("gitweb: accepting SSO tokens from %s (aud=%s)", *vaultIss, *vaultAud)
}
}
var webFS http.FileSystem
if *webDir != "" {
webFS = http.Dir(*webDir)
} else {
sub, err := fs.Sub(embeddedWeb, "web")
if err != nil {
log.Fatalf("gitweb: embed web: %v", err)
}
webFS = http.FS(sub)
}
- srv, err := server.New(server.Config{Store: st, Web: webFS, Vault: vault})
+ srv, err := server.New(server.Config{
+ Store: st,
+ Web: webFS,
+ Vault: vault,
+ CacheDir: *cacheDir,
+ CacheBudgetBytes: *cacheMB << 20,
+ MaxRepoBytes: *maxRepoMB << 20,
+ })
if err != nil {
log.Fatalf("gitweb: init server: %v", err)
}
httpSrv := &http.Server{
Addr: *addr,
Handler: srv.Handler(),
ReadHeaderTimeout: 10 * time.Second,
}
log.Printf("gitweb: serving on %s", *addr)
if *certFile == "" || *keyFile == "" {
log.Fatal(httpSrv.ListenAndServe()) // dev only (e.g. localhost)
}
httpSrv.TLSConfig = &tls.Config{
MinVersion: tls.VersionTLS12,
GetCertificate: (&certReloader{certFile: *certFile, keyFile: *keyFile}).get,
}
log.Fatal(httpSrv.ListenAndServeTLS("", ""))
}
// certReloader serves a file-based TLS certificate, reloading it when the files
// change so certbot renewals are picked up without a restart. (Copied from the
// vault/markdown services, which share this deployment model.)
type certReloader struct {
certFile, keyFile string
mu sync.RWMutex
cert *tls.Certificate
modTime time.Time
}
func (r *certReloader) get(*tls.ClientHelloInfo) (*tls.Certificate, error) {
mt := newest(r.certFile, r.keyFile)
r.mu.RLock()
cert, loaded := r.cert, r.modTime
r.mu.RUnlock()
if cert == nil || mt.After(loaded) {
c, err := tls.LoadX509KeyPair(r.certFile, r.keyFile)
if err != nil {
if cert != nil {
log.Printf("gitweb: cert reload failed, keeping previous: %v", err)
return cert, nil
}
return nil, err
}
r.mu.Lock()
r.cert, r.modTime = &c, mt
r.mu.Unlock()
cert = &c
}
return cert, nil
}
func newest(files ...string) time.Time {
var t time.Time
for _, f := range files {
if fi, err := os.Stat(f); err == nil && fi.ModTime().After(t) {
t = fi.ModTime()
}
}
return t
}