1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
|
package gitrepo
import (
"container/list"
"errors"
"log"
"os"
"path/filepath"
"sync"
"github.com/go-git/go-billy/v5/osfs"
"github.com/go-git/go-git/v5"
"github.com/go-git/go-git/v5/plumbing/cache"
"github.com/go-git/go-git/v5/plumbing/storer"
"github.com/go-git/go-git/v5/storage/filesystem"
"gitweb/internal/store"
)
const (
// defaultCacheBudget bounds the total bytes of materialized repos kept on
// disk; least-recently-used repos are evicted past this.
defaultCacheBudget = 512 << 20 // 512 MiB
// objectCacheSize is the single shared go-git decompressed-object LRU,
// replacing the 96 MiB-per-repo default. Objects are keyed by content
// hash, so one cache is safely shared across all repos and pushes.
objectCacheSize = 32 * cache.MiByte
)
// CacheOptions configures a Cache.
type CacheOptions struct {
// Dir is the cache directory. Required. ALL of its children are deleted
// when the cache is created (crash leftovers) and it is managed
// exclusively by the cache thereafter.
Dir string
// BudgetBytes is the target total size of cached repos on disk; 0 means
// the 512 MiB default. It is a soft target: entries still referenced by
// in-flight requests survive eviction until released.
BudgetBytes int64
// MaxRepoBytes caps a single repo's materialized size; 0 means the
// 256 MiB default.
MaxRepoBytes int64
}
// Cache materializes upspin-stored repos into per-entry directories under a
// cache root and caches the opened repositories, keyed by upspin repo path.
// An entry is reused while the repo's store Sequence is unchanged; any write
// to the repo bumps that sequence and forces re-materialization on the next
// Get. Total size is kept near BudgetBytes by LRU eviction.
type Cache struct {
st store.Store
dir string
budget int64
maxRepo int64
objects cache.Object // shared across all storers (content-hash keyed)
mu sync.Mutex
m map[string]*entry // repoPath -> live entry
lru *list.List // front = most recently used; Value = *entry
total int64 // Σ size over live entries
}
// entry is a materialized, opened repo. Lifecycle: LIVE (in m/lru, size
// counted in total) → RETIRED (dropped from m/lru on staleness, eviction, or
// Invalidate; dir still on disk for in-flight holders) → GONE (dir removed,
// once retired and the last reference is released).
type entry struct {
key string
repo *git.Repository
seq int64 // store Sequence at materialization time
dir string // per-entry directory under Cache.dir
size int64 // Σ manifest sizes
refs int // in-flight holders (Get callers who have not released)
retired bool
elem *list.Element // nil once retired
}
// NewCache returns a repo cache backed by st, materializing into opts.Dir.
// Any existing children of opts.Dir are deleted.
func NewCache(st store.Store, opts CacheOptions) (*Cache, error) {
if opts.Dir == "" {
return nil, errors.New("gitrepo: CacheOptions.Dir is required")
}
if err := os.MkdirAll(opts.Dir, 0o700); err != nil {
return nil, err
}
// Sweep leftovers from a previous run. Children only — the root may be a
// mountpoint.
children, err := os.ReadDir(opts.Dir)
if err != nil {
return nil, err
}
for _, ch := range children {
if err := os.RemoveAll(filepath.Join(opts.Dir, ch.Name())); err != nil {
return nil, err
}
}
budget := opts.BudgetBytes
if budget == 0 {
budget = defaultCacheBudget
}
maxRepo := opts.MaxRepoBytes
if maxRepo == 0 {
maxRepo = defaultMaxRepoBytes
}
return &Cache{
st: st,
dir: opts.Dir,
budget: budget,
maxRepo: maxRepo,
objects: cache.NewObjectLRU(objectCacheSize),
m: make(map[string]*entry),
lru: list.New(),
}, nil
}
// Get returns an opened repository for repoPath, materializing it if absent or
// stale, together with a release func the caller must invoke once it is done
// with the repository (defer it). The repo's on-disk backing survives eviction
// and invalidation until every holder has released. The returned repository is
// read-only and safe for concurrent use across requests. release is idempotent
// and nil when err is non-nil.
func (c *Cache) Get(repoPath string) (*git.Repository, func(), error) {
seq, err := c.st.Seq(repoPath)
if err != nil {
return nil, nil, err
}
c.mu.Lock()
if e, ok := c.m[repoPath]; ok {
if e.seq == seq {
e.refs++
c.lru.MoveToFront(e.elem)
c.mu.Unlock()
return e.repo, c.releaseFunc(e), nil
}
// Stale: retire it and re-materialize below.
dir := c.retire(e)
c.mu.Unlock()
removeDir(dir)
} else {
c.mu.Unlock()
}
// Materialize outside the lock (network/IO heavy).
ne, err := c.open(repoPath, seq)
if err != nil {
return nil, nil, err
}
c.mu.Lock()
// Another request may have populated it meanwhile; prefer the existing
// entry if it is current. Ours was never published and has no holders, so
// its directory can be deleted outright.
var dirs []string
if e, ok := c.m[repoPath]; ok {
if e.seq == seq {
e.refs++
c.lru.MoveToFront(e.elem)
c.mu.Unlock()
removeDir(ne.dir)
return e.repo, c.releaseFunc(e), nil
}
if d := c.retire(e); d != "" {
dirs = append(dirs, d)
}
}
ne.refs = 1
ne.elem = c.lru.PushFront(ne)
c.m[repoPath] = ne
c.total += ne.size
dirs = append(dirs, c.evict()...)
c.mu.Unlock()
for _, d := range dirs {
removeDir(d)
}
return ne.repo, c.releaseFunc(ne), nil
}
// GetStorer returns the go-git storer for repoPath plus a release func, with
// the same lifecycle as Get. The git smart-HTTP paths need the storer (not the
// higher-level *git.Repository); upload-pack in particular streams from it for
// the whole response, so release must not be called before the response body
// is fully written. The returned storer is read-only in practice; push uses a
// separate private storer (see push.go).
func (c *Cache) GetStorer(repoPath string) (storer.Storer, func(), error) {
repo, release, err := c.Get(repoPath)
if err != nil {
return nil, nil, err
}
return repo.Storer, release, nil
}
// Invalidate retires any cached entry for repoPath, forcing the next Get to
// re-materialize. Called after a successful push so readers see the new refs.
// The entry's directory survives until in-flight holders release.
func (c *Cache) Invalidate(repoPath string) {
c.mu.Lock()
var dir string
if e, ok := c.m[repoPath]; ok {
dir = c.retire(e)
}
c.mu.Unlock()
removeDir(dir)
}
// open materializes repoPath into a fresh directory under c.dir and opens it
// (no locking). On error the partial directory is removed.
func (c *Cache) open(repoPath string, seq int64) (e *entry, err error) {
dir, err := os.MkdirTemp(c.dir, "repo-")
if err != nil {
return nil, err
}
defer func() {
if err != nil {
removeDir(dir)
}
}()
fs := osfs.New(dir)
man, err := materialize(c.st, repoPath, fs, c.maxRepo)
if err != nil {
return nil, err
}
sto := filesystem.NewStorage(fs, c.objects)
repo, err := git.Open(sto, nil) // nil worktree => bare
if err != nil {
return nil, err
}
var size int64
for _, fi := range man {
size += fi.Size
}
return &entry{key: repoPath, repo: repo, seq: seq, dir: dir, size: size}, nil
}
// releaseFunc returns the idempotent release closure handed to Get callers.
func (c *Cache) releaseFunc(e *entry) func() {
var once sync.Once
return func() {
once.Do(func() {
c.mu.Lock()
e.refs--
var dir string
if e.retired && e.refs == 0 {
dir = e.dir
}
c.mu.Unlock()
removeDir(dir)
})
}
}
// retire drops e from the map and LRU and uncounts its size (caller holds mu).
// It returns e's directory if no holder remains — the caller must delete it
// after unlocking — or "" if deletion is deferred to the last release.
func (c *Cache) retire(e *entry) string {
if e.retired {
return ""
}
e.retired = true
c.lru.Remove(e.elem)
e.elem = nil
delete(c.m, e.key)
c.total -= e.size
if e.refs == 0 {
return e.dir
}
return ""
}
// evict retires least-recently-used entries until the live total fits the
// budget (caller holds mu), returning directories the caller must delete after
// unlocking. At least one entry is always kept, so a single repo larger than
// the whole budget remains servable — the budget is a target, not a hard cap.
func (c *Cache) evict() (dirs []string) {
for c.total > c.budget && c.lru.Len() > 1 {
e := c.lru.Back().Value.(*entry)
if d := c.retire(e); d != "" {
dirs = append(dirs, d)
}
}
return dirs
}
// removeDir deletes a retired entry's directory. A failure leaks disk space
// until the next startup sweep, so it is logged rather than ignored.
func removeDir(dir string) {
if dir == "" {
return
}
if err := os.RemoveAll(dir); err != nil {
log.Printf("gitrepo: removing cached repo dir: %v", err)
}
}
|