gitmote

gitmote  ·  history  ·  branches

feat: serve the ref advertisement without hydrating objects

2a8dcf804bd62a9c290f8510c4dc4d6f6cc7a36c
Atanas Minev <atmin@protonmail.com> · 2026-07-03 15:53

diff --git a/internal/githttp/backend.go b/internal/githttp/backend.go
index e1f84e1..6c4186a 100644
--- a/internal/githttp/backend.go
+++ b/internal/githttp/backend.go
@@ -123,7 +123,15 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
// Read, or a receive-pack advertisement: materialize then hand off to
// http-backend. The receive advertisement needs http.receivepack enabled.
- if !h.materialize(w, r, repoName) {
+ //
+ // The info/refs advertisement only lists refs, so it materializes refs
+ // alone — no object hydration (task 13 / docs/notes/object-hydration.md).
+ // The upload-pack POST that transfers history still full-hydrates.
+ materialized := h.materialize
+ if endpoint == "info/refs" {
+ materialized = h.materializeRefs
+ }
+ if !materialized(w, r, repoName) {
return
}
var extra []string
@@ -170,7 +178,22 @@ func (h *Handler) serveReceivePack(w http.ResponseWriter, r *http.Request, repoN
// materialize builds the on-disk repo, writing the HTTP error itself on failure
// and returning false so the caller stops.
func (h *Handler) materialize(w http.ResponseWriter, r *http.Request, repoName string) bool {
- if _, err := h.mz.Materialize(r.Context(), repoName); err != nil {
+ _, err := h.mz.Materialize(r.Context(), repoName)
+ return h.finishMaterialize(w, r, repoName, err)
+}
+
+// materializeRefs makes the repo's ref advertisement current without hydrating
+// objects — the info/refs path. It has the same signature and error handling as
+// materialize.
+func (h *Handler) materializeRefs(w http.ResponseWriter, r *http.Request, repoName string) bool {
+ _, err := h.mz.MaterializeRefs(r.Context(), repoName)
+ return h.finishMaterialize(w, r, repoName, err)
+}
+
+// finishMaterialize maps a materialize error to the HTTP response, writing a 404
+// for an unknown repo and a 500 otherwise, and returns whether it succeeded.
+func (h *Handler) finishMaterialize(w http.ResponseWriter, r *http.Request, repoName string, err error) bool {
+ if err != nil {
if errors.Is(err, meta.ErrNotFound) {
http.NotFound(w, r)
return false
diff --git a/internal/repo/materialize.go b/internal/repo/materialize.go
index f4356a1..9692dbb 100644
--- a/internal/repo/materialize.go
+++ b/internal/repo/materialize.go
@@ -44,19 +44,71 @@ func (mz *Materializer) Root() string { return mz.root }
//
// Hydration policy is full-hydrate for the MVP (see task 04 /
// docs/notes/object-hydration.md): every object under the repo's store prefix
-// is pulled. gitmote's own repo is tiny, so this is the safe, simple first cut;
-// bounded per-operation closures are a later optimization.
+// is pulled. This is the path for the data POSTs (upload-pack / receive-pack)
+// that actually transfer history; the info/refs advertisement is served
+// objectless via MaterializeRefs. gitmote's own repo is tiny, so full-hydrate
+// on the data path is the safe, simple first cut; bounded per-operation
+// closures are a later optimization.
func (mz *Materializer) Materialize(ctx context.Context, name string) (string, error) {
r, err := mz.meta.GetRepo(ctx, name)
if err != nil {
return "", err
}
- dir, err := mz.repoDir(name)
+ dir, err := mz.ensureRepo(ctx, name)
+ if err != nil {
+ return "", err
+ }
+
+ if err := mz.hydrateObjects(ctx, name, dir); err != nil {
+ return "", err
+ }
+ if err := mz.writeRefs(ctx, dir, r); err != nil {
+ return "", err
+ }
+ return dir, nil
+}
+
+// MaterializeRefs ensures a bare repo for name exists and makes its ref
+// advertisement current — without hydrating a single object. It writes
+// packed-refs directly from the metadata layer (not update-ref, which requires
+// each target object present) and points HEAD at the default branch. This
+// serves the info/refs GET, which lists refs but transfers no history — the
+// first slice of bounded hydration (docs/notes/object-hydration.md). Returns
+// meta.ErrNotFound if the repo is unknown.
+//
+// MVP omits annotated-tag ^{} peel lines: git still advertises the tag ref and
+// merely drops the peel hint (a lost negotiation optimization, not a
+// correctness issue). A refs-only packed-refs and a later full Materialize's
+// loose refs coexist on disk — loose refs override, which is correct.
+func (mz *Materializer) MaterializeRefs(ctx context.Context, name string) (string, error) {
+ r, err := mz.meta.GetRepo(ctx, name)
if err != nil {
return "", err
}
+ dir, err := mz.ensureRepo(ctx, name)
+ if err != nil {
+ return "", err
+ }
+
+ refs, err := mz.meta.ListRefs(ctx, r.ID)
+ if err != nil {
+ return "", err
+ }
+ if err := writePackedRefs(dir, refs); err != nil {
+ return "", err
+ }
+ return dir, setHEAD(ctx, dir, r.DefaultBranch)
+}
+
+// ensureRepo maps name to its cache dir and guarantees a bare repo exists there,
+// creating one on a cache miss (rebuild-on-miss). It touches no refs or objects.
+func (mz *Materializer) ensureRepo(ctx context.Context, name string) (string, error) {
+ dir, err := mz.repoDir(name)
+ if err != nil {
+ return "", err
+ }
if _, err := os.Stat(dir); errors.Is(err, os.ErrNotExist) {
if err := os.MkdirAll(filepath.Dir(dir), 0o755); err != nil {
return "", err
@@ -67,13 +119,6 @@ func (mz *Materializer) Materialize(ctx context.Context, name string) (string, e
} else if err != nil {
return "", err
}
-
- if err := mz.hydrateObjects(ctx, name, dir); err != nil {
- return "", err
- }
- if err := mz.writeRefs(ctx, dir, r); err != nil {
- return "", err
- }
return dir, nil
}
@@ -159,9 +204,56 @@ func (mz *Materializer) writeRefs(ctx context.Context, dir string, r *meta.Repo)
return err
}
}
- // HEAD is derived from default_branch, not stored as a ref (storage.md).
- // symbolic-ref is happy to point at a branch that has no commits yet.
- return runGit(ctx, dir, "symbolic-ref", "HEAD", path.Join("refs/heads", r.DefaultBranch))
+ return setHEAD(ctx, dir, r.DefaultBranch)
+}
+
+// writePackedRefs overwrites the repo's packed-refs file from refs, atomically
+// (temp file + rename, so a concurrent reader never sees a torn file). Each line
+// is "<sha> <refname>"; the MVP writes no ^{} peel lines and no traits header,
+// so git peels from the objects when present and simply omits the peel hint when
+// they are absent — exactly the objectless-advertisement behavior we want. git
+// advertises refs whose target objects are absent, which is the whole point.
+func writePackedRefs(dir string, refs []meta.Ref) error {
+ var b strings.Builder
+ for _, ref := range refs {
+ b.WriteString(ref.SHA)
+ b.WriteByte(' ')
+ b.WriteString(ref.Name)
+ b.WriteByte('\n')
+ }
+ return writeFileAtomic(filepath.Join(dir, "packed-refs"), []byte(b.String()))
+}
+
+// setHEAD points the repo's HEAD at the default branch. HEAD is derived from
+// default_branch, not stored as a ref (storage.md); symbolic-ref is happy to
+// point at a branch that has no commits yet.
+func setHEAD(ctx context.Context, dir, defaultBranch string) error {
+ return runGit(ctx, dir, "symbolic-ref", "HEAD", path.Join("refs/heads", defaultBranch))
+}
+
+// writeFileAtomic writes data to path via a temp file in the same directory
+// followed by a rename, so a reader observes either the old file or the complete
+// new one, never a partial write.
+func writeFileAtomic(dst string, data []byte) error {
+ tmp, err := os.CreateTemp(filepath.Dir(dst), ".packed-refs-*")
+ if err != nil {
+ return err
+ }
+ tmpName := tmp.Name()
+ if _, err := tmp.Write(data); err != nil {
+ _ = tmp.Close()
+ _ = os.Remove(tmpName)
+ return err
+ }
+ if err := tmp.Close(); err != nil {
+ _ = os.Remove(tmpName)
+ return err
+ }
+ if err := os.Rename(tmpName, dst); err != nil {
+ _ = os.Remove(tmpName)
+ return err
+ }
+ return nil
}
// runGit runs a git subcommand, using dir as the working directory when set.
diff --git a/internal/repo/materialize_test.go b/internal/repo/materialize_test.go
index 0460bcb..f1bda57 100644
--- a/internal/repo/materialize_test.go
+++ b/internal/repo/materialize_test.go
@@ -4,6 +4,7 @@ import (
"bytes"
"context"
"errors"
+ "io"
"os"
"os/exec"
"path/filepath"
@@ -168,6 +169,78 @@ func assertRepo(t *testing.T, dir, branch, head string) {
git(t, dir, "fsck", "--strict") // fails the test on any corruption/missing object
}
+// spyStore wraps a Store and counts Get calls, so a test can assert the
+// advertisement path never touches an object.
+type spyStore struct {
+ store.Store
+ gets int
+}
+
+func (s *spyStore) Get(ctx context.Context, key string) (io.ReadCloser, error) {
+ s.gets++
+ return s.Store.Get(ctx, key)
+}
+
+// advertise runs `git upload-pack --advertise-refs` against dir — exactly what
+// the info/refs GET serves — and returns its output, failing on a nonzero exit.
+func advertise(t *testing.T, dir string) string {
+ t.Helper()
+ return git(t, "", "upload-pack", "--advertise-refs", dir)
+}
+
+func TestMaterializeRefsNoObjects(t *testing.T) {
+ ctx := context.Background()
+ m := openMeta(t)
+ s := &spyStore{Store: store.NewMem()}
+ const repoName = "atmin/dotfiles"
+ branch, head := seedRepo(t, m, s, repoName, false)
+
+ // An annotated tag: meta points the ref at the tag object, which is never
+ // hydrated. The advertisement must still list it.
+ tag := "refs/tags/v1"
+ r, err := m.GetRepo(ctx, repoName)
+ if err != nil {
+ t.Fatalf("GetRepo: %v", err)
+ }
+ tagSHA := "1234567890123456789012345678901234567890"
+ if err := m.CASRef(ctx, r.ID, tag, meta.ZeroSHA, tagSHA); err != nil {
+ t.Fatalf("CASRef tag: %v", err)
+ }
+
+ mz := New(m, s, t.TempDir())
+ s.gets = 0 // seedRepo did no Gets, but be explicit.
+ dir, err := mz.MaterializeRefs(ctx, repoName)
+ if err != nil {
+ t.Fatalf("MaterializeRefs: %v", err)
+ }
+ if s.gets != 0 {
+ t.Errorf("MaterializeRefs called store.Get %d times, want 0", s.gets)
+ }
+
+ // The advertisement lists exactly the meta refs and their SHAs, with HEAD
+ // tracking the default branch — all with no objects on disk.
+ adv := advertise(t, dir)
+ for _, want := range []string{
+ head + " refs/heads/" + branch,
+ tagSHA + " " + tag,
+ "symref=HEAD:refs/heads/" + branch,
+ } {
+ if !strings.Contains(adv, want) {
+ t.Errorf("advertisement missing %q\n%s", want, adv)
+ }
+ }
+ // MVP omits the annotated-tag peel line (the tag object is absent).
+ if strings.Contains(adv, tag+"^{}") {
+ t.Errorf("advertisement unexpectedly carries a peel line for %s\n%s", tag, adv)
+ }
+ // No object was ever written to disk.
+ if entries, err := os.ReadDir(filepath.Join(dir, "objects", "pack")); err != nil {
+ t.Fatalf("ReadDir objects/pack: %v", err)
+ } else if len(entries) != 0 {
+ t.Errorf("objects/pack has %d entries, want 0 (no hydration)", len(entries))
+ }
+}
+
func TestMaterializeUnknownRepo(t *testing.T) {
mz := New(openMeta(t), store.NewMem(), t.TempDir())
if _, err := mz.Materialize(context.Background(), "nope/nope"); !errors.Is(err, meta.ErrNotFound) {
diff --git a/tasks/13-advertise-without-hydration.md b/tasks/13-advertise-without-hydration.md
deleted file mode 100644
index 98e0892..0000000
--- a/tasks/13-advertise-without-hydration.md
+++ /dev/null
@@ -1,69 +0,0 @@
-# Serve the ref advertisement without hydrating objects
-
-## Spec
-
-The `info/refs` advertisement — the GET that opens every clone, fetch, and
-`ls-remote` — must not hydrate object history. It only lists refs, which gitmote
-already holds in s3lite. Today a cold advertisement downloads the whole repo just
-to print ref names; it shouldn't touch a single object.
-
-This is the first, separable slice of bounded hydration
-([docs/notes/object-hydration.md](../docs/notes/object-hydration.md)); the full
-per-operation closure for the data POSTs stays deferred (it needs a reachability
-index — see the note).
-
-## Current
-
-[materialize.go](../internal/repo/materialize.go) `Materialize` runs on **every**
-request — including the `info/refs` GET — and always `hydrateObjects` (full
-closure) then `writeRefs` via `git update-ref`, which *requires each ref's target
-object present*. So the advertisement path drags the entire repo onto disk before
-git prints a ref list.
-
-## Why it's safe (spiked)
-
-`git upload-pack --advertise-refs` advertises refs whose target objects are
-**absent** — verified against git 2.50:
-
-- Refs written into `packed-refs` with no objects present → full, correct
- advertisement (SHAs, `symref=HEAD`, capabilities, exit 0).
-- An annotated-tag ref with its tag object absent → advertised fine; git just
- **omits the `^{}` peel line** (correct, only a lost negotiation hint). With a
- pre-computed `^<commit>` peel line in `packed-refs`, it emits the peel too —
- still no object needed.
-
-The advertisement and the later data POST are both served by the *same* stock
-git, so there is no capability/protocol drift risk — we only skip hydration.
-
-## Change
-
-- Add a refs-only path to the Materializer (e.g. `MaterializeRefs(ctx, name)`):
- ensure the bare repo exists, regenerate `packed-refs` **directly from
- `meta.ListRefs`** (write the file — not `update-ref`, which validates object
- presence), set `HEAD` via `symbolic-ref` from `default_branch`. **No object
- hydration.** MVP omits annotated-tag `^{}` peels (advertisement stays correct);
- storing peels in meta for the optimization is an optional follow-up.
-- In [backend.go](../internal/githttp/backend.go) `ServeHTTP`, when
- `endpoint == "info/refs"` (either service), call `MaterializeRefs`. The data
- POSTs (`git-upload-pack`, `git-receive-pack`) keep the full `Materialize` —
- bounding *those* closures is the deferred task.
-- Factor the `packed-refs` generation so the refs-only and full paths agree on
- ref formatting.
-
-**Notes / out of scope:** refs-only `packed-refs` and a later full-`Materialize`'s
-loose refs coexist (loose overrides — fine). Pruning refs deleted from meta is a
-pre-existing cache gap, unchanged here. The data-transfer POSTs still full-hydrate
-until the bounded-closure task lands.
-
-## Verify
-
-- **No objects touched:** an `info/refs` advertisement (both services) served from
- a repo whose objects were never hydrated returns the correct ref list — assert
- via a spy `store` that `Get` is **never** called on the advertisement path, and
- the advertised refs/SHAs match `meta.ListRefs`. Cover a repo with an annotated
- tag (advertises, peel omitted).
-- **Data path intact:** a real clone/fetch and a push still work end-to-end (the
- POSTs full-hydrate as before) — existing githttp read/write tests stay green.
-- **Regression:** `make e2e-local` + `make e2e-restore` green; `go test ./...`.
-- Update materialize.go's "full-hydrate for the MVP" comment to reflect that the
- advertisement is now objectless.
diff --git a/tasks/README.md b/tasks/README.md
index c3d0475..2f7c409 100644
--- a/tasks/README.md
+++ b/tasks/README.md
@@ -12,11 +12,6 @@ starts (litestream restore), and enforces the single writer by a lease (s3lite
`RoleAuto`), so rolling deploys are safe by construction. Speculative directions
live in [../docs/evolution/](../docs/evolution/).
-- [13-advertise-without-hydration.md](13-advertise-without-hydration.md) — serve
- the `info/refs` advertisement from s3lite refs (via `packed-refs`) with zero
- object hydration. First, separable slice of bounded hydration; the full
- per-operation closure for data POSTs is deferred (needs a reachability index —
- see [../docs/notes/object-hydration.md](../docs/notes/object-hydration.md)).
- [14-browse-repo.md](14-browse-repo.md) — read-only repo browsing in the web UI
(admin-gated): tree, blob, raw download, commit log, single-commit diff, ref
switcher — all cheap `git` plumbing over the already-materialized repo. No