gitmote

gitmote  ·  history  ·  branches

feat: repo materialization — bare repo cache from refs + objects

33eaeae179c483c44040b7a3a41625272ef232c8
Atanas Minev <atmin@protonmail.com> · 2026-07-02 17:05

diff --git a/internal/repo/materialize.go b/internal/repo/materialize.go
new file mode 100644
index 0000000..8578fad
--- /dev/null
+++ b/internal/repo/materialize.go
@@ -0,0 +1,171 @@
+// Package repo materializes on-disk bare git repositories from the durable
+// sources of truth: refs from the metadata layer, objects from the object
+// store. A materialized repo is a disposable cache (docs/architecture/storage.md
+// → ephemeral disk), never authoritative — deleting the cache dir and
+// rebuilding must reproduce an equivalent repo.
+package repo
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "io"
+ "os"
+ "os/exec"
+ "path"
+ "path/filepath"
+ "strings"
+
+ "github.com/atmin/gitmote/internal/meta"
+ "github.com/atmin/gitmote/internal/store"
+)
+
+// Materializer builds and refreshes cached bare repos under a root directory.
+type Materializer struct {
+ meta *meta.Metadata
+ store store.Store
+ root string
+}
+
+// New returns a Materializer that caches repos under root.
+func New(m *meta.Metadata, s store.Store, root string) *Materializer {
+ return &Materializer{meta: m, store: s, root: root}
+}
+
+// Materialize ensures a valid on-disk bare repo for name exists under the cache
+// root and returns its path. It creates the repo on a cache miss (rebuild-on-
+// miss), hydrates the object closure from the store, and writes refs from the
+// metadata layer — so a warm repo is refreshed to the current refs and a cold
+// one is built from scratch. Returns meta.ErrNotFound if the repo is unknown.
+//
+// Hydration policy is full-hydrate for the MVP (see task 04 /
+// docs/notes/object-hydration.md): every object under the repo's store prefix
+// is pulled. gitmote's own repo is tiny, so this is the safe, simple first cut;
+// bounded per-operation closures are a later optimization.
+func (mz *Materializer) Materialize(ctx context.Context, name string) (string, error) {
+ r, err := mz.meta.GetRepo(ctx, name)
+ if err != nil {
+ return "", err
+ }
+
+ dir, err := mz.repoDir(name)
+ if err != nil {
+ return "", err
+ }
+
+ if _, err := os.Stat(dir); errors.Is(err, os.ErrNotExist) {
+ if err := os.MkdirAll(filepath.Dir(dir), 0o755); err != nil {
+ return "", err
+ }
+ if err := runGit(ctx, "", "init", "--bare", dir); err != nil {
+ return "", err
+ }
+ } else if err != nil {
+ return "", err
+ }
+
+ if err := mz.hydrateObjects(ctx, name, dir); err != nil {
+ return "", err
+ }
+ if err := mz.writeRefs(ctx, dir, r); err != nil {
+ return "", err
+ }
+ return dir, nil
+}
+
+// repoDir maps a repo name to its cache path, rejecting names that would escape
+// the cache root (absolute paths, "..", or anything filepath.Clean rewrites).
+func (mz *Materializer) repoDir(name string) (string, error) {
+ if name == "" || filepath.IsAbs(name) ||
+ filepath.ToSlash(filepath.Clean(name)) != name ||
+ strings.HasPrefix(name, "../") {
+ return "", fmt.Errorf("repo: unsafe repo name %q", name)
+ }
+ return filepath.Join(mz.root, filepath.FromSlash(name)), nil
+}
+
+// hydrateObjects copies every object under the repo's store prefix onto disk,
+// mirroring the store's {repo}/objects/… layout into the bare repo's objects/
+// dir. Objects are immutable and content-addressed, so an object already on
+// disk is left untouched.
+func (mz *Materializer) hydrateObjects(ctx context.Context, name, dir string) error {
+ prefix := name + "/objects/"
+ keys, err := mz.store.List(ctx, prefix)
+ if err != nil {
+ return err
+ }
+ for _, key := range keys {
+ rel := strings.TrimPrefix(key, name+"/") // e.g. "objects/ab/cdef…"
+ dst := filepath.Join(dir, filepath.FromSlash(rel))
+ if _, err := os.Stat(dst); err == nil {
+ continue
+ } else if !errors.Is(err, os.ErrNotExist) {
+ return err
+ }
+ if err := mz.copyObject(ctx, key, dst); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+// copyObject streams one store object to dst, writing to a temp file first and
+// renaming into place so a reader never observes a partial object.
+func (mz *Materializer) copyObject(ctx context.Context, key, dst string) error {
+ rc, err := mz.store.Get(ctx, key)
+ if err != nil {
+ return err
+ }
+ defer rc.Close()
+
+ if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil {
+ return err
+ }
+ tmp, err := os.CreateTemp(filepath.Dir(dst), ".hydrate-*")
+ if err != nil {
+ return err
+ }
+ tmpName := tmp.Name()
+ if _, err := io.Copy(tmp, rc); err != nil {
+ _ = tmp.Close()
+ _ = os.Remove(tmpName)
+ return err
+ }
+ if err := tmp.Close(); err != nil {
+ _ = os.Remove(tmpName)
+ return err
+ }
+ if err := os.Rename(tmpName, dst); err != nil {
+ _ = os.Remove(tmpName)
+ return err
+ }
+ return nil
+}
+
+// writeRefs makes the on-disk refs match the metadata layer and points HEAD at
+// the repo's default branch. update-ref validates that each target object is
+// present, so hydration must run first.
+func (mz *Materializer) writeRefs(ctx context.Context, dir string, r *meta.Repo) error {
+ refs, err := mz.meta.ListRefs(ctx, r.ID)
+ if err != nil {
+ return err
+ }
+ for _, ref := range refs {
+ if err := runGit(ctx, dir, "update-ref", ref.Name, ref.SHA); err != nil {
+ return err
+ }
+ }
+ // HEAD is derived from default_branch, not stored as a ref (storage.md).
+ // symbolic-ref is happy to point at a branch that has no commits yet.
+ return runGit(ctx, dir, "symbolic-ref", "HEAD", path.Join("refs/heads", r.DefaultBranch))
+}
+
+// runGit runs a git subcommand, using dir as the working directory when set.
+func runGit(ctx context.Context, dir string, args ...string) error {
+ cmd := exec.CommandContext(ctx, "git", args...)
+ cmd.Dir = dir
+ if out, err := cmd.CombinedOutput(); err != nil {
+ return fmt.Errorf("git %s: %w: %s", strings.Join(args, " "), err, strings.TrimSpace(string(out)))
+ }
+ return nil
+}
diff --git a/internal/repo/materialize_test.go b/internal/repo/materialize_test.go
new file mode 100644
index 0000000..0460bcb
--- /dev/null
+++ b/internal/repo/materialize_test.go
@@ -0,0 +1,189 @@
+package repo
+
+import (
+ "bytes"
+ "context"
+ "errors"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "strings"
+ "testing"
+
+ "github.com/atmin/gitmote/internal/meta"
+ "github.com/atmin/gitmote/internal/store"
+)
+
+// git runs a git command hermetically (no user/system config, fixed identity)
+// and returns its trimmed combined output, failing the test on a nonzero exit.
+func git(t *testing.T, dir string, args ...string) string {
+ t.Helper()
+ return gitStdin(t, dir, nil, args...)
+}
+
+func gitStdin(t *testing.T, dir string, stdin []byte, args ...string) string {
+ t.Helper()
+ cmd := exec.CommandContext(context.Background(), "git", args...)
+ cmd.Dir = dir
+ cmd.Env = append(os.Environ(),
+ "GIT_AUTHOR_NAME=t", "GIT_AUTHOR_EMAIL=t@example.com",
+ "GIT_COMMITTER_NAME=t", "GIT_COMMITTER_EMAIL=t@example.com",
+ "GIT_CONFIG_GLOBAL=/dev/null", "GIT_CONFIG_SYSTEM=/dev/null",
+ )
+ if stdin != nil {
+ cmd.Stdin = bytes.NewReader(stdin)
+ }
+ out, err := cmd.CombinedOutput()
+ if err != nil {
+ t.Fatalf("git %s: %v\n%s", strings.Join(args, " "), err, out)
+ }
+ return strings.TrimSpace(string(out))
+}
+
+// openMeta returns a fresh local-only metadata DB.
+func openMeta(t *testing.T) *meta.Metadata {
+ t.Helper()
+ m, err := meta.Open(context.Background(), meta.Config{
+ LocalPath: filepath.Join(t.TempDir(), "meta.sqlite3"),
+ })
+ if err != nil {
+ t.Fatalf("meta.Open: %v", err)
+ }
+ t.Cleanup(func() { _ = m.Close() })
+ return m
+}
+
+// seedRepo builds a real repo with two commits, loads its objects into the
+// store under name's prefix, records the repo + branch ref in meta, and returns
+// (branch, headSHA). When repack is set the source is packed first, so the
+// store holds a packfile instead of loose objects.
+func seedRepo(t *testing.T, m *meta.Metadata, s store.Store, name string, repack bool) (string, string) {
+ t.Helper()
+ ctx := context.Background()
+
+ src := t.TempDir()
+ git(t, src, "init", "-b", "main", ".")
+ if err := os.WriteFile(filepath.Join(src, "README.md"), []byte("hello\n"), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ git(t, src, "add", "README.md")
+ git(t, src, "commit", "-m", "first")
+ if err := os.WriteFile(filepath.Join(src, "README.md"), []byte("hello\nworld\n"), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ git(t, src, "commit", "-am", "second")
+ if repack {
+ git(t, src, "repack", "-ad")
+ }
+
+ branch := git(t, src, "rev-parse", "--abbrev-ref", "HEAD")
+ head := git(t, src, "rev-parse", "HEAD")
+
+ // Mirror .git/objects/… into the store under {name}/objects/….
+ objRoot := filepath.Join(src, ".git", "objects")
+ err := filepath.WalkDir(objRoot, func(p string, d os.DirEntry, err error) error {
+ if err != nil || d.IsDir() {
+ return err
+ }
+ rel, err := filepath.Rel(objRoot, p)
+ if err != nil {
+ return err
+ }
+ data, err := os.ReadFile(p)
+ if err != nil {
+ return err
+ }
+ key := name + "/objects/" + filepath.ToSlash(rel)
+ return s.Put(ctx, key, bytes.NewReader(data))
+ })
+ if err != nil {
+ t.Fatalf("seed objects: %v", err)
+ }
+
+ r, err := m.CreateRepo(ctx, name, branch)
+ if err != nil {
+ t.Fatalf("CreateRepo: %v", err)
+ }
+ if err := m.CASRef(ctx, r.ID, "refs/heads/"+branch, meta.ZeroSHA, head); err != nil {
+ t.Fatalf("CASRef: %v", err)
+ }
+ return branch, head
+}
+
+func TestMaterialize(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ repack bool
+ }{
+ {"loose", false},
+ {"packed", true},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ ctx := context.Background()
+ m := openMeta(t)
+ s := store.NewMem()
+ const repoName = "atmin/dotfiles"
+ branch, head := seedRepo(t, m, s, repoName, tc.repack)
+
+ mz := New(m, s, t.TempDir())
+ dir, err := mz.Materialize(ctx, repoName)
+ if err != nil {
+ t.Fatalf("Materialize: %v", err)
+ }
+
+ assertRepo(t, dir, branch, head)
+
+ // Cold rebuild: drop the cache dir and rematerialize — must
+ // reproduce an equivalent repo at the same path.
+ if err := os.RemoveAll(dir); err != nil {
+ t.Fatal(err)
+ }
+ dir2, err := mz.Materialize(ctx, repoName)
+ if err != nil {
+ t.Fatalf("Materialize (cold rebuild): %v", err)
+ }
+ if dir2 != dir {
+ t.Errorf("rebuild path = %q, want %q", dir2, dir)
+ }
+ assertRepo(t, dir2, branch, head)
+ })
+ }
+}
+
+// assertRepo checks the materialized repo the way the transport layer will:
+// the ref resolves, the head object is present and typed, HEAD tracks the
+// branch, and fsck finds no corruption or missing objects.
+func assertRepo(t *testing.T, dir, branch, head string) {
+ t.Helper()
+ if got := git(t, dir, "rev-parse", branch); got != head {
+ t.Errorf("rev-parse %s = %q, want %q", branch, got, head)
+ }
+ check := gitStdin(t, dir, []byte(head+"\n"), "cat-file", "--batch-check")
+ if !strings.HasPrefix(check, head+" commit ") {
+ t.Errorf("cat-file --batch-check = %q, want %q commit …", check, head)
+ }
+ if got := git(t, dir, "symbolic-ref", "HEAD"); got != "refs/heads/"+branch {
+ t.Errorf("HEAD = %q, want refs/heads/%s", got, branch)
+ }
+ git(t, dir, "fsck", "--strict") // fails the test on any corruption/missing object
+}
+
+func TestMaterializeUnknownRepo(t *testing.T) {
+ mz := New(openMeta(t), store.NewMem(), t.TempDir())
+ if _, err := mz.Materialize(context.Background(), "nope/nope"); !errors.Is(err, meta.ErrNotFound) {
+ t.Fatalf("Materialize(unknown) = %v, want meta.ErrNotFound", err)
+ }
+}
+
+func TestMaterializeRejectsUnsafeName(t *testing.T) {
+ ctx := context.Background()
+ m := openMeta(t)
+ // A repo whose name would escape the cache root must never materialize.
+ if _, err := m.CreateRepo(ctx, "../evil", "main"); err != nil {
+ t.Fatalf("CreateRepo: %v", err)
+ }
+ mz := New(m, store.NewMem(), t.TempDir())
+ if _, err := mz.Materialize(ctx, "../evil"); err == nil {
+ t.Fatal("Materialize(../evil) succeeded, want unsafe-name error")
+ }
+}
diff --git a/tasks/04-repo-materialization.md b/tasks/04-repo-materialization.md
deleted file mode 100644
index 2d1e2d7..0000000
--- a/tasks/04-repo-materialization.md
+++ /dev/null
@@ -1,34 +0,0 @@
-# 04 — Repo materialization / cache
-
-Depends on: 02, 03.
-
-## Spec
-
-Produce a valid on-disk **bare** repo for an operation — a disposable cache,
-never the source of truth ([storage.md](../docs/architecture/storage.md) →
-ephemeral disk). Refs come from s3lite; objects are hydrated from S3 into the
-closure the operation needs.
-
-## Current
-
-Nothing yet.
-
-## Change
-
-- `internal/repo/materialize.go` — given a repo name: create a temp bare repo
- (`git init --bare`), write refs from s3lite, hydrate objects from the `Store`.
-- **Hydration policy: full-hydrate for the MVP** — a write hydrates the target
- branch's full history (needed for the fast-forward / connectivity check), a
- read hydrates the closure it serves. Partial-clone / promisor is a later
- optimization (see
- [notes/object-hydration.md](../docs/notes/object-hydration.md)); gitmote's own
- repo is tiny, so full-hydrate is the safe, simple first cut.
-- Per-repo cache dir with rebuild-on-miss; eviction is best-effort (disposable).
-
-## Verify
-
-- Given a seeded repo (S3 objects + s3lite refs), materialize then assert with
- real git: `git -C <dir> rev-parse <ref>` and `git cat-file --batch-check`
- resolve; `git fsck` is clean.
-- A cold rebuild after deleting the cache dir reproduces an equivalent repo.
-- Non-breaking: internal helper, no route yet.
diff --git a/tasks/README.md b/tasks/README.md
index bead2f2..6994327 100644
--- a/tasks/README.md
+++ b/tasks/README.md
@@ -6,7 +6,6 @@ lands** (commits are the record — this list never becomes a changelog).
Ordered by dependency; each task lands in a non-breaking, tested state.
-- [04-repo-materialization.md](04-repo-materialization.md) — Build ephemeral bare repos from refs + hydrated objects.
- [05-read-path-clone.md](05-read-path-clone.md) — Smart-HTTP `upload-pack`; `git clone` works.
- [06-auth-pat-acl.md](06-auth-pat-acl.md) — Bearer PATs + per-repo ACL enforcement.
- [07-write-path-cas.md](07-write-path-cas.md) — `receive-pack` + pre-receive RPC + object PUT + ref CAS; `git push` works.