diff --git a/cmd/gitmote/main.go b/cmd/gitmote/main.go
index eb7f17a..2c2dd25 100644
--- a/cmd/gitmote/main.go
+++ b/cmd/gitmote/main.go
@@ -343,6 +343,15 @@ func buildGitHandler(ctx context.Context, logger *slog.Logger) (http.Handler, *w
logger.Warn("lost the writer lease; now read-only", "error", err)
})
+ // Fail fast on the refs-without-objects split (typically a wrong or removed
+ // GITMOTE_S3_PREFIX): the metadata restored, so refs would be advertised, but
+ // the objects are unreachable in the store. A loud crash at startup beats
+ // silently serving clones/pushes/browses that cannot succeed.
+ if err := verifyObjectStore(ctx, md, objs); err != nil {
+ _ = md.Close()
+ return nil, nil, nil, nil, noop, fmt.Errorf("object store consistency: %w", err)
+ }
+
// First-run auto-bootstrap: a fresh empty-bucket instance is usable without a
// second command. Runs before anything serves; non-fatal (logged, not returned).
maybeAutoBootstrap(ctx, md, logger)
@@ -472,6 +481,48 @@ func buildGitHandler(ctx context.Context, logger *slog.Logger) (http.Handler, *w
return handler, ui, reportAPI, md.IsLeader, cleanup, nil
}
+// verifyObjectStore fails fast on the "refs without objects" split: the
+// metadata restored (so refs will be advertised) but the object store holds no
+// objects for a repo that has refs. Serving in that state advertises refs it
+// cannot fulfil — clones get upload-pack "not our ref", pushes fail with
+// "unpacker error", browses 404. The usual cause is a wrong or removed
+// GITMOTE_S3_PREFIX: object keys are {prefix}/{repo}/objects/…, but the metadata
+// replica is prefix-independent, so a bad prefix strands every object while refs
+// still restore. A fresh instance (no repo has refs yet) passes — absent objects
+// are correct there.
+func verifyObjectStore(ctx context.Context, md *meta.Metadata, objs store.Store) error {
+ repos, err := md.ListRepos(ctx)
+ if err != nil {
+ return fmt.Errorf("list repos: %w", err)
+ }
+ for _, r := range repos {
+ refs, err := md.ListRefs(ctx, r.ID)
+ if err != nil {
+ return fmt.Errorf("list refs for %q: %w", r.Name, err)
+ }
+ if len(refs) == 0 {
+ continue // no history yet — the store is correctly empty for this repo
+ }
+ // This repo has refs, so a correctly-addressed store holds at least one
+ // object for it. One repo suffices: the object prefix is global, so an
+ // empty result here means every repo's objects are unreachable.
+ keys, err := objs.List(ctx, r.Name+"/objects/")
+ if err != nil {
+ return fmt.Errorf("list objects for %q: %w", r.Name, err)
+ }
+ if len(keys) == 0 {
+ return fmt.Errorf("repo %q has %d ref(s) in metadata but no objects in the store "+
+ "under %q/objects/ — the objects are unreachable at the current "+
+ "GITMOTE_S3_PREFIX (object keys are {prefix}/{repo}/objects/…; the meta replica "+
+ "is prefix-independent, so refs restored while objects did not). Verify "+
+ "GITMOTE_S3_PREFIX matches the prefix the objects were written under",
+ r.Name, len(refs), r.Name)
+ }
+ return nil // a repo with refs has objects — the store is reachable
+ }
+ return nil // no repo has refs yet — fresh instance, nothing to verify
+}
+
// buildUI constructs the management UI (always on). Its session cookie key is
// GITMOTE_COOKIE_KEY when set, else an auto-generated key persisted in meta and
// restored across restart / scale-to-zero — so a session stays valid through an
diff --git a/cmd/gitmote/main_test.go b/cmd/gitmote/main_test.go
index 77a44a2..3f4a764 100644
--- a/cmd/gitmote/main_test.go
+++ b/cmd/gitmote/main_test.go
@@ -621,3 +621,68 @@ func TestRunFailsOnBadAddr(t *testing.T) {
t.Errorf("run error = %q, want a listen error", err)
}
}
+
+// TestVerifyObjectStore covers the startup guard against the refs-without-objects
+// split (the wrong/removed GITMOTE_S3_PREFIX regression): a fresh instance and a
+// healthy store pass, but refs whose objects are unreachable fail loud.
+func TestVerifyObjectStore(t *testing.T) {
+ ctx := context.Background()
+ const repoName = "gitmote"
+ const head = "aa1c7cbe7b72b787270f456d61fe0046a46cba12"
+
+ openMeta := func(t *testing.T) *meta.Metadata {
+ t.Helper()
+ m, err := meta.Open(ctx, meta.Config{LocalPath: filepath.Join(t.TempDir(), "meta.sqlite3")})
+ if err != nil {
+ t.Fatalf("meta.Open: %v", err)
+ }
+ t.Cleanup(func() { _ = m.Close() })
+ return m
+ }
+ withRef := func(t *testing.T) *meta.Metadata {
+ t.Helper()
+ m := openMeta(t)
+ r, err := m.CreateRepo(ctx, repoName, "master")
+ if err != nil {
+ t.Fatalf("CreateRepo: %v", err)
+ }
+ if err := m.CASRef(ctx, r.ID, "refs/heads/master", meta.ZeroSHA, head); err != nil {
+ t.Fatalf("CASRef: %v", err)
+ }
+ return m
+ }
+
+ t.Run("fresh instance with no refs passes", func(t *testing.T) {
+ m := openMeta(t)
+ if _, err := m.CreateRepo(ctx, repoName, "master"); err != nil {
+ t.Fatalf("CreateRepo: %v", err)
+ }
+ // A repo exists but nothing was pushed — an empty store is correct.
+ if err := verifyObjectStore(ctx, m, store.NewMem()); err != nil {
+ t.Errorf("fresh instance rejected: %v", err)
+ }
+ })
+
+ t.Run("refs with objects present passes", func(t *testing.T) {
+ m := withRef(t)
+ s := store.NewMem()
+ if err := s.Put(ctx, repoName+"/objects/aa/1c7cbe", bytes.NewReader([]byte("x"))); err != nil {
+ t.Fatalf("Put: %v", err)
+ }
+ if err := verifyObjectStore(ctx, m, s); err != nil {
+ t.Errorf("healthy store rejected: %v", err)
+ }
+ })
+
+ t.Run("refs without objects fails loud", func(t *testing.T) {
+ m := withRef(t)
+ // Refs restored but the store is empty at the prefix — the regression.
+ err := verifyObjectStore(ctx, m, store.NewMem())
+ if err == nil {
+ t.Fatal("expected an error for refs-without-objects, got nil")
+ }
+ if !strings.Contains(err.Error(), "GITMOTE_S3_PREFIX") {
+ t.Errorf("error must name the likely cause GITMOTE_S3_PREFIX; got: %v", err)
+ }
+ })
+}
diff --git a/docs/ops.md b/docs/ops.md
index d52ee5d..ad7f704 100644
--- a/docs/ops.md
+++ b/docs/ops.md
@@ -115,7 +115,7 @@ one of these turns a trigger on — see [CI substrate](#ci-substrate)).
| `GITMOTE_DATA` | base dir for the db (`meta.sqlite3`), cache, and socket; **preset to `/data` in the image** — mount a volume there to keep the cache across restarts. Ephemeral by design (restored from S3 on cold start) |
| `GITMOTE_DB_REPLICA` | override the derived `s3://{bucket}/meta` replica target |
| `GITMOTE_DB` / `GITMOTE_CACHE` / `GITMOTE_SOCK` | override the individual paths that otherwise derive from `GITMOTE_DATA` |
-| `GITMOTE_S3_PREFIX` | key prefix for git objects inside the bucket (does **not** affect the `meta` replica path) |
+| `GITMOTE_S3_PREFIX` | key prefix for git objects inside the bucket (does **not** affect the `meta` replica path). The code default is empty (objects at `{repo}/objects/…`), which is valid for a *fresh* bucket — but **prod set `objects`** to get the `objects/` + `meta/` [one-bucket layout](#infrastructure), so every object was written under keys like `objects/{repo}/objects/…`. ⚠️ **This is a data-address, not a preference: once objects exist under it, the prefix is load-bearing — never remove or change it on a running instance.** Reverting to the default doesn't move the data, it just makes the server look at the wrong (empty) keys. Because the `meta` replica is prefix-independent, refs still restore — so the instance comes up advertising refs it cannot serve (clone → `not our ref`, push → `unpacker error`, browse → 404) |
| `GITMOTE_HOOK` / `GITMOTE_RUNNER` | hook / CI-runner binary paths (default beside the server) |
| `GITMOTE_ADMIN_HANDLE` | first-run auto-bootstrap admin handle (default `admin`) |
@@ -227,6 +227,7 @@ scw container container create \
port=8080 \
environment-variables.GITMOTE_S3_BUCKET=gitmote \
environment-variables.GITMOTE_S3_ENDPOINT=https://s3.fr-par.scw.cloud \
+ environment-variables.GITMOTE_S3_PREFIX=objects \
environment-variables.GITMOTE_DATA=/tmp/gitmote \
environment-variables.AWS_REGION=fr-par \
secret-environment-variables.AWS_ACCESS_KEY_ID=<KEY> \