diff --git a/cmd/gitmote/main.go b/cmd/gitmote/main.go
index a8a35d0..f4cd85f 100644
--- a/cmd/gitmote/main.go
+++ b/cmd/gitmote/main.go
@@ -12,6 +12,7 @@ import (
"os"
"os/signal"
"path/filepath"
+ "strings"
"syscall"
"time"
@@ -116,7 +117,7 @@ func run(ctx context.Context, logger *slog.Logger, addr string) error {
ctx, stop := signal.NotifyContext(ctx, syscall.SIGINT, syscall.SIGTERM)
defer stop()
- gitHandler, ui, reportAPI, closeMeta, err := buildGitHandler(ctx, logger)
+ gitHandler, ui, reportAPI, isLeader, closeMeta, err := buildGitHandler(ctx, logger)
if err != nil {
return err
}
@@ -128,7 +129,7 @@ func run(ctx context.Context, logger *slog.Logger, addr string) error {
srv := &http.Server{
Addr: addr,
- Handler: newHandler(gitHandler, ui, reportAPI),
+ Handler: newHandler(gitHandler, ui, reportAPI, isLeader),
}
if reportAPI != nil {
@@ -181,7 +182,11 @@ func reconcileLoop(ctx context.Context, reportAPI *ci.ReportAPI, logger *slog.Lo
// management UI under /ui/ and /login (more specific patterns that win over the
// git catch-all). gitHandler, when non-nil, serves the git smart-HTTP endpoints
// at "/"; the exact health/version routes stay more specific.
-func newHandler(gitHandler http.Handler, ui *webui.Handler, reportAPI *ci.ReportAPI) http.Handler {
+//
+// isLeader gates every metadata-derived response on the writer lease (see
+// leaderGate): a follower — the brief post-deploy window before it promotes —
+// serves a frozen, stale snapshot, so it must not answer with stale refs.
+func newHandler(gitHandler http.Handler, ui *webui.Handler, reportAPI *ci.ReportAPI, isLeader func() bool) http.Handler {
mux := http.NewServeMux()
mux.HandleFunc("GET /healthz", func(w http.ResponseWriter, _ *http.Request) {
fmt.Fprintln(w, "ok")
@@ -198,7 +203,35 @@ func newHandler(gitHandler http.Handler, ui *webui.Handler, reportAPI *ci.Report
if gitHandler != nil {
mux.Handle("/", gitHandler)
}
- return mux
+ return leaderGate(mux, isLeader)
+}
+
+// leaderGate serves stale-free: only the writer (leader) answers requests that
+// read or write the metadata DB. A follower restored a snapshot at startup and
+// does not catch up until it promotes (s3lite RoleAuto), so its refs can lag the
+// true tip — a browse would show a just-pushed file as missing, a fetch would
+// miss commits. Rather than serve that, a follower returns 503 + Retry-After.
+//
+// Exceptions that must stay up on a follower: the liveness probes (Scaleway's
+// health check — gating them would deadlock a rolling deploy, since the new
+// instance can't promote until the old one drains) and static assets (no
+// metadata). A nil isLeader (unreplicated dev, RoleOff) means always-leader.
+func leaderGate(next http.Handler, isLeader func() bool) http.Handler {
+ return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if alwaysServable(r.URL.Path) || isLeader == nil || isLeader() {
+ next.ServeHTTP(w, r)
+ return
+ }
+ w.Header().Set("Retry-After", "2")
+ http.Error(w, "gitmote is warming up (not the writer yet) — retry shortly",
+ http.StatusServiceUnavailable)
+ })
+}
+
+// alwaysServable reports whether a path is answerable by a follower: the liveness
+// probes and static assets, which carry no metadata.
+func alwaysServable(p string) bool {
+ return p == "/healthz" || p == "/version" || strings.HasPrefix(p, "/ui/static/")
}
// buildGitHandler wires the git read + write paths when an object store is
@@ -214,17 +247,17 @@ func newHandler(gitHandler http.Handler, ui *webui.Handler, reportAPI *ci.Report
// repos cache under GITMOTE_CACHE (default a temp dir). The push path listens on
// GITMOTE_SOCK (default a temp path) and installs the pre-receive hook binary
// at GITMOTE_HOOK (default gitmote-hook alongside this binary).
-func buildGitHandler(ctx context.Context, logger *slog.Logger) (http.Handler, *webui.Handler, *ci.ReportAPI, func() error, error) {
+func buildGitHandler(ctx context.Context, logger *slog.Logger) (http.Handler, *webui.Handler, *ci.ReportAPI, func() bool, func() error, error) {
noop := func() error { return nil }
if os.Getenv("GITMOTE_S3_BUCKET") == "" {
logger.Warn("GITMOTE_S3_BUCKET unset; git endpoints disabled (health/version only)")
- return nil, nil, nil, noop, nil
+ return nil, nil, nil, nil, noop, nil
}
objs, err := store.NewS3FromEnv(ctx)
if err != nil {
- return nil, nil, nil, noop, fmt.Errorf("object store: %w", err)
+ return nil, nil, nil, nil, noop, fmt.Errorf("object store: %w", err)
}
// RoleAuto: this instance becomes the writer when it can acquire the lease,
@@ -234,7 +267,7 @@ func buildGitHandler(ctx context.Context, logger *slog.Logger) (http.Handler, *w
// unchanged.
md, err := meta.Open(ctx, metaConfigFromEnv(logger, s3lite.RoleAuto))
if err != nil {
- return nil, nil, nil, noop, fmt.Errorf("metadata: %w", err)
+ return nil, nil, nil, nil, noop, fmt.Errorf("metadata: %w", err)
}
md.OnPromote(func() {
logger.Info("became writer: acquired the lease", "generation", md.Generation())
@@ -247,7 +280,7 @@ func buildGitHandler(ctx context.Context, logger *slog.Logger) (http.Handler, *w
writer, err := githttp.NewWriter(md, objs, hookBinaryPath(), sockPath, logger)
if err != nil {
_ = md.Close()
- return nil, nil, nil, noop, fmt.Errorf("write path: %w", err)
+ return nil, nil, nil, nil, noop, fmt.Errorf("write path: %w", err)
}
cacheRoot := envOr("GITMOTE_CACHE", filepath.Join(os.TempDir(), "gitmote-repos"))
materializer := repo.New(md, objs, cacheRoot)
@@ -260,7 +293,7 @@ func buildGitHandler(ctx context.Context, logger *slog.Logger) (http.Handler, *w
if err != nil {
_ = writer.Close()
_ = md.Close()
- return nil, nil, nil, noop, fmt.Errorf("ci secrets: %w", err)
+ return nil, nil, nil, nil, noop, fmt.Errorf("ci secrets: %w", err)
}
secretsSvc := secrets.NewService(keyring, md)
@@ -279,7 +312,7 @@ func buildGitHandler(ctx context.Context, logger *slog.Logger) (http.Handler, *w
if workerSecret == "" || gitmoteURL == "" {
_ = writer.Close()
_ = md.Close()
- return nil, nil, nil, noop, fmt.Errorf("SCW_CI_JOB_DEFINITION_ID is set but WORKER_SECRET or GITMOTE_URL is missing")
+ return nil, nil, nil, nil, noop, fmt.Errorf("SCW_CI_JOB_DEFINITION_ID is set but WORKER_SECRET or GITMOTE_URL is missing")
}
// SCW_REGION falls back to the AWS_REGION already set for the object store.
trigger = scaleway.NewJobsClient(os.Getenv("SCW_SECRET_KEY"), envOr("SCW_REGION", os.Getenv("AWS_REGION")), jobDefID)
@@ -344,15 +377,15 @@ func buildGitHandler(ctx context.Context, logger *slog.Logger) (http.Handler, *w
})
if err != nil {
_ = cleanup()
- return nil, nil, nil, noop, err
+ return nil, nil, nil, nil, noop, err
}
ui, err := buildUI(md, materializer, objs, guard, secretsSvc, logger)
if err != nil {
_ = cleanup()
- return nil, nil, nil, noop, err
+ return nil, nil, nil, nil, noop, err
}
- return handler, ui, reportAPI, cleanup, nil
+ return handler, ui, reportAPI, md.IsLeader, cleanup, nil
}
// buildUI constructs the management UI when GITMOTE_COOKIE_KEY is set; otherwise
diff --git a/cmd/gitmote/main_test.go b/cmd/gitmote/main_test.go
index d48d4e3..c214e28 100644
--- a/cmd/gitmote/main_test.go
+++ b/cmd/gitmote/main_test.go
@@ -13,7 +13,7 @@ import (
)
func TestHealthz(t *testing.T) {
- srv := httptest.NewServer(newHandler(nil, nil, nil))
+ srv := httptest.NewServer(newHandler(nil, nil, nil, nil))
defer srv.Close()
resp, err := http.Get(srv.URL + "/healthz")
@@ -28,7 +28,7 @@ func TestHealthz(t *testing.T) {
}
func TestVersion(t *testing.T) {
- srv := httptest.NewServer(newHandler(nil, nil, nil))
+ srv := httptest.NewServer(newHandler(nil, nil, nil, nil))
defer srv.Close()
resp, err := http.Get(srv.URL + "/version")
@@ -43,7 +43,7 @@ func TestVersion(t *testing.T) {
}
func TestUnknownRouteNotFound(t *testing.T) {
- srv := httptest.NewServer(newHandler(nil, nil, nil))
+ srv := httptest.NewServer(newHandler(nil, nil, nil, nil))
defer srv.Close()
resp, err := http.Get(srv.URL + "/nope")
@@ -63,7 +63,7 @@ func TestGitHandlerMountedAtRoot(t *testing.T) {
gitHandler := http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.WriteHeader(http.StatusTeapot)
})
- srv := httptest.NewServer(newHandler(gitHandler, nil, nil))
+ srv := httptest.NewServer(newHandler(gitHandler, nil, nil, nil))
defer srv.Close()
health, err := http.Get(srv.URL + "/healthz")
@@ -85,6 +85,44 @@ func TestGitHandlerMountedAtRoot(t *testing.T) {
}
}
+func TestLeaderGate(t *testing.T) {
+ // A follower must not answer metadata-derived requests with a stale snapshot;
+ // the git/read catch-all is 503'd. Health/version stay up so a rolling deploy
+ // can still promote (gating them would deadlock it).
+ gitHandler := http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ w.WriteHeader(http.StatusTeapot)
+ })
+ leader := false
+ srv := httptest.NewServer(newHandler(gitHandler, nil, nil, func() bool { return leader }))
+ defer srv.Close()
+
+ get := func(path string) int {
+ resp, err := http.Get(srv.URL + path)
+ if err != nil {
+ t.Fatalf("GET %s: %v", path, err)
+ }
+ resp.Body.Close()
+ return resp.StatusCode
+ }
+
+ // As a follower: probes stay up, the git read path is refused.
+ if code := get("/healthz"); code != http.StatusOK {
+ t.Errorf("follower /healthz = %d, want 200 (must stay up for the deploy)", code)
+ }
+ if code := get("/version"); code != http.StatusOK {
+ t.Errorf("follower /version = %d, want 200", code)
+ }
+ if code := get("/some/repo/info/refs"); code != http.StatusServiceUnavailable {
+ t.Errorf("follower git read = %d, want 503", code)
+ }
+
+ // After promotion, the same route is served.
+ leader = true
+ if code := get("/some/repo/info/refs"); code != http.StatusTeapot {
+ t.Errorf("leader git read = %d, want %d (served)", code, http.StatusTeapot)
+ }
+}
+
func TestRunShutsDownOnCancel(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
logger := slog.New(slog.DiscardHandler)
diff --git a/docs/architecture/safety.md b/docs/architecture/safety.md
index 2cdc030..19379c6 100644
--- a/docs/architecture/safety.md
+++ b/docs/architecture/safety.md
@@ -11,8 +11,14 @@
is now enforced by a **lease**, not by procedure. The server opens s3lite in
`RoleAuto`: it writes only while it holds the lease (litestream's `s3.Leaser`,
a conditional-write CAS on `lock.json` in the replica bucket) and otherwise runs
- as a read-only follower. gitmote gates receive-pack on `IsLeader()` — a follower
- refuses pushes with a retryable `503` and still serves reads. A rolling deploy's
+ as a read-only follower. gitmote gates **all metadata-derived requests** on
+ `IsLeader()` — a follower refuses them with a retryable `503`; only the liveness
+ probes and static assets stay up (gating those would deadlock a rolling deploy,
+ since the new instance can't promote until the old drains). Writes are gated for
+ *safety*, reads for *freshness*: a follower serves only the snapshot it restored
+ at startup and doesn't catch up until it promotes, so serving a read would return
+ a stale ref — a just-pushed file appearing missing, a `fetch` missing commits.
+ A rolling deploy's
brief old+new overlap is therefore one writer + one follower: the new instance
boots as a follower, the old releases the lease on its graceful SIGTERM `Close`,
and the new promotes on its next lease poll. **Overlap is safe by construction**,