diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 2bcb774..83a37dd 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -67,6 +67,21 @@ jobs:
sudo curl -fsSL "$SCW_URL" -o /usr/local/bin/scw
sudo chmod +x /usr/local/bin/scw
+ # Stop-first: drain the running writer to zero BEFORE swapping the image, so
+ # the rolling deploy doesn't briefly run two litestream writers (safety.md
+ # §1). POST /admin/quit self-SIGTERMs the instance → graceful flush + exit.
+ # We then wait (NOT poll — a request would re-wake it) for it to exit; the
+ # image swap below then starts a fresh instance with no old one to overlap.
+ # Residual: a request racing this ~20s window can re-wake the old image;
+ # negligible for an operator-triggered deploy (docs/ops.md).
+ - name: Drain the running writer
+ env:
+ DEPLOY_KEY: ${{ secrets.GITMOTE_DEPLOY_KEY }}
+ run: |
+ curl -fsS -m 15 -X POST -H "Authorization: Bearer $DEPLOY_KEY" \
+ https://gitmote.atmin.net/admin/quit || true
+ sleep 20
+
# A unique (SHA-tagged) image reference makes this a clean replacement, and
# max-scale=1 (the single-writer invariant) is re-asserted every deploy.
# min-scale=0 idles to zero between uses. --wait blocks until the new
diff --git a/cmd/gitmote/main.go b/cmd/gitmote/main.go
index 1546563..83d83c5 100644
--- a/cmd/gitmote/main.go
+++ b/cmd/gitmote/main.go
@@ -3,6 +3,7 @@ package main
import (
"context"
+ "crypto/subtle"
"errors"
"flag"
"fmt"
@@ -12,6 +13,7 @@ import (
"os"
"os/signal"
"path/filepath"
+ "strings"
"syscall"
"time"
@@ -119,7 +121,7 @@ func run(ctx context.Context, logger *slog.Logger, addr string) error {
srv := &http.Server{
Addr: addr,
- Handler: newHandler(gitHandler, ui),
+ Handler: newHandler(gitHandler, ui, os.Getenv("GITMOTE_DEPLOY_KEY")),
}
errCh := make(chan error, 1)
@@ -149,8 +151,9 @@ func run(ctx context.Context, logger *slog.Logger, addr string) error {
// newHandler returns the server's HTTP routes. ui, when non-nil, registers the
// management UI under /ui/ and /login (more specific patterns that win over the
// git catch-all). gitHandler, when non-nil, serves the git smart-HTTP endpoints
-// at "/"; the exact health/version routes stay more specific.
-func newHandler(gitHandler http.Handler, ui *webui.Handler) http.Handler {
+// at "/"; the exact health/version routes stay more specific. deployKey, when
+// non-empty, enables POST /admin/quit — the deploy-time self-drain.
+func newHandler(gitHandler http.Handler, ui *webui.Handler, deployKey string) http.Handler {
mux := http.NewServeMux()
mux.HandleFunc("GET /healthz", func(w http.ResponseWriter, _ *http.Request) {
fmt.Fprintln(w, "ok")
@@ -158,6 +161,16 @@ func newHandler(gitHandler http.Handler, ui *webui.Handler) http.Handler {
mux.HandleFunc("GET /version", func(w http.ResponseWriter, _ *http.Request) {
fmt.Fprintln(w, version)
})
+ if deployKey != "" {
+ // Deploy-time stop-first: the pipeline drains the running writer before
+ // swapping the image, so the rolling deploy never runs two litestream
+ // writers (docs/ops.md, safety.md §1). Self-SIGTERM drops into the same
+ // graceful-shutdown path that flushes replication.
+ mux.HandleFunc("POST /admin/quit", adminQuitHandler(deployKey, func() {
+ time.Sleep(200 * time.Millisecond) // let the response flush first
+ _ = syscall.Kill(syscall.Getpid(), syscall.SIGTERM)
+ }))
+ }
if ui != nil {
ui.Register(mux)
}
@@ -167,6 +180,24 @@ func newHandler(gitHandler http.Handler, ui *webui.Handler) http.Handler {
return mux
}
+// adminQuitHandler authenticates a deploy-key bearer token in constant time and,
+// on success, acknowledges and triggers quit (graceful shutdown). quit is a seam
+// so tests exercise auth without killing the test process.
+func adminQuitHandler(deployKey string, quit func()) http.HandlerFunc {
+ return func(w http.ResponseWriter, r *http.Request) {
+ got := strings.TrimPrefix(r.Header.Get("Authorization"), "Bearer ")
+ if subtle.ConstantTimeCompare([]byte(got), []byte(deployKey)) != 1 {
+ http.Error(w, "unauthorized", http.StatusUnauthorized)
+ return
+ }
+ _, _ = io.WriteString(w, "draining\n")
+ if f, ok := w.(http.Flusher); ok {
+ f.Flush()
+ }
+ go quit()
+ }
+}
+
// buildGitHandler wires the git read + write paths when an object store is
// configured (GITMOTE_S3_BUCKET). Without it, the server runs health/version
// only — a dev convenience — and the returned close func is a no-op.
diff --git a/cmd/gitmote/main_test.go b/cmd/gitmote/main_test.go
index 141b7d8..ce3cc70 100644
--- a/cmd/gitmote/main_test.go
+++ b/cmd/gitmote/main_test.go
@@ -12,8 +12,58 @@ import (
"time"
)
+func TestAdminQuit(t *testing.T) {
+ const key = "s3cr3t-deploy-key"
+
+ // Unauthorized: wrong/absent bearer → 401, quit never fires.
+ for _, auth := range []string{"", "Bearer wrong", "Basic " + key} {
+ called := false
+ h := adminQuitHandler(key, func() { called = true })
+ rec := httptest.NewRecorder()
+ req := httptest.NewRequest(http.MethodPost, "/admin/quit", nil)
+ if auth != "" {
+ req.Header.Set("Authorization", auth)
+ }
+ h(rec, req)
+ if rec.Code != http.StatusUnauthorized {
+ t.Errorf("auth %q: status = %d, want 401", auth, rec.Code)
+ }
+ if called {
+ t.Errorf("auth %q: quit fired on unauthorized request", auth)
+ }
+ }
+
+ // Authorized: 200, acknowledges, and triggers quit.
+ done := make(chan struct{})
+ h := adminQuitHandler(key, func() { close(done) })
+ rec := httptest.NewRecorder()
+ req := httptest.NewRequest(http.MethodPost, "/admin/quit", nil)
+ req.Header.Set("Authorization", "Bearer "+key)
+ h(rec, req)
+ if rec.Code != http.StatusOK {
+ t.Fatalf("authorized status = %d, want 200", rec.Code)
+ }
+ select {
+ case <-done:
+ case <-time.After(time.Second):
+ t.Fatal("quit was not triggered on authorized request")
+ }
+
+ // The route only exists when a deploy key is configured.
+ srv := httptest.NewServer(newHandler(nil, nil, ""))
+ defer srv.Close()
+ resp, err := http.Post(srv.URL+"/admin/quit", "", nil)
+ if err != nil {
+ t.Fatalf("POST /admin/quit: %v", err)
+ }
+ defer resp.Body.Close()
+ if resp.StatusCode != http.StatusNotFound {
+ t.Errorf("quit with no deploy key = %d, want 404 (route absent)", resp.StatusCode)
+ }
+}
+
func TestHealthz(t *testing.T) {
- srv := httptest.NewServer(newHandler(nil, nil))
+ srv := httptest.NewServer(newHandler(nil, nil, ""))
defer srv.Close()
resp, err := http.Get(srv.URL + "/healthz")
@@ -28,7 +78,7 @@ func TestHealthz(t *testing.T) {
}
func TestVersion(t *testing.T) {
- srv := httptest.NewServer(newHandler(nil, nil))
+ srv := httptest.NewServer(newHandler(nil, nil, ""))
defer srv.Close()
resp, err := http.Get(srv.URL + "/version")
@@ -43,7 +93,7 @@ func TestVersion(t *testing.T) {
}
func TestUnknownRouteNotFound(t *testing.T) {
- srv := httptest.NewServer(newHandler(nil, nil))
+ srv := httptest.NewServer(newHandler(nil, nil, ""))
defer srv.Close()
resp, err := http.Get(srv.URL + "/nope")
@@ -63,7 +113,7 @@ func TestGitHandlerMountedAtRoot(t *testing.T) {
gitHandler := http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.WriteHeader(http.StatusTeapot)
})
- srv := httptest.NewServer(newHandler(gitHandler, nil))
+ srv := httptest.NewServer(newHandler(gitHandler, nil, ""))
defer srv.Close()
health, err := http.Get(srv.URL + "/healthz")
diff --git a/docs/architecture/safety.md b/docs/architecture/safety.md
index e9fa207..24ed33a 100644
--- a/docs/architecture/safety.md
+++ b/docs/architecture/safety.md
@@ -9,7 +9,13 @@
collaborators. A few invited users and the web UI's service commits all funnel
into one container. This suits the scale-to-zero shape: 0↔1 instances is fine.
**The one operational rule: never run two writer instances** — no overlapping
- deploys of the writer.
+ deploys of the writer. A rolling deploy would violate this (it briefly runs
+ old + new), so the deploy is **stop-first**: it drains the running writer to
+ exit before starting the replacement (see [ops.md](../ops.md) — deploy). The
+ airtight version — read replicas plus a single leased writer, so overlap is
+ safe by construction — is the planned evolution
+ ([reader-writer-split](../evolution/reader-writer-split.md)); the current
+ drain closes the window in practice, not by proof.
2. **In-process mutex linearizes writes.** Git's own lockfiles can't be trusted
across a synced/object backend, so a per-repo mutex in the process does the
diff --git a/docs/evolution/reader-writer-split.md b/docs/evolution/reader-writer-split.md
new file mode 100644
index 0000000..671518d
--- /dev/null
+++ b/docs/evolution/reader-writer-split.md
@@ -0,0 +1,68 @@
+# Reader/writer split with a leased writer (speculative)
+
+> **Status: evolution note — speculative, not a commitment.** Where gitmote could
+> go, captured so the reasoning survives a fresh session. Nothing here is planned
+> or specced; don't cite it as a spec.
+
+## The problem it solves
+
+[safety.md §1](../architecture/safety.md) requires **one writer, ever** — two
+litestream writers on the same WAL can corrupt the metadata replica. Today that's
+enforced by running a single container (`max-scale = 1`).
+
+But a **deploy** momentarily breaks it. Scaleway Serverless Containers do rolling
+deploys — the new instance boots while the old drains — so instance count spikes
+to 2 (observed in Cockpit), i.e. two writers. Scaleway won't let you scale to 0
+(`max-scale` min is 1), so it can't stop-first on its own.
+
+The shipped interim ([ops.md](../ops.md) — deploy) is **stop-first via self-drain**:
+CI `POST /admin/quit`s the running writer (self-SIGTERM → graceful flush + exit)
+before swapping the image, so the swap starts a fresh instance with no old one to
+overlap. That closes the window *in practice* but not *by proof* — a request
+racing the ~20 s drain→swap window can re-wake the old image. It also can't serve
+reads during the handoff (brief), and cold-start-from-zero must just-write.
+
+## The idea
+
+Stop conflating "an instance is running" with "an instance is the writer." Let
+**many instances run and serve reads**; let **exactly one hold a writer lease**.
+
+- Every instance boots as a **read replica**: serves `upload-pack` (clone/fetch),
+ UI reads, `/healthz`. It restores the metadata DB from the replica and follows
+ it read-only; it does **not** replicate or accept pushes.
+- Exactly one instance holds a **writer lease** and is the litestream leader: it
+ accepts `receive-pack`, does the ref CAS, and replicates the WAL. A push to a
+ non-leader is rejected/redirected.
+- **Handoff on deploy:** the new instance comes up as a reader; when the old
+ releases the lease (on SIGTERM drain), the new acquires it and promotes. Zero
+ read downtime, a brief write pause, and — because promotion is gated on the
+ lease — *never* two writers, by construction.
+
+This is the split [safety.md §2](../architecture/safety.md) already anticipates
+("in-process now, shared coordination later… the SQL CAS keeps the door open")
+and the "litestream read replicas" the read-scaling
+[open question](../architecture/open-questions.md) names.
+
+## Why it isn't shipped
+
+The lease needs a coordination substrate with **atomic compare-and-swap**, which
+is exactly what **Scaleway Object Storage lacks** (no `If-Match`/`If-None-Match`).
+So it requires two real changes:
+
+1. **Move the metadata WAL/lease to conditional-write storage** — Cloudflare R2 or
+ AWS S3 support preconditions; Scaleway doesn't. Compute can stay on Scaleway.
+2. **Leader election in s3lite** — acquire/refresh a lease (fencing token), and a
+ litestream lifecycle that only replicates while leader. gitmote grows a
+ reader/leader mode and promotion.
+
+The cold-start subtlety that motivates the lease: a lone instance waking on a
+push can't tell "I'm the only one, safe to write" from "I'm a new deploy while
+the old still writes" — they look identical from inside. Only an external lease
+answers "am I allowed to be the writer?".
+
+## Trigger to build it
+
+When any of: reads get heavy enough to want replicas; deploy-time write pauses or
+the residual overlap window become unacceptable; or `max-scale` needs to exceed 1
+for throughput. Until then the single-writer pin plus stop-first drain is
+sufficient and far simpler.
diff --git a/docs/ops.md b/docs/ops.md
index fb25003..9764415 100644
--- a/docs/ops.md
+++ b/docs/ops.md
@@ -31,18 +31,24 @@ in-process-race concern: it is a data-integrity precondition.
- **`max-scale = 1`** on the container, asserted on every deploy (the CI deploy
step sets it; it is not left to a console default). Autoscaling above 1 opens
the forbidden state. `min-scale = 0` (idle to zero) is fine — it only changes
- cost/latency, not the invariant; `max-scale = 1` is what guards it.
-- **No two instances during a transition.** Each deploy pushes a uniquely
- (SHA-)tagged image and `update … --wait`s for the new deployment to serve. With
- `max-scale = 1` Scaleway should replace the single instance rather than run two;
- the same must hold when waking from zero. **Confirm on the first deploy and the
- first wake-from-idle** (watch `scw container container get` — instance count
- must not exceed 1). If Scaleway is ever seen to overlap, switch the deploy to
- scale the container to zero, wait for termination, then bring up the new image.
-- Because pushes are operator-initiated and infrequent, deploy while no push is
- in flight. The content-before-pointer ordering ([safety.md §3](architecture/safety.md))
- bounds any residual window to orphan objects (garbage `gc` reclaims), never a
- ref pointing at a missing object.
+ cost/latency, not the invariant; `max-scale = 1` is what guards *steady state*.
+- **Deploys are stop-first, because `max-scale = 1` does *not* cover the deploy
+ swap.** Scaleway's rolling deploy briefly runs old + new together (observed:
+ instance count spikes to 2 in Cockpit), and Scaleway won't let you scale to 0
+ (`max-scale` min is 1), so it can't stop-first on its own. Instead the pipeline
+ drains the running writer itself before swapping the image: `POST /admin/quit`
+ (bearer `GITMOTE_DEPLOY_KEY`) self-SIGTERMs the instance → graceful flush + exit
+ → the image swap then starts a fresh instance with no old one to overlap. The
+ drain step **waits** rather than polls, because any request (a health poll
+ included) would re-wake the instance.
+- **Residual gap (accepted):** if a request lands in the ~20 s drain→swap window
+ it can re-wake the old image and reintroduce a brief overlap. Negligible for an
+ operator-triggered deploy with no concurrent push. The *airtight* fix is a
+ reader/writer split with a writer lease — deliberately deferred, written up in
+ [reader-writer-split](evolution/reader-writer-split.md). And by the
+ content-before-pointer ordering ([safety.md §3](architecture/safety.md)) even a
+ hit window can only leak orphan objects (garbage `gc` reclaims), never a ref
+ pointing at a missing object.
Note: Scaleway Object Storage has **no conditional writes** (`If-Match` /
`If-None-Match`). This does **not** affect gitmote — the ref CAS is a SQL
@@ -50,10 +56,13 @@ transaction in the metadata DB, not an S3 precondition. (It does rule out the
safety.md §4 escape hatch of moving refs to a conditional-PUT on S3 *while on
Scaleway*; that would need a provider with preconditions.)
-## Runtime env vars
+## Runtime env vars (on the container)
-Set on the Scaleway Serverless Container (secrets as
-`secret-environment-variables`):
+These configure the **running server** and are set **on the Scaleway Serverless
+Container** (via `scw container container create/update`, secrets as
+`secret-environment-variables`). They are **not** the CI credentials — the deploy
+pipeline's Scaleway API keys are separate GitHub Actions secrets (next section).
+Setting these on the container does nothing for CI, and vice versa.
| Variable | Value / meaning |
|----------|-----------------|
@@ -66,6 +75,7 @@ Set on the Scaleway Serverless Container (secrets as
| `GITMOTE_CACHE` | `/tmp/gitmote/cache` — ephemeral; materialized repos rebuild from S3 |
| `GITMOTE_SOCK` | `/tmp/gitmote/gitmote.sock` — pre-receive hook RPC socket |
| `GITMOTE_COOKIE_KEY` | secret — signs management-UI session cookies (enables `/ui`) |
+| `GITMOTE_DEPLOY_KEY` | secret — bearer token for `POST /admin/quit` (deploy stop-first drain); enables the route |
| `AWS_REGION` | `fr-par` |
| `AWS_ACCESS_KEY_ID` / `AWS_SECRET_ACCESS_KEY` | secret — Scaleway API key pair; covers both the object store and the litestream replica |
@@ -81,13 +91,42 @@ Memory: **1 GB** (`memory-limit-bytes=1GB` — the CLI requires a `G`/`GB` unit)
gives headroom and, at `min-scale=0`, is idle-billed. Tune from logs.
Scale: **`min-scale = 0`** (idle to zero). A remote pushed to occasionally is idle
-almost always, and always-on at 512 MB exceeds Scaleway's free tier (~1.3M GB-s/mo
+almost always, and always-on at 1 GB far exceeds Scaleway's free tier (~2.6M GB-s/mo
vs 400k), so scale-to-zero is the cheaper default. The cost is a few seconds of
cold start on the first request after idle (restore + materialize, the path above);
scale-down is a graceful SIGTERM, so the shutdown `Sync` flushes the WAL — no lost
writes. Set `min-scale = 1` only if instant first-push latency is worth the
always-on cost. `max-scale` stays **1** regardless (single-writer invariant).
+## CI secrets (GitHub Actions)
+
+Separate from the container env above: these are the Scaleway API credentials the
+deploy pipeline uses, set in the **GitHub repo** → Settings → Secrets and variables
+→ **Actions** → **Repository secrets**. A missing one doesn't error at reference
+time — GitHub substitutes an empty string, so a wrong name surfaces later as e.g.
+`docker login` → *"Password required"*.
+
+A Scaleway API key has two halves — don't swap them: the **access key** is
+`SCWXXXXXXXXXXXXXXXXX` (starts with `SCW`), the **secret key** is a UUID
+(`xxxxxxxx-xxxx-…`). The registry password is the **secret key**. Both must come
+from the same Organization/Project as the `atmin` registry namespace, or registry
+login fails with `api_key … not_found`.
+
+| Secret | Value |
+|--------|-------|
+| `SCW_SECRET_KEY` | Scaleway API **secret key** — the UUID, not the `SCW…` access key (registry login + deploy) |
+| `SCW_ACCESS_KEY` | Scaleway **access key** (`SCW…`) |
+| `SCW_REGISTRY_ENDPOINT` | `rg.fr-par.scw.cloud/atmin` |
+| `SCW_ORGANIZATION_ID` | Scaleway organization ID |
+| `SCW_PROJECT_ID` | Scaleway project ID |
+| `SCW_CONTAINER_ID` | the gitmote Serverless Container ID |
+| `GITMOTE_DEPLOY_KEY` | bearer token the deploy sends to `/admin/quit` — must equal the container's `GITMOTE_DEPLOY_KEY` env |
+
+If a secret reads as empty despite being set, check: it's under **Secrets** not
+**Variables**; it's a **Repository** secret (a job with no `environment:` can't see
+Environment secrets); the name matches exactly (case-sensitive); and, if kept as an
+**Organization** secret, the `gitmote` repo is granted access to it.
+
## One-time Scaleway setup
```bash
@@ -120,8 +159,13 @@ scw container container create \
environment-variables.GITMOTE_SOCK=/tmp/gitmote/gitmote.sock \
environment-variables.AWS_REGION=fr-par \
secret-environment-variables.GITMOTE_COOKIE_KEY="$(openssl rand -base64 32)" \
+ secret-environment-variables.GITMOTE_DEPLOY_KEY=<DEPLOY_KEY> \
secret-environment-variables.AWS_ACCESS_KEY_ID=<KEY> \
secret-environment-variables.AWS_SECRET_ACCESS_KEY=<SECRET>
+# <DEPLOY_KEY>: generate once (`openssl rand -base64 32`) and use the SAME value
+# here and as the GITMOTE_DEPLOY_KEY GitHub Actions secret — the deploy's drain
+# call authenticates against it. GITMOTE_COOKIE_KEY is container-only, so a fresh
+# random inline is fine for it.
# 4. Custom domain — two parts:
# (a) DNS: add a CNAME gitmote.atmin.net → <container-endpoint>.scw.cloud
@@ -133,9 +177,8 @@ scw container domain list container-id=<CONTAINER_ID> # status pending → r
# Note: gitmote has no route at "/", so http(s)://gitmote.atmin.net/ returns 404
# by design; check /healthz for liveness. Git lives under /<owner>/<repo>/….
-# 5. GitHub secrets for the deploy pipeline (same set as atmin.net):
-# SCW_ACCESS_KEY, SCW_SECRET_KEY, SCW_ORGANIZATION_ID, SCW_PROJECT_ID,
-# SCW_REGISTRY_ENDPOINT (rg.fr-par.scw.cloud/atmin), SCW_CONTAINER_ID.
+# 5. Set the GitHub Actions repository secrets for the deploy pipeline — these are
+# the CI credentials, NOT container env vars. See "CI secrets (GitHub Actions)".
```
## Bootstrap (run once, before the server is live)
@@ -161,8 +204,9 @@ it refuses to clobber an existing admin.
Automatic on every green push to `master` (`.github/workflows/ci.yml`, `deploy`
job): `ci` (lint/test/build) → `e2e` (local push/clone + litestream restore) →
-build+push the amd64 image → `scw container container update … min-scale=0
-max-scale=1 --wait`.
+build+push the amd64 image → **drain the running writer** (`POST /admin/quit`,
+stop-first — see the single-writer section) → `scw container container update …
+min-scale=0 max-scale=1 --wait`.
```bash
git push origin master