From 58e570f7d0debd59c61a057f2728aff3036585c8 Mon Sep 17 00:00:00 2001 From: Tomaz Zaman Date: Tue, 11 Aug 2026 23:35:13 +0200 Subject: [PATCH 35/72] mono-update: signed, verified, automated release flow The update client decided freshness and anti-rollback on latest.json, which was unsigned - only the per-release sha256sums was signed, and its filenames carry no version. Anyone able to serve the update URL's bytes could advertise an inflated tag pointing at an old, still-validly-signed image and force a downgrade to a vulnerable release, unattended, with no signing key (it replays old valid signatures). Bind the version into signed material: latest.json now carries format_version and is itself usign-signed (latest.json.sig), so the client verifies the tag and per-board image hash it acts on. Refactor signing/publishing into composable scripts while keeping the nightly patch-release path fully automated: - mono-update.sh builds + stages, then auto-signs+publishes when MONO_SIGN_KEY is present, or stages + prints the two manual steps when it isn't (a new minor, or off-host signing). - mono-sign-release.sh signs sha256sums + latest.json where the key lives. - mono-publish-release.sh rsyncs + pushes, and REFUSES unless both signatures VALIDATE against the fleet's baked keys (mono-release.pub / mono-rotation.pub) - not merely exist. A wrong or rotated signing key, or a stale latest.json.sig, fails here (the trap drops the tag; retry + alert) instead of shipping a release every device would reject. The client that consumes this manifest is the following commit. Co-Authored-By: Claude Opus 4.8 --- scripts/mono-publish-release.sh | 80 ++++++++++++++++++++++++++++++++ scripts/mono-sign-release.sh | 32 +++++++++++++ scripts/mono-update.sh | 82 +++++++++++++++------------------ 3 files changed, 149 insertions(+), 45 deletions(-) create mode 100755 scripts/mono-publish-release.sh create mode 100755 scripts/mono-sign-release.sh diff --git a/scripts/mono-publish-release.sh b/scripts/mono-publish-release.sh new file mode 100755 index 0000000000..2b26be1533 --- /dev/null +++ b/scripts/mono-publish-release.sh @@ -0,0 +1,80 @@ +#!/bin/sh +# Publish an already-signed, staged Mono release: rsync the artifacts and the +# signed latest.json to MONO_PUBLISH_DEST, then push the branch + tag and +# mirror on the forge. REFUSES unless both usign signatures are present, so an +# unsigned or tampered staging directory can never reach the fleet. +# +# Env: MONO_PUBLISH_DEST (rsync destination). A 'mono' git remote enables the +# branch/tag push and the gh forge mirror. +# +# Usage: scripts/mono-publish-release.sh +set -eu + +cd "$(dirname "$0")/.." +RELTAG=${1:?usage: mono-publish-release.sh } +BRANCH=mono +OUT="releases/$RELTAG" + +[ -d "$OUT" ] || { echo "mono-publish: $OUT not staged - build it first" >&2; exit 1; } +[ -f releases/latest.json ] || { echo "mono-publish: releases/latest.json missing - stage the release first" >&2; exit 1; } +[ -f "$OUT/sha256sums.sig" ] || { echo "mono-publish: $OUT/sha256sums.sig missing - run scripts/mono-sign-release.sh $RELTAG" >&2; exit 1; } +[ -f releases/latest.json.sig ] || { echo "mono-publish: releases/latest.json.sig missing - run scripts/mono-sign-release.sh $RELTAG" >&2; exit 1; } + +# Verify the signatures actually VALIDATE against the keys the fleet bakes into +# every image - not merely that a .sig file exists. A wrong or rotated signing +# key, or a stale latest.json.sig left from a prior release, yields a +# valid-looking file that every device would reject; catch it here, before it +# ships, instead of freezing the fleet. Accept if ANY baked key verifies (the +# rotation key is trusted alongside the release key). +USIGN=${MONO_USIGN:-staging_dir/host/bin/usign} +[ -x "$USIGN" ] || command -v "$USIGN" >/dev/null 2>&1 || { + echo "mono-publish: usign not found ($USIGN); set MONO_USIGN" >&2; exit 1; } +KEYS="package/mono/mono-update-check/files/mono-release.pub package/mono/mono-update-check/files/mono-rotation.pub" +verify() { # : true if any baked key validates it + for k in $KEYS; do + [ -f "$k" ] || continue + "$USIGN" -V -q -m "$1" -x "$2" -p "$k" 2>/dev/null && return 0 + done + return 1 +} +verify "$OUT/sha256sums" "$OUT/sha256sums.sig" || { + echo "mono-publish: $OUT/sha256sums.sig fails to verify against the fleet's baked keys (wrong signing key?) - refusing" >&2; exit 1; } +verify releases/latest.json releases/latest.json.sig || { + echo "mono-publish: releases/latest.json.sig fails to verify (wrong key or stale sig?) - refusing" >&2; exit 1; } + +# latest.json must describe THIS release (guards against a stale manifest from +# a later staging run being shipped with an older release's signature). +json_tag=$(sed -n 's/.*"tag": *"\([^"]*\)".*/\1/p' releases/latest.json | head -1) +[ "$json_tag" = "$RELTAG" ] || { + echo "mono-publish: latest.json tag '$json_tag' != $RELTAG - refusing" >&2; exit 1; } + +if [ -n "${MONO_PUBLISH_DEST:-}" ]; then + echo "mono-publish: publishing to $MONO_PUBLISH_DEST" + rsync -a "$OUT" releases/latest.json releases/latest.json.sig "$MONO_PUBLISH_DEST/" +else + echo "mono-publish: MONO_PUBLISH_DEST unset, nothing rsynced" >&2 +fi + +# Push the rebased branch and release tag when a 'mono' remote is configured +# (machine-local in .git/config; nothing committed here). +if git remote get-url mono >/dev/null 2>&1; then + echo "mono-publish: pushing $BRANCH and $RELTAG" + git push --force-with-lease mono "$BRANCH" + git push -f mono "$RELTAG" + + # Mirror the release on the forge with the flashables + signed manifest + # attached. Repo derived from the remote URL; failure is reported, not fatal. + if command -v gh >/dev/null 2>&1; then + REPO=$(git remote get-url mono | sed -E 's#(git@[^:]+:|https://[^/]+/)##; s#\.git$##') + gh release create "$RELTAG" --repo "$REPO" \ + --title "$RELTAG" \ + --notes "Automated release: $RELTAG." \ + "$OUT"/*-sysupgrade.bin "$OUT"/*-emmc.img.gz \ + "$OUT/sha256sums" "$OUT/sha256sums.sig" \ + releases/latest.json releases/latest.json.sig \ + "$OUT/flash-mono-gateway.sh" \ + || echo "mono-publish: WARNING: GitHub release failed" >&2 + fi +fi + +echo "mono-publish: published $RELTAG" diff --git a/scripts/mono-sign-release.sh b/scripts/mono-sign-release.sh new file mode 100755 index 0000000000..f2bf0126a5 --- /dev/null +++ b/scripts/mono-sign-release.sh @@ -0,0 +1,32 @@ +#!/bin/sh +# Sign a staged Mono release. Runs where the signing key lives - ideally NOT +# the build/publish host, so a compromise of that host is not also a signing +# compromise. Given a release tag it usign-signs both the per-release hash +# list and the latest.json manifest that devices verify before flashing. +# +# Requires MONO_SIGN_KEY (path to the usign secret key). usign is taken from +# the OpenWrt host tree by default; override with MONO_USIGN if signing on a +# host without a build tree. +# +# Usage: scripts/mono-sign-release.sh +set -eu + +cd "$(dirname "$0")/.." +RELTAG=${1:?usage: mono-sign-release.sh } +: "${MONO_SIGN_KEY:?set MONO_SIGN_KEY to the usign secret key path}" +USIGN=${MONO_USIGN:-staging_dir/host/bin/usign} +[ -x "$USIGN" ] || command -v "$USIGN" >/dev/null 2>&1 || { + echo "mono-sign: usign not found ($USIGN); set MONO_USIGN" >&2; exit 1; } + +OUT="releases/$RELTAG" +[ -f "$OUT/sha256sums" ] || { echo "mono-sign: $OUT/sha256sums not found - stage the release first" >&2; exit 1; } +[ -f releases/latest.json ] || { echo "mono-sign: releases/latest.json not found - stage the release first" >&2; exit 1; } + +# The manifest must actually describe this release, or we'd sign a stale one. +json_tag=$(sed -n 's/.*"tag": *"\([^"]*\)".*/\1/p' releases/latest.json | head -1) +[ "$json_tag" = "$RELTAG" ] || { + echo "mono-sign: latest.json tag '$json_tag' != $RELTAG - stale manifest, refusing" >&2; exit 1; } + +"$USIGN" -S -m "$OUT/sha256sums" -s "$MONO_SIGN_KEY" -x "$OUT/sha256sums.sig" +"$USIGN" -S -m releases/latest.json -s "$MONO_SIGN_KEY" -x releases/latest.json.sig +echo "mono-sign: signed $RELTAG (sha256sums + latest.json)" diff --git a/scripts/mono-update.sh b/scripts/mono-update.sh index 36b6745b34..fd8aa6c3d6 100755 --- a/scripts/mono-update.sh +++ b/scripts/mono-update.sh @@ -3,10 +3,15 @@ # and stage release artifacts. Designed for cron: quiet no-op when # already current, hard stop (nonzero, clean tree) on conflicts. # -# No infrastructure knowledge lives here. Publishing only happens when -# the environment provides: -# MONO_PUBLISH_DEST rsync destination (e.g. host:/srv/openwrt) -# MONO_PUBLISH_URL public base URL written into latest.json +# Builds and stages releases//, then releases it. When MONO_SIGN_KEY is +# set (the nightly patch-release host) it auto-signs and publishes via: +# scripts/mono-sign-release.sh (needs MONO_SIGN_KEY) +# scripts/mono-publish-release.sh (needs MONO_PUBLISH_DEST; re-verifies, refuses unsigned) +# When MONO_SIGN_KEY is unset it stages only and prints those two steps, to run +# where the key lives (a new minor, or an off-host signing setup). Publishing +# re-verifies the signatures against the fleet's baked keys, so a wrong-key run +# fails and drops the tag instead of shipping something every device rejects. +# MONO_PUBLISH_URL is the public base URL baked into latest.json. # # Usage: scripts/mono-update.sh [--dry-run] [--force] # --force: release the current base even when already up to date @@ -76,11 +81,11 @@ fi # (mono-update-check reads it at build time). Dropped again on failure. git tag -f "$RELTAG" -# From here on ANY nonzero exit - build, artifact staging, publish, push - -# removes the tag, so the next run recomputes this same revision and -# retries instead of seeing a tag at branch head and skipping. Without -# this a failed rsync would leave the tag and wedge the release. set -e -# turns a failed command into an exit, which fires the trap. +# From here on ANY nonzero exit - build, staging, sign, or publish - removes +# the tag, so the next run recomputes this same revision and retries instead +# of seeing a tag at branch head and skipping. Without this a failed publish +# would leave the tag and wedge the release. set -e turns a failed command +# into an exit, which fires the trap. on_exit() { rc=$? [ "$rc" -eq 0 ] && return 0 @@ -108,15 +113,10 @@ cp "$BINDIR"/openwrt-layerscape-armv8_64b-mono_*-ext4-emmc.img.gz \ git format-patch --quiet -o "$OUT/patches" "$LATEST..$BRANCH" (cd "$OUT" && sha256sum *.img.gz *.bin > sha256sums) -# Sign the hash list. This - not the sha in latest.json - is the trust -# anchor devices verify against the baked public key before flashing. -USIGN=staging_dir/host/bin/usign -if [ -n "${MONO_SIGN_KEY:-}" ] && [ -x "$USIGN" ]; then - "$USIGN" -S -m "$OUT/sha256sums" -s "$MONO_SIGN_KEY" -x "$OUT/sha256sums.sig" - echo "mono-update: signed sha256sums" -else - echo "mono-update: WARNING: unsigned release (set MONO_SIGN_KEY); auto-update devices will refuse it" >&2 -fi +# NOTE: this script no longer signs. Signing happens on the key host via +# scripts/mono-sign-release.sh, and publishing (scripts/mono-publish-release.sh) +# refuses to run without the signatures. Keeping the key off the build/publish +# host means a compromise of this host is not also a signing compromise. # A real flashing tool, shipped with the release (not a two-step dd in prose). cat > "$OUT/flash-mono-gateway.sh" <<'FLASH' @@ -151,36 +151,28 @@ for name, p in prof.items(): continue h = hashlib.sha256(open(os.path.join(out, img), "rb").read()).hexdigest() devices[boards[0]] = {"sysupgrade": f"{urlbase}/{reltag}/{img}", "sha256": h} -json.dump({"tag": reltag, +# format_version lets the client reject a manifest shape it doesn't understand. +json.dump({"format_version": 1, + "tag": reltag, "date": datetime.datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ"), "devices": devices}, sys.stdout, indent=2) PY -if [ -n "${MONO_PUBLISH_DEST:-}" ]; then - echo "mono-update: publishing to $MONO_PUBLISH_DEST" - rsync -a "$OUT" releases/latest.json "$MONO_PUBLISH_DEST/" +# Release it. On the nightly patch-release host the signing key is present, so +# sign + publish run automatically. mono-publish-release.sh re-verifies both +# signatures against the fleet's baked keys before shipping, so a wrong or +# rotated key fails here (set -e -> trap drops the tag -> retry + alert) rather +# than pushing a release every device would reject. Without the key on this +# host, stage only and print the two steps to run where the key lives. +if [ -n "${MONO_SIGN_KEY:-}" ]; then + scripts/mono-sign-release.sh "$RELTAG" + scripts/mono-publish-release.sh "$RELTAG" + echo "mono-update: done - $RELTAG (published)" else - echo "mono-update: MONO_PUBLISH_DEST unset, artifacts staged in $OUT only" + cat >&2 </dev/null 2>&1; then - echo "mono-update: pushing $BRANCH and $RELTAG" - git push --force-with-lease mono "$BRANCH" - git push -f mono "$RELTAG" - - # Mirror the release on the forge with the flashables attached. - # Repo derived from the remote URL; failure is reported, not fatal. - if command -v gh >/dev/null 2>&1; then - REPO=$(git remote get-url mono | sed -E 's#(git@[^:]+:|https://[^/]+/)##; s#\.git$##') - gh release create "$RELTAG" --repo "$REPO" \ - --title "$RELTAG" \ - --notes "Automated release: OpenWrt $LATEST base, branch $(git rev-parse --short "$BRANCH")." \ - "$OUT"/*-sysupgrade.bin "$OUT"/*-emmc.img.gz \ - "$OUT/sha256sums" "$OUT"/sha256sums.sig "$OUT/flash-mono-gateway.sh" \ - || echo "mono-update: WARNING: GitHub release failed" >&2 - fi -fi - -echo "mono-update: done - $RELTAG" -- 2.47.3