From c66060bc552bff349613b7bb236ac8035215a54c Mon Sep 17 00:00:00 2001 From: Tomaz Zaman Date: Wed, 12 Aug 2026 15:06:02 +0200 Subject: [PATCH 53/59] build: run OpenWrt builds in the pinned Nix FHS env (canonical) Adopt cvandesande's flake.nix as mono's canonical build environment, ahead of him co-maintaining OpenWrt: both maintainers now build against a byte-identical pinned toolchain instead of a drifting host apt package set. - flake.nix + flake.lock at the source-tree root (from cvd's tree, which he already maintains). shell.nix intentionally skipped -- it is upstream OpenWrt's post-25.12 file plus his local drift, and flake.nix is self-contained. Validated here: `nix run . -- -c '...'` gives GCC 15.2.0 / Make 4.4.1 / bison at the FHS /usr/bin OpenWrt expects. - scripts/mono-update.sh runs the three make steps via `nix run . -- -c 'set -e; make ...'`; only the build is wrapped, so git, publish and signing stay on the host (their tools are not in the flake). Use `nix run`, never `nix develop -c` / `nix-shell --run`, which hang on the env's shellHook exec. - BUILDING.md documents the prereqs, interactive vs scripted use, the `nix develop --command` hang, and that Nix only sees git-tracked files. Nix (Determinate 3.15.2) already installed on the build host. Co-Authored-By: Claude Fable 5 --- BUILDING.md | 50 ++++++++++++++++++ flake.lock | 27 ++++++++++ flake.nix | 117 +++++++++++++++++++++++++++++++++++++++++ scripts/mono-update.sh | 18 ++++--- 4 files changed, 206 insertions(+), 6 deletions(-) create mode 100644 BUILDING.md create mode 100644 flake.lock create mode 100644 flake.nix diff --git a/BUILDING.md b/BUILDING.md new file mode 100644 index 0000000000..8df5da9199 --- /dev/null +++ b/BUILDING.md @@ -0,0 +1,50 @@ +# Building the Mono Gateway image + +The build runs inside a pinned **Nix FHS environment** (`flake.nix`) so the host +toolchain is byte-identical on every machine — there is no `apt install` list to +drift, and a build is reproducible across machines and over time. + +## Prerequisites + +- **Nix with flakes** — [Determinate Nix](https://determinate.systems/nix) + (flakes on by default), or upstream Nix with + `experimental-features = nix-command flakes`. +- **Unprivileged user namespaces** — enabled by default on stock Debian/Ubuntu + kernels; the FHS env uses `bubblewrap`, which needs them. (Some hardened or + corp-locked kernels disable them.) + +Nothing else. All build dependencies (gcc, make, bison, flex, …) come from the +flake, pinned by `flake.lock`. + +## Interactive + +```sh +nix develop # drops you into the FHS shell: gcc, make, bison … on PATH +make menuconfig +make -j"$(nproc)" world +``` + +## Scripted / CI / nightly + +**Do not** use `nix develop --command …` or `nix-shell --run …` — the env's +`shellHook` `exec`s into the FHS shell, so the `--command`/`--run` payload never +runs and the invocation hangs. Use `nix run`, which enters the FHS env directly: + +```sh +nix run . -- -c 'make -j"$(nproc)" world' +``` + +The nightly (`scripts/mono-update.sh`) builds this way. Only the `make` steps run +inside the env; git, publishing and release signing stay on the host. + +## The pin + +`flake.lock` pins `nixpkgs` to an exact revision — that is what makes builds +reproducible. Update it deliberately: + +```sh +nix flake update # bump nixpkgs; then rebuild + test before committing the lock +``` + +Note: Nix only sees **git-tracked** files, so `flake.nix`/`flake.lock` (and any +new source) must be `git add`ed before `nix run`/`nix develop` will pick them up. diff --git a/flake.lock b/flake.lock new file mode 100644 index 0000000000..8a2f2c7a4c --- /dev/null +++ b/flake.lock @@ -0,0 +1,27 @@ +{ + "nodes": { + "nixpkgs": { + "locked": { + "lastModified": 1785491804, + "narHash": "sha256-p00vplVOyfKGlhju0+eGGPAxyYYKaY0lpyuoHLFIx2Y=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "5b4f72e1705a63aace42900610c4db82cb4af0ec", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-26.05", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "nixpkgs": "nixpkgs" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 0000000000..28a69cd82b --- /dev/null +++ b/flake.nix @@ -0,0 +1,117 @@ +{ + description = "OpenWrt FHS build environment (NixOS)"; + + inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-26.05"; + + outputs = { self, nixpkgs }: + let + system = "x86_64-linux"; + pkgs = import nixpkgs { inherit system; }; + + # gcc's cross-prefixed binutils wrappers (e.g. x86_64-unknown-linux-gnu-gcc-ar) + # aren't symlinked under their unprefixed names on PATH by default; some + # host tool builds (e.g. zstd) invoke $AR/$NM/$RANLIB as plain gcc-ar/gcc-nm/ + # gcc-ranlib, so provide those names explicitly. + fixWrapper = pkgs.runCommand "fix-wrapper" { } '' + mkdir -p $out/bin + for i in ${pkgs.gcc.cc}/bin/*-gnu-gcc*; do + ln -s ${pkgs.gcc}/bin/gcc $out/bin/$(basename "$i") + done + for i in ${pkgs.gcc.cc}/bin/*-gnu-{g++,c++}*; do + ln -s ${pkgs.gcc}/bin/g++ $out/bin/$(basename "$i") + done + ln -sf ${pkgs.gcc.cc}/bin/{,*-gnu-}gcc-{ar,nm,ranlib} $out/bin + ''; + + fhs = pkgs.buildFHSEnv { + name = "openwrt-env"; + runScript = "bash"; + targetPkgs = pkgs: with pkgs; [ + bash + bc + bison + bzip2 + ccache + coreutils + diffutils + file + findutils + fixWrapper + flex + gawk + gcc + gettext + git + glibc.static + gnumake + gnugrep + gnused + gnutar + gzip + ncurses + openssl + patch + perl + pkg-config + (python3.withPackages (ps: [ ps.setuptools ])) + rsync + subversion + swig + unzip + util-linux + wget + which + xz + libxcrypt + zlib + zlib.static + zstd + ]; + multiPkgs = null; + extraOutputsToInstall = [ "dev" ]; + profile = '' + export hardeningDisable=all + export NIX_HARDENING_ENABLE= + export NIX_HARDENING_ENABLE_x86_64_unknown_linux_gnu= + export AR=gcc-ar + export NM=gcc-nm + export RANLIB=gcc-ranlib + export FAKEROOTDONTTRYCHOWN=1 + # nixpkgs' ld-wrapper injects -rpath $out/lib into every host link. + # Outside a real nix build $out resolves to a relative "outputs/out", + # so host binaries get a dead rpath into the tree instead of a usable + # one -- 52 of 77 staging_dir/hostpkg/bin binaries carry it. Harmless + # for anything needing only nix-store libs, fatal for any + # hostpkg->hostpkg library dependency. + # + # Hit on 2026-08-01 during a cold build: libselinux/host staged + # libselinux.so.1, gettext-full/host configured minutes later, linked + # against it (its HOST_BUILD_DEPENDS does not order the two), and + # every msg* tool then failed to load it, breaking + # policycoreutils/host. Only cold builds lose that race. + # + # Point the loader at the host staging tree. Guarded on rules.mk so + # it applies only when the shell is entered from an OpenWrt tree. + if [ -f "$PWD/rules.mk" ]; then + export LD_LIBRARY_PATH="$PWD/staging_dir/hostpkg/lib''${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" + fi + ''; + }; + in { + # Non-interactive/scripted use: nix run . -- -c 'make -j$(nproc) target/linux/compile' + packages.${system}.default = fhs; + apps.${system}.default = { + type = "app"; + program = "${fhs}/bin/openwrt-env"; + }; + + # Interactive use: nix develop + devShells.${system}.default = pkgs.mkShell { + name = "openwrt-env-wrapper"; + packages = [ fhs ]; + shellHook = '' + exec ${fhs}/bin/openwrt-env + ''; + }; + }; +} diff --git a/scripts/mono-update.sh b/scripts/mono-update.sh index c217733b09..595e81b56f 100755 --- a/scripts/mono-update.sh +++ b/scripts/mono-update.sh @@ -106,12 +106,18 @@ BINDIR=bin/targets/layerscape/armv8_64b rm -rf "$BINDIR" cp configs/mono_gateway-dk.seed .config -make defconfig -# mono-update-check bakes the release tag into /etc/mono_release at its -# build time; force it fresh or every image ships the stale identity of -# the package's first build (and auto-mode devices re-flash forever). -make package/mono/mono-update-check/clean package/mono/mono-update-check/compile -make -j"$(nproc)" world + +# Build inside the pinned Nix FHS env (flake.nix) so the toolchain is identical +# on every machine. Use `nix run` -- NOT `nix develop -c` / `nix-shell --run`, +# which hang on the env's shellHook exec. git, publish and signing stay on the +# host (their tools are not in the flake's package set). +# mono-update-check is force-rebuilt (clean+compile) so /etc/mono_release carries +# THIS release tag; otherwise every image ships the stale identity of the +# package's first build and auto-mode devices re-flash forever. +nix run . -- -c 'set -e + make defconfig + make package/mono/mono-update-check/clean package/mono/mono-update-check/compile + make -j"$(nproc)" world' # Verify the image actually baked THIS release's identity. The OTA client and # the on-device anti-rollback floor trust /etc/mono_release, so a stale value -- 2.47.3