diff --git a/MODULE.bazel b/MODULE.bazel index 68ded81bb..0be68a91d 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -39,12 +39,26 @@ bazel_dep(name = "rules_cc", version = "0.2.17") # scripts/bazel_hermetic_cc_test.go guards both. deb_sysroot = use_repo_rule("//third_party/cc_toolchain:deb_sysroot.bzl", "deb_sysroot") +# Where every pinned Ubuntu 24.04 package below comes from, tried in order +# (the sha256 pins, not the source, decide what is accepted): +# - gascity's content-addressed mirror, a release holding each pinned +# .deb as .deb (gascity's tools/cc_toolchain/mirror_debs.sh +# publishes it; gascity pins the same packages), so a cold fetch does not +# depend on Canonical's servers; +# - the snapshot these packages were pinned from (immutable, but a single +# host that returns 503 for minutes at a time); +# - Launchpad's librarian, which keeps every file ever published to the +# primary archive, superseded versions included; +# - archive.ubuntu.com, which drops superseded versions. +NOBLE_DEB_URLS = [ + "https://github.com/gastownhall/gascity/releases/download/toolchain-noble-debs-20261001/{sha256}.deb", + "https://snapshot.ubuntu.com/ubuntu/20261001T000000Z/{path}", + "https://launchpad.net/ubuntu/+archive/primary/+files/{file}", + "https://archive.ubuntu.com/ubuntu/{path}", +] + deb_sysroot( name = "cc_sysroot_noble_amd64", - mirrors = [ - "https://snapshot.ubuntu.com/ubuntu/20261001T000000Z/", - "https://archive.ubuntu.com/ubuntu/", - ], packages = { "pool/main/g/gcc-13/libgcc-13-dev_13.3.0-6ubuntu2~24.04.1_amd64.deb": "cd689db2691edaa10f37329307292796bb599e722e0505c79e14caaa1fe9a93a", "pool/main/g/gcc-13/libstdc++-13-dev_13.3.0-6ubuntu2~24.04.1_amd64.deb": "ee5633e863e19c3381ed97842ce35ed32ede96a3d1ae4e94c051d3036fe21347", @@ -70,6 +84,7 @@ deb_sysroot( "usr/lib/gcc/x86_64-linux-gnu/13/libquadmath*", "usr/lib/gcc/x86_64-linux-gnu/13/libstdc++*.a", ], + urls = NOBLE_DEB_URLS, ) LLVM_VERSION = "22.1.8" diff --git a/scripts/bazel_hermetic_cc_test.go b/scripts/bazel_hermetic_cc_test.go index 335d0e648..fec515bff 100644 --- a/scripts/bazel_hermetic_cc_test.go +++ b/scripts/bazel_hermetic_cc_test.go @@ -33,9 +33,14 @@ const ( var ( sha256HexRE = regexp.MustCompile(`^[0-9a-f]{64}$`) - // snapshot.ubuntu.com serves each timestamp's archive state forever; a - // moving mirror (archive.ubuntu.com) may only follow it as a fallback. - ubuntuSnapshotRE = regexp.MustCompile(`^https://snapshot\.ubuntu\.com/ubuntu/\d{8}T\d{6}Z/$`) + // snapshot.ubuntu.com serves each timestamp's archive state forever, so + // the package pins stay resolvable at the snapshot they were taken from. + ubuntuSnapshotRE = regexp.MustCompile(`^https://snapshot\.ubuntu\.com/ubuntu/\d{8}T\d{6}Z/\{path\}$`) + // archive.ubuntu.com drops superseded versions: a fallback, never first. + ubuntuMovingMirrorRE = regexp.MustCompile(`^https?://([a-z0-9-]+\.)*archive\.ubuntu\.com/`) + // deb_sysroot's URL templates: {path} (pool path), {file} (its basename) + // and {sha256} (the pin) are the only placeholders. + debURLPlaceholderRE = regexp.MustCompile(`\{[^}]*\}`) // A sliced llvm_dist archive is a repack (repack_llvm.sh), not an // upstream release, so it may only come from a gastownhall release: // beads' own mirror or gascity's original asset (same bytes, same sha256). @@ -114,36 +119,96 @@ func checkHermeticCCModule(module string) []error { errs = append(errs, errors.New("llvm_dist must pin its archive by sha256")) } - if sysroot, err := moduleCall(module, "deb_sysroot", `name = "cc_sysroot_noble_amd64"`); err != nil { + if _, err := moduleCall(module, "deb_sysroot", `name = "cc_sysroot_noble_amd64"`); err != nil { errs = append(errs, err) + } + for _, m := range regexp.MustCompile(`(?ms)^deb_sysroot\(\n(.*?)^\)`).FindAllStringSubmatch(module, -1) { + errs = append(errs, checkDebSysroot(module, m[1])...) + } + return errs +} + +// checkDebSysroot: every package is an amd64 pool .deb pinned by sha256 that +// ships no ICU, and its URL templates give each package more than one host, +// keep the snapshot the pins were taken from, and never start with a moving +// mirror. +func checkDebSysroot(module, call string) []error { + var errs []error + name := "deb_sysroot" + if m := regexp.MustCompile(`name = "([^"]+)"`).FindStringSubmatch(call); m != nil { + name += " " + m[1] + } + urls, err := debSysrootURLs(module, call) + if err != nil { + errs = append(errs, errors.New(name+": "+err.Error())) } else { - mirrors := regexp.MustCompile(`(?s)mirrors = \[\s*"([^"]+)"`).FindStringSubmatch(sysroot) - if mirrors == nil || !ubuntuSnapshotRE.MatchString(mirrors[1]) { - errs = append(errs, errors.New("deb_sysroot's first mirror must be an immutable snapshot.ubuntu.com/ubuntu// root")) + hosts := map[string]bool{} + snapshot := false + for _, u := range urls { + holes := debURLPlaceholderRE.FindAllString(u, -1) + if len(holes) != 1 || (holes[0] != "{path}" && holes[0] != "{file}" && holes[0] != "{sha256}") { + errs = append(errs, errors.New(name+": url "+u+" must name the package by exactly one of {path}, {file} or {sha256}")) + } + snapshot = snapshot || ubuntuSnapshotRE.MatchString(u) + host, _, _ := strings.Cut(strings.TrimPrefix(u, "https://"), "/") + hosts[host] = true } - body := regexp.MustCompile(`(?s)packages = \{(.*?)\}`).FindStringSubmatch(sysroot) - if body == nil { - errs = append(errs, errors.New("deb_sysroot has no packages")) - } else { - pkgs := quotedPairs(body[1]) - if len(pkgs) == 0 { - errs = append(errs, errors.New("deb_sysroot has no packages")) - } - for path, sum := range pkgs { - if !strings.HasPrefix(path, "pool/") || !strings.HasSuffix(path, "_amd64.deb") || !sha256HexRE.MatchString(sum) { - errs = append(errs, errors.New("deb_sysroot package "+path+" must be an amd64 pool .deb pinned by sha256")) - } - // ICU policy (engdocs/ICU-POLICY.md): nothing links libicu, - // so the sysroot offers no ICU to link against. - if strings.Contains(path, "/icu/") { - errs = append(errs, errors.New("deb_sysroot package "+path+" ships ICU, which beads must never link (engdocs/ICU-POLICY.md)")) - } - } + if !snapshot { + errs = append(errs, errors.New(name+": urls must include an immutable https://snapshot.ubuntu.com/ubuntu//{path}")) + } + if len(hosts) < 2 { + errs = append(errs, errors.New(name+": urls must span at least two hosts, so one outage cannot fail a cold fetch")) + } + if len(urls) > 0 && ubuntuMovingMirrorRE.MatchString(urls[0]) { + errs = append(errs, errors.New(name+": the first url must not be a moving mirror (archive.ubuntu.com drops superseded packages)")) + } + } + body := regexp.MustCompile(`(?s)packages = \{(.*?)\}`).FindStringSubmatch(call) + if body == nil { + return append(errs, errors.New(name+" has no packages")) + } + pkgs := quotedPairs(body[1]) + if len(pkgs) == 0 { + errs = append(errs, errors.New(name+" has no packages")) + } + for path, sum := range pkgs { + if !strings.HasPrefix(path, "pool/") || !strings.HasSuffix(path, "_amd64.deb") || !sha256HexRE.MatchString(sum) { + errs = append(errs, errors.New(name+" package "+path+" must be an amd64 pool .deb pinned by sha256")) + } + // ICU policy (engdocs/ICU-POLICY.md): nothing links libicu, + // so the sysroot offers no ICU to link against. + if strings.Contains(path, "/icu/") { + errs = append(errs, errors.New(name+" package "+path+" ships ICU, which beads must never link (engdocs/ICU-POLICY.md)")) } } return errs } +// debSysrootURLs returns a deb_sysroot call's url templates: an inline list, +// or a top-level MODULE.bazel list constant. +func debSysrootURLs(module, call string) ([]string, error) { + m := regexp.MustCompile(`(?s)urls = (\[.*?\]|[A-Z][A-Z0-9_]*),`).FindStringSubmatch(call) + if m == nil { + return nil, errors.New("no urls") + } + list := m[1] + if !strings.HasPrefix(list, "[") { + def := regexp.MustCompile(`(?ms)^` + list + ` = (\[.*?^\])`).FindStringSubmatch(module) + if def == nil { + return nil, errors.New("urls names " + list + ", which MODULE.bazel does not define as a list") + } + list = def[1] + } + var urls []string + for _, q := range regexp.MustCompile(`"([^"]+)"`).FindAllStringSubmatch(list, -1) { + urls = append(urls, q[1]) + } + if len(urls) == 0 { + return nil, errors.New("no urls") + } + return urls, nil +} + // checkHermeticLLVMSlice: a sliced llvm_dist downloads only from a // gastownhall release, and its members equal the repack script's list, so // the pinned slice can be rebuilt from upstream and checked file by file. @@ -240,12 +305,16 @@ func TestBazelHermeticCCToolchainGuards(t *testing.T) { module := `bazel_dep(name = "rules_go", version = "0.63.0") bazel_dep(name = "toolchains_llvm", version = "1.11.0") +NOBLE_DEB_URLS = [ + "https://github.com/gastownhall/gascity/releases/download/toolchain-noble-debs-20261001/{sha256}.deb", + "https://snapshot.ubuntu.com/ubuntu/20261001T000000Z/{path}", + "https://launchpad.net/ubuntu/+archive/primary/+files/{file}", + "https://archive.ubuntu.com/ubuntu/{path}", +] + deb_sysroot( name = "cc_sysroot_noble_amd64", - mirrors = [ - "https://snapshot.ubuntu.com/ubuntu/20261001T000000Z/", - "https://archive.ubuntu.com/ubuntu/", - ], + urls = NOBLE_DEB_URLS, packages = { "pool/main/g/glibc/libc6-dev_2.39-0ubuntu8.9_amd64.deb": "` + sum + `", }, @@ -293,7 +362,14 @@ llvm.sysroot( "other-host dist version": strings.Replace(module, "llvm_version = LLVM_VERSION", `llvm_version = "17.0.6"`, 1), "other-host dist versions": strings.Replace(module, `llvm_versions = {"": LLVM_VERSION},`, "", 1), "unpinned LLVM": strings.Replace(module, ` sha256 = "`+sum+`",`+"\n)", "\n)", 1), - "moving mirror first": strings.Replace(module, `"https://snapshot.ubuntu.com/ubuntu/20261001T000000Z/",`, "", 1), + "moving mirror first": strings.Replace(module, `"https://github.com/gastownhall/gascity/releases/download/toolchain-noble-debs-20261001/{sha256}.deb",`, `"https://archive.ubuntu.com/ubuntu/{path}",`, 1), + "no snapshot": strings.Replace(module, `"https://snapshot.ubuntu.com/ubuntu/20261001T000000Z/{path}",`, "", 1), + "moving snapshot": strings.Replace(module, "/ubuntu/20261001T000000Z/{path}", "/ubuntu/{path}", 1), + "single host": strings.NewReplacer(`"https://github.com/gastownhall/gascity/releases/download/toolchain-noble-debs-20261001/{sha256}.deb",`, "", `"https://launchpad.net/ubuntu/+archive/primary/+files/{file}",`, "", `"https://archive.ubuntu.com/ubuntu/{path}",`, "").Replace(module), + "package-blind url": strings.Replace(module, "/+files/{file}", "/+files/libc6.deb", 1), + "unknown placeholder": strings.Replace(module, "/+files/{file}", "/+files/{name}", 1), + "undefined url list": strings.Replace(module, "urls = NOBLE_DEB_URLS,", "urls = OTHER_URLS,", 1), + "no urls": strings.Replace(module, "urls = NOBLE_DEB_URLS,", "", 1), "unpinned package": strings.Replace(module, `_amd64.deb": "`+sum+`"`, `_amd64.deb": ""`, 1), "ICU in sysroot": strings.Replace(module, " packages = {\n", " packages = {\n \"pool/main/i/icu/libicu74_74.2-1ubuntu3.1_amd64.deb\": \""+sum+"\",\n", 1), "sysroot other targets": strings.Replace(module, "label = \""+hermeticCCSysrootLabel+"\",\n targets = [\"linux-x86_64\"]", "label = \""+hermeticCCSysrootLabel+"\",\n targets = []", 1), diff --git a/third_party/cc_toolchain/deb_sysroot.bzl b/third_party/cc_toolchain/deb_sysroot.bzl index ae96aa392..809476f93 100644 --- a/third_party/cc_toolchain/deb_sysroot.bzl +++ b/third_party/cc_toolchain/deb_sysroot.bzl @@ -1,7 +1,9 @@ """deb_sysroot: a C/C++ sysroot assembled from pinned Debian/Ubuntu packages. Every package is fetched by sha256, so the sysroot is byte-identical on every -host. It gives the hermetic LLVM toolchain (MODULE.bazel) the glibc, +host whichever source served it; the sources are only an availability chain. +All packages download concurrently (Bazel bounds it by +--http_max_parallel_downloads), each trying its URLs in order. It gives the hermetic LLVM toolchain (MODULE.bazel) the glibc, libstdc++ and ICU headers and link libraries that cgo targets need, instead of whatever the host has under /usr. """ @@ -23,20 +25,30 @@ grep -rlI --include='*.so' 'GROUP' usr/lib | while IFS= read -r s; do done """ +def _package_urls(templates, path, sha256): + return [ + t.format(path = path, file = path.rsplit("/", 1)[-1], sha256 = sha256) + for t in templates + ] + def _deb_sysroot_impl(rctx): + pending = [] for path, sha256 in rctx.attr.packages.items(): - name = path.rsplit("/", 1)[-1] - deb = "_debs/" + name - rctx.download( - url = [mirror + path for mirror in rctx.attr.mirrors], + deb = "_debs/" + path.rsplit("/", 1)[-1] + pending.append((deb, rctx.download( + url = _package_urls(rctx.attr.urls, path, sha256), output = deb, sha256 = sha256, - ) - unpacked = "_debs/" + name + ".d" + block = False, + ))) + for _, download in pending: + download.wait() + for deb, _ in pending: + unpacked = deb + ".d" rctx.extract(deb, output = unpacked) data = [f for f in rctx.path(unpacked).readdir() if f.basename.startswith("data.tar")] if len(data) != 1: - fail("%s: expected one data.tar.* member, found %s" % (name, data)) + fail("%s: expected one data.tar.* member, found %s" % (deb, data)) rctx.extract(data[0], output = ".") rctx.delete("_debs") @@ -67,11 +79,12 @@ deb_sysroot = repository_rule( attrs = { "packages": attr.string_dict( mandatory = True, - doc = "Pool path (relative to each mirror) to the .deb's sha256.", + doc = "Pool path (pool//...) to the .deb's sha256.", ), - "mirrors": attr.string_list( + "urls": attr.string_list( mandatory = True, - doc = "Archive roots tried in order; the first is an immutable snapshot.", + doc = "URL templates tried in order for every package: {path} is the pool " + + "path, {file} its basename and {sha256} its pinned digest.", ), "remove": attr.string_list( doc = "Bash globs (globstar) dropped after unpacking: docs, unused runtimes and archives.",