diff --git a/.github/scripts/refresh-launchers.sh b/.github/scripts/refresh-launchers.sh new file mode 100755 index 0000000000..96d49f1454 --- /dev/null +++ b/.github/scripts/refresh-launchers.sh @@ -0,0 +1,35 @@ +#!/bin/bash +# +# The contents of this file are subject to the terms of the Common Development and +# Distribution License (the License). You may not use this file except in compliance with the +# License. +# +# You can obtain a copy of the License at legal/CDDLv1.0.txt. See the License for the +# specific language governing permission and limitations under the License. +# +# When distributing Covered Software, include this CDDL Header Notice in each file and include +# the License file at legal/CDDLv1.0.txt. If applicable, add the following below the CDDL +# Header, with the fields enclosed by brackets [] replaced by your own identifying +# information: "Portions copyright [year] [name of copyright owner]". +# +# Copyright 2026 3A Systems, LLC. + +# Usage: refresh-launchers.sh BUILT LIB +# +# Copies each BUILT/*.exe over its namesake in LIB, unless the two differ only in the +# toolchain stamp (same-pe-code.py). The Package/Deploy workflow runs it before it +# commits LIB; the Build workflow runs it on known launcher pairs before merge. +set -e +here=$(dirname "$0") +for built in "$1"/*.exe; do + committed=$2/$(basename "$built") + # Exit 1 (the code differs) and 2 (not readable as a PE image) both take the rebuilt + # file: when in doubt, refresh. + if [ -f "$committed" ] && python3 "$here/same-pe-code.py" "$committed" "$built"; then + if ! cmp -s "$committed" "$built"; then + echo "$committed differs from the rebuilt one only in the toolchain stamp - keeping it." + fi + continue + fi + cp "$built" "$committed" +done diff --git a/.github/scripts/same-pe-code.py b/.github/scripts/same-pe-code.py new file mode 100755 index 0000000000..d497a2aa8e --- /dev/null +++ b/.github/scripts/same-pe-code.py @@ -0,0 +1,163 @@ +#!/usr/bin/env python3 +# +# The contents of this file are subject to the terms of the Common Development and +# Distribution License (the License). You may not use this file except in compliance with the +# License. +# +# You can obtain a copy of the License at legal/CDDLv1.0.txt. See the License for the +# specific language governing permission and limitations under the License. +# +# When distributing Covered Software, include this CDDL Header Notice in each file and include +# the License file at legal/CDDLv1.0.txt. If applicable, add the following below the CDDL +# Header, with the fields enclosed by brackets [] replaced by your own identifying +# information: "Portions copyright [year] [name of copyright owner]". +# +# Copyright 2026 3A Systems, LLC. + +"""Tell whether two PE images differ in anything but the stamp of the toolchain. + +Usage: same-pe-code.py COMMITTED REBUILT + +Exits 0 when the two files are identical once the build stamp and the header padding +are left out of both, 1 when they differ anywhere else, and 2 when either one cannot be +read as a PE image. + +The Windows launchers are linked with /Brepro, so their bytes are a function of the +inputs - and the inputs include the build numbers of cl, link and cvtres. Two runner +images a patch release of Visual Studio apart (14.51.36256 and 14.51.36257, say) turn +out byte-for-byte the same code, data and resources, yet different files: the Rich +header lists those build numbers, the REPRO debug entry holds a hash over them, and +the COFF and debug directory timestamps and the PE checksum are derived from that +hash. GitHub rolls a new image out over days, so the Windows job lands on either one +and a byte comparison refreshes the committed launchers back and forth on every push. + +What is left out of the comparison: +- the Rich header; +- the COFF and debug directory timestamps and the PE checksum; +- the REPRO hash, and the PDB GUID and age of an RSDS CodeView entry; +- the zero padding the linker puts after the DOS stub, up to the PE header, and after + the section table, up to SizeOfHeaders, and e_lfanew, which only says where the first + of them ends. A patch release may pad differently: + 14.51.36252 put the PE header at 0x100, 14.51.36256 at 0xf0, around the same code. +The DOS stub, the PE headers, the section table and every byte from SizeOfHeaders on +are still compared, so a change of source or of code generation still shows. A Rich +header that gains or loses an entry no longer does on its own: it comes with a change +of the objects linked in, which moves the code as well. +""" + +import struct +import sys + +DEBUG_TYPE_CODEVIEW = 2 +DEBUG_TYPE_REPRO = 16 +DEBUG_ENTRY_SIZE = 28 + + +class NotPE(Exception): + pass + + +def u16(b, off): + return struct.unpack_from(" len(b): + raise NotPE("field at 0x%x runs past the end of the file" % off) + b[off:off + length] = bytes(length) + + +def normalize(data): + b = bytearray(data) + try: + if b[:2] != b"MZ": + raise NotPE("no MZ signature") + pe = u32(b, 0x3C) + if b[pe:pe + 4] != b"PE\0\0": + raise NotPE("no PE signature") + + # The Rich header sits between the DOS stub and the PE header: "DanS" XOR key, + # the (prodId, build, count) records XOR key, then "Rich" and the key itself. + rich = b.find(b"Rich", 0x40, pe) + if rich >= 0: + key = b[rich + 4:rich + 8] + dans = bytes(x ^ y for x, y in zip(b"DanS", key)) + start = b.rfind(dans, 0x40, rich) + if start < 0: + raise NotPE("Rich header without its DanS marker") + blank(b, start, rich + 8 - start) + + coff = pe + 4 + sections = u16(b, coff + 2) + opt_size = u16(b, coff + 16) + blank(b, coff + 4, 4) # TimeDateStamp + + opt = coff + 20 + magic = u16(b, opt) + if magic == 0x10B: + data_dirs = opt + 96 + elif magic == 0x20B: + data_dirs = opt + 112 + else: + raise NotPE("unknown optional header magic 0x%x" % magic) + blank(b, opt + 64, 4) # CheckSum + headers_end = u32(b, opt + 60) # SizeOfHeaders + if not pe < headers_end <= len(b): + raise NotPE("SizeOfHeaders 0x%x out of range" % headers_end) + + table = opt + opt_size + spans = [] + for i in range(sections): + s = table + 40 * i + spans.append((u32(b, s + 12), u32(b, s + 8), u32(b, s + 20))) # va, vsize, raw + + def file_offset(rva): + for va, vsize, raw in spans: + if va <= rva < va + vsize: + return raw + rva - va + raise NotPE("RVA 0x%x lies in no section" % rva) + + debug_rva = u32(b, data_dirs + 6 * 8) + debug_size = u32(b, data_dirs + 6 * 8 + 4) + if debug_rva and debug_size: + base = file_offset(debug_rva) + for i in range(debug_size // DEBUG_ENTRY_SIZE): + entry = base + DEBUG_ENTRY_SIZE * i + blank(b, entry + 4, 4) # TimeDateStamp + kind = u32(b, entry + 12) + size = u32(b, entry + 16) + raw = u32(b, entry + 24) + if kind == DEBUG_TYPE_REPRO: + blank(b, raw, size) # the hash itself + elif kind == DEBUG_TYPE_CODEVIEW and b[raw:raw + 4] == b"RSDS": + blank(b, raw + 4, 20) # PDB GUID and age + except struct.error as e: + raise NotPE(str(e)) + # Compare what the padding surrounds, wherever it ends (see above). The Rich header + # is blanked by now, so it goes with the padding after the DOS stub; e_lfanew goes + # too, since it only says where that padding ends. + return (bytes(b[:0x3C]) + bytes(b[0x40:pe]).rstrip(b"\0") + + bytes(b[pe:headers_end]).rstrip(b"\0") + bytes(b[headers_end:])) + + +def main(argv): + if len(argv) != 3: + print(__doc__.strip().splitlines()[2], file=sys.stderr) + return 2 + try: + images = [] + for path in argv[1:]: + with open(path, "rb") as f: + images.append(normalize(f.read())) + except (OSError, NotPE) as e: + print("same-pe-code: %s" % e, file=sys.stderr) + return 2 + return 0 if images[0] == images[1] else 1 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/.github/scripts/test-refresh-launchers.sh b/.github/scripts/test-refresh-launchers.sh new file mode 100755 index 0000000000..50c1e42f3b --- /dev/null +++ b/.github/scripts/test-refresh-launchers.sh @@ -0,0 +1,121 @@ +#!/bin/bash +# +# The contents of this file are subject to the terms of the Common Development and +# Distribution License (the License). You may not use this file except in compliance with the +# License. +# +# You can obtain a copy of the License at legal/CDDLv1.0.txt. See the License for the +# specific language governing permission and limitations under the License. +# +# When distributing Covered Software, include this CDDL Header Notice in each file and include +# the License file at legal/CDDLv1.0.txt. If applicable, add the following below the CDDL +# Header, with the fields enclosed by brackets [] replaced by your own identifying +# information: "Portions copyright [year] [name of copyright owner]". +# +# Copyright 2026 3A Systems, LLC. + +# Checks same-pe-code.py and refresh-launchers.sh against launchers committed in the +# history of master, so that neither can start keeping a changed launcher, or refreshing +# one that differs only in the toolchain stamp, while CI stays green. Needs the full +# history (fetch-depth: 0). Run from the root of the repository. +set -e +here=$(dirname "$0") +t=$(mktemp -d) +trap 'rm -rf "$t"' EXIT + +g() { git show "$1:opendj-server-legacy/lib/$2.exe" > "$t/$2-$3.exe"; } +expect() { + local rc=0 + python3 "$here/same-pe-code.py" "$t/$2" "$t/$3" || rc=$? + if [ "$rc" != "$1" ]; then + echo "::error::same-pe-code.py $2 $3 exited $rc, expected $1 ($4)" + exit 1 + fi + echo "ok: $2 $3 -> $rc ($4)" +} + +for f in launcher_administrator winlauncher opendj_service; do + g a611c10c46^ $f 36252 + g a611c10c46 $f 36256 + g a8a18f000d $f 36257 + g 74cb4f5a84 $f old-code + expect 0 $f-36256.exe $f-36257.exe "the toolchain stamp only" + expect 0 $f-36257.exe $f-36256.exe "the toolchain stamp only" + expect 1 $f-old-code.exe $f-36257.exe "before and after a change of the launcher sources" +done +# winlauncher.exe has CheckSum 0 in both builds, launcher_administrator.exe does not. +for f in launcher_administrator winlauncher; do + expect 0 $f-36252.exe $f-36256.exe "the same code, the PE header at 0x100 and at 0xf0" +done +expect 1 opendj_service-36252.exe opendj_service-36256.exe "a code change of the 05.09 refresh" +expect 1 winlauncher-36257.exe launcher_administrator-36257.exe "two different launchers" +# A real change moves the headers too; this one does not, so the sections must be compared. +python3 - "$t/winlauncher-36257.exe" "$t/winlauncher-text-byte.exe" <<'EOF' +import struct, sys +b = bytearray(open(sys.argv[1], "rb").read()) +pe = struct.unpack_from(" "$t/truncated.exe" +expect 2 truncated.exe winlauncher-36257.exe "a file that is not a PE image" + +# The loop: keeps a launcher that differs only in the stamp, refreshes one whose code +# changed, adds one that is not committed yet, and refreshes one whose committed file +# is not a PE image. +mkdir "$t/built" "$t/lib" +cp "$t/winlauncher-36257.exe" "$t/built/winlauncher.exe" +cp "$t/winlauncher-36256.exe" "$t/lib/winlauncher.exe" +cp "$t/launcher_administrator-36257.exe" "$t/built/launcher_administrator.exe" +cp "$t/launcher_administrator-old-code.exe" "$t/lib/launcher_administrator.exe" +cp "$t/opendj_service-36257.exe" "$t/built/opendj_service.exe" +cp "$t/winlauncher-36257.exe" "$t/built/unreadable.exe" +cp "$t/truncated.exe" "$t/lib/unreadable.exe" +bash "$here/refresh-launchers.sh" "$t/built" "$t/lib" +cmp "$t/lib/winlauncher.exe" "$t/winlauncher-36256.exe" +cmp "$t/lib/launcher_administrator.exe" "$t/built/launcher_administrator.exe" +cmp "$t/lib/opendj_service.exe" "$t/built/opendj_service.exe" +cmp "$t/lib/unreadable.exe" "$t/built/unreadable.exe" +echo "ok: refresh-launchers.sh keeps, refreshes and adds as expected" diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8469aec276..61ef64ef2e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -56,6 +56,12 @@ jobs: with: fetch-depth: 0 submodules: recursive + # deploy.yml runs these two scripts only after a push build, so this is the check they + # get before merge. It needs the history above: it compares launchers committed on master. + - name: Check same-pe-code.py and refresh-launchers.sh against committed launchers + if: runner.os == 'Linux' && matrix.java == '11' + shell: bash + run: bash .github/scripts/test-refresh-launchers.sh - name: Java ${{ matrix.Java }} (${{ matrix.os }}) uses: actions/setup-java@de7274f081f381c8f8158605e0321c36c376e2e6 # v6.0.1 with: @@ -86,9 +92,10 @@ jobs: git status # Also the source of truth for the committed opendj-server-legacy/lib/*.exe: on a # successful push build, deploy.yml downloads windows-exe-11 from this very run and - # commits its contents back to the branch. Nothing here compares them with what is - # committed - an MSVC toolchain bump on the runner image changes the bytes on its own, - # so a byte-for-byte gate would fire without a source change. + # commits each launcher that differs from the committed one in more than the toolchain + # stamp (.github/scripts/refresh-launchers.sh). Nothing here compares them with what + # is committed - an MSVC toolchain bump on the runner image changes the bytes on its + # own, so a byte-for-byte gate would fire without a source change. - name: Upload Windows exe artifacts if: runner.os == 'Windows' uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index f1baf9d350..6d4b5bb34b 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -76,10 +76,17 @@ jobs: # # This only works because the Makefile passes /Brepro to both cl and link: the output # is a function of the sources, not of the build time. Without it every run would - # produce different bytes and this would commit on every push. An MSVC toolchain bump - # on the runner image does change them, and that refresh commit is correct - the - # committed binary then matches what CI verifies. Pushes made with GITHUB_TOKEN do - # not start new workflow runs, so this cannot loop; a PAT would break that. + # produce different bytes and this would commit on every push. The inputs it hashes + # include the build numbers of the tools, though, so a Visual Studio patch release + # on the runner image changes the stamp of the files - Rich header, REPRO hash, + # timestamps, checksum, header padding - while the code comes out the same. GitHub + # rolls a new image out over days, and while old and new both serve windows-latest + # the launchers were refreshed back and forth on every push (a8a18f000d, then + # c6919eaaf4, which undid it). refresh-launchers.sh compares the files with that + # stamp left out (same-pe-code.py), so only a change of the code, data or resources + # is committed - from a source change or from a toolchain that really generates + # different code. Pushes made with GITHUB_TOKEN do not start new workflow runs, so + # this cannot loop; a PAT would break that. - name: Download the launchers built by the triggering Build run continue-on-error: true uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 @@ -103,7 +110,13 @@ jobs: echo "::warning title=No launcher binaries from the Build run::windows-exe-11 could not be downloaded, leaving opendj-server-legacy/lib/*.exe as committed." exit 0 fi - cp "$BUILT"/*.exe opendj-server-legacy/lib/ + # A checked-out branch that predates the script (sustaining/4.10.x, say) takes + # the rebuilt files as they are, which is the old behaviour. + if [ -f .github/scripts/refresh-launchers.sh ]; then + bash .github/scripts/refresh-launchers.sh "$BUILT" opendj-server-legacy/lib + else + cp "$BUILT"/*.exe opendj-server-legacy/lib/ + fi # status --porcelain, not diff: it reports a brand-new launcher that was never # git-added just as well as a modified one. if [ -z "$(git status --porcelain -- opendj-server-legacy/lib)" ]; then diff --git a/opendj-server-legacy/src/build-tools/windows/Makefile b/opendj-server-legacy/src/build-tools/windows/Makefile index 2f73f590a1..6399ebfd5a 100644 --- a/opendj-server-legacy/src/build-tools/windows/Makefile +++ b/opendj-server-legacy/src/build-tools/windows/Makefile @@ -39,8 +39,9 @@ LAUNCHER_ADMINISTRATOR_PROGNAME=launcher_administrator.exe WINLAUNCHER_PROGNAME=winlauncher.exe # /Brepro makes the outputs reproducible (content-hash PE timestamps instead of the # build time). The Package/Deploy workflow commits these binaries back to the branch -# whenever their bytes differ from the committed ones; without /Brepro every build -# would differ and it would commit on every push. +# whenever they differ from the committed ones in more than the toolchain stamp +# (.github/scripts/same-pe-code.py); without /Brepro every build would differ and it +# would commit on every push. LINKER=link -nologo /machine:x86 /Brepro LIBS=advapi32.lib