From 19dbabf41386e5c993924238f1036e7f5d620bfc Mon Sep 17 00:00:00 2001 From: Valera V Harseko Date: Thu, 24 Sep 2026 18:45:03 +0300 Subject: [PATCH 1/3] Refresh the Windows launchers only when their code changes, not their toolchain stamp /Brepro hashes the build numbers of cl, link and cvtres into the output, so two runner images a Visual Studio patch release apart (14.51.36256 and 36257) build the same code into different files: only the Rich header, the REPRO hash, the timestamps derived from it and the PE checksum differ. While GitHub rolled the new windows-2025-vs2026 image out, the Windows job landed on either one, and the Package/Deploy workflow committed the launchers back and forth (a8a18f000d, then c6919eaaf4, which undid it). Compare the committed and rebuilt launchers with that stamp blanked, and keep the committed ones when nothing else differs. A new launcher, a file that does not parse as a PE image, or a branch that predates the script still takes the rebuilt file, as before. --- .github/scripts/same-pe-code.py | 148 ++++++++++++++++++ .github/workflows/deploy.yml | 26 ++- .../src/build-tools/windows/Makefile | 5 +- 3 files changed, 173 insertions(+), 6 deletions(-) create mode 100755 .github/scripts/same-pe-code.py diff --git a/.github/scripts/same-pe-code.py b/.github/scripts/same-pe-code.py new file mode 100755 index 0000000000..5e356ea9ec --- /dev/null +++ b/.github/scripts/same-pe-code.py @@ -0,0 +1,148 @@ +#!/usr/bin/env python3 +# +# The contents of this file are subject to the terms of the Common Development and +# Distribution License (the License). You may not use this file except in compliance with the +# License. +# +# You can obtain a copy of the License at legal/CDDLv1.0.txt. See the License for the +# specific language governing permission and limitations under the License. +# +# When distributing Covered Software, include this CDDL Header Notice in each file and include +# the License file at legal/CDDLv1.0.txt. If applicable, add the following below the CDDL +# Header, with the fields enclosed by brackets [] replaced by your own identifying +# information: "Portions copyright [year] [name of copyright owner]". +# +# Copyright 2026 3A Systems, LLC. + +"""Tell whether two PE images differ in anything but the stamp of the toolchain. + +Usage: same-pe-code.py COMMITTED REBUILT + +Exits 0 when the two files are identical once the build stamp is blanked in both, +1 when they differ anywhere else, and 2 when either one cannot be read as a PE image. + +The Windows launchers are linked with /Brepro, so their bytes are a function of the +inputs - and the inputs include the build numbers of cl, link and cvtres. Two runner +images a patch release of Visual Studio apart (14.51.36256 and 14.51.36257, say) turn +out byte-for-byte the same code, data and resources, yet different files: the Rich +header lists those build numbers, the REPRO debug entry holds a hash over them, and +the COFF and debug directory timestamps and the PE checksum are derived from that +hash. GitHub rolls a new image out over days, so the Windows job lands on either one +and a byte comparison refreshes the committed launchers back and forth on every push. +The fields blanked here are exactly those; everything a change of source or of code +generation can move is still compared. + +A Rich header that gains or loses an entry changes length and shifts every offset +after it, which then compares as a difference: the refresh is committed. That is the +safe way to be wrong. +""" + +import struct +import sys + +DEBUG_TYPE_CODEVIEW = 2 +DEBUG_TYPE_REPRO = 16 +DEBUG_ENTRY_SIZE = 28 + + +class NotPE(Exception): + pass + + +def u16(b, off): + return struct.unpack_from(" len(b): + raise NotPE("field at 0x%x runs past the end of the file" % off) + b[off:off + length] = bytes(length) + + +def normalize(data): + b = bytearray(data) + try: + if b[:2] != b"MZ": + raise NotPE("no MZ signature") + pe = u32(b, 0x3C) + if b[pe:pe + 4] != b"PE\0\0": + raise NotPE("no PE signature") + + # The Rich header sits between the DOS stub and the PE header: "DanS" XOR key, + # the (prodId, build, count) records XOR key, then "Rich" and the key itself. + rich = b.find(b"Rich", 0x40, pe) + if rich >= 0: + key = b[rich + 4:rich + 8] + dans = bytes(x ^ y for x, y in zip(b"DanS", key)) + start = b.rfind(dans, 0x40, rich) + if start < 0: + raise NotPE("Rich header without its DanS marker") + blank(b, start, rich + 8 - start) + + coff = pe + 4 + sections = u16(b, coff + 2) + opt_size = u16(b, coff + 16) + blank(b, coff + 4, 4) # TimeDateStamp + + opt = coff + 20 + magic = u16(b, opt) + if magic == 0x10B: + data_dirs = opt + 96 + elif magic == 0x20B: + data_dirs = opt + 112 + else: + raise NotPE("unknown optional header magic 0x%x" % magic) + blank(b, opt + 64, 4) # CheckSum + + table = opt + opt_size + spans = [] + for i in range(sections): + s = table + 40 * i + spans.append((u32(b, s + 12), u32(b, s + 8), u32(b, s + 20))) # va, vsize, raw + + def file_offset(rva): + for va, vsize, raw in spans: + if va <= rva < va + vsize: + return raw + rva - va + raise NotPE("RVA 0x%x lies in no section" % rva) + + debug_rva = u32(b, data_dirs + 6 * 8) + debug_size = u32(b, data_dirs + 6 * 8 + 4) + if debug_rva and debug_size: + base = file_offset(debug_rva) + for i in range(debug_size // DEBUG_ENTRY_SIZE): + entry = base + DEBUG_ENTRY_SIZE * i + blank(b, entry + 4, 4) # TimeDateStamp + kind = u32(b, entry + 12) + size = u32(b, entry + 16) + raw = u32(b, entry + 24) + if kind == DEBUG_TYPE_REPRO: + blank(b, raw, size) # the hash itself + elif kind == DEBUG_TYPE_CODEVIEW and b[raw:raw + 4] == b"RSDS": + blank(b, raw + 4, 20) # PDB GUID and age + except struct.error as e: + raise NotPE(str(e)) + return bytes(b) + + +def main(argv): + if len(argv) != 3: + print(__doc__.strip().splitlines()[2], file=sys.stderr) + return 2 + try: + images = [] + for path in argv[1:]: + with open(path, "rb") as f: + images.append(normalize(f.read())) + except (OSError, NotPE) as e: + print("same-pe-code: %s" % e, file=sys.stderr) + return 2 + return 0 if images[0] == images[1] else 1 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index f1baf9d350..ed9b2a63cd 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -76,9 +76,15 @@ jobs: # # This only works because the Makefile passes /Brepro to both cl and link: the output # is a function of the sources, not of the build time. Without it every run would - # produce different bytes and this would commit on every push. An MSVC toolchain bump - # on the runner image does change them, and that refresh commit is correct - the - # committed binary then matches what CI verifies. Pushes made with GITHUB_TOKEN do + # produce different bytes and this would commit on every push. The inputs it hashes + # include the build numbers of the tools, though, so a Visual Studio patch release + # on the runner image changes the stamp of the files - Rich header, REPRO hash, + # timestamps, checksum - while the code comes out the same. GitHub rolls a new image + # out over days, and while old and new both serve windows-latest the launchers were + # refreshed back and forth on every push (a8a18f000d, then c6919eaaf4, which undid + # it). same-pe-code.py compares the files with that stamp blanked, so only a change + # of the code, data or resources is committed - from a source change or from a + # toolchain that really generates different code. Pushes made with GITHUB_TOKEN do # not start new workflow runs, so this cannot loop; a PAT would break that. - name: Download the launchers built by the triggering Build run continue-on-error: true @@ -103,7 +109,19 @@ jobs: echo "::warning title=No launcher binaries from the Build run::windows-exe-11 could not be downloaded, leaving opendj-server-legacy/lib/*.exe as committed." exit 0 fi - cp "$BUILT"/*.exe opendj-server-legacy/lib/ + for built in "$BUILT"/*.exe; do + committed=opendj-server-legacy/lib/$(basename "$built") + # Exit 1 (the code differs) and 2 (not readable as a PE image) both take the + # rebuilt file: when in doubt, refresh. So does a checked-out branch that predates + # the script - python exits 2 on a missing file too - which is the old behaviour. + if [ -f "$committed" ] && python3 .github/scripts/same-pe-code.py "$committed" "$built"; then + if ! cmp -s "$committed" "$built"; then + echo "$committed differs from the rebuilt one only in the toolchain stamp - keeping it." + fi + continue + fi + cp "$built" "$committed" + done # status --porcelain, not diff: it reports a brand-new launcher that was never # git-added just as well as a modified one. if [ -z "$(git status --porcelain -- opendj-server-legacy/lib)" ]; then diff --git a/opendj-server-legacy/src/build-tools/windows/Makefile b/opendj-server-legacy/src/build-tools/windows/Makefile index 2f73f590a1..6399ebfd5a 100644 --- a/opendj-server-legacy/src/build-tools/windows/Makefile +++ b/opendj-server-legacy/src/build-tools/windows/Makefile @@ -39,8 +39,9 @@ LAUNCHER_ADMINISTRATOR_PROGNAME=launcher_administrator.exe WINLAUNCHER_PROGNAME=winlauncher.exe # /Brepro makes the outputs reproducible (content-hash PE timestamps instead of the # build time). The Package/Deploy workflow commits these binaries back to the branch -# whenever their bytes differ from the committed ones; without /Brepro every build -# would differ and it would commit on every push. +# whenever they differ from the committed ones in more than the toolchain stamp +# (.github/scripts/same-pe-code.py); without /Brepro every build would differ and it +# would commit on every push. LINKER=link -nologo /machine:x86 /Brepro LIBS=advapi32.lib From db02041c1af657f326804a675bd5bb14e486dc97 Mon Sep 17 00:00:00 2001 From: Valera V Harseko Date: Fri, 25 Sep 2026 11:41:09 +0300 Subject: [PATCH 2/3] Leave the header padding out of the comparison, and check both scripts before merge 14.51.36252 put the PE header at 0x100 and 14.51.36256 at 0xf0 around the same code, so launcher_administrator.exe and winlauncher.exe of the 05.09 refresh still compared as different. same-pe-code.py now drops the zero padding after the DOS stub and after the section table, and the Rich header with it; the docstring lists every field it leaves out, the CodeView GUID and age included. The keep-or-copy loop moves into refresh-launchers.sh, and the Linux Java 11 build runs test-refresh-launchers.sh on launchers committed on master: it fails when either script starts keeping a changed launcher or refreshing one that differs only in the stamp. deploy.yml falls back to a plain copy on a branch without the script. --- .github/scripts/refresh-launchers.sh | 35 +++++++++++ .github/scripts/same-pe-code.py | 25 ++++++-- .github/scripts/test-refresh-launchers.sh | 75 +++++++++++++++++++++++ .github/workflows/build.yml | 13 +++- .github/workflows/deploy.yml | 35 +++++------ 5 files changed, 154 insertions(+), 29 deletions(-) create mode 100755 .github/scripts/refresh-launchers.sh create mode 100755 .github/scripts/test-refresh-launchers.sh diff --git a/.github/scripts/refresh-launchers.sh b/.github/scripts/refresh-launchers.sh new file mode 100755 index 0000000000..96d49f1454 --- /dev/null +++ b/.github/scripts/refresh-launchers.sh @@ -0,0 +1,35 @@ +#!/bin/bash +# +# The contents of this file are subject to the terms of the Common Development and +# Distribution License (the License). You may not use this file except in compliance with the +# License. +# +# You can obtain a copy of the License at legal/CDDLv1.0.txt. See the License for the +# specific language governing permission and limitations under the License. +# +# When distributing Covered Software, include this CDDL Header Notice in each file and include +# the License file at legal/CDDLv1.0.txt. If applicable, add the following below the CDDL +# Header, with the fields enclosed by brackets [] replaced by your own identifying +# information: "Portions copyright [year] [name of copyright owner]". +# +# Copyright 2026 3A Systems, LLC. + +# Usage: refresh-launchers.sh BUILT LIB +# +# Copies each BUILT/*.exe over its namesake in LIB, unless the two differ only in the +# toolchain stamp (same-pe-code.py). The Package/Deploy workflow runs it before it +# commits LIB; the Build workflow runs it on known launcher pairs before merge. +set -e +here=$(dirname "$0") +for built in "$1"/*.exe; do + committed=$2/$(basename "$built") + # Exit 1 (the code differs) and 2 (not readable as a PE image) both take the rebuilt + # file: when in doubt, refresh. + if [ -f "$committed" ] && python3 "$here/same-pe-code.py" "$committed" "$built"; then + if ! cmp -s "$committed" "$built"; then + echo "$committed differs from the rebuilt one only in the toolchain stamp - keeping it." + fi + continue + fi + cp "$built" "$committed" +done diff --git a/.github/scripts/same-pe-code.py b/.github/scripts/same-pe-code.py index 5e356ea9ec..81c8f825d4 100755 --- a/.github/scripts/same-pe-code.py +++ b/.github/scripts/same-pe-code.py @@ -29,12 +29,18 @@ the COFF and debug directory timestamps and the PE checksum are derived from that hash. GitHub rolls a new image out over days, so the Windows job lands on either one and a byte comparison refreshes the committed launchers back and forth on every push. -The fields blanked here are exactly those; everything a change of source or of code -generation can move is still compared. -A Rich header that gains or loses an entry changes length and shifts every offset -after it, which then compares as a difference: the refresh is committed. That is the -safe way to be wrong. +What is left out of the comparison: +- the Rich header; +- the COFF and debug directory timestamps and the PE checksum; +- the REPRO hash, and the PDB GUID and age of an RSDS CodeView entry; +- the zero padding the linker puts after the DOS stub, up to the PE header, and after + the section table, up to SizeOfHeaders. A patch release may pad differently: + 14.51.36252 put the PE header at 0x100, 14.51.36256 at 0xf0, around the same code. +The DOS stub, the PE headers, the section table and every byte from SizeOfHeaders on +are still compared, so a change of source or of code generation still shows. A Rich +header that gains or loses an entry no longer does on its own: it comes with a change +of the objects linked in, which moves the code as well. """ import struct @@ -97,6 +103,9 @@ def normalize(data): else: raise NotPE("unknown optional header magic 0x%x" % magic) blank(b, opt + 64, 4) # CheckSum + headers_end = u32(b, opt + 60) # SizeOfHeaders + if not pe < headers_end <= len(b): + raise NotPE("SizeOfHeaders 0x%x out of range" % headers_end) table = opt + opt_size spans = [] @@ -126,7 +135,11 @@ def file_offset(rva): blank(b, raw + 4, 20) # PDB GUID and age except struct.error as e: raise NotPE(str(e)) - return bytes(b) + # Compare what the padding surrounds, wherever it ends (see above). The Rich header + # is blanked by now, so it goes with the padding after the DOS stub; e_lfanew goes + # too, since it only says where that padding ends. + return (bytes(b[:0x3C]) + bytes(b[0x40:pe]).rstrip(b"\0") + + bytes(b[pe:headers_end]).rstrip(b"\0") + bytes(b[headers_end:])) def main(argv): diff --git a/.github/scripts/test-refresh-launchers.sh b/.github/scripts/test-refresh-launchers.sh new file mode 100755 index 0000000000..fb0865df7d --- /dev/null +++ b/.github/scripts/test-refresh-launchers.sh @@ -0,0 +1,75 @@ +#!/bin/bash +# +# The contents of this file are subject to the terms of the Common Development and +# Distribution License (the License). You may not use this file except in compliance with the +# License. +# +# You can obtain a copy of the License at legal/CDDLv1.0.txt. See the License for the +# specific language governing permission and limitations under the License. +# +# When distributing Covered Software, include this CDDL Header Notice in each file and include +# the License file at legal/CDDLv1.0.txt. If applicable, add the following below the CDDL +# Header, with the fields enclosed by brackets [] replaced by your own identifying +# information: "Portions copyright [year] [name of copyright owner]". +# +# Copyright 2026 3A Systems, LLC. + +# Checks same-pe-code.py and refresh-launchers.sh against launchers committed in the +# history of master, so that neither can start keeping a changed launcher, or refreshing +# one that differs only in the toolchain stamp, while CI stays green. Needs the full +# history (fetch-depth: 0). Run from the root of the repository. +set -e +here=$(dirname "$0") +t=$(mktemp -d) +trap 'rm -rf "$t"' EXIT + +g() { git show "$1:opendj-server-legacy/lib/$2.exe" > "$t/$2-$3.exe"; } +expect() { + local rc=0 + python3 "$here/same-pe-code.py" "$t/$2" "$t/$3" || rc=$? + if [ "$rc" != "$1" ]; then + echo "::error::same-pe-code.py $2 $3 exited $rc, expected $1 ($4)" + exit 1 + fi + echo "ok: $2 $3 -> $rc ($4)" +} + +for f in launcher_administrator winlauncher opendj_service; do + g a611c10c46^ $f 36252 + g a611c10c46 $f 36256 + g a8a18f000d $f 36257 + g 74cb4f5a84 $f old-code + expect 0 $f-36256.exe $f-36257.exe "the toolchain stamp only" + expect 0 $f-36257.exe $f-36256.exe "the toolchain stamp only" + expect 1 $f-old-code.exe $f-36257.exe "before and after a change of the launcher sources" +done +# winlauncher.exe has CheckSum 0 in both builds, launcher_administrator.exe does not. +for f in launcher_administrator winlauncher; do + expect 0 $f-36252.exe $f-36256.exe "the same code, the PE header at 0x100 and at 0xf0" +done +expect 1 opendj_service-36252.exe opendj_service-36256.exe "a code change of the 05.09 refresh" +expect 1 winlauncher-36257.exe launcher_administrator-36257.exe "two different launchers" +# A real change moves the headers too; this one does not, so the sections must be compared. +python3 - "$t/winlauncher-36257.exe" "$t/winlauncher-text-byte.exe" <<'EOF' +import struct, sys +b = bytearray(open(sys.argv[1], "rb").read()) +pe = struct.unpack_from(" Date: Fri, 25 Sep 2026 15:47:06 +0300 Subject: [PATCH 3/3] Check a change of the headers only, the CodeView blank and exit 2 before merge test-refresh-launchers.sh now also checks: - a flip of the NX_COMPAT flag, of a byte of the DOS stub and of the flags of the first section, each expected to differ: every real change in the history also moves the sections, so the DOS stub, the COFF and optional headers or the section table could be left out of the comparison with the check green; - two copies of winlauncher.exe whose PDB GUID and age differ, with the POGO debug entry retyped as CodeView, expected to compare equal: no committed launcher has a PDB; - a file cut to 64 bytes, expected to exit 2, and the loop refreshing over it. The docstring of same-pe-code.py names e_lfanew and the header padding in what exit 0 leaves out. --- .github/scripts/same-pe-code.py | 8 ++-- .github/scripts/test-refresh-launchers.sh | 48 ++++++++++++++++++++++- 2 files changed, 52 insertions(+), 4 deletions(-) diff --git a/.github/scripts/same-pe-code.py b/.github/scripts/same-pe-code.py index 81c8f825d4..d497a2aa8e 100755 --- a/.github/scripts/same-pe-code.py +++ b/.github/scripts/same-pe-code.py @@ -18,8 +18,9 @@ Usage: same-pe-code.py COMMITTED REBUILT -Exits 0 when the two files are identical once the build stamp is blanked in both, -1 when they differ anywhere else, and 2 when either one cannot be read as a PE image. +Exits 0 when the two files are identical once the build stamp and the header padding +are left out of both, 1 when they differ anywhere else, and 2 when either one cannot be +read as a PE image. The Windows launchers are linked with /Brepro, so their bytes are a function of the inputs - and the inputs include the build numbers of cl, link and cvtres. Two runner @@ -35,7 +36,8 @@ - the COFF and debug directory timestamps and the PE checksum; - the REPRO hash, and the PDB GUID and age of an RSDS CodeView entry; - the zero padding the linker puts after the DOS stub, up to the PE header, and after - the section table, up to SizeOfHeaders. A patch release may pad differently: + the section table, up to SizeOfHeaders, and e_lfanew, which only says where the first + of them ends. A patch release may pad differently: 14.51.36252 put the PE header at 0x100, 14.51.36256 at 0xf0, around the same code. The DOS stub, the PE headers, the section table and every byte from SizeOfHeaders on are still compared, so a change of source or of code generation still shows. A Rich diff --git a/.github/scripts/test-refresh-launchers.sh b/.github/scripts/test-refresh-launchers.sh index fb0865df7d..50c1e42f3b 100755 --- a/.github/scripts/test-refresh-launchers.sh +++ b/.github/scripts/test-refresh-launchers.sh @@ -59,17 +59,63 @@ b[struct.unpack_from(" "$t/truncated.exe" +expect 2 truncated.exe winlauncher-36257.exe "a file that is not a PE image" # The loop: keeps a launcher that differs only in the stamp, refreshes one whose code -# changed, and adds one that is not committed yet. +# changed, adds one that is not committed yet, and refreshes one whose committed file +# is not a PE image. mkdir "$t/built" "$t/lib" cp "$t/winlauncher-36257.exe" "$t/built/winlauncher.exe" cp "$t/winlauncher-36256.exe" "$t/lib/winlauncher.exe" cp "$t/launcher_administrator-36257.exe" "$t/built/launcher_administrator.exe" cp "$t/launcher_administrator-old-code.exe" "$t/lib/launcher_administrator.exe" cp "$t/opendj_service-36257.exe" "$t/built/opendj_service.exe" +cp "$t/winlauncher-36257.exe" "$t/built/unreadable.exe" +cp "$t/truncated.exe" "$t/lib/unreadable.exe" bash "$here/refresh-launchers.sh" "$t/built" "$t/lib" cmp "$t/lib/winlauncher.exe" "$t/winlauncher-36256.exe" cmp "$t/lib/launcher_administrator.exe" "$t/built/launcher_administrator.exe" cmp "$t/lib/opendj_service.exe" "$t/built/opendj_service.exe" +cmp "$t/lib/unreadable.exe" "$t/built/unreadable.exe" echo "ok: refresh-launchers.sh keeps, refreshes and adds as expected"