DSCI

name: bundle-dll description: >- Stage the three notcurses DLLs into `bundle/`, recursively walk MSYS2's ldd to copy sibling DLL deps into the same dir, then belt-and-braces sweep for ffmpeg codec + text-rendering DLLs that ldd sometimes misses on CLANGARM64. Caller must already have: - notcurses built under $NOTCURSES_SRC_DIR/build/ (the value written by scripts/ci/fetch-notcurses-source.sh into $GITHUB_ENV) - The MSYS2 default shell active (msys2 {0}) - `bundle/` writable - msys2 ldd + timeout + sed on PATH inputs: extra-search-path: description: >- Additional directory to search for DLLs, ahead of the msystem bin dirs. Workspace-relative POSIX path (absolute is also accepted). _build-windows.yml points this at the bin/ of its source-built codec + ffmpeg prefix: ffmpeg's mingw32 target installs shared objects under $PREFIX/bin, which is on no default search path, so without this neither ldd (which resolves via PATH) nor the objdump backstop (which resolves via find_dll_by_name) can locate avcodec-.dll and friends. The directory is also prepended to PATH for the duration of the step so ldd's own resolution works. Empty default = search only the msystem bin dirs, i.e. the pre-source-build behaviour. required: false default: '' runs: using: composite steps: - name: Stage + bundle sibling DLLs shell: 'msys2 {0}' env: EXTRA_SEARCH_PATH: ${{ inputs.extra-search-path }} run: | set -euxo pipefail : "${NOTCURSES_SRC_DIR:?must be set — call scripts/ci/fetch-notcurses-source.sh first}" # Convert any \ path GHA may have put in NOTCURSES_SRC_DIR # (the env propagation goes through GitHub's Windows-style # interpolation) into POSIX form for MSYS2's bash. if command -v cygpath >/dev/null 2>&1; then NOTCURSES_SRC_DIR=$(cygpath -u "$NOTCURSES_SRC_DIR") fi workspace="$PWD" mkdir -p "$workspace/bundle" # Resolve the caller's extra search dir (if any) before we # leave the workspace. Fail loudly on a path that doesn't # exist: the caller asserting "the DLLs are over here" and # being wrong is exactly the case where silently falling back # to the msystem bin dirs would produce a bundle missing every # source-built library, which then only shows up as a # confusing ldd wedge or a runtime load failure on a user's # machine. extra_dll_dir="" if [[ -n "${EXTRA_SEARCH_PATH:-}" ]]; then case "$EXTRA_SEARCH_PATH" in /*) extra_dll_dir="$EXTRA_SEARCH_PATH" ;; *) extra_dll_dir="$workspace/$EXTRA_SEARCH_PATH" ;; esac if [[ ! -d "$extra_dll_dir" ]]; then echo "❌ extra-search-path '$EXTRA_SEARCH_PATH' resolves to" echo " '$extra_dll_dir', which is not a directory." exit 1 fi # ldd resolves imports through the Windows DLL search order, # which reads PATH — find_dll_by_name below only covers the # cases where ldd has already given up. export PATH="$extra_dll_dir:$PATH" echo "Extra DLL search dir: $extra_dll_dir" fi cd "$NOTCURSES_SRC_DIR/build" # MinGW DLL naming varies: libfoo-N.dll, libfoo.dll, foo.dll. # BUT: `notcurses` is a prefix of `notcurses-core` and # `notcurses-ffi`, so a loose *notcurses*.dll glob matches # all three and head -1 picks the same file every iteration. # Restrict: only allow `.` or `-` after the name. for lib in notcurses notcurses-core notcurses-ffi; do found=$(find . -type f $ \ -iname "${lib}.dll" -o \ -iname "lib${lib}.dll" -o \ -iname "${lib}-[0-9]*.dll" -o \ -iname "lib${lib}-[0-9]*.dll" \ $ | head -1) if [[ -z "$found" ]]; then echo "❌ Could not locate built DLL for $lib" find . -iname "*${lib}*" -type f exit 1 fi cp "$found" "$workspace/bundle/lib${lib}.dll" echo "Staged lib${lib}.dll (from $(basename $found))" done cd "$workspace" # On Windows, DLL search order finds siblings in the same # directory as the DLL being loaded — no rpath needed. # We just need to ensure every transitive dep is present # in bundle/. MSYS2's `ldd` works on Windows DLLs, with the # objdump backstop below covering the cases where ldd lies # (notably on CLANGARM64). # # Single source of truth for "is system DLL": the # scripts/ci/win-system-dll.sh helper. Covers API Set # forwarders (api-ms-*, ext-ms-*, ms-win-*), versioned # runtime DLLs (vcruntime140*.dll, d3dcompiler_47.dll), # the long-standing Win32/GDI/COM/security/networking/ # multimedia DLLs by exact name, and any path under # C:\Windows as a belt-and-braces fallback. source scripts/ci/win-system-dll.sh # Fallback: search known DLL dirs by name when ldd gives us a # bogus path. CLANGARM64's ldd is unreliable — often reports # directories or partial paths instead of the actual DLL # location. Silently skipping those would lose legit codec # DLLs. The caller's extra dir goes first so a source-built # library always wins over a same-named pacman one. # Explicit `if` rather than `[[ … ]] && arr+=( … )`: the # AND-list form returns non-zero when the guard is false, # which is only survivable under `set -e` because bash # exempts the left side of an && list. Not worth relying on. DLL_SEARCH_DIRS=() if [[ -n "$extra_dll_dir" ]]; then DLL_SEARCH_DIRS+=( "$extra_dll_dir" ) fi DLL_SEARCH_DIRS+=( /clangarm64/bin /ucrt64/bin /mingw64/bin /usr/bin ) find_dll_by_name() { local name="$1" local search for search in "${DLL_SEARCH_DIRS[@]}"; do if [[ -f "$search/$name" ]]; then echo "$search/$name" return 0 fi done return 1 } # Track DLLs we've already inspected so a cyclic ldd graph # (or one that lies the same way twice) can't loop forever. declare -A SEEN_DEPS=() process_deps() { local dll="$1" local depth="${2:-0}" # Hard recursion cap. The real DLL graph is ~3 deep; # anything past 12 is a cycle or a buggy ldd, not real. if (( depth > 12 )); then echo "process_deps: depth cap hit at $dll, stopping" return 0 fi local key="$(basename "$dll")" [[ -n "${SEEN_DEPS[$key]:-}" ]] && return 0 SEEN_DEPS[$key]=1 # Wrap ldd in `timeout` — on Windows ARM, ldd # subprocesses periodically wedge inside xtajit64 # emulation and never return. 30s is generous; a healthy # ldd call on these DLLs finishes in well under a second. # Retry a few times: most xtajit64 deadlocks clear on a # fresh process. If it stays stuck after 4 tries, fail # the job — silently skipping would ship a bundle missing # this DLL's transitive deps, which then explodes at # runtime on a user's machine. A failed CI job is much # cheaper than a broken release. local ldd_out="" local attempt for attempt in 1 2 3 4; do if ldd_out="$(timeout 30s ldd "$dll" 2>/dev/null)"; then break fi echo "process_deps: ldd attempt $attempt timed out on $dll, retrying..." sleep 2 ldd_out="" done if [[ -z "$ldd_out" ]]; then echo "❌ ldd permanently wedged on $dll after 4 attempts." echo " Almost certainly a xtajit64 deadlock on Windows ARM —" echo " restart the workflow run. If it keeps failing on the" echo " same DLL, that DLL itself is broken or the runner is" echo " in a bad state." exit 1 fi local line name path while IFS=$'\t' read -r name path; do [[ -z "$path" ]] && continue if is_system_dll "$name" "$path"; then continue fi if [[ ! -f "$path" ]]; then if resolved=$(find_dll_by_name "$name"); then path="$resolved" echo "Resolved $name via fallback: $path" else echo "Skipping $name: unresolvable (ldd path='$path', not in known bin dirs)" continue fi fi if [[ ! -f "bundle/$name" ]]; then cp "$path" "bundle/$name" echo "Bundled: $name (from $path)" process_deps "bundle/$name" $((depth + 1)) fi done < <(printf '%s\n' "$ldd_out" \ | sed -n 's/ (.*)$//; s/^[[:space:]]*$[^ ]\+$ => $.\+$$/\1\t\2/p') } for dll in bundle/libnotcurses.dll bundle/libnotcurses-core.dll bundle/libnotcurses-ffi.dll; do process_deps "$dll" done # Belt-and-braces sweep: some msystem variants (notably # CLANGARM64) have ldd failures that cause deps to be # silently missed. If the DLLs exist in one of the search # dirs, copy them anyway — can't hurt (they're already # dynamic deps somewhere in the graph), and catches the cases # where ldd lies. Process each copied lib through # process_deps for its own transitive deps. # # Names not found are silently skipped; that resilience is # the point of the list, and it is also why the list has to # be curated rather than left to grow. A stale entry costs # nothing at runtime but is a lie about what the bundle # contains, and the entries below were read as documentation # more than once. # # This list lost ~20 entries when the lane stopped installing # MSYS2's GPL ffmpeg and started source-building an # LGPL decode-only one. Everything removed was reachable only # through that package's dependency closure and cannot exist # in the bundle any more: # * encoders we never shipped a decoder path for — # libx264-165, libx265-215, libSvtAv1Enc-4, libmp3lame-0, # libvorbisenc-2 (the GPL/encoder tree this conversion # exists to remove); # * external codecs our configure does not enable, so # ffmpeg uses its internal decoder instead — libvorbis-0, # libogg-0, libopenjp2-7; # * ffmpeg subsystems we compile out — liblzma-5, libbz2-1 # (--disable-lzma/--disable-bzlib), libxml2-*, libexpat-1 # (--disable-libxml2), and the libass/drawtext font stack # (libfreetype-6, libfontconfig-1, libharfbuzz-0, # libgraphite2, libbrotli*); # * image libraries that only ever came in via ffmpeg's # optional wrappers — libpng16-16, libjpeg-8, libwebp-7, # libwebpmux-3, libsharpyuv-0, libgif-7, libtiff-6; # * libvpx-1, which no longer exists in any form on # Windows: libvpx's configure refuses to build a DLL off # ELF/OS-2/Darwin, so build-libvpx.sh links it statically # into avcodec here. See build-libvpx.sh's header. # # What stayed: notcurses' own non-codec deps — ncurses (still # pacman's) and libunistring (source-built as of the # third-party-manifest work, so it now resolves out of the # caller's extra search dir like the codec DLLs do), the # libiconv the latter and avcodec both import (source-built # too as of r11, and resolved from the same extra dir), # winpthreads, zlib (ffmpeg --enable-zlib), and the three # codec libraries the source-built chain still emits as DLLs — # libdeflate, libdav1d, libopus. # # r11 removed four more names, and those removals are # decisions rather than tidy-ups. A PE import-reachability # closure computed from libnotcurses{,-core,-ffi}.dll over # every DLL in both published r10 packs found each of them # reachable from nothing at all: # * libintl-8.dll — GNU gettext's message-translation # runtime, in both packs, imported by NOTHING. Not # avcodec, not ncursesw, not libunistring, not even # libiconv (the traffic goes the other way: libintl # imports libiconv). It was there solely because this list # named it. LGPL and pacman-sourced, so we were # redistributing a copyleft binary whose corresponding # source we cannot produce, in exchange for nothing. # * libstdc++-6.dll and libgomp-1.dll — imported by nothing # in the UCRT64 pack, and absent from the CLANGARM64 one. # notcurses is configured -DUSE_CXX=OFF and nothing in the # decoder chain uses OpenMP, so neither ever had a caller. # * libgcc_s_seh-1.dll — imported only by libstdc++-6.dll, # i.e. reachable only from another orphan. gcc links the # unwinder statically into everything else here, which is # why no av*/sw* DLL references it. # The last three are GPL-3.0-with-GCC-exception; the exception # makes redistributing them fine, but redistributing a # compiler runtime nothing loads is still ~2 MB of a user's # download spent on nothing. # # None of the four has a component in # resources/third-party.json any more, and the reachability # gate below fails the lane on any bundled DLL nothing # imports. If one comes back, something has started importing # it — that is a real dependency change to understand, not a # line to restore here. # ffmpeg's own av*/sw* DLLs are deliberately NOT listed: they # are direct imports of libnotcurses-core, so the ldd walk # and the objdump backstop both reach them, and hardcoding # their SONAME majors here would rot on every ffmpeg bump. # # libunistring-2.dll left with the same broom that swept the # encoder tree: our pin (build-libunistring.sh, 1.4.x) has # libtool -version-info 7:R:2, i.e. SONAME major 5, so a # `-2` could only ever come from a package-manager build we # no longer install. resources/third-party.json's Windows # pattern for the component spells the same expectation, and # scripts/ci/audit-third-party.sh fails the lane if the DLL # that lands doesn't match it. libiconv-2.dll's major is # stable for the same kind of reason (-version-info 9:1:7 # gives 2), but unlike libunistring its pacman twin stays # installed — the toolchain group depends on it — so the # caller's extra search dir being first in DLL_SEARCH_DIRS is # what decides which one this line copies. _build-windows.yml # re-checks that decision by hash after the sweep. echo "--- belt-and-braces sweep ---" for name in \ libncursesw6.dll libncurses6.dll \ libunistring-5.dll libiconv-2.dll \ libwinpthread-1.dll \ zlib1.dll \ libdeflate.dll libdav1d-7.dll libopus-0.dll \ ; do if [[ ! -f "bundle/$name" ]]; then if resolved=$(find_dll_by_name "$name"); then cp "$resolved" "bundle/$name" echo "Swept in: $name (from $resolved)" process_deps "bundle/$name" fi # If not found, silently skip — this variant's ffmpeg # simply doesn't ship it, either statically baked in # or unsupported on this platform. fi done # Backstop pass: walk every bundled DLL's PE import table # via `objdump -p`. Catches the failure mode where ldd's # walk lies (well-documented on CLANGARM64 — xtajit64 # emulation quirks cause ldd to silently return incomplete # output) and the belt-and-braces sweep doesn't have the # missing DLL in its hardcoded list. objdump reads the PE # header directly without invoking the binary, so it's # immune to ldd's runtime issues. # # Uses the same `is_system_dll` whitelist sourced at the # top of the step, so what's classified as "system" here # matches what the audit step uses — both can't disagree. echo "--- objdump-based backstop walk ---" declare -A SEEN_OBJDUMP=() process_deps_objdump() { local dll="$1" local depth="${2:-0}" if (( depth > 12 )); then echo "process_deps_objdump: depth cap hit at $dll" return 0 fi local key="$(basename "$dll")" [[ -n "${SEEN_OBJDUMP[$key]:-}" ]] && return 0 SEEN_OBJDUMP[$key]=1 local name while IFS= read -r name; do [[ -z "$name" ]] && continue if is_system_dll "$name"; then continue fi if [[ -f "bundle/$name" ]]; then process_deps_objdump "bundle/$name" $((depth + 1)) continue fi if resolved=$(find_dll_by_name "$name"); then cp "$resolved" "bundle/$name" echo "Bundled (objdump): $name (from $resolved)" process_deps_objdump "bundle/$name" $((depth + 1)) else echo "::warning::objdump-walk: '$name' not in known msys2 bin dirs — audit will flag if it isn't a system DLL" fi done < <(objdump -p "$dll" 2>/dev/null | awk '/DLL Name:/ {print $3}') } # Iterate the bundle after both prior passes. Any DLL ldd or # the sweep already grabbed acts as an entry point; we walk # its real PE imports and pick up anything missed. for dll in bundle/*.dll; do [[ -L "$dll" ]] && continue process_deps_objdump "$dll" done echo "--- bundle contents ---" ls -la bundle/ - name: Audit bundle self-containment (objdump imports) shell: 'msys2 {0}' run: | set -euo pipefail # For every bundled .dll, list its PE import table via # `objdump -p` and reject any imported DLL name that isn't # either a Windows system DLL (universally available on # every supported Windows version) or a sibling in bundle/. # Catches the failure mode where ldd's recursive walk # missed a DLL and the belt-and-braces sweep didn't catch # it either — the bundle works in CI (msys2 has the DLL # on PATH) but fails on a user's machine. # Shared whitelist (same as bundling step uses for ldd + # objdump walks) — guaranteed agreement between what's # classified as system and what gets bundled. source scripts/ci/win-system-dll.sh fail=0 for dll in bundle/*.dll; do [[ -L "$dll" ]] && continue # objdump -p output has lines like: # DLL Name: libavcodec-62.dll # Match those, extract the name. while IFS= read -r imp; do [[ -z "$imp" ]] && continue if is_system_dll "$imp"; then continue fi if [[ -f "bundle/$imp" ]]; then continue fi echo "::error file=$dll::imports '$imp' which is neither a system DLL nor a bundle sibling" fail=1 done < <(objdump -p "$dll" 2>/dev/null | awk '/DLL Name:/{print $3}') done if (( fail != 0 )); then echo echo "❌ Bundle audit failed: one or more bundled DLLs import" echo " non-system DLLs that aren't present in bundle/." echo " Those imports would fail on a user's machine. Either" echo " add the missing DLLs to the belt-and-braces sweep in" echo " process_deps's belt-and-braces section above, or fix" echo " ldd's walk to capture them." exit 1 fi echo "✅ Bundle self-containment audit passed: every bundled DLL's" echo " imports resolve inside bundle/ or to Windows system DLLs." - name: Audit bundle over-collection (import reachability) shell: 'msys2 {0}' run: | set -euo pipefail # The mirror image of the step above. That one asks "can every # import be resolved?"; this one asks "does anything actually # import this?". A DLL nothing loads is not a broken pack, so # nothing else in the pipeline notices it — it is just a # binary we redistribute, and therefore a licence we are # responsible for, a notice we owe, and (if it is copyleft) # corresponding source we must be able to produce, all in # exchange for nothing. # # That is not hypothetical. The published r10 packs carried # four such DLLs: libintl-8.dll in both (LGPL, pacman-sourced, # no pinnable source), and libstdc++-6.dll, libgomp-1.dll and # libgcc_s_seh-1.dll in the UCRT64 one. Every one arrived # through the belt-and-braces sweep list, which by design # copies things whether or not anything needs them. This gate # is what stops that list — or a future ldd/objdump walk that # over-reaches — putting an unowned binary in a release again. # # Roots are the three notcurses DLLs, which is what the pack # exists to deliver. The perf shim is a fourth root rather # than a reachable node — Notcurses::Native loads it directly # and nothing imports it — but the caller compiles it into # bundle/ AFTER this action runs, so it is only picked up if # the step order ever changes. Listing it anyway costs # nothing and keeps that reordering from turning into a # baffling "nothing imports the shim" failure. roots=(bundle/libnotcurses.dll bundle/libnotcurses-core.dll bundle/libnotcurses-ffi.dll) for root in "${roots[@]}"; do if [[ ! -f "$root" ]]; then echo "❌ $root is missing — staging did not run." exit 1 fi done if [[ -f bundle/libnotcurses_native_shim.dll ]]; then roots+=( bundle/libnotcurses_native_shim.dll ) fi # The reached set is a space-delimited string of lowercased # basenames rather than an associative array, so this can be # replayed against a published pack under any bash — including # macOS's 3.2, which is where the r10 over-collection was # actually found. Lowercased because a PE import table spells # a name however the linker that wrote it felt at the time # (KERNEL32.dll, libavcodec-62.dll) while the file on disk has # exactly one spelling, and Windows resolves the two against # each other case-insensitively. reachable=' ' walk() { local dll="$1" local depth="${2:-0}" if (( depth > 12 )); then echo "walk: depth cap hit at $dll" return 0 fi local key key="$(basename "$dll" | tr '[:upper:]' '[:lower:]')" case "$reachable" in *" $key "*) return 0 ;; esac reachable="$reachable$key " local name while IFS= read -r name; do [[ -z "$name" ]] && continue # `-f` goes through the Windows filesystem, which is # case-insensitive, so this matches whatever casing the # import table used. if [[ -f "bundle/$name" ]]; then walk "bundle/$name" $(( depth + 1 )) fi done < <(objdump -p "$dll" 2>/dev/null | awk '/DLL Name:/{print $3}') } for root in "${roots[@]}"; do walk "$root" done fail=0 for dll in bundle/*.dll; do [[ -L "$dll" ]] && continue base="$(basename "$dll")" key="$(printf '%s' "$base" | tr '[:upper:]' '[:lower:]')" case "$reachable" in *" $key "*) echo "ok: $base" continue ;; esac echo "::error file=$dll::nothing in this pack imports '$base'" fail=1 done if (( fail != 0 )); then echo echo "❌ The pack contains DLLs that nothing in it loads." echo " Work out why each one arrived — usually a stale name in" echo " the belt-and-braces sweep list — and stop copying it," echo " rather than adding a component to" echo " resources/third-party.json for a library no user will" echo " ever execute. If a name here IS newly needed, then" echo " something started importing it and the reachability walk" echo " would have found it; check that objdump can read the" echo " importer before concluding otherwise." exit 1 fi echo "✅ Bundle over-collection audit passed: every bundled DLL is" echo " reachable from libnotcurses{,-core,-ffi}.dll."