#!/bin/bash
#
# djibian-iso-build — assemble a Djibian installer ISO from a Debian
# base ISO, a meta-package list, and Djibian preseeds. Dependencies and
# Recommends of every meta-package are resolved transitively against the
# configured Debian mirror and the Djibian PPA, then dropped into the
# pool/ tree. The volume label, dists/ indexes and El-Torito records
# are regenerated so the resulting ISO boots BIOS + UEFI like the
# original Debian installer.
#
# Copyright © 2026 friends of OpenPGP (foopgp)
# License: GPL-3.0-only
#

set -euo pipefail

VERSION='0.6.2'
PROGNAME=${0##*/}

# ---- Defaults --------------------------------------------------------

: "${DJIBIAN_PPA:=https://djibian.foopgp.org/debs/}"
: "${DEBIAN_MIRROR:=https://deb.debian.org/debian}"
: "${DEBIAN_SECURITY_MIRROR:=https://deb.debian.org/debian-security}"
: "${DEBIAN_CODENAME:=trixie}"
: "${DEBIAN_COMPONENTS:=main non-free-firmware contrib}"
: "${ARCH:=amd64}"

# Files that must be embedded next to this script. Locations are resolved from
# $DJIBIAN_ISO_BUILD_SHARE, then $PROGDIR/../share, then $PROGDIR (the in-tree
# layout), then /etc/djibian-build/iso (the installed layout -- this is
# configuration the admin overrides), in that order.
DEFAULT_PRESEED_DIR=
DEFAULT_PACKAGES_LIST=
DEFAULT_BRANDING_DIR=

# ---- Help and version -----------------------------------------------

usage() {
    cat <<EOF
Usage: $PROGNAME [OPTIONS]... -i BASE_ISO -o OUTPUT_ISO

Assemble a Djibian installer ISO from a Debian base ISO and a list of
meta-packages. Every Depends / Recommends of those meta-packages is
resolved against the configured Debian mirror plus the Djibian PPA,
.deb files are downloaded into the pool, the dists/ indexes are
regenerated, and a bootable hybrid ISO (BIOS + UEFI) is written out.

Required:
  -i, --base-iso FILE        Debian netinst/full ISO to start from
                             (volume + bootloader + debian-installer
                             are taken from this image; the pool is
                             rebuilt from scratch).
  -o, --output FILE          Output ISO path.

Optional:
  -p, --packages-list FILE   List of meta-packages, one per line.
                             '#' comments and blank lines are skipped.
                             Their full transitive Depends + Recommends
                             are pulled.
                             (default: \$DJIBIAN_ISO_BUILD_SHARE/packages.list
                             or ../share/packages.list relative to this
                             script, or ./packages.list in the cwd).
  -P, --preseeds DIR         Directory of preseed*.cfg files to inject
                             at /preseeds/ inside the ISO.
                             (default: \$DJIBIAN_ISO_BUILD_SHARE/preseeds
                             or ../share/preseeds relative to this
                             script, or ./preseeds in the cwd).
  -B, --branding DIR         Djibian branding overlay applied verbatim
                             on top of the extracted ISO tree (splash
                             image, isolinux/menu.cfg, boot/grub themes,
                             …). Any occurrence of '@@VERSION@@' inside
                             text files is replaced by --version-tag.
                             (default: --preseeds dir's sibling
                             'branding' if any, or ./branding).
  -n, --name NAME            ISO volume label
                             (default: "Djibian <version>", where
                             <version> is taken from --version-tag).
  -V, --version-tag TAG      Djibian release tag, e.g. "1.2026.6";
                             used in the volume label and changes
                             report (default: today's YYYY.M).
  -d, --djibian-ppa URL      Djibian PPA root (default: $DJIBIAN_PPA).
  -m, --mirror URL           Debian mirror (default: $DEBIAN_MIRROR).
  -s, --security-mirror URL  Debian security mirror
                             (default: $DEBIAN_SECURITY_MIRROR).
  -c, --codename NAME        Debian suite codename
                             (default: $DEBIAN_CODENAME).
  -C, --components LIST      Debian components (space-separated, in one
                             arg) (default: "$DEBIAN_COMPONENTS").
  -a, --arch ARCH            Target architecture (default: $ARCH).
  -b, --previous-iso FILE    Previous ISO to diff against — produces
                             a Release_note.md in the release
                             directory next to the output ISO.
  -S, --sign-key KEY         GPG key id / fingerprint used to detach-sign
                             the output ISO (default: gpg's default-key).
                             A non-zero exit from gpg only emits a
                             warning — the build still succeeds and the
                             ISO ships without an .asc.
  -w, --workdir DIR          Working directory (default: a fresh
                             mktemp -d under \$TMPDIR or /tmp).
  -k, --keep-workdir         Keep the working directory after build
                             (useful for debugging; default: cleaned).
  -1, --one-provider         Resolve the way apt would: one provider per
                             virtual package, one alternative per "a | b"
                             group. Without it, apt-cache --recurse takes
                             every provider and every alternative, which is
                             how a desktop ISO ends up carrying
                             enlightenment, three soundfonts and a Haskell
                             compiler that nothing on it declares. Measured
                             on 2026.9.2: 5608 packages / 8.35 GB against
                             1844 / 2.77 GB.
  -j, --jobs N               Parallel apt-get download workers
                             (default: 8).

  -h, --help                 Show this help and exit.
  -v, --version              Show program version and exit.

Example:
  $PROGNAME \\
      -i debian-13.5.0-amd64-netinst.iso \\
      -p packages.list \\
      -V 1.2026.6 \\
      -o debian-13.5.0_djibian-1.2026.6-amd64.iso \\
      -b djibian-13.3.0_djibian-1.1.0-amd64.iso

Environment variables (overriden by their option twins):
  DJIBIAN_PPA, DEBIAN_MIRROR, DEBIAN_SECURITY_MIRROR, DEBIAN_CODENAME,
  DEBIAN_COMPONENTS, ARCH.

Exit codes:
  0   ISO built successfully.
  1   Usage / argument error.
  2   Missing dependency on the host.
  3   apt resolution or download failed.
  4   ISO assembly failed.
EOF
}

show_version() {
    printf '%s %s\n' "$PROGNAME" "$VERSION"
}

# ---- Logging --------------------------------------------------------

log()   { printf '[%s] %s[%d]: Info: %s\n' "$(date --iso-8601=seconds)" "$PROGNAME" $$ "$*" >&2; }
warn()  { printf '[%s] %s[%d]: Warning: %s\n' "$(date --iso-8601=seconds)" "$PROGNAME" $$ "$*" >&2; }
# "$1", not "$*": every caller passes the exit status as $2, and $* glued it
# into the sentence -- "base ISO not found: /inexistant 1". Harmless while a
# human read it, wrong now that the message field is meant to be ingested.
die()   { printf '[%s] %s[%d]: Error: %s\n' "$(date --iso-8601=seconds)" "$PROGNAME" $$ "$1" >&2; exit "${2:-1}"; }

# ---- Argument parsing ----------------------------------------------

BASE_ISO=
PACKAGES_LIST=
OUTPUT_ISO=
PRESEED_DIR=
BRANDING_DIR=
VOLUME_NAME=
VERSION_TAG=
PREVIOUS_ISO=
SIGN_KEY=
WORKDIR=
KEEP_WORKDIR=0
ONE_PROVIDER=0
JOBS=8

while [[ $# -gt 0 ]]; do
    case "$1" in
        -i|--base-iso)         BASE_ISO=$2; shift 2;;
        -p|--packages-list)    PACKAGES_LIST=$2; shift 2;;
        -o|--output)           OUTPUT_ISO=$2; shift 2;;
        -P|--preseeds)         PRESEED_DIR=$2; shift 2;;
        -B|--branding)         BRANDING_DIR=$2; shift 2;;
        -n|--name)             VOLUME_NAME=$2; shift 2;;
        -V|--version-tag)      VERSION_TAG=$2; shift 2;;
        -d|--djibian-ppa)      DJIBIAN_PPA=$2; shift 2;;
        -m|--mirror)           DEBIAN_MIRROR=$2; shift 2;;
        -s|--security-mirror)  DEBIAN_SECURITY_MIRROR=$2; shift 2;;
        -c|--codename)         DEBIAN_CODENAME=$2; shift 2;;
        -C|--components)       DEBIAN_COMPONENTS=$2; shift 2;;
        -a|--arch)             ARCH=$2; shift 2;;
        -b|--previous-iso)     PREVIOUS_ISO=$2; shift 2;;
        -S|--sign-key)         SIGN_KEY=$2; shift 2;;
        -w|--workdir)          WORKDIR=$2; shift 2;;
        -k|--keep-workdir)     KEEP_WORKDIR=1; shift;;
        -1|--one-provider)     ONE_PROVIDER=1; shift;;
        -j|--jobs)             JOBS=$2; shift 2;;
        -h|--help)             usage; exit 0;;
        -v|--version)          show_version; exit 0;;
        --)                    shift; break;;
        -*)                    die "unknown option: $1" 1;;
        *)                     die "unexpected positional argument: $1" 1;;
    esac
done

[[ -n $BASE_ISO ]]        || die "missing --base-iso" 1
[[ -f $BASE_ISO ]]        || die "base ISO not found: $BASE_ISO" 1
[[ -n $OUTPUT_ISO ]]      || die "missing --output" 1

# Resolve defaults that depend on the script's own location.
PROGDIR=$(cd "$(dirname -- "$0")" && pwd -P)

resolve_share_path() {
    local sub=$1 candidate
    for candidate in \
        "${DJIBIAN_ISO_BUILD_SHARE:-}/$sub" \
        "$PROGDIR/../share/$sub" \
        "$PROGDIR/$sub" \
        "/etc/djibian-build/iso/$sub" \
        "./$sub"; do
        [[ -n $candidate && -e $candidate ]] && { printf '%s\n' "$candidate"; return; }
    done
}

if [[ -z $PACKAGES_LIST ]]; then
    PACKAGES_LIST=$(resolve_share_path packages.list) \
        || die "no packages list found — pass --packages-list or place one alongside the script" 1
fi
[[ -f $PACKAGES_LIST ]] || die "packages list not found: $PACKAGES_LIST" 1

if [[ -z $PRESEED_DIR ]]; then
    PRESEED_DIR=$(resolve_share_path preseeds) \
        || die "no preseed directory found — pass --preseeds or place one alongside the script" 1
fi
[[ -d $PRESEED_DIR ]] || die "preseed directory not found: $PRESEED_DIR" 1

if [[ -z $BRANDING_DIR ]]; then
    BRANDING_DIR=$(resolve_share_path branding) || true
fi
if [[ -n $BRANDING_DIR && ! -d $BRANDING_DIR ]]; then
    die "branding directory not found: $BRANDING_DIR" 1
fi

if [[ -z $VERSION_TAG ]]; then
    VERSION_TAG=$(date +%Y.%-m)
fi
if [[ -z $VOLUME_NAME ]]; then
    VOLUME_NAME="Djibian $VERSION_TAG"
fi

# ---- Host dependency check -----------------------------------------

REQUIRED_BINS=(xorriso apt-get apt-cache apt-ftparchive)
for bin in "${REQUIRED_BINS[@]}"; do
    command -v "$bin" >/dev/null || die "missing host dependency: $bin" 2
done

# ---- Working directory ---------------------------------------------

if [[ -z $WORKDIR ]]; then
    WORKDIR=$(mktemp -d -t djibian-iso-build.XXXXXX)
fi
mkdir -p "$WORKDIR"
WORKDIR=$(cd "$WORKDIR" && pwd -P)
log "working directory: $WORKDIR"

cleanup() {
    if (( KEEP_WORKDIR == 0 )); then
        log "cleaning up working directory"
        rm -rf "$WORKDIR"
    else
        log "keeping working directory: $WORKDIR"
    fi
}
trap cleanup EXIT

EXTRACT_DIR=$WORKDIR/extracted
DEBS_DIR=$WORKDIR/debs
APT_DIR=$WORKDIR/apt
mkdir -p "$EXTRACT_DIR" "$DEBS_DIR" \
         "$APT_DIR/etc" "$APT_DIR/state/lists" "$APT_DIR/cache/archives/partial"

# ---- Extract base ISO ----------------------------------------------

log "extracting base ISO into $EXTRACT_DIR"
xorriso -osirrox on -indev "$BASE_ISO" -extract / "$EXTRACT_DIR" >/dev/null 2>&1 \
    || die "xorriso extraction failed" 4

# xorriso preserves ISO-9660 file permissions verbatim — every file
# comes out 0444, every directory 0555. We need write access to drop
# the old pool/, overlay branding, regenerate indexes, and so on.
chmod -R u+w "$EXTRACT_DIR"

# Save the base ISO's /pool aside before wiping the working tree —
# the Debian netinst ships a few hundred .deb and .udeb that we will
# happily reuse from the tier-1.5 cache rather than re-fetching.
BASE_POOL=$WORKDIR/base-pool
if [[ -d $EXTRACT_DIR/pool ]]; then
    mv "$EXTRACT_DIR/pool" "$BASE_POOL"
else
    BASE_POOL=
fi

# Drop the pre-existing dists so we rebuild from scratch — keeps the
# .iso shape (boot/, EFI/, install.amd/, isolinux/, …) but lets us
# control exactly which .deb ship.
mkdir -p "$EXTRACT_DIR/pool"
rm -rf "$EXTRACT_DIR/dists"
mkdir -p "$EXTRACT_DIR/dists/$DEBIAN_CODENAME"

# ---- Configure user-space apt --------------------------------------

cat > "$APT_DIR/etc/sources.list" <<EOF
deb $DEBIAN_MIRROR $DEBIAN_CODENAME $DEBIAN_COMPONENTS
deb $DEBIAN_SECURITY_MIRROR ${DEBIAN_CODENAME}-security $DEBIAN_COMPONENTS
EOF
cat > "$APT_DIR/etc/apt.conf" <<EOF
Dir::Etc::sourcelist  "$APT_DIR/etc/sources.list";
Dir::Etc::sourceparts "/dev/null";
Dir::Etc::trusted     "/etc/apt/trusted.gpg";
Dir::Etc::trustedparts "/etc/apt/trusted.gpg.d";
Dir::State::Lists     "$APT_DIR/state/lists";
Dir::Cache::Archives  "$APT_DIR/cache/archives";
Dir::State::Status    "/dev/null";
APT::Architecture     "$ARCH";
APT::Architectures    "$ARCH";
APT::Install-Recommends "true";
EOF
export APT_CONFIG=$APT_DIR/etc/apt.conf

log "refreshing apt indexes against $DEBIAN_MIRROR / ${DEBIAN_CODENAME}"
apt-get update >/dev/null 2>&1 || die "apt-get update failed" 3

# ---- Resolve transitive Depends + Recommends -----------------------

# Read the meta-package list, strip comments and blanks.
mapfile -t METAS < <(grep -vE '^\s*(#|$)' "$PACKAGES_LIST")
(( ${#METAS[@]} > 0 )) || die "packages list is empty: $PACKAGES_LIST" 1

# Seed the resolution with every Priority: required / important
# package. apt-cache depends starts from explicit Depends edges only,
# but debootstrap installs the whole "required" set unconditionally
# during base-install, so anything in that set MUST be present in
# pool/ — otherwise dpkg --install fails inside the chroot ("'sh'
# not found", "Failure trying to run …/base-passwd_*.deb"). Same
# story for "important": the user expects vim/debconf/etc on a
# freshly-installed system without extra apt-get install.
log "seeding required + important Debian packages"
ESSENTIAL_SEED=$WORKDIR/essential-seed.txt
apt-cache dumpavail | awk '
    /^Package:/  { pkg = $2 }
    /^Priority:/ {
        if ($2 == "required" || $2 == "important") print pkg
    }
' | sort -u > "$ESSENTIAL_SEED"
log "$(wc -l < "$ESSENTIAL_SEED") essential packages seeded"

mapfile -t ESSENTIALS < "$ESSENTIAL_SEED"

log "resolving Depends + Recommends transitively for ${#METAS[@]} meta-packages and ${#ESSENTIALS[@]} essentials"
# apt-cache depends with --recurse walks transitively. We accept
# Depends, Recommends, PreDepends; we explicitly cut Suggests and
# Conflicts/Breaks. --installed lookups are disabled so apt does not
# consult the host's /var/lib/dpkg/status.
RESOLVED_REAL=$WORKDIR/resolved-packages.txt
# Header lines at column 0 in `apt-cache depends` output are either
# a real package name (kept) or a virtual one wrapped in '<...>'
# (skipped — '<' is not alnum). All indented Depends/Recommends rows
# stay hidden too. apt-cache walks the providers of each virtual
# anyway, so concrete provider names still land in the output via
# their own header line further down.
# ---- Resolving a closure -------------------------------------------
#
# Two ways, and they differ by more than style.
#
# `apt-cache depends --recurse` walks *every* provider of a virtual
# package and *every* alternative of an "a | b" group. apt installs one.
# On a desktop closure that difference is not marginal: `xinit` recommends
# the virtual x-window-manager, so enlightenment arrives with its 400 MB of
# data; something wants sf2-soundfont-gm, so all three soundfonts arrive;
# ghc arrives because it provides haskell-compiler. Measured on Djibian
# 2026.9.2: 5608 packages and 8.35 GB against 1844 and 2.77 GB.
#
# --one-provider asks apt itself, once per root, and unions the answers.
# Per root because the roots conflict with each other by design -- grub-pc
# and grub-efi-amd64 cannot be installed together, but the pool needs both.
#
# Both ways resolve against an empty status file -- Dir::State::Status is set
# to /dev/null in the apt.conf written above -- so neither describes whatever
# happens to be installed on the machine doing the building.
resolve_closure() {
    local out=$1 ; shift
    if [[ $ONE_PROVIDER -eq 0 ]]; then
        apt-cache depends \
            --recurse \
            --no-suggests \
            --no-conflicts --no-breaks --no-replaces --no-enhances \
            "$@" \
            | awk '/^[[:alnum:]]/ { print $1 }' \
            | sort -u > "$out"
        return
    fi
    : > "$out.raw"
    local r
    for r in "$@"; do
        LC_ALL=C apt-get install --simulate --allow-unauthenticated \
            -o APT::Install-Recommends=true \
            "$r" 2>/dev/null \
            | awk '/^Inst /{ print $2 }' >> "$out.raw" || true
    done
    sort -u "$out.raw" > "$out"
    rm -f "$out.raw"
    # An empty closure means the resolution failed, not that nothing is
    # needed. Carrying on would write a bootable ISO with no packages in
    # it -- which is exactly the shape of failure this tool is prone to.
    [[ -s $out ]] || die "resolving produced nothing -- apt could not answer"
}

resolve_closure "$RESOLVED_REAL" "${METAS[@]}" "${ESSENTIALS[@]}"

log "$(wc -l < "$RESOLVED_REAL") real packages to download"

# ---- Build the apt package index (one TSV row per binary package) -
#
# Used by fetch_packages() below to look up canonical Filename and
# SHA256 without one apt-cache invocation per query. apt-cache
# dumpavail prints the same Packages stanzas that apt-get download
# would parse internally.

PKG_INDEX=$WORKDIR/apt-index.tsv
log "building apt package index for cache lookups"
apt-cache dumpavail | awk '
    /^Package:/  { pkg = $2 }
    /^Filename:/ { fn  = $2 }
    /^SHA256:/   { sha = $2 }
    /^$/         {
        if (pkg && fn && sha) print pkg "\t" fn "\t" sha
        pkg=fn=sha=""
    }
    END {
        if (pkg && fn && sha) print pkg "\t" fn "\t" sha
    }
' | sort -u > "$PKG_INDEX"
log "$(wc -l < "$PKG_INDEX") binary packages in the apt index"

# ---- Extract previous-ISO pool (tier-1 cache) ---------------------

PREV_POOL=
if [[ -n $PREVIOUS_ISO ]]; then
    PREV_POOL=$WORKDIR/previous-pool
    mkdir -p "$PREV_POOL"
    log "extracting pool/ from previous ISO ($PREVIOUS_ISO) for cache"
    if ! xorriso -osirrox on -indev "$PREVIOUS_ISO" \
            -extract /pool "$PREV_POOL" >/dev/null 2>&1; then
        warn "could not extract previous ISO pool — tier-1 cache disabled"
        PREV_POOL=
    else
        chmod -R u+w "$PREV_POOL"
    fi
fi

# ---- Download helpers --------------------------------------------

ERR_LOG=$WORKDIR/download-errors.log
FAIL_LIST=$WORKDIR/download-failed.txt
: > "$ERR_LOG"; : > "$FAIL_LIST"

APT_CACHE_DIR=/var/cache/apt/archives
BATCH_SIZE=${BATCH_SIZE:-50}

# try_cache_copy BASENAME EXPECTED_SHA256 TARGET CACHE_DIR
#   Returns 0 and copies if a file named BASENAME inside CACHE_DIR has
#   the expected SHA256, otherwise returns 1. CACHE_DIR is searched
#   recursively (the pool tree under it is several levels deep).
try_cache_copy() {
    local bn=$1 sha=$2 target=$3 dir=$4 cand
    [[ -n $dir && -d $dir ]] || return 1
    cand=$(find "$dir" -name "$bn" -print -quit 2>/dev/null)
    [[ -n $cand ]] || return 1
    [[ $(sha256sum "$cand" | awk '{print $1}') == "$sha" ]] || return 1
    cp "$cand" "$target"
}

# fetch_packages PACKAGE_LIST_FILE
#
# Four-tier .deb fetcher (cheapest first):
#   1.   previous ISO pool        (tier 1,   --previous-iso)
#   1.5  base ISO pool            (tier 1.5, pool/ pulled from --base-iso)
#   2.   /var/cache/apt/archives  (tier 2,   host apt cache)
#   3.   apt-get download         (tier 3,   batched, bisect on failure)
#
# Every local hit is SHA256-verified against the apt index so we never
# ship a tampered or stale .deb. Packages still missing after step 3
# land in $FAIL_LIST.
fetch_packages() {
    local list=$1 t1=0 t1b=0 t2=0 needs_dl=$WORKDIR/needs-dl.$$
    : > "$needs_dl"

    # Resolve each requested package to its (filename, sha256) via
    # a single awk join with the pre-built apt index.
    local resolved=$WORKDIR/resolved.$$
    awk -F'\t' '
        NR==FNR { idx[$1] = $2 "\t" $3; next }
        {
            if ($1 in idx) print $1 "\t" idx[$1]
            else           print $1 > "/dev/stderr"
        }
    ' "$PKG_INDEX" "$list" > "$resolved" 2>"$WORKDIR/index-misses.$$"
    if [[ -s $WORKDIR/index-misses.$$ ]]; then
        warn "$(wc -l < "$WORKDIR/index-misses.$$") packages absent from apt index — listed in $WORKDIR/index-misses.$$"
    fi

    # Walk resolved packages, try caches in cheapest-first order.
    while IFS=$'\t' read -r pkg fn sha; do
        local bn=${fn##*/} target="$DEBS_DIR/${fn##*/}"
        [[ -f $target ]] && continue

        try_cache_copy "$bn" "$sha" "$target" "$PREV_POOL" \
            && { t1=$((t1 + 1)); continue; }
        try_cache_copy "$bn" "$sha" "$target" "$BASE_POOL" \
            && { t1b=$((t1b + 1)); continue; }
        try_cache_copy "$bn" "$sha" "$target" "$APT_CACHE_DIR" \
            && { t2=$((t2 + 1)); continue; }

        printf '%s\n' "$pkg" >> "$needs_dl"
    done < "$resolved"

    local need_count=0
    [[ -s $needs_dl ]] && need_count=$(wc -l < "$needs_dl")
    log "cache: $t1 prev-ISO + $t1b base-ISO + $t2 host-apt, $need_count to download"

    # Tier 3 — batch apt-get download with bisect-on-failure.
    if (( need_count > 0 )); then
        pushd "$DEBS_DIR" >/dev/null
        mapfile -t names < "$needs_dl"
        local pos=0 n=${#names[@]}
        while (( pos < n )); do
            local end=$((pos + BATCH_SIZE))
            (( end > n )) && end=$n
            local -a chunk=("${names[@]:pos:end-pos}")
            if ! apt-get download "${chunk[@]}" >/dev/null 2>>"$ERR_LOG"; then
                bisect_download "${chunk[@]}"
            fi
            pos=$end
        done
        popd >/dev/null
    fi

    rm -f "$resolved" "$needs_dl" "$WORKDIR/index-misses.$$"
}

# Recursive bisect: on apt-get download failure, split the failing
# batch in two and retry. Singletons that still fail land in
# $FAIL_LIST — those are the ones a human needs to investigate.
bisect_download() {
    local -a batch=("$@")
    local n=${#batch[@]}
    if (( n == 1 )); then
        printf '%s\n' "${batch[0]}" >> "$FAIL_LIST"
        return
    fi
    local mid=$((n / 2))
    local -a left=("${batch[@]:0:mid}") right=("${batch[@]:mid}")
    apt-get download "${left[@]}"  >/dev/null 2>>"$ERR_LOG" || bisect_download "${left[@]}"
    apt-get download "${right[@]}" >/dev/null 2>>"$ERR_LOG" || bisect_download "${right[@]}"
}

# ---- Download all .deb files ---------------------------------------

log "downloading .deb files into $DEBS_DIR (cache + batched mirror)"
fetch_packages "$RESOLVED_REAL"

if [[ -s $FAIL_LIST ]]; then
    warn "$(wc -l < "$FAIL_LIST") packages could not be downloaded — see $FAIL_LIST"
fi

# ---- Download .udeb files for the debian-installer ---------------
#
# `apt-get download` ignores .udeb — they live in a separate
# "debian-installer" component that apt does not enable for normal
# operations. Pull them straight from the Packages index of
# dists/$DEBIAN_CODENAME/main/debian-installer/binary-$ARCH, which
# every Debian mirror ships unsigned alongside the regular index.

log "fetching .udeb files for the debian-installer"
UDEB_DIR=$DEBS_DIR/udeb
mkdir -p "$UDEB_DIR"
UDEB_INDEX_URL="$DEBIAN_MIRROR/dists/$DEBIAN_CODENAME/main/debian-installer/binary-$ARCH/Packages.gz"
UDEB_INDEX=$WORKDIR/di-Packages
if curl -fsSL "$UDEB_INDEX_URL" -o "$WORKDIR/di-Packages.gz" 2>>"$ERR_LOG"; then
    gunzip -c "$WORKDIR/di-Packages.gz" > "$UDEB_INDEX"

    # Build a (Filename, SHA256) list for the udebs — same shape
    # try_cache_copy() expects when looking inside the previous/base
    # ISO pool trees. With RS="" each record is one full stanza, so
    # we split on newlines inside the action and pull the two fields
    # we need.
    UDEB_TSV=$WORKDIR/udeb-index.tsv
    awk 'BEGIN { RS = "" }
    {
        fn = sha = ""
        n = split($0, lines, "\n")
        for (j = 1; j <= n; j++) {
            if (match(lines[j], /^Filename:[[:space:]]+pool\//)) {
                fn = substr(lines[j], RSTART + RLENGTH - 5)  # keep "pool/" onward
            } else if (match(lines[j], /^SHA256:[[:space:]]+[0-9a-f]{64}/)) {
                sha = substr(lines[j], RSTART + RLENGTH - 64)
            }
        }
        if (fn && sha) print fn "\t" sha
    }' "$UDEB_INDEX" > "$UDEB_TSV"
    log "$(wc -l < "$UDEB_TSV") udebs listed in the d-i index"

    udeb_t1=0; udeb_t1b=0; udeb_dl=0
    while IFS=$'\t' read -r fn sha; do
        bn=${fn##*/}
        target="$UDEB_DIR/$bn"
        [[ -f $target ]] && continue
        try_cache_copy "$bn" "$sha" "$target" "$PREV_POOL" \
            && { udeb_t1=$((udeb_t1 + 1)); continue; }
        try_cache_copy "$bn" "$sha" "$target" "$BASE_POOL" \
            && { udeb_t1b=$((udeb_t1b + 1)); continue; }
        # Pull from the mirror — udebs are not visible to apt-get
        # download, so curl is what we have.
        if curl -fsSL "$DEBIAN_MIRROR/$fn" -o "$target" 2>>"$ERR_LOG"; then
            udeb_dl=$((udeb_dl + 1))
        else
            warn "udeb fetch failed: $fn"
        fi
    done < "$UDEB_TSV"
    log "udebs: $udeb_t1 prev-ISO + $udeb_t1b base-ISO + $udeb_dl downloaded"
else
    warn "could not fetch debian-installer Packages index — the resulting ISO may fail at the 'load components' stage"
fi

# ---- Pull Djibian PPA packages -------------------------------------

log "fetching Djibian PPA index from $DJIBIAN_PPA"
PPA_INDEX=$WORKDIR/ppa-index.html
PPA_DEBS_DIR=$DEBS_DIR/extra
mkdir -p "$PPA_DEBS_DIR"

if curl -fsSL "$DJIBIAN_PPA" -o "$PPA_INDEX" 2>>"$ERR_LOG"; then
    grep -oE 'href="[^"]+\.deb"' "$PPA_INDEX" \
        | sed 's/href="//; s/"$//' \
        | while read -r f; do
            curl -fsSL "$DJIBIAN_PPA$f" -o "$PPA_DEBS_DIR/$f" 2>>"$ERR_LOG" \
                || warn "failed to fetch $f from Djibian PPA"
        done
    log "$(ls "$PPA_DEBS_DIR"/*.deb 2>/dev/null | wc -l) Djibian PPA packages fetched"
else
    warn "could not fetch Djibian PPA index; the resulting ISO will not include Djibian-specific packages"
fi

# ---- Resolve transitive deps of Djibian PPA packages too -----------
#
# The Djibian PPA is a flat directory, not an apt-cacheable repo, so
# `apt-cache depends` cannot follow the dependencies of djibian-*
# packages. Without this pass, a Djibian package whose Depends point
# at a Debian binary that the user-listed meta-packages don't pull
# in (e.g. djibian-core → scdaemon, when nothing else needs it) ends
# up missing from pool/, and base-installer / pkgsel fails with
# "Unable to satisfy dependencies" on the target system.
#
# Extract every Depends / Pre-Depends / Recommends from each PPA
# .deb, strip versions and alternatives, feed them back through
# apt-cache depends --recurse to resolve them transitively against
# the Debian mirror, then apt-get download the ones still missing.

if compgen -G "$PPA_DEBS_DIR/*.deb" >/dev/null; then
    log "resolving Depends / Recommends of Djibian PPA packages"
    PPA_DEPS=$WORKDIR/ppa-deps.txt
    : > "$PPA_DEPS"
    for deb in "$PPA_DEBS_DIR"/*.deb; do
        dpkg-deb -f "$deb" Depends Pre-Depends Recommends 2>/dev/null
    done \
        | sed -E 's/^(Depends|Pre-Depends|Recommends): //' \
        | tr ',' '\n' \
        | awk -F'|' '{ for (i=1; i<=NF; i++) print $i }' \
        | sed -E 's/\([^)]*\)//; s/^[ \t]+//; s/[ \t]+$//' \
        | grep -vE '^djibian-|^bashlibs-|^sshwgpg$|^onak$|^pgpid$|^appimages-' \
        | grep -E '^[a-z0-9]' \
        | sort -u > "$PPA_DEPS"

    log "$(wc -l < "$PPA_DEPS") direct Debian deps from PPA packages"

    # Transitive resolution through the Debian mirror. We keep
    # Recommends here too — a Djibian PPA package typically pulls a
    # Debian subsystem (e.g. djibian-core → scdaemon) and its
    # Recommends are part of what the user expects to "just work"
    # on a freshly-installed Djibian box (no apt-get install needed
    # post-install). Skip --important: it would silently strip
    # Recommends. Same awk filter — virtual names start with '<' and
    # get dropped.
    PPA_REAL=$WORKDIR/ppa-resolved.txt
    resolve_closure "$PPA_REAL" $(cat "$PPA_DEPS")

    # Download anything not already in $DEBS_DIR, going through the
    # same three-tier cache as the meta-package walk.
    MISSING=$WORKDIR/ppa-missing.txt
    ls "$DEBS_DIR"/*.deb 2>/dev/null \
        | xargs -I{} dpkg-deb -f {} Package 2>/dev/null \
        | sort -u > "$WORKDIR/already-have.txt"
    comm -23 "$PPA_REAL" "$WORKDIR/already-have.txt" > "$MISSING"
    log "$(wc -l < "$MISSING") additional Debian packages to fetch for PPA deps"

    if [[ -s $MISSING ]]; then
        fetch_packages "$MISSING"
    fi
fi

# ---- Fail fast on truly missing packages -------------------------
#
# Anything that landed in $FAIL_LIST is a name apt-cache claimed to
# know but apt-get download could not retrieve — likely a transitional
# package, mirror desync, or a component not enabled on the configured
# mirror. Shipping the ISO anyway would have base-installer / pkgsel
# fail at install time, so refuse to assemble.

if [[ -s $FAIL_LIST ]]; then
    missed=$(wc -l < "$FAIL_LIST")
    cat >&2 <<EOF

error: $missed package(s) could not be downloaded:

$(sed 's/^/  - /' "$FAIL_LIST")

These names showed up in apt-cache depends from the meta-packages or
the Djibian PPA but apt-get download was unable to fetch them.
Inspect $ERR_LOG for the underlying error, and either remove them
from packages.list / from a PPA package's Depends, or switch to a
mirror that ships them.
EOF
    exit 3
fi

# ---- Lay out the pool tree -----------------------------------------

log "arranging .deb files under pool/"

# Convention: Debian-mirror packages land in pool/<component>/<letter>/<source>/,
# with <letter> being the first letter of <source> (or "lib" for lib*).
# We compute <source> from the package's Source: field when present,
# otherwise from its name.
place_deb() {
    local deb=$1 dest_root=$2
    local control source name letter dest
    control=$(dpkg-deb -f "$deb" Package Source 2>/dev/null)
    name=$(awk -F': ' '/^Package: /{print $2}' <<< "$control")
    source=$(awk -F': ' '/^Source: /{print $2}' <<< "$control" | awk '{print $1}')
    [[ -n $source ]] || source=$name
    if [[ $source == lib* ]]; then
        letter=${source:0:4}
    else
        letter=${source:0:1}
    fi
    dest="$dest_root/$letter/$source"
    mkdir -p "$dest"
    cp -a "$deb" "$dest/"
}

POOL_ROOT=$EXTRACT_DIR/pool
for component in $DEBIAN_COMPONENTS; do
    mkdir -p "$POOL_ROOT/$component"
done

for deb in "$DEBS_DIR"/*.deb; do
    [[ -e $deb ]] || continue
    # Most Debian packages live in main; non-free-firmware can be
    # detected by the package Section: field.
    section=$(dpkg-deb -f "$deb" Section 2>/dev/null | tr -d ' ')
    component=main
    case "$section" in
        non-free-firmware/*|non-free-firmware) component=non-free-firmware;;
        contrib/*|contrib)                     component=contrib;;
    esac
    place_deb "$deb" "$POOL_ROOT/$component"
done

# Udeb files go into pool/main/<letter>/<source>/ following the same
# convention as their .deb cousins. d-i finds them through the
# dists/$CODENAME/main/debian-installer/binary-$ARCH/Packages index
# regenerated further down.
if compgen -G "$UDEB_DIR/*.udeb" >/dev/null; then
    for udeb in "$UDEB_DIR"/*.udeb; do
        place_deb "$udeb" "$POOL_ROOT/main"
    done
fi

# Djibian PPA debs go into pool/extra/.
if compgen -G "$PPA_DEBS_DIR/*.deb" >/dev/null; then
    mkdir -p "$POOL_ROOT/extra"
    cp -a "$PPA_DEBS_DIR"/*.deb "$POOL_ROOT/extra/"
fi

# ---- Generate dists/ indexes ---------------------------------------

log "regenerating dists/$DEBIAN_CODENAME indexes"

DIST_ROOT=$EXTRACT_DIR/dists/$DEBIAN_CODENAME
COMPONENTS_FOR_INDEX="$DEBIAN_COMPONENTS"
[[ -d $POOL_ROOT/extra ]] && COMPONENTS_FOR_INDEX="$COMPONENTS_FOR_INDEX extra"

cd "$EXTRACT_DIR"
for component in $COMPONENTS_FOR_INDEX; do
    bin_dir="$DIST_ROOT/$component/binary-$ARCH"
    mkdir -p "$bin_dir"
    apt-ftparchive packages "pool/$component" > "$bin_dir/Packages.full"
    # main/binary-amd64/Packages must list .deb only — debootstrap reads
    # it during base-system install and stumbles when a *-udeb variant
    # ships the same files as its .deb sibling (collision on
    # /usr/share/keyrings/debian-archive-*-automatic.pgp).
    if [[ $component == main ]]; then
        awk -v RS= -v ORS='\n\n' '!/Filename:[^\n]*\.udeb/' \
            "$bin_dir/Packages.full" > "$bin_dir/Packages"
    else
        cp "$bin_dir/Packages.full" "$bin_dir/Packages"
    fi
    rm -f "$bin_dir/Packages.full"
    gzip -fk9 "$bin_dir/Packages"
    xz -fk9    "$bin_dir/Packages"
    cat > "$bin_dir/Release" <<EOF
Archive: stable
Component: $component
Origin: Djibian
Label: Djibian
Architecture: $ARCH
EOF
done

# Filter pool/main udeb entries into the dedicated debian-installer
# index. d-i loads its components from there, not from main/Packages.
# IMPORTANT: pass a RELATIVE pool path to apt-ftparchive — it copies
# the path verbatim into Filename:, and cdrom-retriever then looks
# for the udeb at "<cd>/<Filename>". An absolute builder path here
# produces an unbootable installer ("package retrieval failed" at
# "Charger des composants").
if compgen -G "$POOL_ROOT/main/*/*/*.udeb" >/dev/null; then
    DI_DIR="$DIST_ROOT/main/debian-installer/binary-$ARCH"
    mkdir -p "$DI_DIR"
    apt-ftparchive packages "pool/main" \
        | awk -v RS= -v ORS='\n\n' '/Filename:[^\n]*\.udeb/' \
        > "$DI_DIR/Packages"
    gzip -fk9 "$DI_DIR/Packages"
    cat > "$DI_DIR/Release" <<EOF
Archive: stable
Origin: Djibian
Label: Djibian
Component: main
Architecture: $ARCH
EOF

    # cdrom-retriever scans <component>/debian-installer/binary-<arch>/
    # for every component declared in Release. Generate empty
    # Packages.gz indexes for the non-main components so we don't
    # emit a flood of warnings in the install log.
    for component in $DEBIAN_COMPONENTS extra; do
        [[ $component == main ]] && continue
        empty_dir="$DIST_ROOT/$component/debian-installer/binary-$ARCH"
        mkdir -p "$empty_dir"
        : > "$empty_dir/Packages"
        gzip -fk9 "$empty_dir/Packages"
        cat > "$empty_dir/Release" <<EOF
Archive: stable
Origin: Djibian
Label: Djibian
Component: $component
Architecture: $ARCH
EOF
    done
fi

cat > "$DIST_ROOT/Release" <<EOF
Origin: Djibian
Label: Djibian
Suite: stable
Codename: $DEBIAN_CODENAME
Date: $(date -Ru)
Architectures: $ARCH
Components: $COMPONENTS_FOR_INDEX
Description: Djibian $VERSION_TAG built from Debian $DEBIAN_CODENAME
EOF
# Append checksum sections (apt-ftparchive can do release too, but we
# stick to the same shape the original ISO uses).
{
    printf 'SHA256:\n'
    find dists/$DEBIAN_CODENAME -type f \
        ! -name Release ! -name Release.gpg ! -name InRelease \
        | while read -r f; do
            rel=${f#dists/$DEBIAN_CODENAME/}
            printf ' %s %s %s\n' \
                "$(sha256sum "$f" | awk '{print $1}')" \
                "$(stat -c%s "$f")" \
                "$rel"
          done
} >> "$DIST_ROOT/Release"

cd - >/dev/null

# ---- Inject preseeds -----------------------------------------------

log "injecting preseeds from $PRESEED_DIR"
mkdir -p "$EXTRACT_DIR/preseeds"
cp -a "$PRESEED_DIR"/* "$EXTRACT_DIR/preseeds/"

# ---- Apply Djibian branding overlay -------------------------------
#
# The base Debian netinst ships its own splash, grub theme and isolinux
# menus. We replace them by the Djibian look-and-feel: custom
# splash-djibian.png, isolinux menu pointing at the Djibian
# auto-install entries, grub themes for the nested submenus. Every
# '@@VERSION@@' placeholder inside text files is replaced by
# $VERSION_TAG so the right version appears on the first installer
# screen the user sees.

if [[ -n $BRANDING_DIR ]]; then
    log "applying branding overlay from $BRANDING_DIR (version $VERSION_TAG)"
    cp -a "$BRANDING_DIR"/. "$EXTRACT_DIR/"
    # Substitute the version placeholder in text files only — leaves
    # PNG/JPG/font binaries untouched.
    while IFS= read -r -d '' f; do
        if grep -lq '@@VERSION@@' "$f" 2>/dev/null; then
            sed -i "s|@@VERSION@@|${VERSION_TAG}|g" "$f"
        fi
    done < <(find "$EXTRACT_DIR" -type f \
        \( -name '*.cfg' -o -name '*.conf' -o -name '*.txt' \
           -o -path '*/boot/grub/theme/djibian*' -o -path '*/.disk/info' \) -print0)
fi

if [[ -d $EXTRACT_DIR/.disk ]]; then
    printf '%s\n' "${VOLUME_NAME} (Debian ${DEBIAN_CODENAME} base) - amd64 DVD Binary-1 with firmware - $(date -u +%Y%m%d-%H:%M)" \
        > "$EXTRACT_DIR/.disk/info"
fi

# ---- Lay out the release directory --------------------------------
#
# Group the assembled artefacts under <out_dir>/djibian-<version_tag>/
# so the whole release is one `scp -r` away. --output's basename is
# preserved inside the release directory.

ORIG_OUTPUT_PARENT=$(dirname -- "$OUTPUT_ISO")
ORIG_OUTPUT_BASE=$(basename -- "$OUTPUT_ISO")
RELEASE_DIR=$ORIG_OUTPUT_PARENT/djibian-$VERSION_TAG
mkdir -p "$RELEASE_DIR"
OUTPUT_ISO=$RELEASE_DIR/$ORIG_OUTPUT_BASE

# ---- Reassemble bootable ISO ---------------------------------------

log "assembling output ISO: $OUTPUT_ISO"

# We detect El-Torito boot record location by reading the base ISO's
# boot catalog. For Debian installer ISOs the conventional paths are:
#   isolinux/isolinux.bin  (BIOS)
#   boot/grub/efi.img      (UEFI)
# When either is missing in the extracted tree we fall back to flags
# that xorriso can still resolve, but we never invent a path.

BIOS_BOOT=
[[ -f $EXTRACT_DIR/isolinux/isolinux.bin ]] && BIOS_BOOT=isolinux/isolinux.bin
EFI_BOOT=
[[ -f $EXTRACT_DIR/boot/grub/efi.img    ]] && EFI_BOOT=boot/grub/efi.img

XORRISO_ARGS=(
    -as mkisofs
    -volid "$VOLUME_NAME"
    -joliet -rock
    -o "$OUTPUT_ISO"
)
if [[ -n $BIOS_BOOT ]]; then
    XORRISO_ARGS+=(
        -b "$BIOS_BOOT"
        -c isolinux/boot.cat
        -no-emul-boot -boot-load-size 4 -boot-info-table
    )
fi
if [[ -n $EFI_BOOT ]]; then
    XORRISO_ARGS+=(
        -eltorito-alt-boot
        -e "$EFI_BOOT"
        -no-emul-boot
        -isohybrid-gpt-basdat
    )
fi
XORRISO_ARGS+=("$EXTRACT_DIR")

xorriso "${XORRISO_ARGS[@]}" >/dev/null 2>&1 \
    || die "xorriso assembly failed" 4

# Detach-sign the ISO with gpg. We do NOT abort the build on failure —
# unattended runs (cron, CI) typically can't unlock the smartcard, and
# the .iso itself is still useful (the checksum is implicit in the
# transport's TLS + the dists/trixie/Release SHA256 already inside it).
out_dir=$(dirname -- "$OUTPUT_ISO")
out_base=$(basename -- "$OUTPUT_ISO")

GPG_ARGS=(--batch --yes --armor --detach-sign --output "$OUTPUT_ISO.asc")
[[ -n $SIGN_KEY ]] && GPG_ARGS+=(--local-user "$SIGN_KEY")
GPG_ARGS+=("$OUTPUT_ISO")

if gpg "${GPG_ARGS[@]}" 2>>"$WORKDIR/gpg-sign.log"; then
    log "wrote detached signature: $OUTPUT_ISO.asc"
else
    warn "gpg --detach-sign failed — shipping ISO without .asc (see $WORKDIR/gpg-sign.log)"
    rm -f "$OUTPUT_ISO.asc"
fi

# ---- Optional: file_changes report ---------------------------------

if [[ -n $PREVIOUS_ISO ]]; then
    log "writing Release_note.md against $PREVIOUS_ISO"
    # Reuse the pool already extracted for the tier-1 cache. xorriso
    # -extract /pool $DST drops files DIRECTLY under $DST (no extra
    # "pool/" prefix), so paths here are like
    #   $PREV_POOL/main/a/acl/acl_2.3.2_amd64.deb
    if [[ -z $PREV_POOL || ! -d $PREV_POOL ]]; then
        warn "previous-pool unavailable — skipping Release_note.md"
    else
        changes_path="$RELEASE_DIR/Release_note.md"

        {
            printf '# Djibian %s — release notes\n\n' "$VERSION_TAG"
            printf 'Diff against %s.\n\n' "$(basename -- "$PREVIOUS_ISO")"
        } > "$changes_path"

        prev_list=$(mktemp); curr_list=$(mktemp)
        (cd "$PREV_POOL"          && find . -name '*.deb' | sort) > "$prev_list"
        (cd "$EXTRACT_DIR/pool"   && find . -name '*.deb' | sort) > "$curr_list"

        # Normalise to "<pool-path-without-version>\t<full-display-path>".
        norm() { sed -E 's|/([^/_]+)_[^_/]+_[^_/]+\.deb$|/\1|; s|^\./|./pool/|' "$1"; }

        prev_norm=$(mktemp); curr_norm=$(mktemp)
        paste <(norm "$prev_list") <(sed 's|^\./|./pool/|' "$prev_list") | sort -t$'\t' -k1,1 > "$prev_norm"
        paste <(norm "$curr_list") <(sed 's|^\./|./pool/|' "$curr_list") | sort -t$'\t' -k1,1 > "$curr_norm"

        added=$(comm -13 <(cut -f1 "$prev_norm") <(cut -f1 "$curr_norm") | wc -l)
        removed=$(comm -23 <(cut -f1 "$prev_norm") <(cut -f1 "$curr_norm") | wc -l)
        join -t$'\t' -1 1 -2 1 "$prev_norm" "$curr_norm" \
            | awk -F'\t' '$2 != $3' > "$WORKDIR/updates.tsv"
        updated=$(wc -l < "$WORKDIR/updates.tsv")

        {
            # bash printf treats a format that starts with '-' as an
            # option attempt; always pass `--` to lock that down.
            printf -- '## Summary\n\n'
            printf -- '- Added: %d\n'   "$added"
            printf -- '- Updated: %d\n' "$updated"
            printf -- '- Removed: %d\n\n' "$removed"
            printf -- '## Updated packages\n\n```\n'
            awk -F'\t' '{ printf "U %s\n  -> %s\n", $2, $3 }' "$WORKDIR/updates.tsv"
            printf -- '```\n\n## Added packages\n\n```\n'
            comm -13 <(cut -f1 "$prev_norm") <(cut -f1 "$curr_norm") \
                | while read -r key; do
                    grep -m1 -F "$key"$'\t' "$curr_norm" | awk -F'\t' '{ printf "+ %s\n", $2 }'
                  done
            printf -- '```\n\n## Removed packages\n\n```\n'
            comm -23 <(cut -f1 "$prev_norm") <(cut -f1 "$curr_norm") \
                | while read -r key; do
                    grep -m1 -F "$key"$'\t' "$prev_norm" | awk -F'\t' '{ printf "- %s\n", $2 }'
                  done
            printf -- '```\n'
        } >> "$changes_path"
        log "wrote $changes_path"
        rm -f "$prev_list" "$curr_list" "$prev_norm" "$curr_norm"
    fi
fi

log "done — release directory: $RELEASE_DIR ($(du -sh "$RELEASE_DIR" | cut -f1))"
