[Merge] ~jibel/livecd-rootfs/+git/add_multi_layered_squashfses_support:ubuntu/master into livecd-rootfs:ubuntu/master
Michael Hudson-Doyle
mwhudsonlp at fastmail.fm
Wed Jan 30 01:12:22 UTC 2019
More comments. Definitely getting closer though, mostly this is typo level stuff.
Diff comments:
> diff --git a/debian/tests/default-bootstraps b/debian/tests/default-bootstraps
> index 0c0062a..3399932 100755
> --- a/debian/tests/default-bootstraps
> +++ b/debian/tests/default-bootstraps
> @@ -23,6 +23,7 @@ ALL_TRIPLETS="
> lubuntu::
> mythbuntu::
> ubuntu::
> + ubuntu:ubiquity-ng:
I guess this should be ubuntu:canary: now?
> ubuntu-base::
> ubuntu-budgie::
> ubuntu-budgie-desktop::
> diff --git a/live-build/auto/build b/live-build/auto/build
> index 70287ba..e9f2096 100755
> --- a/live-build/auto/build
> +++ b/live-build/auto/build
> @@ -387,93 +320,89 @@ deb file:/var/lib/preinstalled-pool/ $LB_DISTRIBUTION $LB_PARENT_ARCHIVE_AREAS
> #
> @@EOF
>
> - cp chroot/etc/apt/sources.list chroot/etc/apt/sources.list.orig
> - cp chroot/etc/apt/sources.list.preinstall chroot/etc/apt/sources.list
> -
> - echo "Waiting on gnupg ("$GPG_PROCESS") to finish generating a key."
> - wait $GPG_PROCESS
> -
> - R_ORIGIN=$(lsb_release -i -s)
> - R_CODENAME=$(lsb_release -c -s)
> - R_VERSION=$(lsb_release -r -s)
> - R_PRETTYNAME=$(echo $R_CODENAME | sed -e 's/^\(.\)/\U\1/')
> -
> - apt-ftparchive -o APT::FTPArchive::Release::Origin=$R_ORIGIN \
> - -o APT::FTPArchive::Release::Label=$R_ORIGIN \
> - -o APT::FTPArchive::Release::Suite=$R_CODENAME-local \
> - -o APT::FTPArchive::Release::Version=$R_VERSION \
> - -o APT::FTPArchive::Release::Codename=$R_CODENAME \
> - -o APT::FTPArchive::Release::Description="$R_ORIGIN $R_PRETTYNAME Local" \
> - release chroot/var/lib/preinstalled-pool/dists/$R_CODENAME/ \
> - > config/gnupg/Release
> -
> - gpg --home config/gnupg --detach-sign --armor config/gnupg/Release
> - mv config/gnupg/Release \
> - chroot/var/lib/preinstalled-pool/dists/$R_CODENAME/Release
> - mv config/gnupg/Release.asc \
> - chroot/var/lib/preinstalled-pool/dists/$R_CODENAME/Release.gpg
> - apt-key --keyring chroot/etc/apt/trusted.gpg add config/gnupg/pubring.gpg
> - find chroot/var/lib/preinstalled-pool/ -name Packages | xargs rm
> -
> - Chroot chroot "apt-get update"
> - cat chroot/etc/apt/sources.list.preinstall chroot/etc/apt/sources.list.orig \
> - > chroot/etc/apt/sources.list
> - rm chroot/etc/apt/sources.list.preinstall chroot/etc/apt/sources.list.orig
> - fi
> - case $PROJECT:$SUBPROJECT in
> - *)
> - if [ -e "config/seeded-snaps" ]; then
> - snap_list=$(cat config/seeded-snaps)
> - preinstall_snaps $snap_list
> - fi
> - ;;
> - esac
> -
> - if [ "$PROJECT" = "ubuntu-touch" ] || [ "$PROJECT" = "ubuntu-touch-custom" ]; then
> - if [ "$ARCH" = "armhf" ]; then
> - INFO_DESC="$(lsb_release -d -s)"
> - echo "$INFO_DESC - $ARCH ($BUILDSTAMP)" >chroot/etc/media-info
> - mkdir -p chroot/var/log/installer
> - Chroot chroot "ln -s /etc/media-info /var/log/installer/media-info"
> + cp chroot/etc/apt/sources.list chroot/etc/apt/sources.list.orig
> + cp chroot/etc/apt/sources.list.preinstall chroot/etc/apt/sources.list
> +
> + echo "Waiting on gnupg ("$GPG_PROCESS") to finish generating a key."
> + wait $GPG_PROCESS
> +
> + R_ORIGIN=$(lsb_release -i -s)
> + R_CODENAME=$(lsb_release -c -s)
> + R_VERSION=$(lsb_release -r -s)
> + R_PRETTYNAME=$(echo $R_CODENAME | sed -e 's/^\(.\)/\U\1/')
> +
> + apt-ftparchive -o APT::FTPArchive::Release::Origin=$R_ORIGIN \
> + -o APT::FTPArchive::Release::Label=$R_ORIGIN \
> + -o APT::FTPArchive::Release::Suite=$R_CODENAME-local \
> + -o APT::FTPArchive::Release::Version=$R_VERSION \
> + -o APT::FTPArchive::Release::Codename=$R_CODENAME \
> + -o APT::FTPArchive::Release::Description="$R_ORIGIN $R_PRETTYNAME Local" \
> + release chroot/var/lib/preinstalled-pool/dists/$R_CODENAME/ \
> + > config/gnupg/Release
> +
> + gpg --home config/gnupg --detach-sign --armor config/gnupg/Release
> + mv config/gnupg/Release \
> + chroot/var/lib/preinstalled-pool/dists/$R_CODENAME/Release
> + mv config/gnupg/Release.asc \
> + chroot/var/lib/preinstalled-pool/dists/$R_CODENAME/Release.gpg
> + apt-key --keyring chroot/etc/apt/trusted.gpg add config/gnupg/pubring.gpg
> + find chroot/var/lib/preinstalled-pool/ -name Packages | xargs rm
> +
> + Chroot chroot "apt-get update"
> + cat chroot/etc/apt/sources.list.preinstall chroot/etc/apt/sources.list.orig \
> + > chroot/etc/apt/sources.list
> + rm chroot/etc/apt/sources.list.preinstall chroot/etc/apt/sources.list.orig
> fi
> - fi
> - if [ "$PROJECT" = "ubuntu-cpc" ]; then
> - if [ "${SUBPROJECT:-}" = minimized ]; then
> - BUILD_NAME=minimal
> - else
> - BUILD_NAME=server
> + case $PROJECT:$SUBPROJECT in
> + ubuntu-server:live)
> + lb chroot_resolv install
I'm not sure why you changed this bit. We still want the base layer of ubuntu-server:live to have the snaps from the server seed.
> + snap_prepare chroot
> + lb chroot_resolv remove
> + ;;
> + *)
> + if [ -e "config/seeded-snaps" ]; then
> + snap_list=$(cat config/seeded-snaps)
> + preinstall_snaps $snap_list
> + fi
> + ;;
> + esac
> +
> + if [ "$PROJECT" = "ubuntu-touch" ] || [ "$PROJECT" = "ubuntu-touch-custom" ]; then
> + if [ "$ARCH" = "armhf" ]; then
> + INFO_DESC="$(lsb_release -d -s)"
> + echo "$INFO_DESC - $ARCH ($BUILDSTAMP)" >chroot/etc/media-info
> + mkdir -p chroot/var/log/installer
> + Chroot chroot "ln -s /etc/media-info /var/log/installer/media-info"
> + fi
> fi
> - cat > chroot/etc/cloud/build.info << EOF
> + if [ "$PROJECT" = "ubuntu-cpc" ]; then
> + if [ "${SUBPROJECT:-}" = minimized ]; then
> + BUILD_NAME=minimal
> + else
> + BUILD_NAME=server
> + fi
> + cat > chroot/etc/cloud/build.info << EOF
> build_name: $BUILD_NAME
> serial: $BUILDSTAMP
> EOF
> + fi
> +
> + configure_network_manager
> +
> + echo "===== Checking size of /usr/share/doc ====="
> + echo BEGIN docdirs
> + (cd chroot && find usr/share/doc -maxdepth 1 -type d | xargs du -s | sort -nr)
> + echo END docdirs
> +
> + /usr/share/livecd-rootfs/minimize-manual chroot
> fi
>
> - # If the image pre-installs network-manager, let it manage all devices by
> - # default. Installing NM on an existing system only manages wifi and wwan via
> - # /usr/lib/NetworkManager/conf.d/10-globally-managed-devices.conf. When setting
> - # the global backend to NM, netplan overrides that file.
> - if [ -e chroot/usr/sbin/NetworkManager ]; then
> - echo "===== Enabling all devices in NetworkManager ===="
> - mkdir -p chroot/etc/netplan
> - cat <<EOF > chroot/etc/netplan/01-network-manager-all.yaml
> -# Let NetworkManager manage all devices on this system
> -network:
> - version: 2
> - renderer: NetworkManager
> -EOF
> + if [ -n "${PASSES}" ]; then
> + PATH="config/:$PATH" lb binary_layered "$@"
> else
> - echo "==== NetworkManager not installed ===="
> + lb binary "$@"
> fi
>
> - echo "===== Checking size of /usr/share/doc ====="
> - echo BEGIN docdirs
> - (cd chroot && find usr/share/doc -maxdepth 1 -type d | xargs du -s | sort -nr)
> - echo END docdirs
> -
> - /usr/share/livecd-rootfs/minimize-manual chroot
> -
> - lb binary "$@"
> touch binary.success
> ) 2>&1 | tee binary.log
>
> diff --git a/live-build/auto/config b/live-build/auto/config
> index 65f5df5..c391eb1 100755
> --- a/live-build/auto/config
> +++ b/live-build/auto/config
> @@ -58,43 +120,113 @@ add_task ()
> # probably a lurking timebomb that we need to fix. In the meantime,
> # the Architecture restriction at least saves us from abject
> # failure.
> + #
> + # We want as well to grab the snap list for each PASS. Resolve for all
> + # given task, and deduplicate them to generate snaps for the PASS.
>
> for task; do
> # We need a ridiculous number of backslashes to protect
> # parentheses from eval.
> echo "!chroot chroot apt-cache dumpavail | grep-dctrl -nsPackage \\\\\\( -XFArchitecture $ARCH -o -XFArchitecture all \\\\\\) -a -wFTask $task" >> "config/package-lists/livecd-rootfs.list.chroot_$pass"
> +
> + curseed=$(seed_from_task ${task})
> + if [ -z "${curseed}" ]; then
> + echo "W: No seed matching task ${task}"
> + continue
> + fi
> + snap_list_file="config/package-lists/seed.${curseed}.snaplist.full"
> + snap_from_seed "${curseed}" $snap_list_file
> + if [ -e "$snap_list_file" ]; then
> + snap_list_files="${snap_list_files} $snap_list_file"
> + fi
> done
> + # The snap list is one line, and could be duplicated between seeds via inheritance.
> + # Uniquely sort them and store them back in one line.
> + if [ -n "${snap_list_files}" ]; then
> + cat ${snap_list_files}|xargs -n1|sort -u > "config/package-lists/livecd-rootfs.snaplist.chroot_${pass}.full"
> + rm ${snap_list_files}
> + fi
> }
>
> add_package ()
> {
> + # Adds a pass named pass_name composed of packages to install
> + # $1 pass
> + # $@ list of packages
> +
> local pass="$1"
> shift
> local pkg
>
> + _check_immutable_passes_to_layers
> + _register_pass "$pass"
> +
> for pkg; do
> echo "$pkg" >> "config/package-lists/livecd-rootfs.list.chroot_$pass"
> done
> }
>
> -OPTS=
> -COMPONENTS=
> -BINARY_REMOVE_LINUX=:
> -BINARY_IMAGES=none
> -MEMTEST=none
> -SOURCE='--source false'
> -BOOTLOADER=none
> -BOOTAPPEND_LIVE=
> -LIVE_TASK=
> -PREINSTALLED=false
> -PREINSTALL_POOL=
> -PREINSTALL_POOL_SEEDS=
> -PREFIX="livecd.$PROJECT${SUBARCH:+-$SUBARCH}"
> +remove_package ()
> +{
> + # Adds a pass named pass_name composed of packages to remove
> + # $1 pass
> + # $@ list of packages
>
> -CHROOT_HOOKS=
> -BINARY_HOOKS=
> + local pass="$1"
> + shift
> + local pkg
>
> -APT_OPTIONS=" --yes -oDebug::pkgDepCache::AutoInstall=yes "
> + _check_immutable_passes_to_layers
> + _check_layers_only_API "remove_package"
> + _register_pass "$pass"
> +
> + for pkg; do
> + echo "$pkg" >> "config/package-lists/livecd-rootfs.removal-list.chroot_$pass"
> + done
> +}
> +
> +add_packages_from_seed_regexp () {
> + # Creates one or more passes, depending on base_pass_name, from any seeds matching seed_regexp.
> + # $1 base pass
> + # $2 seeds (regexp)
wonky indentation
> + local pass
> +
> + _check_immutable_passes_to_layers
> + _check_layers_only_API "add_packages_from_seed_regexp"
> +
> + for seed in $(ls config/germinate-output/|grep -P "$2"); do
> + pass=${1}.${seed}
> + _register_pass "$pass"
> + list_packages_from_seed ${seed} >> config/package-lists/livecd-rootfs.list.chroot_$pass
> + done
> +}
> +
> +remove_packages_from_seed_regexp() {
> + # Creates one or more passes, based on base_pass_name, from any seed matching seed_regexp.
> + # This package list is a list of packages to remove (and included in a previous dependent
> + # pass then), resulting from base_seed - {current_seed_match_from_regexp}.
> + # $1 base pass
> + # $2 base seed
> + # $3 seeds to remove from base seed (regexp). If empty, a no-<base-seed> sublayer is generated.
Wonky indentation here. I know the indentation is all kinds of inconsistent all through the codebase but lets at least try to keep it consistent in a function :)
> + local pass
> +
> + _check_immutable_passes_to_layers
> + _check_layers_only_API "remove_packages_from_seed_regexp"
> +
> + local seed_regexp="$3"
> + if [ -z "${seed_regexp}" ]; then
> + pass="${1}.no-${2}"
> + _register_pass "$pass"
> + subtract_package_lists ${2} "" >> config/package-lists/livecd-rootfs.removal-list.chroot_$pass
> + return
> + fi
> +
> + for seed in $(ls config/germinate-output/|grep -P "$seed_regexp"); do
> + pass="${1}.${seed}"
> + _register_pass "$pass"
> + subtract_package_lists ${2} ${seed} >> config/package-lists/livecd-rootfs.removal-list.chroot_$pass
> + done
> +}
>
> add_chroot_hook ()
> {
> @@ -106,6 +238,44 @@ add_binary_hook ()
> BINARY_HOOKS="${BINARY_HOOKS:+$BINARY_HOOKS }$1"
> }
>
> +_sanitize_passes ()
> +{
> + # Returns an uniquely ordered list of passes and ensure dependency tree is coherent
> + # $1 list of passes
> + local passes="$1"
> + [ -z "$passes" ] && return
> +
> + passes=$(echo $passes | tr ' ' '\n' | sort -u)
> + for pass in $passes; do
> + parent=$(get_parent_pass $pass)
> + # if root pass, no parent to find
> + [ -z "$parent" ] && continue
> + if [ $(echo "$passes"|grep -cE "^$parent\$") -ne 1 ]; then
> + echo "ERROR: '$parent' is required by '$pass' but is missing. Registered passes are:\n$passes"
> + exit 1
> + fi
> + done
> +
> + # return the list of passes
> + echo $passes
> +}
> +
> +_get_live_passes ()
> +{
> + # Returns a list of all passes that ends with .live for automated live passes detection
> + # $1 list of passes
> + local passes="$1"
> + local livepasses=""
> + [ -z "$passes" ] && return
> +
> + for pass in $passes; do
> + if echo $pass | grep -Eq '\.live*'; then
Surely this should be '\.live$' not '\.live*'
> + livepasses="$pass $livepasses"
> + fi
> + done
> + echo $livepasses
> +}
> +
> if [ -z "${IMAGEFORMAT:-}" ]; then
> case $PROJECT:${SUBPROJECT:-} in
> ubuntu-cpc:*)
> @@ -359,10 +510,32 @@ esac
>
> case $PROJECT in
> ubuntu|ubuntu-dvd)
> - add_task install minimal standard ubuntu-desktop
> - LIVE_TASK='ubuntu-live'
> - case $ARCH in
> - amd64) add_package live $SIGNED_KERNEL_PACKAGE ;;
> +
> + case ${SUBPROJECT:-} in
> + canary)
> + PASSES_TO_LAYERS="true"
> + add_task minimal minimal standard ubuntu-desktop-minimal ubuntu-desktop-minimal-default-languages
> + add_task minimal.standard ubuntu-desktop ubuntu-desktop-default-languages
> + add_task minimal.standard.live ubuntu-live
> + add_package minimal.standard.live lupin-casper
> +
> + case $ARCH in
> + amd64) add_package minimal.standard.live $SIGNED_KERNEL_PACKAGE ;;
> + esac
> +
> + # LANG PASS for minimal and standard
> + remove_packages_from_seed_regexp minimal desktop-minimal-default-languages '^desktop-minimal-(?!default-languages)[^.]+$'
> + remove_packages_from_seed_regexp minimal desktop-minimal-default-languages '' # none (if no default langpack is selected)
> + remove_packages_from_seed_regexp minimal.standard desktop-default-languages '^desktop-(?!default-languages|minimal|common)[^.]+$'
> + remove_packages_from_seed_regexp minimal.standard desktop-default-languages '' # none (if no default langpack is selected)
> + ;;
> + *)
> + LIVE_TASK='ubuntu-live'
> + add_task install minimal standard ubuntu-desktop ubuntu-desktop-minimal-default-languages ubuntu-desktop-default-languages
This adds a bunch of tasks to the non-canary case (ubuntu-desktop-minimal-default-languages ubuntu-desktop-default-languages), is that intentional?
> + case $ARCH in
> + amd64) add_package live $SIGNED_KERNEL_PACKAGE ;;
> + esac
> + ;;
> esac
> ;;
>
> @@ -892,9 +1046,23 @@ lb config noauto \
> $OPTS \
> "$@"
>
> +PASSES=$(_sanitize_passes "$PASSES")
> +LIVE_PASSES=${LIVE_PASSES:-$(_get_live_passes "$PASSES")}
> +
> +if [ -n "$PASSES" ] && [ -z "$LIVE_PASSES" ]; then
> + Echo_warning "Multi-layered mode is enabled, but we didn't find any live pass." \
I don't think this deserves a warning, at least ubuntu-server:live won't have a live pass.
> + "Either set \$LIVE_PASSES or add a pass ending with '.live'."
> +fi
> +
> echo "LB_CHROOT_HOOKS=\"$CHROOT_HOOKS\"" >> config/chroot
> echo "SUBPROJECT=\"${SUBPROJECT:-}\"" >> config/chroot
> echo "LB_DISTRIBUTION=\"$SUITE\"" >> config/chroot
> +if [ -n "$PASSES" ]; then
> + echo "PASSES=\"$PASSES\"" >> config/common
> +fi
> +if [ -n "$LIVE_PASSES" ]; then
> + echo "LIVE_PASSES=\"$LIVE_PASSES\"" >> config/common
> +fi
> echo "LB_BINARY_HOOKS=\"$BINARY_HOOKS\"" >> config/binary
> echo "BUILDSTAMP=\"$NOW\"" >> config/binary
> echo "SUBPROJECT=\"${SUBPROJECT:-}\"" >> config/binary
> diff --git a/live-build/functions b/live-build/functions
> index 2cbe3e0..30aaadf 100644
> --- a/live-build/functions
> +++ b/live-build/functions
> @@ -501,3 +546,224 @@ snap_preseed() {
> fi
> _snap_preseed $CHROOT_ROOT $SNAP $CHANNEL
> }
> +
> +snap_from_seed() {
> + local base_seed=$1
> + local out=$2
> + local all_snaps
> + local seeds_expanded
> +
> + seeds_expanded=$(inheritance ${base_seed})
> + for seed in ${seeds_expanded}; do
> + echo "snap: considering ${seed}"
> + file=config/germinate-output/${seed}.snaps
> + [ -e "${file}" ] || continue
> + # extract the first column (snap package name) from germinate's output
> + # translate the human-readable "foo (classic)" into a
> + # more machine readable "foo/classic"
> + seed_snaps=$(sed -rn '1,/-----/d;/-----/,$d; s/(.*) \|.*/\1/; s, \(classic\),/classic,; p' "${file}")
> + for snap in ${seed_snaps}; do
> + echo "snap: found ${snap}"
> + all_snaps="${all_snaps:+${all_snaps} }${snap}"
> + done
> + done
> + if [ -n "${all_snaps}" ]; then
> + echo "${all_snaps}" > $out
> + fi
> +}
> +
> +seed_from_task ()
> +{
> + # Retrieve the name of the seed from a task name
> + local task=$1
> + local seed
> + local seedfile
> + local seedfiles
> +
> + seedfile="$(grep -lE "^Task-Key: +${task}\$" config/germinate-output/*seedtext|head -1)"
> + if [ -n "$seedfile" ]; then
> + basename $seedfile .seedtext
> + return
> + fi
> +
> + seedfiles="$(grep -lE "^Task-Per-Derivative: *1\$" config/germinate-output/*seedtext)"
> + if [ -n "$seedfiles" ]; then
> + for seed in $(echo $seedfiles | xargs basename -s .seedtext); do
> + if [ ${PROJECT}-${seed} = $task ]; then
> + echo ${seed}
> + return
> + fi
> + done
> + fi
> +}
> +
> +list_packages_from_seed () {
> + # Store all packages for a given seed, including its seed dependency
> + # $1: Name of the seed to expand to a package list
> +
> + local all_seeds="$(inheritance $1)"
> +
> + for seed in $all_seeds; do
> + head -n-2 config/germinate-output/${seed}.seed|tail -n+3|awk '{print $1}'
> + done|sort -u
> +}
> +
> +subtract_package_lists() {
> + # Subtract a package list from another
> + #
> + # $1 source package list
> + # $2 Package list to subtract from source package list
> + local list1=$(mktemp)
> + local list2=$(mktemp)
> +
> + list_packages_from_seed $1 > list1
> + list_packages_from_seed $2 > list2
> + comm -23 list1 list2
> +
> + rm list1
> + rm list2
> +}
> +
> +clean_debian_chroot() {
> + # remove crufty files that shouldn't be left in an image
> + rm -f chroot/var/cache/debconf/*-old chroot/var/lib/dpkg/*-old
> + Chroot chroot apt clean
> +}
> +
> +configure_universe() {
> + if [ -f config/universe-enabled ]; then
> + # This is cargo-culted almost verbatim (with some syntax changes for
> + # preinstalled being slightly different in what it doesn't ask) from
> + # debian-installer's apt-setup:
> +
> + cat > chroot/etc/apt/sources.list << EOF
> +# See http://help.ubuntu.com/community/UpgradeNotes for how to upgrade to
> +# newer versions of the distribution.
> +deb $LB_PARENT_MIRROR_BINARY $LB_DISTRIBUTION main restricted
> +# deb-src $LB_PARENT_MIRROR_BINARY $LB_DISTRIBUTION main restricted
> +
> +## Major bug fix updates produced after the final release of the
> +## distribution.
> +deb $LB_PARENT_MIRROR_BINARY $LB_DISTRIBUTION-updates main restricted
> +# deb-src $LB_PARENT_MIRROR_BINARY $LB_DISTRIBUTION-updates main restricted
> +
> +## N.B. software from this repository is ENTIRELY UNSUPPORTED by the Ubuntu
> +## team. Also, please note that software in universe WILL NOT receive any
> +## review or updates from the Ubuntu security team.
> +deb $LB_PARENT_MIRROR_BINARY $LB_DISTRIBUTION universe
> +# deb-src $LB_PARENT_MIRROR_BINARY $LB_DISTRIBUTION universe
> +deb $LB_PARENT_MIRROR_BINARY $LB_DISTRIBUTION-updates universe
> +# deb-src $LB_PARENT_MIRROR_BINARY $LB_DISTRIBUTION-updates universe
> +
> +## N.B. software from this repository is ENTIRELY UNSUPPORTED by the Ubuntu
> +## team, and may not be under a free licence. Please satisfy yourself as to
> +## your rights to use the software. Also, please note that software in
> +## multiverse WILL NOT receive any review or updates from the Ubuntu
> +## security team.
> +deb $LB_PARENT_MIRROR_BINARY $LB_DISTRIBUTION multiverse
> +# deb-src $LB_PARENT_MIRROR_BINARY $LB_DISTRIBUTION multiverse
> +deb $LB_PARENT_MIRROR_BINARY $LB_DISTRIBUTION-updates multiverse
> +# deb-src $LB_PARENT_MIRROR_BINARY $LB_DISTRIBUTION-updates multiverse
> +
> +## N.B. software from this repository may not have been tested as
> +## extensively as that contained in the main release, although it includes
> +## newer versions of some applications which may provide useful features.
> +## Also, please note that software in backports WILL NOT receive any review
> +## or updates from the Ubuntu security team.
> +deb $LB_PARENT_MIRROR_BINARY $LB_DISTRIBUTION-backports main restricted universe multiverse
> +# deb-src $LB_PARENT_MIRROR_BINARY $LB_DISTRIBUTION-backports main restricted universe multiverse
> +
> +## Uncomment the following two lines to add software from Canonical's
> +## 'partner' repository.
> +## This software is not part of Ubuntu, but is offered by Canonical and the
> +## respective vendors as a service to Ubuntu users.
> +# deb http://archive.canonical.com/ubuntu $LB_DISTRIBUTION partner
> +# deb-src http://archive.canonical.com/ubuntu $LB_DISTRIBUTION partner
> +
> +deb $LB_PARENT_MIRROR_BINARY_SECURITY $LB_DISTRIBUTION-security main restricted
> +# deb-src $LB_PARENT_MIRROR_BINARY_SECURITY $LB_DISTRIBUTION-security main restricted
> +deb $LB_PARENT_MIRROR_BINARY_SECURITY $LB_DISTRIBUTION-security universe
> +# deb-src $LB_PARENT_MIRROR_BINARY_SECURITY $LB_DISTRIBUTION-security universe
> +deb $LB_PARENT_MIRROR_BINARY_SECURITY $LB_DISTRIBUTION-security multiverse
> +# deb-src $LB_PARENT_MIRROR_BINARY_SECURITY $LB_DISTRIBUTION-security multiverse
> +EOF
> +
> +fi
> +}
> +
> +configure_network_manager() {
> + # If the image pre-installs network-manager, let it manage all devices by
> + # default. Installing NM on an existing system only manages wifi and wwan via
> + # /usr/lib/NetworkManager/conf.d/10-globally-managed-devices.conf. When setting
> + # the global backend to NM, netplan overrides that file.
> + if [ -e chroot/usr/sbin/NetworkManager -a ! -f chroot/etc/netplan/01-network-manager-all.yaml ]; then
> + echo "===== Enabling all devices in NetworkManager ===="
> + mkdir -p chroot/etc/netplan
> + cat <<EOF > chroot/etc/netplan/01-network-manager-all.yaml
> +# Let NetworkManager manage all devices on this system
> +network:
> + version: 2
> + renderer: NetworkManager
> +EOF
> + else
> + echo "==== NetworkManager not installed ===="
> + fi
> +}
> +
> +get_parent_pass () {
> + # return parent pass
> + # $1 name of the pass
> + # return parent pass name or '' if pass is root pass.
> + local pass="$1"
> +
> + parent_pass=${pass%.*}
> + if [ "${parent_pass}" = "${pass}" ]; then
> + return
> + fi
> + echo ${pass%.*}
> +}
> +
> +setenv_file () {
> + # Exposes an environment variable in a chroot
> + # $1 Name of the variable
> + # $2 Value of the variable
> + # $3 Path to the environment file of the chroot
wonky indentation here.
> + local var="$1"
> + local val="$2"
> + local file="$3"
> +
> + grep -v "^$var" $file || true > $file.new
I know this is my code, but it should be grepping for "^$var=" not just "$var".
> + echo "${var}=${val}" >> $file.new
> + mv $file.new $file
> +}
> +
> +divert_update_initramfs () {
> + Chroot chroot "dpkg-divert --quiet --add \
> + --divert /usr/sbin/update-initramfs.REAL --rename \
> + /usr/sbin/update-initramfs"
> + cat > chroot/usr/sbin/update-initramfs <<'EOF'
> +#! /bin/sh
> +if [ $# != 1 ] || [ "$1" != -u ]; then
> + exec update-initramfs.REAL "$@"
> +fi
> +echo "update-initramfs: diverted by livecd-rootfs (will be called later)" >&2
> +
> +exit 0
> +EOF
> + chmod +x chroot/usr/sbin/update-initramfs
> +
> + cat > config/hooks/999-undivert-update-initramfs.chroot <<'EOF'
Hm I wonder if this hook being executed in every pass was what resulted in my odd problems with /usr/sbin/update-initramfs disappearing?
> +#! /bin/sh
> +[ ! -f /usr/sbin/update-initramfs.REAL ] && exit 0
> +rm -f /usr/sbin/update-initramfs
> +dpkg-divert --quiet --remove --rename /usr/sbin/update-initramfs
> +EOF
> +}
> +
> +is_root_layer () {
> + local pass=$1
> + if [ -z "$(get_parent_pass $pass)" ]; then
> + return 0
> + fi
> + return 1
> +}
> diff --git a/live-build/lb_binary_layered b/live-build/lb_binary_layered
> new file mode 100755
> index 0000000..91c6404
> --- /dev/null
> +++ b/live-build/lb_binary_layered
> @@ -0,0 +1,136 @@
> +#!/bin/sh
> +
> +## live-build(7) - System Build Scripts
> +## Copyright (C) 2006-2012 Daniel Baumann <daniel at debian.org>
> +##
> +## This program comes with ABSOLUTELY NO WARRANTY; for details see COPYING.
> +## This is free software, and you are welcome to redistribute it
> +## under certain conditions; see COPYING for details.
> +
> +
> +set -e
> +
> +# Including common functions
> +( . "${LIVE_BUILD}/scripts/build.sh" > /dev/null 2>&1 || true ) || . /usr/lib/live/build.sh
> +
> +
> +# Automatically populating config tree
> +if [ -x auto/config ] && [ ! -e .build/config ]
> +then
> + Echo_message "Automatically populating config tree."
> + lb config
> +fi
> +
> +# Setting static variables
> +DESCRIPTION="$(Echo 'build binary images')"
> +HELP=""
> +USAGE="${PROGRAM} [--force]"
> +
> +Arguments "${@}"
> +
> +# Reading configuration files
> +Read_conffiles config/all config/common config/bootstrap config/chroot config/binary config/source
> +Set_defaults
> +
> +# Setup cleanup function
> +Setup_cleanup
> +
> +. config/functions
> +
> +lb_binary_includes () {
> + # Copying includes from pass subdirectory
> + local pass="$1"
> +
> + if [ ! -d config/includes.binary.${pass} ]; then
> + return
> + fi
> +
> + cd config/includes.binary.${pass}
> + find . | cpio -dmpu --no-preserve-owner "${OLDPWD}"/chroot
> + cd "${OLDPWD}"
> +}
> +
> +build_layered_squashfs () {
> + local pass=$1
> + shift 1 # restore ${*}
> +
> + Echo_message "lb_binary_layered: treating pass $pass"
> +
> + # Building squashfs filesystem & manifest
> + local overlay_dir="overlay.${pass}"
> + base="${PWD}/livecd.${PROJECT}.${pass}"
> + squashfs_f="${base}.squashfs"
> +
> + # We have already treated that pass
> + if [ -f "${squashfs_f}" ]; then
> + return
> + fi
> +
> + rm -f .build/binary_chroot
> +
> + mkdir -p "$overlay_dir/"
> + lowerdirs=$(get_lowerdirs_for_pass $pass)
> + if [ -n "$lowerdirs" ]; then
> + mkdir -p chroot/
> + mount_overlay "$lowerdirs" "$overlay_dir" chroot/
> + else
> + ln -s "$overlay_dir/" chroot
> + fi
I wonder if this little sequence (from 1204-1210 in the diff) could be its own function in functions?
> +
> + export PASS=${pass}
> + setenv_file PASS "${pass}" config/environment.chroot
> +
> + # Cleanup root filesystem
> + lb binary_chroot ${*}
> +
> + lb_binary_includes $pass ${*}
> + lb binary_hooks ${*}
> +
> + # Full manifest until that PASS
> + squashfs_f_manifest="${base}.manifest"
> + create_manifest "chroot" "${squashfs_f_manifest}.full"
> +
> + # Delta manifest
> + diff -NU0 ${PWD}/livecd.${PROJECT}.$(get_parent_pass $pass).manifest.full ${squashfs_f_manifest}.full|grep -v ^@ > $squashfs_f_manifest
> +
> + squashfs_f_size="${base}.size"
> + du -B 1 -s "overlay.${pass}/" | cut -f1 > "${squashfs_f_size}"
> +
> + # We take first live path for "global" ISO properties (used by installers and checkers):
live pass?
> + # Prepare initrd + kernel
> + # Main manifest and size files
> + prefix="livecd.$PROJECT${SUBARCH:+-$SUBARCH}"
> + if [ ! -e "${prefix}.manifest" ]; then
> + for livepass in $LIVE_PASSES; do
> + [ "$livepass" != "$pass" ] && continue
> + lb binary_linux-image ${*}
> + cp ${squashfs_f_size} "${prefix}.size"
> + cp "${squashfs_f_manifest}.full" "${prefix}.manifest"
This means that means livecd.ubuntu.{size,manifest} will just be the size and manifest of the live layer, no?
> + break
> + done
> + fi
> +
> + (cd "overlay.${pass}/" &&
> + mksquashfs . ${squashfs_f} \
> + -no-progress -xattrs -comp xz )
> +
> + if [ -n "$lowerdirs" ]; then
> + umount chroot
> + rmdir chroot
> + else
> + rm chroot
> + fi
> +}
> +
> +for _PASS in $PASSES
> +do
> + build_layered_squashfs "${_PASS}" ${*}
> +done
> +
> +# Ubiquity-compatible removal manifest for ISO not using a layered-aware installer
So there is code here to create the manifest-remove and then conditional code somewhere else to remove it again? Not sure that makes sense any more?
> +if [ -n "$(ls livecd.${PROJECT}.*install.live.manifest.full 2>/dev/null)" ] && \
> + [ -n "$(ls livecd.${PROJECT}.*install.manifest.full 2>/dev/null)" ]; then
> + echo "$(diff livecd.${PROJECT}.*install.live.manifest.full livecd.${PROJECT}.*install.manifest.full | awk '/^< / { print $2 }')" > livecd.${PROJECT}-manifest-remove
> +fi
> +
> +chmod 644 *.squashfs *.manifest* *.size
> diff --git a/live-build/lb_chroot_layered b/live-build/lb_chroot_layered
> new file mode 100755
> index 0000000..b1bf31f
> --- /dev/null
> +++ b/live-build/lb_chroot_layered
> @@ -0,0 +1,280 @@
> +#!/bin/sh
> +
> +## live-build(7) - System Build Scripts
> +## Copyright (C) 2006-2012 Daniel Baumann <daniel at debian.org>
> +##
> +## This program comes with ABSOLUTELY NO WARRANTY; for details see COPYING.
> +## This is free software, and you are welcome to redistribute it
> +## under certain conditions; see COPYING for details.
> +
> +## This is a fork of lb_chroot for layered live system.
> +## We don't want leaking host configuratino in each layer, and so,
configuratino
> +## we clean and setup the chroot each time.
> +## In addition, we create the squashfs for each layer, but top one (live)
> +## which still can be configured after lb chroot call.
> +
> +set -e
> +
> +# Including common functions
> +( . "${LIVE_BUILD}/scripts/build.sh" > /dev/null 2>&1 || true ) || . /usr/lib/live/build.sh
> +
> +# Automatically populating config tree
> +if [ -x auto/config ] && [ ! -e .build/config ]
> +then
> + Echo_message "Automatically populating config tree."
> + lb config
> +fi
> +
> +# Setting static variables
> +DESCRIPTION="$(Echo 'customize the Debian system')"
> +HELP=""
> +USAGE="${PROGRAM} [--force]"
> +
> +Arguments "${@}"
> +
> +# Reading configuration files
> +Read_conffiles config/all config/common config/bootstrap config/chroot config/binary config/source
> +Set_defaults
> +
> +# Setup cleanup function
> +Setup_cleanup
> +
> +. config/functions
> +
> +lb_chroot_remove_packages () {
> + # Remove packages from the chroot specific to this layer
> + #
> + # $1: Name of the pass*
> + local pass=$1
> +
> + Expand_packagelist "$(basename config/package-lists/*.removal-list.chroot_${pass})" "config/package-lists" \
> + >> chroot/root/packages.chroot.removal
> + Chroot chroot "xargs --arg-file=/root/packages.chroot.removal apt-get ${APT_OPTIONS} autoremove --purge"
> + rm -f chroot/root/packages.chroot.removal
> +}
> +
> +# Create the snap list specific to this layer
> +lb_chroot_snap_lists () {
> + local pass=$1
> +
> + # This assumes that the prefix is unique for a given project
> + local snap_for_pass=$(ls config/package-lists/*.snaplist.chroot_${pass}.full 2>/dev/null || true)
> + parent_pass=$(get_parent_pass $pass)
> + local snap_for_parent_pass=$(ls config/package-lists/*.snaplist.chroot_${parent_pass}.full 2>/dev/null || true)
> +
> + if [ -z "${snap_for_pass}" ]; then
> + return
> + fi
> +
> + if [ -z "${snap_for_parent_pass}" ]; then
> + cp ${snap_for_pass} ${snap_for_pass%.full}
> + return
> + fi
> +
> + # Generate a list of snaps added to a layer.
> + diff -NU0 ${snap_for_parent_pass} ${snap_for_pass}|grep -Ev '^(---|\+\+\+|@@)'|cut -c2- > ${snap_for_pass%.full}
> +}
> +
> +lb_chroot_install_snaps () {
> + # Prepare the snap environment and install snaps into a chroot
> + #
> + # $1: Name of the pass
> +
> + local snaplist_file=$(ls config/package-lists/*.snaplist.chroot_${1} 2>/dev/null || true)
> +
> + if [ -z "${snaplist_file}" ]; then
> + return
> + fi
> +
> + snap_prepare chroot
> +
> + while read snap; do
> + snap_preseed chroot "${snap}"
> + done < $snaplist_file
> +}
> +
> +lb_chroot_includes () {
> + # Copying includes from pass subdirectory
> + local pass="$1"
> +
> + if [ ! -d config/includes.chroot.${pass} ]; then
> + return
> + fi
> +
> + cd config/includes.chroot.${pass}
> + find . | cpio -dmpu --no-preserve-owner "${OLDPWD}"/chroot
> + cd "${OLDPWD}"
> +}
> +
> +reduce_pass_size () {
> + # Remove duplicated files between parent and currant pass
"current" not "currant" (which is a fruit ;-p)
> + # Note the empty directories created in a child pass are not removed
> + local pass=$1
> + local parent="$(get_parent_pass $pass)"
> +
> + $(is_root_layer $pass) && return
> +
> + pass_dir="overlay.${pass}"
> + parent_pass_dir="overlay.${parent}"
> +
> + local list_pass=$(mktemp)
> + local list_parent=$(mktemp)
indentation is inconsistent in this function
> +
> + (cd $pass_dir && find . ! -type d -printf "%h/%f|%s|%y|%U|%G|%m\n"|sort > $list_pass)
> + (cd $parent_pass_dir && find . ! -type d -printf "%h/%f|%s|%y|%U|%G|%m\n"|sort > $list_parent)
> +
> + # Only iterate on common files with same type, owner, permission and size
> + comm -12 $list_pass $list_parent|cut -d'|' -f1|while read f; do
> + # If file contents are different, keep it
> + if ! diff --brief --no-dereference "$pass_dir/$f" "$parent_pass_dir/$f" >/dev/null; then
> + continue
> + fi
> + # Files are strictly identical between the 2 passes (only mod or access times differs). No need for unused delta.
> + Echo_message "reduce_pass_size: '$f' is strictly identical between $parent and $pass. Removing."
> + rm "$pass_dir/$f"
> + done
> +
> + rm $list_pass
> + rm $list_parent
> +}
> +
> +create_chroot_pass () {
> + local pass=$1
> + shift 1 # restore ${*}
> +
> + Echo_message "lb_chroot_layered: treating pass $pass"
> +
> + # We have already treated that pass just return.
> + local overlay_dir="overlay.${pass}"
> + if [ -d "$overlay_dir/" ]; then
> + return
> + fi
> +
> + # Only get some function executed on root passes
> + # Copy bootstrap on root layers
> + if $(is_root_layer $pass); then
> + rm -f .build/chroot_linux-image .build/chroot_preseed .build/chroot_hacks
> + cp -a chroot.bootstrap/ "$overlay_dir/"
> + fi
> + # Others have to be executed on every pass
> + rm -f .build/chroot_early_hooks .build/chroot_hooks .build/chroot_interactive
> +
> + mkdir -p "$overlay_dir/"
> + lowerdirs=$(get_lowerdirs_for_pass $pass)
> + if [ -n "$lowerdirs" ]; then
> + mkdir -p chroot/
> + mount_overlay "$lowerdirs" "$overlay_dir" chroot/
> + else
> + ln -s "$overlay_dir/" chroot
> + fi
> +
> + export PASS=${pass}
> + setenv_file PASS "${pass}" config/environment.chroot
> +
> + # Configuring chroot
> + lb chroot_devpts install ${*}
> + lb chroot_proc install ${*}
> + lb chroot_sysfs install ${*}
> + # We run chroot_hacks only on root layers (update-initramfs diverted)
See below but is there any reason not to run this on every layer? I think this whole question about whether/when to run hacks is pretty confused actually. I'm not sure that we need to run it at all? AFAICT the only meaningful thing it does for any of our configs is run update-initramfs and that's diverted away.
> + if $(is_root_layer $pass); then
> + divert_update_initramfs
> + fi
> + lb chroot_debianchroot install ${*}
> + lb chroot_dpkg install ${*}
> + lb chroot_tmpfs install ${*}
> + lb chroot_hosts install ${*}
> + lb chroot_resolv install ${*}
> + lb chroot_hostname install ${*}
> + lb chroot_apt install ${*}
> + # Note: this triggers an upgrade + dist-ugprade; which may impact sublayers with more
> + # diff content than desired. So only running this on root pass.
> + # Also configure universe
> + # Only configure universe on root passes
I guess with reduce_pass_size this is no longer so needed but is probably still sensible? At least until/unless per-layer archives become needed.
Comment has a nearly-duplicate line.
> + if $(is_root_layer $pass); then
> + lb chroot_archives chroot install ${*}
> + configure_universe
> + fi
> +
> + # Customizing chroot
> + lb chroot_linux-image ${*}
> + lb chroot_preseed ${*}
> + lb chroot_early_hooks ${*}
> +
> + lb chroot_package-lists ${pass} ${*}
> + lb chroot_install-packages ${pass} ${*}
> + lb_chroot_remove_packages ${pass} ${*}
> +
> + # Snap management
> + lb_chroot_snap_lists ${pass} ${*}
> + lb_chroot_install_snaps ${pass} ${*}
> +
> + configure_network_manager
> +
> + # Mark kernel headers as autoremovable
> + Chroot chroot "dpkg -l linux-headers-3* linux-headers-4*" 2>/dev/null \
> + | awk '/^i/ {print $2}' > chroot.headers
> + for i in $(cat chroot.headers); do
> + Chroot chroot "apt-mark auto $i"
> + done
> +
> + Chroot chroot "apt-get --purge -y autoremove"
> +
> + # Add live packages to live layers
> + for livepass in $LIVE_PASSES; do
> + [ "$livepass" != "$pass" ] && continue
> + lb chroot_live-packages ${*}
> + break
> + done
> +
> + # Run includes by pass
> + lb_chroot_includes ${pass} ${*}
> +
> + lb chroot_hooks ${*}
> +
> + # Run chroot_hacks only on root layers.
> + # chroot_hacks changes the mode of boot/initrd*. The side effect in
> + # layered mode is to create an initrd on each layer with a significant
> + # impact on image size (+30MB per layer). This is an issue even with
> + # update-initramfs disabled.
> + lb chroot_hacks ${*}
Code and comment disagree here -- I'm guessing that the change to call reduce_pass_size means that it's now OK to call chroot_hacks on each layer?
> +
> + lb chroot_interactive ${*}
> +
> + # Misc ubuntu cleanup and post-layer configuration
> + clean_debian_chroot
> + /usr/share/livecd-rootfs/minimize-manual chroot
> +
> + Chroot chroot "dpkg-query -W" > chroot.packages.${pass}
> +
> + # Deconfiguring chroot
> + if $(is_root_layer $pass); then
> + lb chroot_archives chroot remove ${*}
> + fi
> + lb chroot_apt remove ${*}
> + lb chroot_hostname remove ${*}
> + lb chroot_resolv remove ${*}
> + lb chroot_hosts remove ${*}
> + lb chroot_tmpfs remove ${*}
> + lb chroot_dpkg remove ${*}
> + lb chroot_debianchroot remove ${*}
> + lb chroot_sysfs remove ${*}
> + lb chroot_proc remove ${*}
> + lb chroot_devpts remove ${*}
> +
> + if [ -n "$lowerdirs" ]; then
> + umount chroot
> + rmdir chroot
> + else
> + rm chroot
> + fi
> +
> + reduce_pass_size $pass
> +}
> +
> +if [ ! -d chroot.bootstrap/ ]; then
> + mv chroot/ chroot.bootstrap/
> +fi
> +for _PASS in $PASSES
> +do
> + create_chroot_pass "$_PASS" ${*}
> +done
--
https://code.launchpad.net/~jibel/livecd-rootfs/+git/add_multi_layered_squashfses_support/+merge/360878
Your team Ubuntu Core Development Team is subscribed to branch livecd-rootfs:ubuntu/master.
More information about the Ubuntu-reviews
mailing list