[Merge] ~jibel/livecd-rootfs/+git/add_multi_layered_squashfses_support:ubuntu/master into livecd-rootfs:ubuntu/master

[Didier Roche]

Diff comments:

> diff --git a/live-build/lb_chroot_layered b/live-build/lb_chroot_layered
> new file mode 100755
> index 0000000..89b8472
> --- /dev/null
> +++ b/live-build/lb_chroot_layered
> @@ -0,0 +1,246 @@
> +#!/bin/sh
> +
> +## live-build(7) - System Build Scripts
> +## Copyright (C) 2006-2012 Daniel Baumann <daniel at debian.org>
> +##
> +## This program comes with ABSOLUTELY NO WARRANTY; for details see COPYING.
> +## This is free software, and you are welcome to redistribute it
> +## under certain conditions; see COPYING for details.
> +
> +## This is a fork of lb_chroot for layered live system.
> +## We don't want leaking host configuratino in each layer, and so,

ack.

> +## we clean and setup the chroot each time.
> +## In addition, we create the squashfs for each layer, but top one (live)
> +## which still can be configured after lb chroot call.
> +
> +set -e
> +
> +# Including common functions
> +( . "${LIVE_BUILD}/scripts/build.sh" > /dev/null 2>&1 || true ) || . /usr/lib/live/build.sh
> +
> +# Automatically populating config tree
> +if [ -x auto/config ] && [ ! -e .build/config ]
> +then
> +	Echo_message "Automatically populating config tree."
> +	lb config
> +fi
> +
> +# Setting static variables
> +DESCRIPTION="$(Echo 'customize the Debian system')"
> +HELP=""
> +USAGE="${PROGRAM} [--force]"
> +
> +Arguments "${@}"
> +
> +# Reading configuration files
> +Read_conffiles config/all config/common config/bootstrap config/chroot config/binary config/source
> +Set_defaults
> +
> +# Setup cleanup function
> +Setup_cleanup
> +
> +. config/functions
> +
> +lb_chroot_remove_packages() {
> +	# Remove packages from the chroot specific to this layer
> +	#
> +	# $1: Name of the pass*
> +	local pass=$1
> +
> +	Expand_packagelist "$(basename config/package-lists/*.removal-list.chroot_${pass})" "config/package-lists" \
> +			>> chroot/root/packages.chroot.removal
> +	Chroot chroot "xargs --arg-file=/root/packages.chroot.removal apt-get ${APT_OPTIONS} autoremove --purge"
> +	rm -f chroot/root/packages.chroot.removal
> +}
> +
> +# Create the snap list specific to this layer
> +lb_chroot_snap_lists() {
> +	local pass=$1
> +	local prevpass=$2
> +
> +	# This assumes that the prefix is unique for a given project
> +	local snap_for_pass=$(ls config/package-lists/*.snaplist.chroot_${pass}.full 2>/dev/null || true)
> +	local snap_for_prevpass=$(ls config/package-lists/*.snaplist.chroot_${prevpass}.full 2>/dev/null || true)
> +
> +	if [ -z "${snap_for_pass}" ]; then
> +		return
> +	fi
> +
> +	if [ -z "${snap_for_prevpass}" ]; then
> +		cp ${snap_for_pass} ${snap_for_pass%.full}
> +		return
> +	fi
> +
> +	# Generate a list of snaps added to a layer.
> +	diff -NU0 ${snap_for_prevpass} ${snap_for_pass}|grep -Ev '^(---|\+\+\+|@@)'|cut -c2- > ${snap_for_pass%.full}
> +}
> +
> +lb_chroot_install_snaps() {
> +	# Prepare the snap environment and install snaps into a chroot
> +	#
> +	# $1: Name of the pass
> +
> +	local snaplist_file=$(ls config/package-lists/*.snaplist.chroot_${1} 2>/dev/null || true)
> +
> +	if [ -z "${snaplist_file}" ]; then
> +		return
> +	fi
> +
> +	snap_prepare chroot
> +
> +	while read snap; do
> +		snap_preseed chroot "${snap}"
> +	done < $snaplist_file
> +}
> +
> +lb_chroot_includes() {
> +	# Copying includes from pass subdirectory
> +	local pass="$1"
> +
> +	if [ ! -d config/includes.chroot.${pass} ]; then
> +		return
> +	fi
> +
> +	cd config/includes.chroot.${pass}
> +	find . | cpio -dmpu --no-preserve-owner "${OLDPWD}"/chroot
> +	cd "${OLDPWD}"
> +}
> +
> +create_chroot_pass() {
> +	local pass=$1
> +	local prevpass=$2
> +	local lowerlayers=$3
> +	local passtype=$4  # "first"|"last"|"" empty string
> +	shift 4 # restore ${*}
> +
> +	# We have already treated that pass
> +	if [ -d "overlay.${pass}/" ]; then
> +		return
> +	fi
> +
> +	export PASS=${pass}

ack, thanks for spotting!

> +
> +	if [ "${passtype}" != "first" ]; then
> +		mkdir overlay.${pass}
> +		mount_overlay ${lowerlayers} "overlay.${pass}/" chroot/
> +	fi
> +
> +	# Configuring chroot
> +	lb chroot_cache restore ${*}
> +	lb chroot_devpts install ${*}
> +	lb chroot_proc install ${*}
> +	lb chroot_sysfs install ${*}
> +	lb chroot_debianchroot install ${*}
> +	lb chroot_dpkg install ${*}
> +	lb chroot_tmpfs install ${*}
> +	lb chroot_hosts install ${*}
> +	lb chroot_resolv install ${*}
> +	lb chroot_hostname install ${*}
> +	lb chroot_apt install ${*}
> +	# Note: this triggers an upgrade + dist-ugprade; which may impact sublayers with more
> +	# diff content than desired. However, we still need to setup the archive and teardown
> +	# for each layer.
> +	# We could modify livebuild if necessary to have conditional upgrade (first pass only).
> +	lb chroot_archives chroot install ${*}
> +
> +	if [ "${passtype}" = "first" ]; then
> +		configure_universe
> +	fi
> +
> +	# Customizing chroot
> +	lb chroot_linux-image ${*}

This is indeed annoying. Is that needed for every layer or only one with linux kernel?

> +	lb chroot_preseed ${*}
> +	lb chroot_early_hooks ${*}
> +
> +	lb chroot_package-lists ${pass} ${*}
> +	lb chroot_install-packages ${pass} ${*}
> +	lb_chroot_remove_packages ${pass} ${*}
> +
> +	# Snap management
> +	lb_chroot_snap_lists ${pass} ${prevpass}
> +	lb_chroot_install_snaps ${pass} ${*}
> +
> +	# Kernel should be in first layer

Actually, this was already ran on the first layer (even in the ubuntu-server live use-case) and seems to be a nooop if network-manager isn't installed or any kernel. So maybe, let's live right now with this assumption?

> +	if [ "${passtype}" = "first" ]; then
> +		configure_network_manager
> +		Chroot chroot "dpkg -l linux-headers-3* linux-headers-4*" 2>/dev/null \
> +			| awk '/^i/ {print $2}' > chroot.headers
> +		for i in $(cat chroot.headers); do
> +			Chroot chroot "apt-mark auto $i"
> +		done
> +	fi
> +
> +	Chroot chroot "apt-get --purge -y autoremove"
> +
> +	# Add live packages to top layer
> +	if [ "${passtype}" = "last" ]; then
> +		lb chroot_live-packages ${*}

Indeed, this is opening a bigger discussion. We'll write that as an opened question in the spec documentation so that we can iterate over it.

> +	fi
> +
> +	# Run includes by pass
> +	lb_chroot_includes ${pass} ${*}
> +
> +	lb chroot_hooks ${*}
> +	lb chroot_hacks ${*}
> +	lb chroot_interactive ${*}
> +
> +	# Misc ubuntu cleanup and post-layer configuration
> +	clean_debian_chroot
> +	/usr/share/livecd-rootfs/minimize-manual chroot
> +
> +	Chroot chroot "dpkg-query -W" > chroot.packages.${pass}
> +
> +	# Deconfiguring chroot
> +	lb chroot_archives chroot remove ${*}
> +	lb chroot_apt remove ${*}
> +	lb chroot_hostname remove ${*}
> +	lb chroot_resolv remove ${*}
> +	lb chroot_hosts remove ${*}
> +	lb chroot_tmpfs remove ${*}
> +	lb chroot_dpkg remove ${*}
> +	lb chroot_debianchroot remove ${*}
> +	lb chroot_sysfs remove ${*}
> +	lb chroot_proc remove ${*}
> +	lb chroot_devpts remove ${*}
> +	lb chroot_cache save ${*}
> +
> +	if [ "${passtype}" = "first" ]; then
> +		mv chroot overlay.${pass}
> +		mkdir chroot
> +	else
> +		umount chroot
> +	fi
> +
> +	# Handle direct sublayer of current one
> +	# Extract the name of the pass corresponding to the sublayer
> +	for subpass in $(ls config/package-lists/*list.chroot_${pass}_* 2>/dev/null | sed -e "s/.*list\.chroot_\(${pass}_[^_]\+\).*/\1/"); do
> +		lowerlayers_for_subpass="overlay.${pass}:${lowerlayers}"
> +		lowerlayers_for_subpass="${lowerlayers_for_subpass%:}"
> +		create_chroot_pass "${subpass}" "${pass}" "${lowerlayers_for_subpass}" "" ${*}
> +	done
> +}
> +
> +PASSES="${PASSES:-install live}"
> +CURPASS=1
> +PREVPASS=""
> +LASTPASS=$(echo $PASSES|wc -w)
> +LOWER_LAYERS=""
> +for _PASS in $PASSES
> +do
> +	PASSTYPE=""
> +	if [ $CURPASS -eq 1 ]; then
> +		PASSTYPE="first"
> +	elif [ $CURPASS -eq $LASTPASS ]; then
> +		PASSTYPE="last"
> +	fi
> +
> +	create_chroot_pass "$_PASS" "$PREVPASS" "$LOWER_LAYERS" "$PASSTYPE" ${*}
> +	
> +	LOWER_LAYERS="overlay.${_PASS}:$LOWER_LAYERS"
> +	LOWER_LAYERS="${LOWER_LAYERS%:}"
> +	PREVPASS=${_PASS}
> +	
> +	CURPASS=$(( CURPASS + 1 ))
> +done
> +
> +rmdir chroot


-- 
https://code.launchpad.net/~jibel/livecd-rootfs/+git/add_multi_layered_squashfses_support/+merge/360878
Your team Ubuntu Core Development Team is subscribed to branch livecd-rootfs:ubuntu/master.