[UNSTABLE][PATCH 1/2] UBUNTU: [Packaging] Remove in-tree abi checks
Dimitri John Ledkov
dimitri.ledkov at canonical.com
Fri Feb 2 15:43:11 UTC 2024
linux-buildinfo packages are now externally compared by swm, with
results approving or rejecting updates based on the stable
tracker. Those checks also allow hints and overrides to accept
intentional changes.
Also these are done on the correct pair-wise comparisons, especially
when two streams are being cranked.
The above eliminates the need to identify previous build abi,
download, extract it, vendor it in, and assert it at build time.
Signed-off-by: Dimitri John Ledkov <dimitri.ledkov at canonical.com>
---
debian/rules | 8 +-
debian/rules.d/0-common-vars.mk | 1 -
debian/rules.d/4-checks.mk | 14 +-
debian/scripts/checks/abi-check | 139 ------------------
debian/scripts/checks/final-checks | 44 ------
debian/scripts/checks/module-check | 131 -----------------
debian/scripts/misc/getabis | 222 -----------------------------
7 files changed, 2 insertions(+), 557 deletions(-)
delete mode 100755 debian/scripts/checks/abi-check
delete mode 100755 debian/scripts/checks/module-check
delete mode 100755 debian/scripts/misc/getabis
diff --git a/debian/rules b/debian/rules
index 1319a5f5d8..f7d3c10a0b 100755
--- a/debian/rules
+++ b/debian/rules
@@ -109,12 +109,6 @@ ifneq ($(do_tools),true)
do_tools_host=
endif
-$(foreach _m,$(all_built-in_dkms_modules), \
- $(if $(filter true,$(do_$(_m))),, \
- $(eval do_$(_m)_disable := $$(shell for m in $$$$(cat $(DROOT)/$(_m)-modules.ignore); do grep -qxF $$$$m $(prev_abidir)/../modules.ignore 2>/dev/null || echo $$$$m >> $(prev_abidir)/../modules.ignore; done)) \
- ) \
-)
-
# Either tools package needs the common source preparation
do_any_tools=$(sort $(filter-out false,$(do_linux_tools) $(do_cloud_tools)))
@@ -132,7 +126,7 @@ clean: debian/control debian/canonical-certs.pem debian/canonical-revoked-certs.
dh_clean
# normal build junk
- rm -rf $(DEBIAN)/abi/$(release)-$(revision)
+ rm -rf $(DEBIAN)/abi
rm -rf $(builddir)
rm -f $(stampdir)/stamp-*
rm -rf debian/linux-*/
diff --git a/debian/rules.d/0-common-vars.mk b/debian/rules.d/0-common-vars.mk
index 0f6108d7ca..d64349e2ab 100644
--- a/debian/rules.d/0-common-vars.mk
+++ b/debian/rules.d/0-common-vars.mk
@@ -88,7 +88,6 @@ export gcc?=gcc-13
GCC_BUILD_DEPENDS=\ $(gcc), $(gcc)-aarch64-linux-gnu [arm64] <cross>, $(gcc)-arm-linux-gnueabihf [armhf] <cross>, $(gcc)-powerpc64le-linux-gnu [ppc64el] <cross>, $(gcc)-riscv64-linux-gnu [riscv64] <cross>, $(gcc)-s390x-linux-gnu [s390x] <cross>, $(gcc)-x86-64-linux-gnu [amd64] <cross>,
abidir := $(CURDIR)/$(DEBIAN)/__abi.current/$(arch)
-prev_abidir := $(CURDIR)/$(DEBIAN)/abi/$(arch)
commonconfdir := $(CURDIR)/$(DEBIAN)/config
archconfdir := $(CURDIR)/$(DEBIAN)/config/$(arch)
sharedconfdir := $(CURDIR)/debian.master/config
diff --git a/debian/rules.d/4-checks.mk b/debian/rules.d/4-checks.mk
index 6e2d4f8524..7cb84e9f71 100644
--- a/debian/rules.d/4-checks.mk
+++ b/debian/rules.d/4-checks.mk
@@ -1,15 +1,3 @@
-# Check ABI for package against last release (if not same abinum)
-abi-check-%: $(stampdir)/stamp-install-%
- @echo Debug: $@
- $(DROOT)/scripts/checks/abi-check "$*" \
- "$(prev_abidir)" "$(abidir)" $(do_skip_checks)
-
-# Check the module list against the last release (always)
-module-check-%: $(stampdir)/stamp-install-%
- @echo Debug: $@
- $(DROOT)/scripts/checks/module-check "$*" \
- "$(prev_abidir)" "$(abidir)" $(do_skip_checks)
-
# Check the signature of staging modules
module-signature-check-%: $(stampdir)/stamp-install-%
@echo Debug: $@
@@ -18,7 +6,7 @@ module-signature-check-%: $(stampdir)/stamp-install-%
"$(DROOT)/$(mods_extra_pkg_name)-$*" \
$(do_skip_checks)
-checks-%: module-check-% module-signature-check-% abi-check-%
+checks-%: module-signature-check-%
@echo Debug: $@
# Check the config against the known options list.
diff --git a/debian/scripts/checks/abi-check b/debian/scripts/checks/abi-check
deleted file mode 100755
index f29ce638ff..0000000000
--- a/debian/scripts/checks/abi-check
+++ /dev/null
@@ -1,139 +0,0 @@
-#!/usr/bin/python3
-#
-# Check ABI changes
-#
-# To skip the ABI checks, add a file
-# debian.<foo>/abi/<arch>/ignore.abi
-# or
-# debian.<foo>/abi/<arch>/<flavor>.ignore.abi
-#
-# To ignore a list of symbols, add the symbols to the file
-# debian.<foo>/abi/abi.ignore
-#
-
-import os
-import re
-import sys
-
-def decode_symline(line):
- comps = re.sub(r'\s+', ' ', line).split(' ')
- if comps[0].startswith('EXPORT_SYMBOL'):
- stype, sloc, shash, sname = comps
- else:
- stype, shash, sname, sloc = comps[1:]
- return sname, {'type': stype, 'loc': sloc, 'hash': shash}
-
-if len(sys.argv) < 4 or len(sys.argv) > 5:
- print('Usage: abi-check <flavor> <prev_abidir> <abidir> [<skipabi>]')
- sys.exit(2)
-
-flavor, prev_abidir, abidir = sys.argv[1:4] # pylint: disable=W0632
-if len(sys.argv) > 4:
- skipabi = sys.argv[4].lower() in ['1', 'true', 'yes']
-else:
- skipabi = False
-
-print('II: Checking ABI for {}...'.format(flavor), end='')
-
-if ((os.path.exists('{}/ignore.abi'.format(prev_abidir)) or
- os.path.exists('{}/{}.ignore.abi'.format(prev_abidir, flavor)))):
- print('WW: Explicitly ignoring ABI')
- print('II: Done')
- sys.exit(0)
-
-curr_abi = '{}/{}'.format(abidir, flavor)
-prev_abi = '{}/{}'.format(prev_abidir, flavor)
-if not os.path.exists(curr_abi) or not os.path.exists(prev_abi):
- print('II: Previous or current ABI file missing!')
- print(' {}'.format(curr_abi))
- print(' {}'.format(prev_abi))
- if skipabi:
- print('WW: Explicitly asked to ignore failures')
- print('II: Done')
- sys.exit(0)
- print('EE: Missing ABI file')
- sys.exit(1)
-
-print()
-
-symbols = {}
-symbols_ignore = {}
-
-# See if we have any ignores
-print(' Reading symbols to ignore...', end='')
-ignore = 0
-prev_abi_ignore = '{}/../abi.ignore'.format(prev_abidir)
-if os.path.exists(prev_abi_ignore):
- with open(prev_abi_ignore) as fh:
- for sym in fh:
- sym = sym.strip()
- if sym.startswith('#'):
- continue
- symbols_ignore[sym] = 1
- ignore += 1
-print('read {} symbols.'.format(ignore))
-
-# Read new symbols first
-print(' Reading new symbols...', end='')
-new_count = 0
-with open('{}/{}'.format(abidir, flavor)) as fh:
- for line in fh:
- sym, vals = decode_symline(line.strip())
- symbols[sym] = vals
- new_count += 1
-print('read {} symbols.'.format(new_count))
-
-# Now the old symbols
-print(' Reading old symbols...', end='')
-old_count = 0
-with open('{}/{}'.format(prev_abidir, flavor)) as fh:
- for line in fh:
- sym, vals = decode_symline(line.strip())
- if sym not in symbols:
- symbols[sym] = {}
- symbols[sym]['old'] = vals
- old_count += 1
-print('read {} symbols.'.format(old_count))
-
-print('II: Checking for ABI changes...')
-changed_loc = 0
-changed_type = 0
-error = False
-for sym, vals in symbols.items():
- # Ignore new symbols
- if 'old' not in vals or 'loc' not in vals:
- continue
-
- # Changes in location don't hurt us, but log it anyway
- if vals['loc'] != vals['old']['loc']:
- changed_loc += 1
- print(' MOVE : {:40} : {} => {}'.format(sym, vals['old']['loc'],
- vals['loc']))
-
- # Changes from GPL to non-GPL are bad
- if ((vals['old']['type'] == 'EXPORT_SYMBOL_GPL' and
- vals['type'] != 'EXPORT_SYMBOL_GPL')):
- changed_type += 1
- if sym in symbols_ignore or vals['loc'] in symbols_ignore:
- ignored = ' (ignore)'
- else:
- ignored = ''
- error = True
- print(' TYPE : {:40} : {} => {}{}'.format(sym, vals['old']['type'],
- vals['type'], ignored))
-
-if changed_loc > 0:
- print('II: {} symbols changed location'.format(changed_loc))
-
-if changed_type > 0:
- print('II: {} symbols changed export type'.format(changed_type))
-
-if error:
- if skipabi:
- print('WW: Explicitly asked to ignore failures')
- else:
- print('EE: Symbol types changed')
- sys.exit(1)
-
-print('II: Done')
-sys.exit(0)
diff --git a/debian/scripts/checks/final-checks b/debian/scripts/checks/final-checks
index 4602cb5c34..98986b341b 100755
--- a/debian/scripts/checks/final-checks
+++ b/debian/scripts/checks/final-checks
@@ -3,12 +3,6 @@
debian="$1"
abi="$2"
-skipabi=${3:-}
-case "${skipabi,,}" in
- 1|true|yes) skipabi=true ;;
- *) skipabi=false ;;
-esac
-
archs=$(awk '/^Architecture:/ { $1=""; for (i=1; i<=NF; i++) { if ($i != "all") { print $i }}}' debian/control | sort -u)
fail=0
@@ -19,41 +13,6 @@ failure()
fail=1
}
-abi_check()
-{
- local abidir="$1"
- local arch="$2"
- local flavour="$3"
-
- local abidir="$abidir/$arch"
-
- if [ ! -f "$abidir/$flavour" ] && \
- [ ! -f "$abidir/$flavour.ignore" ] && \
- [ ! -f "$abidir/ignore" ]
- then
- failure "$arch/$flavour ABI symbol file missing"
- fi
-
- if [ ! -f "$abidir/$flavour.modules" ] && \
- [ ! -f "$abidir/$flavour.ignore.modules" ] && \
- [ ! -f "$abidir/ignore.modules" ]
- then
- failure "$arch/$flavour ABI modules file missing"
- fi
-}
-
-abi_version="$debian/abi/version"
-if ! [ -f "$abi_version" ]
-then
- failure "$abi_version ABI version file missing"
-fi
-
-version=$(cat "$abi_version")
-if [ "$abi" != "$version" ]
-then
- failure "$abi_version ABI version mismatch ($abi != $version)"
-fi
-
for arch in $archs
do
if [ ! -f "$debian/rules.d/$arch.mk" ]; then
@@ -82,9 +41,6 @@ do
failure "'CONFIG_SYSTEM_REVOCATION_KEYS=\"debian/canonical-revoked-certs.pem\"' is required"
fi
fi
- if [ "${skipabi}" = "false" ] ; then
- abi_check "$debian/abi" "$arch" "$flavour"
- fi
done
done
diff --git a/debian/scripts/checks/module-check b/debian/scripts/checks/module-check
deleted file mode 100755
index dcad1b33b4..0000000000
--- a/debian/scripts/checks/module-check
+++ /dev/null
@@ -1,131 +0,0 @@
-#!/usr/bin/python3
-#
-# Check modules changes
-#
-# To skip the modules check, add a file
-# debian.<foo>/abi/<arch>/ignore.modules
-# or
-# debian.<foo>/abi/<arch>/<flavor>.ignore.modules
-#
-# To ignore a list of modules, add the modules to the file
-# debian.<foo>/abi/modules.ignore
-#
-
-import os
-import sys
-
-if len(sys.argv) < 4 or len(sys.argv) > 5:
- print('Usage: module-check <flavor> <prev_abidir> <abidir> [<skipmodule>]')
- sys.exit(2)
-
-flavor, prev_abidir, abidir = sys.argv[1:4] # pylint: disable=W0632
-if len(sys.argv) > 4:
- skipmodule = sys.argv[4].lower() in ['1', 'true', 'yes']
-else:
- skipmodule = False
-
-print('II: Checking modules for {}...'.format(flavor), end='')
-
-if ((os.path.exists('{}/ignore.modules'.format(prev_abidir)) or
- os.path.exists('{}/{}.ignore.modules'.format(prev_abidir, flavor)))):
- print('WW: Explicitly ignoring modules')
- print('II: Done')
- sys.exit(0)
-
-curr_modules = '{}/{}.modules'.format(abidir, flavor)
-prev_modules = '{}/{}.modules'.format(prev_abidir, flavor)
-if not os.path.exists(curr_modules) or not os.path.exists(prev_modules):
- print('II: Previous or current modules file missing!')
- print(' {}'.format(curr_modules))
- print(' {}'.format(prev_modules))
- if skipmodule:
- print('WW: Explicitly asked to ignore failures')
- print('II: Done')
- sys.exit(0)
- print('EE: Missing modules file')
- sys.exit(1)
-
-print()
-
-modules = {}
-modules_ignore = {}
-
-# See if we have any ignores
-print(' Reading modules to ignore...', end='')
-ignore = 0
-prev_modules_ignore = '{}/../modules.ignore'.format(prev_abidir)
-if os.path.exists(prev_modules_ignore):
- with open(prev_modules_ignore) as fh:
- for mod in fh:
- mod = mod.strip()
- if mod.startswith('#'):
- continue
- modules_ignore[mod] = 1
- ignore += 1
-print('read {} modules.'.format(ignore))
-
-# Read new modules first
-print(' Reading new modules...', end='')
-new_count = 0
-for f in (curr_modules, curr_modules + '.builtin'):
- if not os.path.exists(f):
- continue
- with open(f) as fh:
- for mod in fh:
- mod = mod.strip()
- modules[mod] = {'new': 1}
- new_count += 1
-print('read {} modules.'.format(new_count))
-
-# Now the old modules
-print(' Reading old modules...', end='')
-old_count = 0
-for f in (prev_modules, prev_modules + '.builtin'):
- if not os.path.exists(f):
- continue
- with open(f) as fh:
- for mod in fh:
- mod = mod.strip()
- if mod not in modules:
- modules[mod] = {}
- modules[mod]['old'] = 1
- old_count += 1
-print('read {} modules.'.format(old_count))
-
-print('II: Checking for modules changes...')
-new = 0
-missing = 0
-missing_ignored = 0
-error = False
-for mod, vals in modules.items():
- # New modules
- if 'old' not in vals:
- print(' NEW: {}'.format(mod))
- new += 1
-
- # Missing modules
- if 'new' not in vals:
- missing += 0
- if mod in modules_ignore:
- ignored = ' (ignored)'
- missing_ignored += 1
- else:
- ignored = ''
- error = True
- print(' MISS : {}{}'.format(mod, ignored))
-
-if new > 0:
- print('II: {} new modules'.format(new))
-
-if missing > 0:
- print('II: {} missing modules ({} ignored)'.format(missing, missing_ignored))
-
-if error:
- if skipmodule:
- print('WW: Explicitly asked to ignore failures')
- else:
- print('EE: Missing modules')
- sys.exit(1)
-
-print('II: Done')
-sys.exit(0)
diff --git a/debian/scripts/misc/getabis b/debian/scripts/misc/getabis
deleted file mode 100755
index 819824f616..0000000000
--- a/debian/scripts/misc/getabis
+++ /dev/null
@@ -1,222 +0,0 @@
-#!/bin/bash
-
-export LC_ALL=C.UTF-8
-
-if [ "$#" = "1" ]; then
- set - $(echo "$1" | sed -e 's/-/ /')
-fi
-if [ "$#" != "2" ]; then
- echo "Usage: $0 <version>" 1>&2
- echo "Usage: $0 <release> <revision>" 1>&2
- exit 1
-fi
-
-if [ "$DEBIAN" = "" ]; then
- . debian/debian.env
-fi
-
-ver=$1
-revision=$2
-abi=${revision%%.*}
-
-verabi=$ver-$abi
-verfull=$ver-$revision
-
-WGET="wget --tries=1 --timeout=10 --quiet -c"
-
-# Check if we use a flat (unversioned) ABI directory
-if [ -f "${DEBIAN}/abi/version" ] || \
- grep -qP '^abidir\s+.*/__abi.current/' debian/rules.d/0-common-vars.mk ; then
- echo "Using flat ABI directory"
- flat_abi=1
- abidir=$(pwd)/${DEBIAN}/abi
-else
- echo "Using versioned ABI directory"
- flat_abi=0
- abidir=$(pwd)/${DEBIAN}/abi/${verfull}
-fi
-
-tmpdir="`pwd`/abi-tmp-$verfull"
-origdir="`pwd`"
-fwinfo=$abidir/fwinfo
-
-test -d $tmpdir || mkdir $tmpdir
-
-package_prefixes() {
- : # no longer used ...
-}
-
-getall() {
- arch=$1
- shift
-
- mkdir -p $abidir/$arch
-
- for sub in $@; do
- if [ -f $abidir/$arch/$sub ]; then
- echo "Existing $sub($arch)..."
- continue
- fi
- echo "Fetching $sub($arch)..."
- getall_set "linux-buildinfo" "$arch" "$sub" || \
- getall_set "linux-image-unsigned linux-modules linux-modules-extra" "$arch" "$sub" || \
- getall_set "linux-image-unsigned linux-modules" "$arch" "$sub" || \
- getall_set "linux-image linux-modules linux-modules-extra" "$arch" "$sub" || \
- getall_set "linux-image linux-modules" "$arch" "$sub" || \
- getall_set "linux-image linux-image-extra" "$arch" "$sub" || \
- getall_set "linux-image" "$arch" "$sub" || \
- { echo "FAILED"; exit 1; }
- done
-}
-getall_set()
-{
- prefixes="$1"
- arch="$2"
- sub="$3"
- (
- echo -n " set:"
- filenames=""
- cd $tmpdir
- found=1
- for prefix in $prefixes
- do
- echo -n " $prefix="
- if [ "$found" = 0 ]; then
- echo -n "-"
- continue
- fi
- filename=${prefix}-${verabi}-${sub}_${verfull}_${arch}.deb
- for r in "${repo_list[@]}"
- do
- if ! [ -f $filename ]; then
- $WGET $r/$filename
- rc="$?"
- # If this was not successful or a valid error
- # return from the server all bets are off, bail.
- [ "$rc" != 0 -a "$rc" != 8 ] && return 2
- fi
- if [ -f $filename ]; then
- echo -n "y"
- filenames="$filenames $filename"
- break
- fi
- done
- if [ ! -f "$filename" ]; then
- echo -n "n"
- found=0
- fi
- done
- echo ""
- if [ "$found" = 0 ]; then
- return 1
- fi
- echo " extracting..."
- for filename in $filenames
- do
- dpkg-deb --extract $filename tmp
- done
- # FORM 1: linux-image et al extracted here.
- if [ -d tmp/boot ]; then
- echo " images..."
- find tmp -name "*.ko" | while read f; do
- modinfo $f | grep ^firmware >> $fwinfo
- done
- if [ -f tmp/boot/abi-* ]; then
- mv tmp/boot/abi-* $abidir/$arch/$sub
- else
- echo " NO ABI FILE"
- fi
- if [ -f tmp/boot/retpoline-* ]; then
- mv tmp/boot/retpoline-* $abidir/$arch/$sub.retpoline
- else
- echo " NO RETPOLINE FILE"
- fi
- (cd tmp; find lib/modules/$verabi-$sub/kernel -name '*.ko') | \
- sed -e 's/.*\/\([^\/]*\)\.ko/\1/' | sort > \
- $abidir/$arch/$sub.modules
- (
- cd tmp;
- # Prevent exposing some errors when called by python scripts. SIGPIPE seems to get
- # exposed when using the `find ...` form of the command.
- ko=$(find lib/modules/$verabi-$sub/kernel \
- -name '*.ko' | head -1)
- readelf -p .comment "$ko" | gawk '
- ($1 == "[") {
- printf("%s", $3);
- for (n=4; n<=NF; n++) {
- printf(" %s", $n);
- }
- print ""
- }' | sort -u >$abidir/$arch/$sub.compiler
- version=`cat $abidir/$arch/$sub.compiler`
- echo " $version"
- )
- # FORM 2: moduleinfo packages
- # cranky fix -- modinfo supported
- else
- echo " buildinfo..."
- base="tmp/usr/lib/linux/${verabi}-${sub}"
- mv "$base/abi" "$abidir/$arch/$sub"
- for comp in 'modules' 'retpoline' 'compiler'
- do
- mv "$base/$comp" "$abidir/$arch/$sub.$comp"
- done
- if [ -e "${base}"/modules.builtin ] ; then
- mv "${base}"/modules.builtin "${abidir}/${arch}/${sub}".modules.builtin
- fi
- cat "$base/fwinfo" >>"$fwinfo"
- if [ -e "${base}"/fwinfo.builtin ] ; then
- cat "${base}"/fwinfo.builtin >> "${fwinfo}".builtin
- fi
- fi
- rm -rf tmp $filenames
- echo " done"
- )
- rc="$?"
- if [ "$rc" = 2 ]; then
- echo "ERROR: downloads are reporting network failures" 1>&2
- exit 1
- fi
- return "$rc"
-}
-
-# MAIN
-
-# Setup abi directory
-rm -rf "${abidir}"
-mkdir -p $abidir
-echo $abi > $abidir/abiname
-if [ ${flat_abi} -eq 1 ] ; then
- echo "${verfull}" > "${abidir}"/version
-fi
-
-# NOTE: The flavours are hardcoded, because they may have changed from the
-# current build.
-
-. $DEBIAN/etc/getabis
-
-# Extract compiler source package version from e.g.:
-# GCC: (Ubuntu/Linaro 4.8.2-19ubuntu1) 4.8.2
-compilers=`sed 's/^.*(.* \(.*\)).*$/\1/' $abidir/*/*.compiler | sort -u | wc -l`
-if [ "$compilers" != 1 ]; then
- echo "WARNING: inconsistent compiler versions detected:" 1>&2
- sort -u $abidir/*/*.compiler | sed 's/^/WARNING: /' 1>&2
-fi
-
-# Sort fwinfo files
-sort < $fwinfo | uniq > fwinfo.tmp
-mv fwinfo.tmp $fwinfo
-if [ -e "${fwinfo}".builtin ] ; then
- sort < "${fwinfo}".builtin | uniq > fwinfo.tmp
- mv fwinfo.tmp "${fwinfo}".builtin
-fi
-
-rmdir $tmpdir
-
-# If this is running in a git repo, add the new ABI directory, remove the old
-if [ -d ".git" ]; then
- git add "${abidir}"
- if [ ${flat_abi} -eq 0 ] ; then
- find "$DEBIAN"/abi/* -maxdepth 0 -type d | grep -v "$verfull" | while read f; do git rm -rf "$f"; done
- fi
-fi
--
2.34.1
More information about the kernel-team
mailing list