diff options
Diffstat (limited to 'make-boot-image.sh')
| -rwxr-xr-x | make-boot-image.sh | 227 |
1 files changed, 127 insertions, 100 deletions
diff --git a/make-boot-image.sh b/make-boot-image.sh index 8852256..51c8bc8 100755 --- a/make-boot-image.sh +++ b/make-boot-image.sh @@ -1,10 +1,11 @@ #!/bin/bash -xe PS4='+ ($LINENO) ' # To ease debugging -VERSION="1.0rc1" +VERSION="1.0rc2" # Config # ########## WORKDIR=./work # Must not be with "nodev" mount option + # sudo rm -r this folder if you want to re-run everything DLDIR=./downloads OUTDIR=./out OUTUSB=/dev/sdb # Will wreck everything here ! @@ -13,9 +14,9 @@ DEVEL_MODE=n # Adds debugging tools in the generated image INCLUDE_TCPDUMP=y # tcpdump costs few Mb with libcrypto ROOTCMD=sudo WGET="wget" # "wget --no-check-certificate" could help but is a security concern -KERNEL_TARBALL_URL=https://cdn.kernel.org/pub/linux/kernel/v4.x/linux-4.9.58.tar.xz -KCONFIGLIB_MAIN_URL=https://raw.githubusercontent.com/ulfalizer/Kconfiglib/7eace27993ad3aa1d6911866d9c60a11f32d36d9/kconfiglib.py -KCONFIGLIB_PATCH_URL=https://raw.githubusercontent.com/ulfalizer/Kconfiglib/7eace27993ad3aa1d6911866d9c60a11f32d36d9/makefile.patch +KERNEL_TARBALL_URL=https://cdn.kernel.org/pub/linux/kernel/v4.x/linux-4.19.17.tar.xz +KCONFIGLIB_MAIN_URL=https://raw.githubusercontent.com/ulfalizer/Kconfiglib/v10.36.0/kconfiglib.py +KCONFIGLIB_PATCH_URL=https://raw.githubusercontent.com/ulfalizer/Kconfiglib/master/makefile.patch #NIC_FIRMWARE_URL=http://fr.archive.ubuntu.com/ubuntu/pool/main/l/linux-firmware/nic-firmware_1.169_all.udeb BUSYBOX_BIN_URL=https://busybox.net/downloads/binaries/1.26.2-defconfig-multiarch/busybox-x86_64 PCI_IDS_URL=https://pci-ids.ucw.cz/v2.2/pci.ids @@ -46,13 +47,13 @@ function mkchroot return 1 fi # Skip files that already exist at target. - [ -f "$dest/$p" ] && continue + [ -s "$dest/$p" ] && continue # Create destination path d=$(echo "$p" | grep -o '.*/') && mkdir -p "$dest/$d" && # Copy file - echo + cp --dereference --preserve=all "$p" "$dest/$p" && - cp --dereference --preserve=all "$p" "$dest/$p" && + echo + cp --dereference --preserve=mode "$p" "$dest/$p" && + cp --dereference --preserve=mode "$p" "$dest/$p" && # Recursively copy shared libraries' shared libraries. mkchroot "$dest" $(ldd "$p" | egrep -o '/.* ') || return $? done @@ -61,36 +62,42 @@ function mkchroot # Environement and dependencies # ################################# codename=$(lsb_release -sc || true) -if [ "x$codename" != "xjessie" ] +if [ "x$codename" != "xstretch" ] then cat >&2 <<EOT -This script is tested only on Debian 8 (aka jessie). +This script is tested only on Debian 9 (aka stretch). The fastest way to have the right environment is : - * download debian live http://cdimage.debian.org/debian-cd/current-live/amd64/iso-hybrid/debian-live-8.5.0-amd64-standard.iso + * download debian live http://cdimage.debian.org/debian-cd/current-live/amd64/iso-hybrid/debian-live-9.6.0-amd64-gnome.iso * burn it or copy it on a USB stick (as raw, with "cp XX.iso /dev/sdX"), alternatively launch a VM with it * download and run this script from there - * grab the generated out/BOOTX64.EFI + * grab the generated out/BOOTX64.EFI file EOT exit 1 fi -[ -d "$WORKDIR" ] || mkdir "$WORKDIR" -[ -d "$DLDIR" ] || mkdir "$DLDIR" -[ -d "$OUTDIR" ] || mkdir "$OUTDIR" +mkdir -p "$WORKDIR" "$DLDIR" "$OUTDIR" +umask 0022 # Needed for embeding initrd without "sudo make" in kernel dir -if [ ! -e "$WORKDIR/apt-done" ] -then if ! ls /var/lib/apt/lists/*_jessie-backports_* >/dev/null +if [ ! -e "$DLDIR/apt-update-done" ] +then + if ! ls /var/lib/apt/lists/*_jessie-backports_* >/dev/null then echo 'deb http://ftp.debian.org/debian jessie-backports main' \ | $ROOTCMD tee /etc/apt/sources.list.d/backports.list fi $ROOTCMD apt-get update + > "$DLDIR/apt-update-done" +fi + +if [ ! -e "$WORKDIR/apt-done" ] +then # Dependencies of this script (assuming default debian install or live) $ROOTCMD apt-get install wget libncurses5-dev coreutils [ "x$LEGACY" == "xy" ] && $ROOTCMD apt-get install mbr syslinux # Dependencies for kernel building $ROOTCMD apt-get build-dep linux-source + $ROOTCMD apt-get install libelf-dev libssl-dev # Dependencies for kernel tools - [ "x$DEVEL_MODE" == "xy" ] && $ROOTCMD apt-get install libelf-dev libunwind-dev \ - libdw-dev libaudit-dev libssl-dev libslang2-dev libiberty-dev flex bison + [ "x$DEVEL_MODE" == "xy" ] && $ROOTCMD apt-get install libunwind-dev \ + libdw-dev libaudit-dev libslang2-dev libiberty-dev flex bison # Optionnally qemu to run the result for santity checking [ "x$DEVEL_MODE" = "xy" ] && $ROOTCMD apt-get install qemu-system-x86 # Dependencies to put into the initrd @@ -110,111 +117,127 @@ fi # Kernel build setup # ###################### kernel_tarball=$DLDIR/$(basename $KERNEL_TARBALL_URL) -[ -f "$kernel_tarball" ] || $WGET -O "$kernel_tarball" "$KERNEL_TARBALL_URL" -if [ ! -d "$WORKDIR/kernel" ] -then mkdir "$WORKDIR/kernel" +[ -s "$kernel_tarball" ] || $WGET -O "$kernel_tarball" "$KERNEL_TARBALL_URL" +if [ ! -s "$WORKDIR/kernel/Makefile" ] +then mkdir -p "$WORKDIR/kernel" tar xf "$kernel_tarball" --strip-components=1 -C "$WORKDIR/kernel" fi -if [ ! -d "$WORKDIR/kernel/scripts/Kconfiglib" ] +if [ ! -s "$WORKDIR/kernel/scripts/Kconfiglib/kconfiglib.py" ] then - [ -f "$DLDIR/kconfiglib.py" ] || $WGET -O "$DLDIR/kconfiglib.py" "$KCONFIGLIB_MAIN_URL" - [ -f "$DLDIR/makefile.patch" ] || $WGET -O "$DLDIR/makefile.patch" "$KCONFIGLIB_PATCH_URL" - mkdir "$WORKDIR/kernel/scripts/Kconfiglib" + [ -s "$DLDIR/kconfiglib.py" ] || $WGET -O "$DLDIR/kconfiglib.py" "$KCONFIGLIB_MAIN_URL" + [ -s "$DLDIR/makefile.patch" ] || $WGET -O "$DLDIR/makefile.patch" "$KCONFIGLIB_PATCH_URL" + mkdir -p "$WORKDIR/kernel/scripts/Kconfiglib" + patch -t -p1 -d "$WORKDIR/kernel" < "$DLDIR/makefile.patch" && \ cp "$DLDIR/kconfiglib.py" "$WORKDIR/kernel/scripts/Kconfiglib/kconfiglib.py" patch -t -p1 -d "$WORKDIR/kernel" < "$DLDIR/makefile.patch" fi cat >"$WORKDIR/kernel/scripts/Kconfiglib/customize.py" <<"EOT" #!/usr/bin/env python -import kconfiglib import sys +from kconfiglib import Kconfig, standard_config_filename, TRI_TO_STR, TRISTATE def sset(sym, value=None): - if not sym.is_modifiable(): - print("%s is not modifiable at all"%(sym.get_name())) + # Default value + if value == None: + if sym.assignable: + # find highest possible assignable value (last item of modifiable sorted tuple) + value = sym.assignable[-1] + else: + print('%s is not modifiable at all for now'%sym.name) + return True + # Sanity check + if isinstance(value, (int, long)) and value not in sym.assignable: + print('%s can\'t be set to %s for now'%(sym.name,TRI_TO_STR[value])) return True - if value is None and sym.get_type() in [ kconfiglib.BOOL, kconfiglib.TRISTATE ]: - value = sym.get_upper_bound() - old_value = sym.get_value() + # Idempotency check + if isinstance(value, (int, long)): + old_value = sym.tri_value + else: + old_value = sym.str_value if old_value == value: + # No more_work return False - print("CONFIG_%s=%s [was: %s]"%(sym.get_name(),value,old_value)) - sym.set_user_value(value) + # Set value + if isinstance(value, (int, long)): + print('%s=%s [was: %s]'%(sym.name,TRI_TO_STR[value],TRI_TO_STR[old_value])) + else: + print('%s=%s [was: %s]'%(sym.name,value,old_value)) + sym.set_value(value) + # plausible more_work to do return True +kconf = Kconfig(sys.argv[1]) +kconf.load_config(standard_config_filename()) debug = '--debug' in sys.argv; -conf = kconfiglib.Config(sys.argv[1]) -conf.load_config('.config') -support_xz = conf['KERNEL_XZ'] is not None -menu_netfs = conf['NETWORK_FILESYSTEMS'] +passes = 5 + +support_xz = 'HAVE_KERNEL_XZ' in kconf.syms +print('support_xz == %r'%support_xz) + i = 0 more_work = True -while more_work and i < 10: +while more_work and i < passes: more_work = False i += 1 - print("Kconfiglib/customize.py pass %i"%i) - - for sym in conf.get_symbols(): - name = sym.get_name() + print('Kconfiglib/customize.py pass %i'%i) + for sym in kconf.defined_syms: # Default hostname is (none) and could make FreeBSD's dhcpd complain because unallowed '()' - if name in ['DEFAULT_HOSTNAME']: + if sym.name == 'DEFAULT_HOSTNAME': more_work = sset(sym, 'eficast') or more_work # Embed initrd in the EFI bootable kernel - if name in ['INITRAMFS_SOURCE']: + if sym.name == 'INITRAMFS_SOURCE': more_work = sset(sym, '../initrd/') or more_work - # Make kernel directly loadable by EFI - if name in ['EFI_STUB', 'EFI_VARS']: + # Make kernel directly loadable by EFI, add USB3, Dell flash + if sym.name in ['EFI_STUB', 'EARLY_PRINTK_EFI', 'EFI_VARS', 'DELL_RBU', 'USB_XHCI_HCD', 'IKCONFIG']: more_work = sset(sym) or more_work - # Support FUSE for ntfs-3g, Dell flash, USB3, PC Speaker, NVMe (PCIe) SSD - if name in ['FUSE_FS', 'DELL_RBU', 'USB_XHCI_HCD', 'IKCONFIG', 'INPUT_PCSPKR', 'BLK_DEV_NVME']: - more_work = sset(sym, 'm') or more_work - # Support soft RAID (linux) and hard RAID (some cards) - if name in ['SCSI_LOWLEVEL', 'MEGARAID_NEWGEN']: + if sym.name in ['DM_RAID', 'SCSI_LOWLEVEL', 'MEGARAID_SAS', 'MEGARAID_NEWGEN']: more_work = sset(sym) or more_work - if name in ['DM_RAID', 'MEGARAID_SAS']: - more_work = sset(sym, 'm') or more_work # If --debug passed as arg, make kernel aware of virtual drivers (used for testing eficast on qemu/kvm) - if debug and name in ['VIRTIO_PCI', 'VIRTIO_MMIO', 'VIRTIO_NET', 'VIRTIO_BLK', 'SCSI_VIRTIO']: - more_work = sset(sym, 'm') or more_work + if debug and sym.name in ['VIRTIO_PCI', 'VIRTIO_MMIO', 'VIRTIO_NET', 'VIRTIO_BLK', 'SCSI_LOWLEVEL', 'SCSI_VIRTIO']: + more_work = sset(sym) or more_work - # Disable all Network Filesystems support except after-boot NFSv3 client - if name not in ['NET_FS', 'NFS_V3'] and menu_netfs in sym.get_referenced_symbols(): - more_work = sset(sym, 'n') or more_work - # Disable thing that are unneeded or annoying for the purpose of disk cloning - if name in ['LOGO', 'SUSPEND', 'HIBERNATION', 'CPU_FREQ', 'PCCARD', 'HAMRADIO', 'WIRELESS', 'RFKILL', 'WLAN', 'SOUND', 'NFS_V2', 'NFS_V4', 'ROOT_NFS', 'SECURITY', 'VIRTUALIZATION']: - more_work = sset(sym, 'n') or more_work + # FIXME Need NFS v3 client + if sym.name in [ 'HAMRADIO', 'HIBERNATION', 'KEYS', 'LOGO', 'NETFILTER', 'NETWORK_FILESYSTEMS', + 'PCCARD', 'RFKILL', 'SECURITY', 'SOUND', 'SUSPEND', 'VIRTUALIZATION', 'WIRELESS', 'WLAN']: + more_work = sset(sym, 0) or more_work # Compress everything with XZ if available (slower, smaller) -#FIXME : marche pas en 3.x if support_xz: - if name in ['KERNEL_XZ']: + if sym.name in ['KERNEL_XZ', 'RD_XZ']: # , 'INITRAMFS_COMPRESSION_XZ']: more_work = sset(sym) or more_work - if name in ['RD_GZIP', 'RD_BZIP2', 'RD_LZMA', 'RD_LZO', 'RD_LZ4']: - more_work = sset(sym, 'n') or more_work + if sym.name in ['RD_GZIP', 'RD_BZIP2', 'RD_LZMA', 'RD_LZO', 'RD_LZ4']: + more_work = sset(sym, 0) or more_work + if sym.name == 'INITRAMFS_COMPRESSION': + more_work = sset(sym, '.xz') or more_work - # Following generic actions are meant for features, not choices - if not sym.is_choice_symbol(): + # Following generic actions should done only on visible TRISTATE symbols + if sym.type == TRISTATE and sym.visibility > 0: # Build all available net/ethernet drivers - if sym.is_modifiable() and True in [ ('drivers/net/ethernet' in filename) for (filename,_) in sym.get_def_locations() ]: - if sym.get_type() == kconfiglib.BOOL: - more_work = sset(sym) or more_work - if sym.get_type() == kconfiglib.TRISTATE: - more_work = sset(sym, 'm') or more_work - -if i == 10: - print("ERROR : can't set some of kernel config symbols after 10 passes") - sys.exit(1) -else: - sys.exit( conf.write_config(".config") ) + if True in [ ('drivers/net/ethernet' in node.filename) for node in sym.nodes ]: + more_work = sset(sym) or more_work + + # Try to get everything in kernel, not as a module (1=='m') + if sym.tri_value == 1 and sym.assignable and 2 in sym.assignable: + more_work = sset(sym) or more_work + +# Write .config even if some symbols are unset +res = kconf.write_config(standard_config_filename()) + +if i == passes: + print('ERROR : can\'t set some of kernel config symbols after %i passes'%passes) + res = 1 +sys.exit(res) + EOT chmod +x "$WORKDIR/kernel/scripts/Kconfiglib/customize.py" @@ -222,7 +245,7 @@ chmod +x "$WORKDIR/kernel/scripts/Kconfiglib/customize.py" ############################### ( cd "$WORKDIR/kernel" - if [ ! -f .config ] + if [ ! -s .config ] then make defconfig if [ "x$DEVEL_MODE" == "xy" ] then extra="SCRIPT_ARG=--debug" @@ -232,17 +255,17 @@ chmod +x "$WORKDIR/kernel/scripts/Kconfiglib/customize.py" fi ) -p="$WORKDIR/kernel/tools/perf/perf" -if [ "x$DEVEL_MODE" == "xy" -a ! -f "$p" ] +if [ "x$DEVEL_MODE" == "xy" -a ! -s "$WORKDIR/kernel/tools/perf/perf" ] then ( cd "$WORKDIR/kernel" - make tools/perf + # Workaround : linux-3.16.57 (and others?) have make tools/perf broken, ignore it + make tools/perf || true ) fi # Initial Ram Disk building (embed in kernel) # ############################################### -if [ ! -d "$WORKDIR/initrd" ] +if [ ! -s "$WORKDIR/initrd/etc/group" ] then mkdir -p "$WORKDIR/initrd/"{bin,dev,etc/rc.d,mnt/nfs,root,proc,root,sbin,sys,run/lock,run,tmp,usr/share/udhcpc,var/log} $ROOTCMD cp -a /dev/{null,console,tty1} "$WORKDIR/initrd/dev/" $ROOTCMD chmod 1777 "$WORKDIR/initrd/run/lock" @@ -257,22 +280,25 @@ fi # XXX workaround, kernel makefile's cpio preseves everything and it is not so cool for us $ROOTCMD chown -R $USER: "$WORKDIR/initrd" -if [ ! -f "$WORKDIR/initrd/bin/busybox" ] -then [ -f "$DLDIR/busybox" ] || $WGET -O "$DLDIR/busybox" "$BUSYBOX_BIN_URL" - cp -a "$DLDIR/busybox" "$WORKDIR/initrd/bin/busybox" +if [ ! -s "$WORKDIR/initrd/bin/busybox" ] +then [ -s "$DLDIR/busybox" ] || $WGET -O "$DLDIR/busybox" "$BUSYBOX_BIN_URL" + cp "$DLDIR/busybox" "$WORKDIR/initrd/bin/busybox" chmod +x "$WORKDIR/initrd/bin/busybox" - ln -s /bin/busybox "$WORKDIR/initrd/init" +fi +if [ ! -L "$WORKDIR/initrd/init" ] +then ln -s /bin/busybox "$WORKDIR/initrd/init" fi -if [ ! -f "$WORKDIR/initrd/etc/keys.bmap" ] +if [ ! -s "$WORKDIR/initrd/etc/keys.bmap" ] then # When using sudo with password auth, ask and cache pass first $ROOTCMD true # The following compound command will suck at asking pass - $ROOTCMD dumpkeys | $ROOTCMD loadkeys -b > "$WORKDIR/initrd/etc/keys.bmap" + # FIXME fails on Debian 9 + #$ROOTCMD dumpkeys | $ROOTCMD loadkeys -b > "$WORKDIR/initrd/etc/keys.bmap" cp -a /etc/localtime "$WORKDIR/initrd/etc/" fi -if [ ! -f "$WORKDIR/initrd/usr/sbin/partclone.restore" ] +if [ ! -s "$WORKDIR/initrd/usr/sbin/partclone.restore" ] then ( set +x PATH="$WORKDIR/kernel/tools/perf:/usr/sbin:/usr/bin:/sbin:/bin" @@ -316,7 +342,7 @@ then ( ) fi -if [ ! -d "$WORKDIR/initrd/usr/man" ] +if [ ! -f "$WORKDIR/initrd/usr/share/groff/current/man.local" ] then mkdir -p "$WORKDIR"/initrd/usr/man/man{1,6,8} "$WORKDIR"/initrd/usr/share/groff/current/font mkdir -p "$WORKDIR"/initrd/etc/groff/ @@ -332,7 +358,7 @@ then mkdir -p "$WORKDIR"/initrd/usr/man/man{1,6,8} "$WORKDIR"/initrd/usr/share/g fi p="$WORKDIR/kernel/tools/perf/perf" -if [ "x$KERNEL_TOOLS" == "xy" -a ! -f "$p" ] +if [ "x$KERNEL_TOOLS" == "xy" -a ! -s "$p" ] then ( cp -a "$p" "$WORKDIR/initrd/sbin/" set +x @@ -340,16 +366,16 @@ then ( ) fi -if [ ! -d "$WORKDIR/initrd/var/lib" ] -then [ -f "$DLDIR/pci.ids" ] || $WGET -O "$DLDIR/pci.ids" "$PCI_IDS_URL" - [ -f "$DLDIR/usb.ids" ] || $WGET -O "$DLDIR/usb.ids" "$USB_IDS_URL" +if [ ! -s "$WORKDIR/initrd/usr/share/misc/pci.ids" ] +then [ -s "$DLDIR/pci.ids" ] || $WGET -O "$DLDIR/pci.ids" "$PCI_IDS_URL" + [ -s "$DLDIR/usb.ids" ] || $WGET -O "$DLDIR/usb.ids" "$USB_IDS_URL" mkdir -p "$WORKDIR/initrd/var/lib/usbutils" "$WORKDIR/initrd/usr/share/misc" cp "$DLDIR/usb.ids" "$WORKDIR/initrd/var/lib/usbutils/" cp "$DLDIR/pci.ids" "$WORKDIR/initrd/usr/share/misc/" fi #if [ ! -d "$WORKDIR/initrd/lib/firmware" ] -#then [ -f "$DLDIR/nic-firmware.deb" ] || $WGET -O "$DLDIR/nic-firmware.deb" "$NIC_FIRMWARE_URL" +#then [ -s "$DLDIR/nic-firmware.deb" ] || $WGET -O "$DLDIR/nic-firmware.deb" "$NIC_FIRMWARE_URL" # dpkg -x "$DLDIR/nic-firmware.deb" "$WORKDIR/initrd/" # find "$WORKDIR/initrd/lib/firmware/" \( -name 'ipw*' -o -name 'brcmfmac*' -o -name '*wifi*' \) -print0 | xargs -r0 rm -v #fi @@ -749,11 +775,12 @@ EOF # This make will produce a kernel with embed initrd without modules make -j $((nproc+1)) # This will complete the inird tree with modules - INSTALL_MOD_PATH="../initrd/" make modules_install + INSTALL_MOD_PATH=../initrd/ make modules_install # XXX workaround, kernel makefile s cpio preseves everything and it is not so cool for us - $ROOTCMD chown -R root: "../initrd" + $ROOTCMD chown -R root: ../initrd # XXX Workaround : some kernel version forget to update embed initramfs in certain cases [ -f usr/initramfs_data.cpio.gz ] && rm usr/initramfs_data.cpio.gz + $ROOTCMD chmod -R go+rX ../initrd/lib/modules # This produce the final image make ) @@ -769,8 +796,8 @@ if [ -n "$OUTUSB" -a -b "${OUTUSB}1" ] then [ -d "$WORKDIR/mountpoint" ] || mkdir "$WORKDIR/mountpoint" mount | grep -E "^${OUTUSB}1" -q && $ROOTCMD umount "${OUTUSB}1" if [ "x$LEGACY" == "xy" ] - then $ROOTCMD install-mbr "${OUTUSB}" - $ROOTCMD sfdisk --activate=1 "$OUTUSB" + then $ROOTCMD install-mbr -f "${OUTUSB}" + $ROOTCMD sfdisk --activate "$OUTUSB" 1 sleep 1 # XXX do a proper udev wait $ROOTCMD mount "${OUTUSB}1" "$WORKDIR/mountpoint" $ROOTCMD tee "$WORKDIR/mountpoint/syslinux.cfg" > /dev/null <<"EOT" |