From 5ff2742d228ce3ae54a969b98867b1abac25dad0 Mon Sep 17 00:00:00 2001 From: Ludovic Pouzenc Date: Thu, 20 Jul 2017 12:04:28 +0200 Subject: first working version with NFS --- make-boot-image.sh | 338 ++++++++++++++++++++++++----------------------------- 1 file changed, 152 insertions(+), 186 deletions(-) (limited to 'make-boot-image.sh') diff --git a/make-boot-image.sh b/make-boot-image.sh index 0ba8ce8..ed1c35d 100755 --- a/make-boot-image.sh +++ b/make-boot-image.sh @@ -3,19 +3,19 @@ PS4='+ ($LINENO) ' # Config # ########## -WORKDIR=./work +WORKDIR=./work # Must not be with "nodev" mount option DLDIR=./downloads OUTDIR=./out -#OUTUSB=/dev/sdb1 -DEVEL_MODE=y +OUTUSB=/dev/sdb # Will wreck everything here ! +LEGACY=y # make USB bootable key compatible with non UEFI-BIOS +DEVEL_MODE=n # Adds debugging tools in the generated image ROOTCMD=sudo -WGET="wget" #"wget --no-check-certificate" +WGET="wget" # "wget --no-check-certificate" could help but is a security concern KERNEL_TARBALL_URL=https://cdn.kernel.org/pub/linux/kernel/v4.x/linux-4.6.tar.xz KCONFIGLIB_MAIN_URL=https://raw.githubusercontent.com/ulfalizer/Kconfiglib/7eace27993ad3aa1d6911866d9c60a11f32d36d9/kconfiglib.py KCONFIGLIB_PATCH_URL=https://raw.githubusercontent.com/ulfalizer/Kconfiglib/7eace27993ad3aa1d6911866d9c60a11f32d36d9/makefile.patch -NIC_FIRMWARE_URL=http://fr.archive.ubuntu.com/ubuntu/pool/main/l/linux-firmware/nic-firmware_1.162_all.udeb -UFTP_TARBALL_URL=http://downloads.sourceforge.net/project/uftp-multicast/source-tar/uftp-4.9.3.tar.gz -BUSYBOX_BIN_URL=https://busybox.net/downloads/binaries/busybox-x86_64 +NIC_FIRMWARE_URL=http://fr.archive.ubuntu.com/ubuntu/pool/main/l/linux-firmware/nic-firmware_1.165_all.udeb +BUSYBOX_BIN_URL=https://busybox.net/downloads/binaries/1.26.2-defconfig-multiarch/busybox-x86_64 PCI_IDS_URL=https://pci-ids.ucw.cz/v2.2/pci.ids USB_IDS_URL=https://usb-ids.gowdy.us/usb.ids @@ -43,8 +43,8 @@ function mkchroot d=$(echo "$p" | grep -o '.*/') && mkdir -p "$dest/$d" && # Copy file - echo + cp --dereference --preserve=mode "$p" "$dest/$p" && - cp --dereference --preserve=mode "$p" "$dest/$p" && + echo + cp --dereference --preserve=all "$p" "$dest/$p" && + cp --dereference --preserve=all "$p" "$dest/$p" && # Recursively copy shared libraries' shared libraries. mkchroot "$dest" $(ldd "$p" | egrep -o '/.* ') || return $? done @@ -59,7 +59,8 @@ This script is tested only on Debian 8 (aka jessie). The fastest way to have the right environment is : * download debian live http://cdimage.debian.org/debian-cd/current-live/amd64/iso-hybrid/debian-live-8.5.0-amd64-standard.iso * burn it or copy it on a USB stick (as raw, with "cp XX.iso /dev/sdX"), alternatively launch a VM with it - * run this script from there + * download and run this script from there + * grab the generated out/BOOTX64.EFI EOT exit 1 fi @@ -71,17 +72,19 @@ fi if [ ! -e "$WORKDIR/apt-done" ] then $ROOTCMD apt-get update # Dependencies of this script (assuming default debian install or live) - $ROOTCMD apt-get install wget libncurses5-dev coreutils syslinux + $ROOTCMD apt-get install wget libncurses5-dev coreutils + [ "x$LEGACY" == "xy" ] && $ROOTCMD apt-get install mbr syslinux # Dependencies for kernel building $ROOTCMD apt-get build-dep linux-source - # Dependancies for kernel tools + # Dependencies for kernel tools [ "x$DEVEL_MODE" == "xy" ] && $ROOTCMD apt-get install libelf-dev libunwind-dev \ - libdw-dev libaudit-dev libssl-dev libslang2-dev libiberty-dev + libdw-dev libaudit-dev libssl-dev libslang2-dev libiberty-dev flex bison # Optionnally qemu to run the result for santity checking [ "x$DEVEL_MODE" = "xy" ] && $ROOTCMD apt-get install qemu-system-x86 # Dependencies to put into the initrd - $ROOTCMD apt-get install dmidecode pciutils usbutils lshw sysstat iftop strace - $ROOTCMD apt-get install partclone sfdisk udpcast gdisk efibootmgr tcpdump + $ROOTCMD apt-get install dmidecode pciutils usbutils lshw sysstat iftop strace \ + partclone util-linux udpcast gdisk efibootmgr tcpdump + # util-linux : sfdisk, lsblk, setterm > "$WORKDIR/apt-done" fi @@ -125,6 +128,7 @@ debug = '--debug' in sys.argv; conf = kconfiglib.Config(sys.argv[1]) conf.load_config('.config') support_xz = conf['KERNEL_XZ'] is not None +menu_netfs = conf['NETWORK_FILESYSTEMS'] i = 0 more_work = True while more_work and i < 10: @@ -155,8 +159,12 @@ while more_work and i < 10: if debug and name in ['VIRTIO_PCI', 'VIRTIO_MMIO', 'VIRTIO_NET', 'VIRTIO_BLK', 'SCSI_LOWLEVEL', 'SCSI_VIRTIO']: more_work = sset(sym) or more_work + # Disable all Network Filesystems support except after-boot NFSv3 client + if name not in ['NET_FS', 'NFS_V3'] and menu_netfs in sym.get_referenced_symbols(): + more_work = sset(sym, 'n') or more_work + # Disable thing that are unneeded or annoying for the purpose of disk cloning - if name in ['LOGO', 'SUSPEND', 'HIBERNATION', 'CPU_FREQ', 'PCCARD', 'HAMRADIO', 'WIRELESS', 'RFKILL', 'WLAN', 'SOUND', 'NETWORK_FILESYSTEMS', 'KEYS', 'SECURITY', 'VIRTUALIZATION']: + if name in ['LOGO', 'SUSPEND', 'HIBERNATION', 'CPU_FREQ', 'PCCARD', 'HAMRADIO', 'WIRELESS', 'RFKILL', 'WLAN', 'SOUND', 'NFS_V2', 'NFS_V4', 'ROOT_NFS', 'SECURITY', 'VIRTUALIZATION']: more_work = sset(sym, 'n') or more_work # Compress everything with XZ if available (slower, smaller) @@ -208,26 +216,10 @@ then ( ) fi -# Build additionnal tools from source # -####################################### -uftp_tarball=$DLDIR/$(basename "$UFTP_TARBALL_URL") -[ -f "$uftp_tarball" ] || $WGET -O "$uftp_tarball" "$UFTP_TARBALL_URL" -if [ ! -d "$WORKDIR/uftp" ] -then mkdir "$WORKDIR/uftp" - tar xf "$uftp_tarball" --strip-components=1 -C "$WORKDIR/uftp" -fi - -if [ ! -f "$WORKDIR/uftp/uftpd" ] -then ( - cd "$WORKDIR/uftp/" - make NO_ENCRYPTION=1 - ) -fi - # Initial Ram Disk building (embed in kernel) # ############################################### if [ ! -d "$WORKDIR/initrd" ] -then mkdir -p "$WORKDIR/initrd/"{bin,dev,etc/init.d,mnt,root,proc,root,sbin,sys,run/lock,run/uftpd,tmp,var/log} +then mkdir -p "$WORKDIR/initrd/"{bin,dev,etc/init.d,mnt/nfs,root,proc,root,sbin,sys,run/lock,run,tmp,usr/share/udhcpc,var/log} $ROOTCMD cp -a /dev/{null,console,tty1} "$WORKDIR/initrd/dev/" $ROOTCMD chmod 1777 "$WORKDIR/initrd/run/lock" ln -s "/proc/mounts" "$WORKDIR/initrd/etc/mtab" @@ -235,9 +227,12 @@ then mkdir -p "$WORKDIR/initrd/"{bin,dev,etc/init.d,mnt,root,proc,root,sbin,sys, ln -s "../run/lock" "$WORKDIR/initrd/var/lock" fi +# XXX workaround, kernel makefile's cpio preseves everything and it is not so cool for us +$ROOTCMD chown -R $USER: "$WORKDIR/initrd" + if [ ! -f "$WORKDIR/initrd/bin/busybox" ] then [ -f "$DLDIR/busybox" ] || $WGET -O "$DLDIR/busybox" "$BUSYBOX_BIN_URL" - cp "$DLDIR/busybox" "$WORKDIR/initrd/bin/busybox" + cp -a "$DLDIR/busybox" "$WORKDIR/initrd/bin/busybox" chmod +x "$WORKDIR/initrd/bin/busybox" ln -s /bin/busybox "$WORKDIR/initrd/init" fi @@ -254,9 +249,9 @@ then ( set +x PATH="$WORKDIR/kernel/tools/perf:/usr/sbin:/usr/bin:/sbin:/bin" # Diagnostic tools - mkchroot "$WORKDIR/initrd" dmidecode iftop iostat lshw lspci lsusb mpstat tcpdump - # Manpage display - mkchroot "$WORKDIR/initrd" strace groff nroff troff grotty gtbl + mkchroot "$WORKDIR/initrd" dmidecode iftop iostat lshw lspci lsblk lsusb mpstat tcpdump + # Console tools and manpages display + mkchroot "$WORKDIR/initrd" setterm strace groff nroff troff grotty gtbl # Cloning tools mkchroot "$WORKDIR/initrd" /usr/sbin/partclone* efibootmgr sfdisk gdisk sgdisk udp-receiver # Some dyn-loaded libraries (ldd will not display them) @@ -276,23 +271,11 @@ fi if [ ! -d "$WORKDIR/initrd/usr/man" ] then mkdir -p "$WORKDIR"/initrd/usr/man/man{1,8} "$WORKDIR"/initrd/usr/share/groff/1.22.2/font "$WORKDIR/initrd/etc/groff/" - cp /usr/share/man/man1/{iostat,mpstat,strace,udp-receiver}* "$WORKDIR/initrd/usr/man/man1/" - cp /usr/share/man/man8/{dmidecode,partclone,efibootmgr,gdisk,iftop,tcpdump}* "$WORKDIR/initrd/usr/man/man8/" - cp -r /usr/share/groff/1.22.2/font/devascii "$WORKDIR/initrd/usr/share/groff/1.22.2/font/" - cp -r /usr/share/groff/1.22.2/tmac "$WORKDIR/initrd/usr/share/groff/1.22.2/" - cp /etc/groff/man.local "$WORKDIR/initrd/usr/share/groff/1.22.2/" -fi - -p="$WORKDIR/initrd/bin/uftp" -if [ ! -f "$p" ] -then ( - cd "$WORKDIR/uftp/" - make DESTDIR="../initrd" install - ) - ( - set +x - mkchroot "$WORKDIR/initrd" $(ldd "$p" | egrep -o '/.* ') - ) + cp -a /usr/share/man/man1/{iostat,mpstat,strace,udp-receiver}* "$WORKDIR/initrd/usr/man/man1/" + cp -a /usr/share/man/man8/{dmidecode,partclone,efibootmgr,gdisk,iftop,tcpdump}* "$WORKDIR/initrd/usr/man/man8/" + cp -ra /usr/share/groff/1.22.2/font/devascii "$WORKDIR/initrd/usr/share/groff/1.22.2/font/" + cp -ra /usr/share/groff/1.22.2/tmac "$WORKDIR/initrd/usr/share/groff/1.22.2/" + cp -a /etc/groff/man.local "$WORKDIR/initrd/usr/share/groff/1.22.2/" fi p="$WORKDIR/kernel/tools/perf/perf" @@ -319,13 +302,33 @@ then [ -f "$DLDIR/nic-firmware.deb" ] || $WGET -O "$DLDIR/nic-firmware.deb" "$NI fi cat > "$WORKDIR/initrd/etc/init.d/funcs" <<"EOF" +# echo_color [prefix_string] +echo_color() { + [ $# -eq 4 ] && echo -n "$3" + setterm --foreground "$1" --background "$2" + [ $# -eq 4 ] && echo -n "$4" || echo -n "$3" + setterm --foreground white --background black + echo +} + rescue_shell() { echo "Something went wrong. Dropping to a shell." - setsid cttyhack /bin/busybox sh + PS1='\h:\w# ' setsid cttyhack /bin/busybox sh sync umount /dev /sys /proc #umount /dev/pts /dev /sys /proc - poweroff -d1 -f + reboot -f +} + +mount_pseudofilesystems() { + # Mount pseudo-filesystems + mount -t proc none /proc || return $? + mount -t sysfs none /sys || return $? + mount -t devtmpfs -o size=1m none /dev || return $? + ln -s /proc/self/fd/2 /dev/stderr + #mkdir /dev/pts + #mount -t devpts none /dev/pts + return 0 } tty_prog() { @@ -344,40 +347,59 @@ network_up() { echo ip link set dev $iface up ip link set dev $iface up done - sleep 8 # PHY link det. + IPv6 DAD & Autoconf } machine_info() { + setterm -bold on + grep -F MemTotal: /proc/meminfo for k in system-manufacturer system-product-name \ baseboard-manufacturer baseboard-product-name \ bios-version bios-release-date do echo $k: $(dmidecode -s $k) done - grep -F MemTotal: /proc/meminfo lspci -nn | cut -d' ' -f2- | sed -ne 's/^Ethernet[^:]*/network-card/p' ip -o l | sed -ne 's/[0-9]*: \([^:]*\):[^\\]*\\\s*link\/ether\s/network-mac-\1: /p' + lsblk -dnl | sed 's/^/disk: /' lsusb 2>/dev/null | grep -vE hub$ | cut -d: -f2- | sed 's/^ ID/usb-device:/' + setterm -bold off +} + +network_conf() { + ip -oneline link | grep LOWER_UP | cut -d: -f2 | grep -v sit | grep -v lo | while read iface + do + udhcpc -b $iface + done } network_show() { + setterm -bold on ip -o addr show | sed -ne 's/[0-9]*:\s*\(\S*\)\s*inet6*\s\(\S*\)\s.*$/\1: \2/p' + setterm -bold off } -start_uftpd() { - # FIXME : -I is there as a bug workaround "address already in use" - uftpd -q -B 2097152 -x2 -F /run/uftpd.csv -L /run/uftpd.log -t -D /run/uftpd -M ff02::42 -I 2/6 - [ $? -eq 0 ] && netstat -nlu | grep -q ':1044 ' && echo -e '\033[42m' ----- UFTPD ready ----- '\033[0m' +nfs_check() { + [ -x /mnt/nfs/autorun.sh ] || return 1 + echo_color white green "----- NFS server ready ------" +} + +nfs_autorun() { + PS1='\h:\w#' setsid cttyhack /mnt/nfs/autorun.sh +} + +# no args, print colored message, wait 1 minute and reboot +eficast_end() { + echo_color white green "----- EFICAST end of execution ------" + sleep 60 + reboot -f } EOF cat > "$WORKDIR/initrd/etc/inittab" <<"EOF" # Custom init scripts ::sysinit:/etc/init.d/rcS -# Executes as root what is received via uftpd (no security at all) -::respawn:/bin/sinkdo /run/uftpd /run/sink # Standard things follow -::ctrlaltdel:/sbin/reboot +::ctrlaltdel:/sbin/reboot -f ::shutdown:/sbin/swapoff -a ::shutdown:/bin/umount -a -r ::restart:/sbin/init @@ -388,140 +410,63 @@ EOF cat > "$WORKDIR/initrd/etc/init.d/rcS" <<"EOF" #!/bin/busybox sh -echo "rcS script started" -# Load helper functions -. /etc/init.d/funcs -# Trace execution -set -v +/bin/busybox sleep 3 # Let the kernel take his time to flood the console +echo -e '\033[37;43m'----- rcS script started -----'\033[0m' # Hint for user about boot steps if its hangs +. /etc/init.d/funcs # Load helper functions +set -v # Trace execution -# Setup links for programs supported by busybox /bin/busybox --install -s || rescue_shell - -# Mount pseudo-filesystems -mount -t proc none /proc || rescue_shell -mount -t sysfs none /sys || rescue_shell -mount -t devtmpfs -o size=1m none /dev || rescue_shell -ln -s /proc/self/fd/2 /dev/stderr -#mkdir /dev/pts || rescue_shell -#mount -t devpts none /dev/pts || rescue_shell +mount_pseudofilesystems || rescue_shell # Allow using most of the RAM for rootfs mount -o remount -o size=80% / -# Load keyboard map +# Load keyboard layout and set screen sleep mode after 60 minutes loadkmap < /etc/keys.bmap +setterm -blank 60 network_up -efibootmgr -v +# Wait for physical link detection + STP + IPv6 DAD & Autoconf +sleep 8 + machine_info +network_conf network_show -start_uftpd + +mount -v -t nfs -o nolock 172.16.2.28:/masters /mnt/nfs || rescue_shell +nfs_check || rescue_shell +nfs_autorun # /mnt/nfs/autorun.sh should never give back control to /etc/init.d/rcS (poweroff or reboot...) +rescue_shell EOF chmod +x "$WORKDIR/initrd/etc/init.d/rcS" -cat > "$WORKDIR/initrd/bin/sinkcat" <<"EOF" -#!/bin/busybox sh - -if [ $# -ne 1 ] -then cat < - Concatenate then delete files as soon they appear in sink-dir to stdout. - If multiple files are found in sink, the first in alphabetical order is choosen. - must not exists, this program must create it (avoiding mistakes). - Dropping an empty file in dir-sink will clean exit this program. -EOT - exit 1 -fi - -SINKDIR=$1 -mkdir "$SINKDIR" && cd "$SINKDIR" -if [ $? -ne 0 ] -then echo "Cannot mkdir/chdir to '$SINKDIR'" >&2 - exit 2 -fi - -while true -do - f=$(ls | grep -v '~' | head -n1) - if [ -n "$f" ] - then if [ -f "$f" -a -r "$f" ] - then size=$(stat -c'%s' -- "$f") - # Do the actual work on the following line - cat -- "$f" && rm -- "$f" >&2 - # Normal exit condition - if [ $size -eq 0 ] - then cd / && rmdir -- "$SINKDIR" >&2 - exit 0 - fi - else echo "'$SINKDIR/$f' is not a readable file" >&2 - exit 3 - fi - fi - sleep 1 -done +cat > "$WORKDIR/initrd/usr/share/udhcpc/default.script" <<"EOF" +#!/bin/sh +case $1 in + bound) + # Configure interface and default gateway + busybox ifconfig $interface ${mtu:+mtu $mtu} $ip netmask $subnet ${broadcast:+broadcast $broadcast} + busybox ip -4 route add default via $router dev $interface + + # Update resolver configuration file + [ -n "$domain" ] && R="domain $domain" || R="" + for i in $dns; do + R="$R + nameserver $i" + done + echo $R > /etc/resolv.conf + + # Update in-kernel-memory hostname + [ -n "$hostname" ] && hostname $hostname + ;; + renew | deconf) echo "no action taken: $1: $interface" >&2 ;; + leasefail | nak) echo "configuration failed: $1: $message" >&2 ;; +esac EOF -chmod +x "$WORKDIR/initrd/bin/sinkcat" +chmod +x "$WORKDIR/initrd/usr/share/udhcpc/default.script" -cat > "$WORKDIR/initrd/bin/sinkdo" <<"EOF" -#!/bin/busybox sh - -if [ $# -ne 2 ] -then cat < -EOT - exit 1 -fi - -LANDDIR=$1 -SINKDIR=$2 -cd "$LANDDIR" || exit 2 - -curtask="(start)" -while true -do - f=$(ls | grep -v '~' | head -n1) - if [ -z "$f" ] - then sleep 1 - continue - fi - if [ ! -f "$f" -o ! -r "$f" ] - then echo "'$LANDDIR/$f' is not a readable file" >&2 - exit 2 - fi - size=$(stat -c'%s' -- "$f") - task=${f:0:2} - if [ "$curtask" == "$task" ] - then # Next file of an already started task - mv -- "$f" "$SINKDIR/" - else # Switch to the next task - if [ -d "$SINKDIR" ] - then # Inform sinkcat about end-of-data - touch -- "$SINKDIR/zz" - wait - res=$? # Error code from "./$f" - # sinkcat always rmdir "$SINKDIR" on normal exit - if [ $res -ne 0 -o -d "$SINKDIR" ] - then echo "Task $task has ran into troubles" >&2 - exit 3 - fi - fi - echo "Switching from task $curtask to $task" >&2 - curtask=$task - if [ "$task" == "99" ] - then # Normal exit condition - rm -- "$f" >&2 - echo "All tasks completed sucessfully" >&2 - exit 0 - else # Start a new task - chmod +x -- "$f" # XXX Checks on $f (is a script ?) - sinkcat $SINKDIR | "./$f" & - while [ ! -d "$SINKDIR" ]; do sleep 1; done - rm -- "$f" >&2 - fi - fi -done -EOF -chmod +x "$WORKDIR/initrd/bin/sinkdo" +# XXX workaround, kernel makefile's cpio preseves everything and it is not so cool for us +$ROOTCMD chown -R root: "$WORKDIR/initrd" # Kernel build (with embed initramfs) # ####################################### @@ -540,10 +485,31 @@ chmod +x "$WORKDIR/initrd/bin/sinkdo" [ -f "$OUTDIR/BOOTX64.EFI" ] && rm -f "$OUTDIR/BOOTX64.EFI" cp "$WORKDIR/kernel/arch/x86/boot/bzImage" "$OUTDIR/BOOTX64.EFI" -if [ -n "$OUTUSB" -a -b "$OUTUSB" ] +# Prepare an USB bootable disk if $OUTUSB is set and is has at least 1 primary partition +if [ -n "$OUTUSB" -a -b "${OUTUSB}1" ] then [ -d "$WORKDIR/mountpoint" ] || mkdir "$WORKDIR/mountpoint" - mount | grep -E "^$OUTUSB" -q && $ROOTCMD umount "$OUTUSB" - $ROOTCMD mount "$OUTUSB" "$WORKDIR/mountpoint" + mount | grep -E "^${OUTUSB}1" -q && $ROOTCMD umount "${OUTUSB}1" + if [ "x$LEGACY" == "xy" ] + then $ROOTCMD install-mbr "${OUTUSB}" + $ROOTCMD sfdisk --activate=1 "$OUTUSB" + sleep 1 # XXX do a proper udev wait + $ROOTCMD mount "${OUTUSB}1" "$WORKDIR/mountpoint" + $ROOTCMD tee "$WORKDIR/mountpoint/syslinux.cfg" > /dev/null <<"EOT" +default eficast +label eficast + kernel /EFI/BOOT/BOOTX64.EFI + append from=syslinux +label debug + kernel /EFI/BOOT/BOOTX64.EFI + append debug ignore_loglevel bootmem_debug apic=debug show_lapic=all acpi.debug_layer=0xffffffff acpi.debug_level=0xffffffff debugpat initcall_debug pnp.debug=1 sched_debug +label noacpi + kernel /EFI/BOOT/BOOTX64.EFI + append noacpi +timeout 10 +prompt 1 +EOT + else $ROOTCMD mount "${OUTUSB}1" "$WORKDIR/mountpoint" + fi [ -d "$WORKDIR/mountpoint/BOOT/EFI" ] || $ROOTCMD mkdir -p "$WORKDIR/mountpoint/EFI/BOOT" $ROOTCMD cp "$OUTDIR/BOOTX64.EFI" "$WORKDIR/mountpoint/EFI/BOOT" $ROOTCMD umount "$WORKDIR/mountpoint" -- cgit v1.2.3