use drivers as modules, and many improvements

- enable syslog
- enable module embeding and loading
- enable module coldplug and hotplug
- make tcpdump optionnal
- include mii-tool and ethtool
- temporary disable some kernel tweaks for 3.x compat
- bug fix error catching code while running tmux-main-pane
- update to linux-4.9.58 (was .39)

2 files changed, 96 insertions, 42 deletions

diff --git a/make-boot-image.sh b/make-boot-image.sh
index c96b19b..39c919b 100755
--- a/make-boot-image.sh
+++ b/make-boot-image.sh
@@ -10,12 +10,13 @@ OUTDIR=./out
 OUTUSB=/dev/sdb		# Will wreck everything here !
 LEGACY=n		# make USB bootable key compatible with non UEFI-BIOS
 DEVEL_MODE=n		# Adds debugging tools in the generated image
+INCLUDE_TCPDUMP=y	# tcpdump costs few Mb with libcrypto
 ROOTCMD=sudo
 WGET="wget"		 # "wget --no-check-certificate" could help but is a security concern
-KERNEL_TARBALL_URL=https://cdn.kernel.org/pub/linux/kernel/v4.x/linux-4.9.39.tar.xz
+KERNEL_TARBALL_URL=https://cdn.kernel.org/pub/linux/kernel/v4.x/linux-4.9.58.tar.xz
 KCONFIGLIB_MAIN_URL=https://raw.githubusercontent.com/ulfalizer/Kconfiglib/7eace27993ad3aa1d6911866d9c60a11f32d36d9/kconfiglib.py
 KCONFIGLIB_PATCH_URL=https://raw.githubusercontent.com/ulfalizer/Kconfiglib/7eace27993ad3aa1d6911866d9c60a11f32d36d9/makefile.patch
-NIC_FIRMWARE_URL=http://fr.archive.ubuntu.com/ubuntu/pool/main/l/linux-firmware/nic-firmware_1.169_all.udeb
+#NIC_FIRMWARE_URL=http://fr.archive.ubuntu.com/ubuntu/pool/main/l/linux-firmware/nic-firmware_1.169_all.udeb
 BUSYBOX_BIN_URL=https://busybox.net/downloads/binaries/1.26.2-defconfig-multiarch/busybox-x86_64
 PCI_IDS_URL=https://pci-ids.ucw.cz/v2.2/pci.ids
 USB_IDS_URL=https://usb-ids.gowdy.us/usb.ids
@@ -89,14 +90,18 @@ then	if ! ls /var/lib/apt/lists/*_jessie-backports_* >/dev/null
 	$ROOTCMD apt-get build-dep linux-source
 	# Dependencies for kernel tools
 	[ "x$DEVEL_MODE" == "xy" ] && $ROOTCMD apt-get install libelf-dev libunwind-dev \
-		libdw-dev libaudit-dev libssl-dev libslang2-dev libiberty-dev flex bison tcpdump
+		libdw-dev libaudit-dev libssl-dev libslang2-dev libiberty-dev flex bison
 	# Optionnally qemu to run the result for santity checking
 	[ "x$DEVEL_MODE" = "xy" ] && $ROOTCMD apt-get install qemu-system-x86
 	# Dependencies to put into the initrd
 	$ROOTCMD apt-get install dmidecode pciutils usbutils lshw sysstat iftop strace \
 		ntfs-3g e2fsprogs partclone util-linux udpcast gdisk efibootmgr pigz \
-		pv bc figlet toilet-fonts ncurses-bin beep
+		pv bc figlet toilet-fonts ncurses-bin beep net-tools ethtool
+	# Optionnal dependencies to put into the initrd
+	[ "x$INCLUDE_TCPDUMP" == "xy" ] && $ROOTCMD apt-get install tcpdump
+
 	# util-linux : fdisk, sfdisk, lsblk, setterm
+	# net-tools : mii-tool
 	# ncurses-bin : tput
 	$ROOTCMD apt-get -t jessie-backports install tmux
 	> "$WORKDIR/apt-done"
@@ -161,21 +166,23 @@ while more_work and i < 10:
         if name in ['INITRAMFS_SOURCE']:
             more_work = sset(sym, '../initrd/') or more_work
 
-        # Make kernel directly loadable by EFI, add Dell flash, USB3, PC Speaker
-        if name in ['EFI_STUB', 'EFI_VARS', 'DELL_RBU', 'USB_XHCI_HCD', 'IKCONFIG', 'INPUT_PCSPKR']:
+        # Make kernel directly loadable by EFI
+        if name in ['EFI_STUB', 'EFI_VARS']:
             more_work = sset(sym) or more_work
 
-        # Support FUSE for ntfs-3g
-        if name in ['FUSE_FS']:
-            more_work = sset(sym) or more_work
+	# Support FUSE for ntfs-3g, Dell flash, USB3, PC Speaker, NVMe (PCIe) SSD
+        if name in ['FUSE_FS', 'DELL_RBU', 'USB_XHCI_HCD', 'IKCONFIG', 'INPUT_PCSPKR', 'BLK_DEV_NVME']:
+            more_work = sset(sym, 'm') or more_work
 
         # Support soft RAID (linux) and hard RAID (some cards)
-        if name in ['DM_RAID', 'SCSI_LOWLEVEL', 'MEGARAID_SAS', 'MEGARAID_NEWGEN']:
+        if name in ['SCSI_LOWLEVEL', 'MEGARAID_NEWGEN']:
             more_work = sset(sym) or more_work
+        if name in ['DM_RAID', 'MEGARAID_SAS']:
+            more_work = sset(sym, 'm') or more_work
 
         # If --debug passed as arg, make kernel aware of virtual drivers (used for testing eficast on qemu/kvm)
-        if debug and name in ['VIRTIO_PCI', 'VIRTIO_MMIO', 'VIRTIO_NET', 'VIRTIO_BLK', 'SCSI_LOWLEVEL', 'SCSI_VIRTIO']:
-            more_work = sset(sym) or more_work
+        if debug and name in ['VIRTIO_PCI', 'VIRTIO_MMIO', 'VIRTIO_NET', 'VIRTIO_BLK', 'SCSI_VIRTIO']:
+            more_work = sset(sym, 'm') or more_work
 
 	# Disable all Network Filesystems support except after-boot NFSv3 client
 	if name not in ['NET_FS', 'NFS_V3'] and menu_netfs in sym.get_referenced_symbols():
@@ -186,9 +193,10 @@ while more_work and i < 10:
             more_work = sset(sym, 'n') or more_work
 
         # Compress everything with XZ if available (slower, smaller)
+#FIXME : marche pas en 3.x
         if support_xz:
             if name in ['KERNEL_XZ']:
-                more_work = sset(sym, 'y') or more_work
+                more_work = sset(sym) or more_work
             if name in ['RD_GZIP', 'RD_BZIP2', 'RD_LZMA', 'RD_LZO', 'RD_LZ4']:
                 more_work = sset(sym, 'n') or more_work
 
@@ -196,13 +204,11 @@ while more_work and i < 10:
         if not sym.is_choice_symbol():
 
             # Build all available net/ethernet drivers
-            if sym.is_modifiable() and sym.get_type() in [ kconfiglib.BOOL, kconfiglib.TRISTATE ] \
-            and True in [ ('drivers/net/ethernet' in filename) for (filename,_) in sym.get_def_locations() ]:
-                more_work = sset(sym) or more_work
-
-            # Try to get everything in kernel, not as a module
-            if sym.get_value() == 'm' and sym.get_upper_bound() == 'y':
-                more_work = sset(sym, 'y') or more_work
+            if sym.is_modifiable() and True in [ ('drivers/net/ethernet' in filename) for (filename,_) in sym.get_def_locations() ]:
+                if sym.get_type() == kconfiglib.BOOL:
+                    more_work = sset(sym) or more_work
+                if sym.get_type() == kconfiglib.TRISTATE:
+                    more_work = sset(sym, 'm') or more_work
 
 if i == 10:
     print("ERROR : can't set some of kernel config symbols after 10 passes")
@@ -278,12 +284,20 @@ then	(
 		mkchroot "$WORKDIR/initrd" mkfs mke2fs /sbin/mkfs.ext* mkntfs mkfs.ntfs mkfs.fat mkexfatfs mkfs.exfat mkfs
 		mkchroot "$WORKDIR/initrd" ntfs-3g mount.ntfs mount.fuse mount.exfat-fuse mount.exfat
 		mkchroot "$WORKDIR/initrd" /sbin/ntfs* /bin/ntfs*
+		# Network tools
+		mkchroot "$WORKDIR/initrd" mii-tool ethtool
 		# Disk tools
 		mkchroot "$WORKDIR/initrd" fdisk gdisk sfdisk sgdisk
 		# Cloning tools
 		mkchroot "$WORKDIR/initrd" /usr/sbin/partclone* efibootmgr pigz udp-receiver
 		# Some dyn-loaded libraries (ldd will not display them)
 		mkchroot "$WORKDIR/initrd" /lib/x86_64-linux-gnu/libusb-1.0.so.0
+
+		if [ "x$INCLUDE_TCPDUMP" == "xy" ]
+		then	# tcpdump costs few Mb with libcrypto
+			mkchroot "$WORKDIR/initrd" tcpdump
+		fi
+
 		# Some needed data files
 		cp -ra /lib/terminfo "$WORKDIR"/initrd/lib/
 		mkdir -p "$WORKDIR"/initrd/usr/lib/locale "$WORKDIR"/initrd/usr/share/figlet
@@ -294,8 +308,6 @@ fi
 
 if [ "x$DEVEL_MODE" == "xy" ]
 then	(
-		# tcpdump costs few Mb with libcrypto
-		mkchroot "$WORKDIR/initrd" tcpdump
 		# Perf tool
 		p="$WORKDIR/kernel/tools/perf/perf"
 		cp -a "$p" "$WORKDIR/initrd/sbin/"
@@ -312,7 +324,7 @@ then	mkdir -p "$WORKDIR"/initrd/usr/man/man{1,6,8} "$WORKDIR"/initrd/usr/share/g
 	cp -a /usr/share/man/man6/figlet* "$WORKDIR"/initrd/usr/man/man6/
 	cp -a /usr/share/man/man8/{dmidecode,iftop,lspci,lsblk,lsusb,partclone,efibootmgr,mkfs}* "$WORKDIR"/initrd/usr/man/man8/
 	cp -a /usr/share/man/man8/{ntfs,mkntfs,mkexfatfs,mount.ntfs,mount.fuse,mount.exfat-fuse}* "$WORKDIR"/initrd/usr/man/man8/
-	cp -a /usr/share/man/man8/{mount.exfat,fdisk,gdisk,sfdisk,sgdisk,tcpdump}* "$WORKDIR"/initrd/usr/man/man8/
+	cp -a /usr/share/man/man8/{mount.exfat,fdisk,gdisk,sfdisk,sgdisk,tcpdump,mii-tool,ethtool}* "$WORKDIR"/initrd/usr/man/man8/
 
 	cp -ra /usr/share/groff/current/font/devascii "$WORKDIR"/initrd/usr/share/groff/current/font/
 	cp -ra /usr/share/groff/current/tmac "$WORKDIR"/initrd/usr/share/groff/current/
@@ -336,11 +348,11 @@ then	[ -f "$DLDIR/pci.ids" ] || $WGET -O "$DLDIR/pci.ids" "$PCI_IDS_URL"
 	cp "$DLDIR/pci.ids" "$WORKDIR/initrd/usr/share/misc/"
 fi
 
-if [ ! -d "$WORKDIR/initrd/lib/firmware" ]
-then	[ -f "$DLDIR/nic-firmware.deb" ] || $WGET -O "$DLDIR/nic-firmware.deb" "$NIC_FIRMWARE_URL"
-	dpkg -x "$DLDIR/nic-firmware.deb" "$WORKDIR/initrd/"
-	find "$WORKDIR/initrd/lib/firmware/" \( -name 'ipw*' -o -name 'brcmfmac*' -o -name '*wifi*' \) -print0 | xargs -r0 rm -v
-fi
+#if [ ! -d "$WORKDIR/initrd/lib/firmware" ]
+#then	[ -f "$DLDIR/nic-firmware.deb" ] || $WGET -O "$DLDIR/nic-firmware.deb" "$NIC_FIRMWARE_URL"
+#	dpkg -x "$DLDIR/nic-firmware.deb" "$WORKDIR/initrd/"
+#	find "$WORKDIR/initrd/lib/firmware/" \( -name 'ipw*' -o -name 'brcmfmac*' -o -name '*wifi*' \) -print0 | xargs -r0 rm -v
+#fi
 
 echo $VERSION > "$WORKDIR/initrd/etc/eficast_version"
 
@@ -411,13 +423,16 @@ autorun() {
 		set -g pane-border-status top \; \
 		set -g pane-border-format " #T " \; \
 		set-window-option status off \; \
-		split-window -t %0 -l1 pigz-watch progression
+		split-window -t %0 -l1 -b pigz-watch progression
 	
-	[ "x$1" == "xnfs" ] && tmux split-window -t %0 -l5 iftop-watch network
-	[ $(tput lines) -gt 30 ] && tmux \
+	if [ $(tput lines) -gt 30 ]
+	then tmux \
+		split-window -t %0 -l5 mpstat-watch cpu-softintr \; \
 		split-window -t %0 -l10 iostat-watch cpu-disk \; \
 		split-window -h -p26 pstree-watch processes \; \
-		split-window -t -1 mpstat-watch cpu-softintr
+		split-window -t -1 iftop-watch network
+	fi
+
 	tmux select-pane -t:.0 \; attach
 
 	# Hack to keep the error flow intact
@@ -504,6 +519,17 @@ mount_pseudofilesystems() {
 	return 0
 }
 
+coldplugging() {
+	echo 4 > /proc/sys/kernel/printk
+	for d in /sys/bus/*/devices/*
+	do
+		cd $d
+		[ -r driver ] || echo add > uevent
+	done
+	sleep 3
+	cut -f4 /proc/sys/kernel/printk > /proc/sys/kernel/printk
+}
+
 network_up() {
 	ip -oneline link | grep DOWN | cut -d: -f2 | grep -v sit | while read iface
 	do
@@ -516,9 +542,11 @@ network_up() {
 set -v
 /bin/busybox --install -s||rescue_shell	# Setup busybox symlinks for all applets
 mount_pseudofilesystems || rescue_shell	# Setup /dev, /proc, /sys and so
-loadkmap < /etc/keys.bmap		# Load keyboard layout
+klogd; syslogd				# Start logging in /var/log/messages
 mount -o remount -o size=80% /		# Allow using most of RAM for rootfs
+coldplugging				# Load modules for cold-plugged peripherials
 network_up				# Bring net interfaces up (no config) 
+loadkmap < /etc/keys.bmap		# Load keyboard layout
 sleep 5 	# Wait for physical link detection + STP + IPv6 DAD & Autoconf
 set +v
 # Hint users about boot steps to help them if it hangs
@@ -556,7 +584,7 @@ fi
 EOF
 add_initrd_script "/etc/rc.d/tmux-main-pane" <<"EOF"
 # Drop a in-tmux rescue shell in case of failure, prevents clearing screen with valuable informations
-. /etc/rc.d/funcs		# Load helper functions
+source /etc/rc.d/funcs		# Load helper functions
 # Helper to trap user interruption and error cases
 # Will exit the entire tmux, leaving last error code availble in a file
 cleanup() {
@@ -593,13 +621,34 @@ case $1 in
 		message nfs autorun
 		echo -ne "\e]2;nfs-autorun.sh\007" # Term title (tmux)
 		cd /mnt/nfs
-		source ./nfs-autorun.sh
+		# don't use source here, exit in inner script will skip cleanup routine (and rescue_shell)
+		./nfs-autorun.sh
 		;;
 	*)	echo "Usage : $0 (nfs|initrd)"
-		false; cleanup ;;
+		false;;
 esac
-# Don't add code here, $? from source ...-autorun.sh is used in cleanup
 cleanup
+# Don't add code between esac and cleanup, $? is used
+EOF
+add_initrd_script "/sbin/hotplug" <<"EOF"
+# Be verbose for PCI cards, be silent for the rest (many many things), log everything
+# No support for change or remove events
+# Could run very early, before busybox links installation
+if [ "x$ACTION" = "xadd" ]
+then	if [ -n "$PCI_ID" ]
+	then	if [ -n "$MODALIAS" ]
+		then	echo "$0: PCI_ID==$PCI_ID, starting 'modprobe $MODALIAS'"
+		else	echo -e "\e[37;43m$0: PCI_ID==$PCI_ID, no MODALIAS found\e[0m"
+		fi
+		modprobe -v $MODALIAS 2>&1 | busybox awk -vT="$0: DEVPATH=$DEVPATH PCI_ID==$PCI_ID " '{print T $0 }' | tee -a /var/log/hotplug-pci.log
+
+	else	if [ -n "$MODALIAS" ]
+		then	modprobe -v $MODALIAS 2>&1 | busybox awk -vT="$0: DEVPATH=$DEVPATH MODALIAS==$MODALIAS " '{print T $0 }' >> /var/log/hotplug-non-pci.log
+		else	echo "$0: DEVPATH=$DEVPATH no MODALIAS found" >> /var/log/hotplug-non-pci.log
+		fi
+	fi
+else	echo "$0: DEVPATH=$DEVPATH : no support for '$ACTION'" >> /var/log/hotplug-unsupported.log
+fi > /dev/console 2>&1
 EOF
 add_initrd_script "/bin/nfs-mount" <<"EOF"
 echo -ne "\e]2;$0\007"	# Term title (tmux)
@@ -679,18 +728,22 @@ while [ -n "$1" ]; do
 done
 EOF
 
-# XXX workaround, kernel makefile's cpio preseves everything and it is not so cool for us
-$ROOTCMD chown -R root: "$WORKDIR/initrd"
-
 # Kernel build (with embed initramfs) #
 #######################################
 (
 	cd "$WORKDIR/kernel"
-	# Workaround : some kernel version forget to update embed initramfs in certain cases
-	[ -f usr/initramfs_data.cpio.gz ] && rm usr/initramfs_data.cpio.gz
 	nproc=$(nproc --all)
 	nproc=${nproc:-4}
+	# This make will produce a kernel with embed initrd without modules
 	make -j $((nproc+1))
+	# This will complete the inird tree with modules
+	INSTALL_MOD_PATH="../initrd/" make modules_install
+	# XXX workaround, kernel makefile s cpio preseves everything and it is not so cool for us
+	$ROOTCMD chown -R root: "../initrd"
+	# XXX Workaround : some kernel version forget to update embed initramfs in certain cases
+	[ -f usr/initramfs_data.cpio.gz ] && rm usr/initramfs_data.cpio.gz
+	# This produce the final image
+	make
 )
 
 # Copy / run result EFI file #
diff --git a/nfs/nfs-autorun.sh b/nfs/nfs-autorun.sh
index 3a0df2f..bb3c602 100755
--- a/nfs/nfs-autorun.sh
+++ b/nfs/nfs-autorun.sh
@@ -6,6 +6,7 @@ scriptdir=/mnt/nfs/scripts/
 hostname=$(hostname)
 
 # Load helper functions
+source /etc/rc.d/funcs		# Load EFIcast helper functions (echo_color, machine_info, network_show, sfx_*)
 source $scriptdir/funcs.sh	# Load some nfs-autorun.sh specific helper functions
 
 # Check all prerequisites and exit if it fail