Merge branch 'master' of pouzenc.fr:eficast

Conflicts:
	make-boot-image.sh

4 files changed, 241 insertions, 82 deletions

diff --git a/make-boot-image.sh b/make-boot-image.sh
index 42dc91f..30b4462 100755
--- a/make-boot-image.sh
+++ b/make-boot-image.sh
@@ -1,17 +1,20 @@
 #!/bin/bash -xe
+PS4='+ ($LINENO) '
+
 # Config #
 ##########
 WORKDIR=./work
 DLDIR=./downloads
 OUTDIR=./out
-OUTUSB=/dev/sdb1
+#OUTUSB=/dev/sdb1
 DEVEL_MODE=y
 ROOTCMD=sudo
-WGET="wget --no-check-certificate"
+WGET="wget" #"wget --no-check-certificate"
 KERNEL_TARBALL_URL=https://cdn.kernel.org/pub/linux/kernel/v4.x/linux-4.6.tar.xz
 KCONFIGLIB_MAIN_URL=https://raw.githubusercontent.com/ulfalizer/Kconfiglib/7eace27993ad3aa1d6911866d9c60a11f32d36d9/kconfiglib.py
 KCONFIGLIB_PATCH_URL=https://raw.githubusercontent.com/ulfalizer/Kconfiglib/7eace27993ad3aa1d6911866d9c60a11f32d36d9/makefile.patch
 NIC_FIRMWARE_URL=http://fr.archive.ubuntu.com/ubuntu/pool/main/l/linux-firmware/nic-firmware_1.162_all.udeb
+UFTP_TARBALL_URL=http://downloads.sourceforge.net/project/uftp-multicast/source-tar/uftp-4.9.2.tar.gz
 BUSYBOX_BIN_URL=https://busybox.net/downloads/binaries/busybox-x86_64
 PCI_IDS_URL=https://pci-ids.ucw.cz/v2.2/pci.ids
 USB_IDS_URL=https://usb-ids.gowdy.us/usb.ids
@@ -40,7 +43,7 @@ function mkchroot
     d=$(echo "$p" | grep -o '.*/') &&
     mkdir -p "$dest/$d" &&
     # Copy file
-    echo ${PS4}cp --dereference --preserve=mode "$p" "$dest/$p" &&
+    echo + cp --dereference --preserve=mode "$p" "$dest/$p" &&
     cp --dereference --preserve=mode "$p" "$dest/$p" &&
     # Recursively copy shared libraries' shared libraries.
     mkchroot "$dest" $(ldd "$p" | egrep -o '/.* ') || return $?
@@ -68,7 +71,7 @@ fi
 if [ ! -e "$WORKDIR/apt-done" ]
 then	$ROOTCMD apt-get update
 	# Dependencies of this script (assuming default debian install or live)
-	$ROOTCMD apt-get install wget libncurses5-dev
+	$ROOTCMD apt-get install wget libncurses5-dev coreutils
 	# Dependencies for kernel building
 	$ROOTCMD apt-get build-dep linux-source
 	# Dependancies for kernel tools
@@ -197,13 +200,30 @@ chmod +x "$WORKDIR/kernel/scripts/Kconfiglib/customize.py"
 	fi
 )
 
-if [ "x$DEVEL_MODE" == "xy" ]
+p="$WORKDIR/kernel/tools/perf/perf"
+if [ "x$DEVEL_MODE" == "xy" -a ! -f "$p" ]
 then	(
 		cd "$WORKDIR/kernel"
 		make tools/perf
 	)
 fi
 
+# Build additionnal tools from source #
+#######################################
+uftp_tarball=$DLDIR/$(basename "$UFTP_TARBALL_URL")
+[ -f "$uftp_tarball" ] || $WGET -O "$uftp_tarball" "$UFTP_TARBALL_URL"
+if [ ! -d "$WORKDIR/uftp" ]
+then	mkdir "$WORKDIR/uftp"
+	tar xf "$uftp_tarball" --strip-components=1 -C "$WORKDIR/uftp"
+fi
+
+if [ ! -f "$WORKDIR/uftp/uftpd" ]
+then	(
+		cd "$WORKDIR/uftp/"
+		make NO_ENCRYPTION=1
+	)
+fi
+
 # Initial Ram Disk building (embed in kernel) #
 ###############################################
 if [ ! -d "$WORKDIR/initrd" ]
@@ -221,21 +241,26 @@ then	[ -f "$DLDIR/busybox" ] || $WGET -O "$DLDIR/busybox" "$BUSYBOX_BIN_URL"
 fi
 
 if [ ! -f "$WORKDIR/initrd/etc/keys.bmap" ]
-then	$ROOTCMD dumpkeys | $ROOTCMD loadkeys -b > "$WORKDIR/initrd/etc/keys.bmap"
+then	# When using sudo with password auth, ask and cache pass first
+	$ROOTCMD true
+	# The following compound command will suck at asking pass
+	$ROOTCMD dumpkeys | $ROOTCMD loadkeys -b > "$WORKDIR/initrd/etc/keys.bmap"
 fi
 
-(
-	set +x
-	PATH="$WORKDIR/kernel/tools/perf:/usr/sbin:/usr/bin:/sbin:/bin"
-	# Diagnostic tools
-	mkchroot "$WORKDIR/initrd" dmidecode iftop iostat lshw lspci lsusb mpstat tcpdump
-	# Manpage display
-	mkchroot "$WORKDIR/initrd" strace groff nroff troff grotty gtbl
-	# Cloning tools
-	mkchroot "$WORKDIR/initrd" /usr/sbin/partclone* efibootmgr gdisk udp-receiver
-	# Some dyn-loaded libraries (ldd will not display them)
-	mkchroot "$WORKDIR/initrd" /lib/x86_64-linux-gnu/libusb-1.0.so.0
-)
+if [ ! -f "$WORKDIR/initrd/partclone-restore" ]
+then	(
+		set +x
+		PATH="$WORKDIR/kernel/tools/perf:/usr/sbin:/usr/bin:/sbin:/bin"
+		# Diagnostic tools
+		mkchroot "$WORKDIR/initrd" dmidecode iftop iostat lshw lspci lsusb mpstat tcpdump
+		# Manpage display
+		mkchroot "$WORKDIR/initrd" strace groff nroff troff grotty gtbl
+		# Cloning tools
+		mkchroot "$WORKDIR/initrd" /usr/sbin/partclone* efibootmgr gdisk udp-receiver
+		# Some dyn-loaded libraries (ldd will not display them)
+		mkchroot "$WORKDIR/initrd" /lib/x86_64-linux-gnu/libusb-1.0.so.0
+	)
+fi
 
 if [ "x$DEVEL_MODE" == "xy" ]
 then	(
@@ -255,6 +280,27 @@ then	mkdir -p "$WORKDIR"/initrd/usr/man/man{1,8} "$WORKDIR"/initrd/usr/share/gro
 	cp /etc/groff/man.local "$WORKDIR/initrd/usr/share/groff/1.22.2/"
 fi
 
+p="$WORKDIR/initrd/bin/uftp"
+if [ ! -f "$p" ]
+then	(
+		cd "$WORKDIR/uftp/"
+		make DESTDIR="../initrd" install
+	)
+	(
+		set +x
+		mkchroot "$WORKDIR/initrd" $(ldd "$p" | egrep -o '/.* ')
+	)
+fi
+
+p="$WORKDIR/kernel/tools/perf/perf"
+if [ "x$KERNEL_TOOLS" == "xy" -a ! -f "$p" ]
+then	(
+		cp -a "$p" "$WORKDIR/initrd/sbin/"
+		set +x
+		mkchroot "$WORKDIR/initrd" $(ldd "$p" | egrep -o '/.* ')
+	)
+fi
+
 if [ ! -d "$WORKDIR/initrd/var/lib" ]
 then	[ -f "$DLDIR/pci.ids" ] || $WGET -O "$DLDIR/pci.ids" "$PCI_IDS_URL"
 	[ -f "$DLDIR/usb.ids" ] || $WGET -O "$DLDIR/usb.ids" "$USB_IDS_URL"
@@ -269,7 +315,7 @@ then	[ -f "$DLDIR/nic-firmware.deb" ] || $WGET -O "$DLDIR/nic-firmware.deb" "$NI
 	find "$WORKDIR/initrd/lib/firmware/" \( -name 'ipw*' -o -name 'brcmfmac*' -o -name '*wifi*' \) -print0 | xargs -r0 rm -v
 fi
 
-cat > "$WORKDIR/initrd/funcs" <<"EOT"
+cat > "$WORKDIR/initrd/funcs" <<"EOF"
 rescue_shell() {
 	echo "Something went wrong. Dropping to a shell."
 	setsid cttyhack sh
@@ -282,8 +328,8 @@ tty_prog() {
 	tty=/dev/tty$1; shift
 	while true
 	do
-		echo "(re)spawning $@ on $tty" >$tty
-		setsid sh -c "exec $@ <$tty >$tty 2>&1"
+		echo "(re)spawning $* on $tty" >$tty
+		setsid sh -c "exec $* <$tty >$tty 2>&1"
 		sleep 2
 	done
 }
@@ -294,6 +340,7 @@ network_up() {
 		echo ip link set dev $iface up
 		ip link set dev $iface up
 	done
+	sleep 8 # PHY link det. + IPv6 DAD & Autoconf
 }
 
 machine_info() {
@@ -308,9 +355,17 @@ machine_info() {
 	ip -o l | sed -ne 's/[0-9]*: \([^:]*\):[^\\]*\\\s*link\/ether\s/network-mac-\1: /p'
 	lsusb 2>/dev/null | grep -vE hub$ | cut -d: -f2- | sed 's/^ ID/usb-device:/'
 }
-EOT
 
-cat > "$WORKDIR/initrd/init" <<"EOT"
+network_show() {
+	ip -o addr show | sed -ne 's/[0-9]*:\s*\(\S*\)\s*inet6*\s\(\S*\)\s.*$/\1: \2/p'
+}
+start_uftpd() {
+	uftpd -q -B 2097152 -x2 -F /run/uftpd.csv -L /run/uftpd.log -T /tmp -D /run/uftpd # -M ff02::1:2:3 -P ff02::4:5:6 Marche pas ???
+	[ $? -eq 0 ] && netstat -nlu | grep -q ':1044 ' && echo -e '\033[42m' ----- UFTPD ready ----- '\033[40m'
+}
+EOF
+
+cat > "$WORKDIR/initrd/init" <<"EOF"
 #!/bin/busybox sh
 echo "Initrd started"
 # Load helper functions
@@ -324,31 +379,133 @@ set -v
 # Mount pseudo-filesystems
 mount -t proc none /proc || rescue_shell
 mount -t sysfs none /sys || rescue_shell
-mount -t devtmpfs none /dev || rescue_shell
+mount -t devtmpfs -o size=1m none /dev || rescue_shell
+
+# Allow using most of the RAM for rootfs
+mount -o remount -o size=80% /
 
 # Load keyboard map
 loadkmap < /etc/keys.bmap || rescue_shell
 
+# Starting sinkdo trojan on tty2 (feed data to uploaded scripts via uftp)
+mkdir /run/uftpd && tty_prog 2 sinkdo /run/uftpd /run/sink &
+
 # Start some debug shells (background)
-for i in 2 3 4 5 6; do tty_prog $i sh & done
+for i in 3 4 5 6; do tty_prog $i sh & done
 
-# Activate networking interfaces
 network_up
-sleep 8 # PHY link detection + IPv6 Duplicate Address Detection & Autoconf
-
-# Dump EFIvars if available
 efibootmgr -v
-
-# Machine basic informations
 machine_info
+network_show
+start_uftpd
+# Remote control on telnet port (23). You could press Alt+F3 to have a shell.
+nc -ll -p 23 -e /bin/sh -i
+EOF
+chmod +x "$WORKDIR/initrd/init"
 
-# Network IP adresses
-ip -o addr show | sed -ne 's/[0-9]*:\s*\(\S*\)\s*inet6*\s\(\S*\)\s.*$/\1: \2/p'
+cat > "$WORKDIR/initrd/bin/sinkcat" <<"EOF"
+#!/bin/busybox sh
 
-# You could press Alt+F2 to have a shell. Remote control on telnet port (TCP 23)
-nc -ll -p 23 -e /bin/sh -i
+if [ $# -ne 1 ]
+then	cat <<EOT
+Usage: $(basename $0) <sink-dir>
+  Concatenate then delete files as soon they appear in sink-dir to stdout.
+  If multiple files are found in sink, the first in alphabetical order is choosen.
+  <sink-dir> must not exists, this program must create it (avoiding mistakes).
+  Dropping an empty file in dir-sink will clean exit this program.
 EOT
-chmod +x "$WORKDIR/initrd/init"
+	exit 1
+fi
+
+SINKDIR=$1
+mkdir "$SINKDIR" && cd "$SINKDIR"
+if [ $? -ne 0 ]
+then	echo "Cannot mkdir/chdir to '$SINKDIR'" >&2
+	exit 2
+fi
+
+while true
+do
+	f=$(ls | head -n1)
+	if [ -n "$f" ]
+	then	if [ -f "$f" -a -r "$f" ]
+		then	size=$(stat -c'%s' -- "$f")
+			# Do the actual work on the following line
+			cat -- "$f" && rm -v -- "$f" >&2
+			# Normal exit condition
+			if [ $size -eq 0 ]
+			then	cd / && rmdir -v -- "$SINKDIR" >&2
+				exit 0
+			fi
+		else	echo "'$SINKDIR/$f' is not a readable file" >&2
+			exit 3
+		fi
+	fi
+	sleep 1
+done
+EOF
+chmod +x "$WORKDIR/initrd/bin/sinkcat"
+
+cat > "$WORKDIR/initrd/bin/sinkdo" <<"EOF"
+#!/bin/busybox sh
+
+if [ $# -ne 2 ]
+then	cat <<EOT
+Usage: $(basename $0) <land-dir> <sink-dir>
+EOT
+	exit 1
+fi
+
+LANDDIR=$1
+SINKDIR=$2
+cd "$LANDDIR" || exit 2
+
+curtask="(start)"
+while true
+do
+	f=$(ls | head -n1)
+	if [ -z "$f" ]
+	then	sleep 1
+		continue
+	fi
+	if [ ! -f "$f" -o ! -r "$f" ]
+	then	echo "'$LANDDIR/$f' is not a readable file" >&2
+		exit 2
+	fi
+	size=$(stat -c'%s' -- "$f")
+	task=${f:0:2}
+	if [ "$curtask" == "$task" ]
+	then	# Next file of an already started task
+		mv "$f" "$SINKDIR/"
+	else	# Switch to the next task
+		if [ -d "$SINKDIR" ]
+		then	# Inform sinkcat about end-of-data
+			touch -- "$SINKDIR/zz"
+			wait
+			# sinkcat always rmdir "$SINKDIR" on normal exit
+			if [ -d "$SINKDIR" ]
+			then	echo "Task $task has ran into troubles" >&2
+				exit 3
+			fi
+		fi
+		echo "Switching from task $curtask to $task" >&2
+		curtask=$task
+		if [ "$task" == "99" ]
+		then	# Normal exit condition
+			rm -v "$f" >&2
+			# reset # Usefull if ncurses on bg, anoying when destroying useful messages
+			echo "All task completed sucessfully" >&2
+			exit 0
+		else	# Start a new task
+			chmod +x -- "$f" # XXX Checks on $f (is a script ?) 
+			sinkcat $SINKDIR | "./$f" &
+			while [ ! -d "$SINKDIR" ]; do sleep 1; done
+			rm -v "$f" >&2
+		fi
+	fi
+done
+EOF
+chmod +x "$WORKDIR/initrd/bin/sinkdo"
 
 # Kernel build (with embed initramfs) #
 #######################################
diff --git a/mcastseed/src/Makefile.am b/mcastseed/src/Makefile.am
index c49cf70..2f2a735 100644
--- a/mcastseed/src/Makefile.am
+++ b/mcastseed/src/Makefile.am
@@ -5,9 +5,7 @@ AM_CFLAGS =\
 	 -Wall \
 	 -Wextra \
 	 -pedantic \
-	 -Wno-format\
-	 -O0\
-	 -g
+	 -Wno-format
 
 bin_PROGRAMS = mcastseed mcastleech random_speed_dd
 
diff --git a/mcastseed/src/dgrambuf.c b/mcastseed/src/dgrambuf.c
index b440390..5d4c302 100644
--- a/mcastseed/src/dgrambuf.c
+++ b/mcastseed/src/dgrambuf.c
@@ -237,7 +237,7 @@ int dgrambuf_have_data_ready_to_write(dgrambuf_t dbuf) {
 	unsigned int next_dgram_seq;
 
 	/* Last write was partial, so there is more to write */
-	if ( dbuf->partial_write_remaining_bytes > 0 ) {
+	if ( dbuf->partial_write_remaining_bytes ) {
 		return 1;
 	}
 
@@ -248,7 +248,7 @@ int dgrambuf_have_data_ready_to_write(dgrambuf_t dbuf) {
 
 	/* Nothing to write if next dgram is not in buffer at all */
 	next_dgram_seq = ((struct indexed_uint *) gl_list_get_at(dbuf->dgram_used_slots, 0))->value;
-	fprintf(stderr, "DEBUG : dgram_seq_base==%u next_dgram_seq == %u\n", dbuf->dgram_seq_base, next_dgram_seq);
+	/*fprintf(stderr, "DEBUG : dgram_seq_base==%u next_dgram_seq == %u\n", dbuf->dgram_seq_base, next_dgram_seq);*/
 	if ( next_dgram_seq != dbuf->dgram_seq_base ) {
 		return 0;
 	}
@@ -262,9 +262,9 @@ int dgrambuf_have_received_everything(dgrambuf_t dbuf) {
 }
 
 ssize_t dgrambuf_write(dgrambuf_t dbuf, int fd, int *info) {
-	size_t dgram_index, i, vlen, used_count;
+	size_t dgram_index, i, vlen, total, len, remain, used_count;
 	unsigned int curr_seq, prev_seq, dgram_len;
-	ssize_t nwrite, total, remain, len;
+	ssize_t nwrite;
 	struct iovec *iov;
 	struct indexed_uint *active_slot;
 	bool pos;
@@ -272,7 +272,7 @@ ssize_t dgrambuf_write(dgrambuf_t dbuf, int fd, int *info) {
 	/* FIXME Info ptr is mandatory */
 	*info = 0;
 
-	if ( dbuf->partial_write_remaining_bytes > 0 ) {
+	if ( dbuf->partial_write_remaining_bytes ) {
 		/* Previous writev() was partial, continue it */
 		iov = dbuf->partial_write_iov;
 		vlen = dbuf->partial_write_remaining_iovcnt;
@@ -364,49 +364,53 @@ ssize_t dgrambuf_write(dgrambuf_t dbuf, int fd, int *info) {
 		return -1;
 	}
 
+	dbuf->partial_write_remaining_bytes = total - nwrite;
 	if ( nwrite > 0 ) {
 		dbuf->stats.write_byte += nwrite;
 		*info |= DGRAMBUF_WRITE_SUCCESS;
-	}
 
-	/* Check if the write was partially done */
-	dbuf->partial_write_remaining_bytes = total - nwrite;
-	if ( dbuf->partial_write_remaining_bytes > 0 ) {
-		*info |= DGRAMBUF_WRITE_PARTIAL;
-		dbuf->stats.write_partial++;
-		/* Find the partially written iov and update it */
-		remain = nwrite;
-		for (i=0; i<vlen; i++) {
-			len = dbuf->iov_write[i].iov_len;
-			if ( remain < len ) {
-				dbuf->partial_write_remaining_iovcnt = vlen - i;
-				dbuf->partial_write_iov = dbuf->iov_write + i;
-
-				dbuf->iov_write[i].iov_base =
-					(uint8_t *) dbuf->iov_write[i].iov_base + remain;
-				dbuf->iov_write[i].iov_len -= remain;
-				break;
+		if ( dbuf->partial_write_remaining_bytes ) {
+			/* If the write was partially done */
+			*info |= DGRAMBUF_WRITE_PARTIAL;
+			dbuf->stats.write_partial++;
+			/* Find the partially written iov and update it */
+			remain = nwrite;
+			for (i=0; i<vlen; i++) {
+				len = dbuf->iov_write[i].iov_len;
+				if ( remain < len ) {
+					dbuf->partial_write_remaining_iovcnt = vlen - i;
+					if ( dbuf->partial_write_iov ) {
+						dbuf->partial_write_iov += i;
+					} else {
+						dbuf->partial_write_iov = dbuf->iov_write + i;
+					}
+
+					dbuf->iov_write[i].iov_base =
+						(uint8_t *) dbuf->iov_write[i].iov_base + remain;
+					dbuf->iov_write[i].iov_len -= remain;
+					break;
+				}
+				remain -= len;
+			}
+			if ( i == vlen ) {
+				fprintf(stderr, "Fatal : failed to find partial iov after partial write\n");
+				return -3;
 			}
-			remain -= len;
-		}
-		if ( i == vlen ) {
-			fprintf(stderr, "Fatal : failed to find partial iov after partial write\n");
-			return -3;
-		}
 
-	} else {
-		/* Full write has happened */
-		for (i=0; i<dbuf->dgram_write_active_slots_count; i++) {
-			active_slot = (struct indexed_uint *) dbuf->dgram_write_active_slots[i];
-			active_slot->value = 0;
-			pos = gl_sortedlist_nx_add(dbuf->dgram_empty_slots, _compare_indexed_uint, active_slot);
-			if ( !pos ) /*TODO: better oom handling */
-				return -4;
+		} else {
+			/* Full write has happened */
+			for (i=0; i<dbuf->dgram_write_active_slots_count; i++) {
+				active_slot = (struct indexed_uint *) dbuf->dgram_write_active_slots[i];
+				active_slot->value = 0;
+				pos = gl_sortedlist_nx_add(dbuf->dgram_empty_slots, _compare_indexed_uint, active_slot);
+				if ( !pos ) /*TODO: better oom handling ? */
+					return -4;
+			}
+			dbuf->dgram_write_active_slots_count = 0;
+			/* Wipe outdated partial_* values */
+			dbuf->partial_write_iov = NULL;
+			dbuf->partial_write_remaining_iovcnt = 0;
 		}
-		dbuf->dgram_write_active_slots_count = 0;
-		/* Wipe outdated partial_* values for clarity in debug mode (only _bytes is use on branching) */
-		dbuf->partial_write_iov = NULL;
-		dbuf->partial_write_remaining_iovcnt = 0;
 	}
 
 	return nwrite;
diff --git a/mcastseed/src/mcastleech.c b/mcastseed/src/mcastleech.c
index 76e1e79..3345665 100644
--- a/mcastseed/src/mcastleech.c
+++ b/mcastseed/src/mcastleech.c
@@ -218,10 +218,10 @@ int receive_data() {
 		return nwrite;
 	}
 
-	fprintf(stderr, "receive_data(): nread == %zi, nwrite == %zi\n", nread, nwrite);
+	/*fprintf(stderr, "receive_data(): nread == %zi, nwrite == %zi\n", nread, nwrite);*/
 
 	/* XXX Crapy dead state detection */
-	if ( nread == 0 && nwrite == 0 ) {
+	if ( nread == 0 /* TEST && nwrite == 0 */ ) {
 		if ( noop_calls_count > 10 ) {
 			return 0;
 		}