Mount /opt/runcvm in-container to /.runcvm/guest

- Facilitates installing RunCVM in a RunCVM VM
- Reduces number of top-level in-container mountpoints to just one
- Introduces $RUNCVM_GUEST variable (="/.runcvm/guest") to be used where
  possible, pursuant to possible future functionality allowing this to be
  configured
- Fixes kernel-selection logic to match docs and comments

Author: struanb

Dockerfile

@@ -109,9 +109,10 @@ RUN apk add --allow-untrusted /tmp/dropbear/dropbear-ssh*.apk /tmp/dropbear/drop
 
 # Patch the binaries and set up symlinks
 COPY build-utils/elf-patcher.sh /usr/local/bin/elf-patcher.sh
-ENV BINARIES="busybox bash jq ip nc mke2fs blkid findmnt dnsmasq xtables-legacy-multi nft xtables-nft-multi nft mount s6-applyuidgid qemu-system-x86_64 qemu-ga /usr/lib/qemu/virtiofsd tput stdbuf coreutils getent dropbear dbclient dropbearkey"
+ENV BINARIES="busybox bash jq ip nc mke2fs blkid findmnt dnsmasq xtables-legacy-multi nft xtables-nft-multi nft mount s6-applyuidgid qemu-system-x86_64 qemu-ga /usr/lib/qemu/virtiofsd tput coreutils getent dropbear dbclient dropbearkey"
 ENV EXTRA_LIBS="/usr/lib/xtables /usr/libexec/coreutils /tmp/dropbear/libepka_file.so /usr/lib/qemu/*.so"
 ENV CODE_PATH="/opt/runcvm"
+ENV EXEC_PATH="/.runcvm/guest"
 RUN /usr/local/bin/elf-patcher.sh && \
     cd $CODE_PATH/bin && \
     for cmd in \

runcvm-init/dumb-init.c

@@ -99,7 +99,7 @@ void forward_signal(int signum) {
 
 pid_t shutdown() {   
    pid_t my_child_pid;
-   char *shutdown_cmd[] = {"/opt/runcvm/scripts/runcvm-ctr-shutdown", NULL};
+   char *shutdown_cmd[] = {"/.runcvm/guest/scripts/runcvm-ctr-shutdown", NULL};
 
     my_child_pid = fork();
     if (my_child_pid < 0) {
@@ -123,7 +123,7 @@ pid_t shutdown() {
 
 void quit(int exit_status) {
     char exit_status_string[4];
-    char *exit_cmd[] = {"/opt/runcvm/scripts/runcvm-ctr-exit", exit_status_string, NULL};
+    char *exit_cmd[] = {"/.runcvm/guest/scripts/runcvm-ctr-exit", exit_status_string, NULL};
 
     sprintf(exit_status_string, "%d", exit_status & 0xFF);
 

runcvm-scripts/runcvm-ctr-defaults

@@ -1,7 +1,7 @@
 #!/bin/bash
 
-RUNCVM=/opt/runcvm
-RUNCVM_PATH=$RUNCVM/usr/sbin:$RUNCVM/usr/bin:$RUNCVM/sbin:$RUNCVM/bin:$RUNCVM/usr/lib/qemu
+RUNCVM_GUEST=${RUNCVM_GUEST:-/.runcvm/guest}
+RUNCVM_PATH=$RUNCVM_GUEST/usr/sbin:$RUNCVM_GUEST/usr/bin:$RUNCVM_GUEST/sbin:$RUNCVM_GUEST/bin:$RUNCVM_GUEST/usr/lib/qemu
 
 QEMU_VIRTIOFSD_SOCKET=/run/.virtiofs.sock
 QEMU_GUEST_AGENT=/run/.qemu-guest-agent
@@ -12,6 +12,7 @@ SSHD_PORT=22222
 clean_env() {
   export -n \
   RUNCVM_BREAK RUNCVM_INIT \
+  RUNCVM_GUEST \
   RUNCVM_RUNTIME_DEBUG RUNCVM_BIOS_DEBUG RUNCVM_KERNEL_DEBUG \
   RUNCVM_KERNEL RUNCVM_KERNEL_ROOT RUNCVM_KERNEL_APPEND RUNCVM_KERNEL_INITRAMFS_PATH RUNCVM_KERNEL_PATH RUNCVM_DISKS \
   RUNCVM_UIDGID RUNCVM_VM_MOUNTPOINT RUNCVM_TMPFS \
@@ -27,4 +28,30 @@ load_network() {
   [ -d /.runcvm/network/devices ] && [ -s /.runcvm/network/devices/$if ] || return 1
   read -r DOCKER_IF DOCKER_IF_MAC DOCKER_IF_MTU DOCKER_IF_IP DOCKER_IF_IP_NETPREFIX  DOCKER_IF_IP_GW </.runcvm/network/devices/$if
   return 0
-}
+}
+
+which() {
+  local cmd="$1"
+  local WHICH_PATH="${RUNCVM_PATH//:/ }" # Replace ':' with ' '
+  for p in $WHICH_PATH; do [ -x "$p/$cmd" ] && echo "$p/$cmd" && return 0; done
+  return 1
+}
+
+create_aliases() {
+  for cmd in \
+      bash \
+      busybox awk cat chgrp chmod cut grep head hostname init ln ls mkdir poweroff ps rm sh sysctl touch tr \
+      ip jq \
+      dnsmasq \
+      blkid findmnt getent mke2fs mount nc \
+      xtables-nft-multi xtables-legacy-multi \
+      qemu-system-x86_64 qemu-ga \
+      dbclient dropbear dropbearkey \
+      s6-applyuidgid \
+      tput
+  do
+    eval "$cmd() { $(which $cmd) \"\$@\"; }"
+  done
+}
+
+create_aliases

runcvm-scripts/runcvm-ctr-entrypoint

@@ -1,4 +1,4 @@
-#!/opt/runcvm/bin/bash
+#!/.runcvm/guest/bin/bash
 
 # DEBUG
 if [[ "$RUNCVM_BREAK" =~ (prenet|postnet) ]]; then set -x; fi
@@ -55,7 +55,7 @@ printf "%s\n" "${args[@]}" >/.runcvm/entrypoint
 # If not, then we set HOME to the requested user's default homedir in accordance with https://github.com/moby/moby/issues/2968.
 
 if [ "$RUNCVM_HAS_HOME" == "0" ]; then
-  HOME=$(/opt/runcvm/usr/bin/getent passwd "${RUNCVM_UIDGID%%:*}" | /opt/runcvm/bin/cut -d':' -f6)
+  HOME=$($RUNCVM_GUEST/usr/bin/getent passwd "${RUNCVM_UIDGID%%:*}" | $RUNCVM_GUEST/bin/cut -d':' -f6)
 fi
 
 # SAVE ENVIRONMENT
@@ -64,10 +64,10 @@ export -n SHLVL OLDPWD
 export >/.runcvm/config
 
 # NOW LOAD DEFAULT ENV AND PATH
-. /opt/runcvm/scripts/runcvm-ctr-defaults && PATH="$RUNCVM_PATH"
+. $RUNCVM_GUEST/scripts/runcvm-ctr-defaults
 
 # LOAD IP MANIPULATION FUNCTIONS
-. $RUNCVM/scripts/runcvm-ip-functions
+. $RUNCVM_GUEST/scripts/runcvm-ip-functions
 
 # SAVE PWD
 busybox pwd >/.runcvm/pwd
@@ -138,18 +138,18 @@ do
     ip route add default via $DOCKER_GW_IF_IP dev $QEMU_BRIDGE
 
     # Accept DNS requests for $RUNCVM_DNS_IP; these will be passed to dnsmasq
-    XTABLES_LIBDIR=/opt/runcvm/lib64/usr/lib/xtables/ /opt/runcvm/sbin/xtables-nft-multi iptables -t nat -A PREROUTING -d $RUNCVM_DNS_IP/32 -p udp -m udp --dport 53 -j REDIRECT
+    XTABLES_LIBDIR=$RUNCVM_GUEST/usr/lib/xtables xtables-nft-multi iptables -t nat -A PREROUTING -d $RUNCVM_DNS_IP/32 -p udp -m udp --dport 53 -j REDIRECT
 
     # Match UDP port 53 traffic, outgoing via the QEMU bridge, from the bridge's own IP:
     # -> Masquerade as if from the VM's IP.
     #    This allows outgoing DNS requests from the VM to be received by dnsmasq running in the container.
-    XTABLES_LIBDIR=/opt/runcvm/lib64/usr/lib/xtables/ /opt/runcvm/sbin/xtables-nft-multi iptables -t nat -A POSTROUTING -o $QEMU_BRIDGE -s $QEMU_BRIDGE_IP/32 -p udp -m udp --sport 53 -j SNAT --to-source $DOCKER_IF_IP
-    XTABLES_LIBDIR=/opt/runcvm/lib64/usr/lib/xtables/ /opt/runcvm/sbin/xtables-nft-multi iptables -t nat -A POSTROUTING -o $QEMU_BRIDGE -s $QEMU_BRIDGE_IP/32 -p udp -m udp --dport 53 -j SNAT --to-source $DOCKER_IF_IP
+    XTABLES_LIBDIR=$RUNCVM_GUEST/usr/lib/xtables xtables-nft-multi iptables -t nat -A POSTROUTING -o $QEMU_BRIDGE -s $QEMU_BRIDGE_IP/32 -p udp -m udp --sport 53 -j SNAT --to-source $DOCKER_IF_IP
+    XTABLES_LIBDIR=$RUNCVM_GUEST/usr/lib/xtables xtables-nft-multi iptables -t nat -A POSTROUTING -o $QEMU_BRIDGE -s $QEMU_BRIDGE_IP/32 -p udp -m udp --dport 53 -j SNAT --to-source $DOCKER_IF_IP
 
     # Match traffic on TCP port $SSHD_PORT, outgoing via the QEMU bridge, from the bridge's own IP:
     # -> Masquerade it as if from the DNS_IP.
     #    This is necessary to allow SSH from within the container to the VM.
-    XTABLES_LIBDIR=/opt/runcvm/lib64/usr/lib/xtables/ /opt/runcvm/sbin/xtables-nft-multi iptables -t nat -A POSTROUTING -o $QEMU_BRIDGE -s $QEMU_BRIDGE_IP/32 -p tcp -m tcp --dport $SSHD_PORT -j SNAT --to-source $RUNCVM_DNS_IP
+    XTABLES_LIBDIR=$RUNCVM_GUEST/usr/lib/xtables xtables-nft-multi iptables -t nat -A POSTROUTING -o $QEMU_BRIDGE -s $QEMU_BRIDGE_IP/32 -p tcp -m tcp --dport $SSHD_PORT -j SNAT --to-source $RUNCVM_DNS_IP
   fi
 
 done
@@ -165,11 +165,11 @@ echo "$RESOLV_CONF_NEW" >/vm/etc/resolv.conf
 dnsmasq -u root --no-hosts
 
 # LAUNCH VIRTIOFSD
-/opt/runcvm/scripts/runcvm-ctr-virtiofsd &
+$RUNCVM_GUEST/scripts/runcvm-ctr-virtiofsd &
 
 # DEBUG
 if [[ "$RUNCVM_BREAK" =~ postnet ]]; then bash; fi
 
 # LAUNCH INIT SUPERVISING QEMU
 # FIXME: Add -v to debug
-exec /opt/runcvm/sbin/runcvm-init -c /opt/runcvm/scripts/runcvm-ctr-qemu
+exec $RUNCVM_GUEST/sbin/runcvm-init -c $RUNCVM_GUEST/scripts/runcvm-ctr-qemu

runcvm-scripts/runcvm-ctr-exec

@@ -1,12 +1,12 @@
-#!/opt/runcvm/bin/bash -e
+#!/.runcvm/guest/bin/bash -e
 
 # See https://qemu-project.gitlab.io/qemu/interop/qemu-ga-ref.html
 
-. /opt/runcvm/scripts/runcvm-ctr-defaults
+# Load original environment
+. /.runcvm/config
 
-busybox() {
-  $RUNCVM/bin/busybox "$@"
-}
+# Load defaults and aliases
+. $RUNCVM_GUEST/scripts/runcvm-ctr-defaults
 
 env() {
   busybox env "$@"
@@ -19,14 +19,15 @@ to_bin() {
 
 # Expects:
 # - To be run as root
+# - To be given env vars
 # - To be given arguments
 #   $1 <uid>:<gid>:<additionalGids>
 #   $2 <cwd>
 #   $3 <ENV-HOME-boolean>
 #   $4 <wantsTerminal-boolean>
 #   $(5...) <command> <args>
 
-command="$RUNCVM/scripts/runcvm-vm-exec"
+command="$RUNCVM_GUEST/scripts/runcvm-vm-exec"
 uidgid="$1"
 cwd="$2"
 hasHome="$3"
@@ -53,7 +54,7 @@ fi
 
 if [ "$hasHome" != "1" ]; then
   # Either this script needs to look up uid's HOME or else runcvm-vm-exec does; for now, we do it here.
-  HOME=$($RUNCVM/usr/bin/getent passwd "$uid" | $RUNCVM/bin/cut -d':' -f6)
+  HOME=$(getent passwd "$uid" | cut -d':' -f6)
 fi
 
 # Clean RUNCVM env vars
@@ -75,4 +76,4 @@ if ! [ -s /.runcvm/dropbear/key ] || ! load_network; then
   exit 1
 fi
 
-exec $RUNCVM/usr/bin/dbclient ${opts[@]} -p $SSHD_PORT -y -y -i /.runcvm/dropbear/key root@$DOCKER_IF_IP "$command '$uidgid' '$(echo -n $cwd | to_bin)' '$args_bin' '$env_bin'"
+exec $RUNCVM_GUEST/usr/bin/dbclient "${opts[@]}" -p $SSHD_PORT -y -y -i /.runcvm/dropbear/key root@$DOCKER_IF_IP "$command '$uidgid' '$(echo -n $cwd | to_bin)' '$args_bin' '$env_bin'"

runcvm-scripts/runcvm-ctr-exit

@@ -1,6 +1,10 @@
-#!/opt/runcvm/bin/sh
+#!/.runcvm/guest/bin/bash
 
-. /opt/runcvm/scripts/runcvm-ctr-defaults && PATH="$RUNCVM_PATH"
+# Load original environment
+. /.runcvm/config
+
+# Load defaults and aliases
+. $RUNCVM_GUEST/scripts/runcvm-ctr-defaults
 
 # runcvm-init execs this script when it exits.
 # It:

runcvm-scripts/runcvm-ctr-qemu

@@ -1,15 +1,18 @@
-#!/opt/runcvm/bin/bash
+#!/.runcvm/guest/bin/bash
 
-# TODO: Clean up ENV vars
+# Exit on errors
+set -o errexit -o pipefail
+
+# Load original environment
 . /.runcvm/config
 
-# Load defaults after exports (so that PATH is overridden)
-. /opt/runcvm/scripts/runcvm-ctr-defaults && PATH="$RUNCVM_PATH"
+# Load defaults
+. $RUNCVM_GUEST/scripts/runcvm-ctr-defaults && unset PATH
 
-QEMU_IFUP="$RUNCVM/scripts/runcvm-ctr-qemu-ifup"
-QEMU_IFDOWN="$RUNCVM/scripts/runcvm-ctr-qemu-ifdown"
+QEMU_IFUP="$RUNCVM_GUEST/scripts/runcvm-ctr-qemu-ifup"
+QEMU_IFDOWN="$RUNCVM_GUEST/scripts/runcvm-ctr-qemu-ifdown"
 
-INIT="init=/opt/runcvm/scripts/runcvm-vm-init"
+INIT="init=$RUNCVM_GUEST/scripts/runcvm-vm-init"
 
 error() {
   echo "$1" >&2
@@ -115,10 +118,10 @@ fi
 # - Consider using '-device pvpanic'
 
 if [ "$RUNCVM_ARCH" = "arm64" ]; then
-  CMD="qemu-system-aarch64"
+  CMD="$(which qemu-system-aarch64)"
   MACHINE+=(-cpu max -machine virt,gic-version=max,usb=off)
 else
-  CMD="qemu-system-x86_64"
+  CMD="$(which qemu-system-x86_64)"
   MACHINE+=(-enable-kvm -cpu host,pmu=off -machine q35,accel=kvm,usb=off,sata=off -device isa-debug-exit)
 fi
 
@@ -249,6 +252,6 @@ ARGS=(
   -append "$RUNCVM_KERNEL_ROOT $INIT rw ${APPEND[*]} $RUNCVM_KERNEL_APPEND"
 )
 
-if [[ "$RUNCVM_BREAK" =~ preqemu ]]; then echo Preparing to run: $CMD "${ARGS[@]@Q}"; bash; fi
+if [[ "$RUNCVM_BREAK" =~ preqemu ]]; then echo Preparing to run: '$CMD' "${ARGS[@]@Q}"; bash; fi
 
-exec $CMD "${ARGS[@]}"
+exec "$CMD" "${ARGS[@]}"

runcvm-scripts/runcvm-ctr-qemu-ifdown

@@ -1,6 +1,10 @@
-#!/opt/runcvm/bin/bash
+#!/.runcvm/guest/bin/bash
 
-. /opt/runcvm/scripts/runcvm-ctr-defaults && PATH="$RUNCVM_PATH"
+# Load original environment
+. /.runcvm/config
+
+# Load defaults and aliases
+. $RUNCVM_GUEST/scripts/runcvm-ctr-defaults
 
 ip link set dev "$1" down || true
 exit 0

runcvm-scripts/runcvm-ctr-qemu-ifup

@@ -1,6 +1,10 @@
-#!/opt/runcvm/bin/bash
+#!/.runcvm/guest/bin/bash
 
-. /opt/runcvm/scripts/runcvm-ctr-defaults && PATH="$RUNCVM_PATH"
+# Load original environment
+. /.runcvm/config
+
+# Load defaults and aliases
+. $RUNCVM_GUEST/scripts/runcvm-ctr-defaults
 
 tap="$1"
 if="$(busybox sed 's/tap-//' <<<$tap)"

runcvm-scripts/runcvm-ctr-qemu-poweroff

@@ -1,5 +1,9 @@
-#!/opt/runcvm/bin/bash
+#!/.runcvm/guest/bin/bash
 
-. /opt/runcvm/scripts/runcvm-ctr-defaults && PATH="$RUNCVM_PATH"
+# Load original environment
+. /.runcvm/config
 
-echo "system_powerdown" | nc -w 1 -U $QEMU_MONITOR_SOCKET
+# Load defaults and aliases
+. $RUNCVM_GUEST/scripts/runcvm-ctr-defaults
+
+echo "system_powerdown" | nc -w 1 -U $QEMU_MONITOR_SOCKET