New experimental --env=RUNCVM_HUGETLB=1 option

- New experimental `--env=RUNCVM_HUGETLB=1` option enables use of
  preallocated hugetlb memory backend, which can improve performance in
  some scenarios

Author: struanb

README.md

@@ -339,6 +339,10 @@ Enable breakpoints (falling to bash shell) during the RunCVM container/VM boot p
 
 `<values>` must be a comma-separated list of: `prenet`, `postnet`, `preqemu`.
 
+### `--env=RUNCVM_HUGETLB=1`
+
+**[EXPERIMENTAL]** Enable use of preallocated hugetlb memory backend, which can improve performance in some scenarios.
+
 ## Advanced usage
 
 ### Running Docker in a RunCVM container/VM

runcvm-scripts/runcvm-ctr-defaults

@@ -13,7 +13,8 @@ clean_env() {
   RUNCVM_RUNTIME_DEBUG RUNCVM_BIOS_DEBUG RUNCVM_KERNEL_DEBUG \
   RUNCVM_KERNEL RUNCVM_KERNEL_ROOT RUNCVM_KERNEL_APPEND RUNCVM_KERNEL_INITRAMFS_PATH RUNCVM_KERNEL_PATH RUNCVM_DISKS \
   RUNCVM_UIDGID RUNCVM_VM_MOUNTPOINT RUNCVM_TMPFS \
-  RUNCVM_CPUS RUNCVM_MEM_SIZE RUNCVM_HAS_HOME
+  RUNCVM_CPUS RUNCVM_MEM_SIZE RUNCVM_HUGETLB \
+  RUNCVM_HAS_HOME
 
   # May be set in VM by busybox init process
   export -n USER

runcvm-scripts/runcvm-ctr-qemu

@@ -142,18 +142,31 @@ fi
 # Disable IPv6, which is currently unsupported, at kernel boot time
 APPEND+=(ipv6.disable=1)
 
+# 16-64 works well and is more performant than 1024 in some scenarios.
+# For now, stick with original figure.
+VIRTIOFS_QUEUE_SIZE=1024
+
+if [ "$RUNCVM_HUGETLB" != "1" ]; then
+  # Tests suggests prealloc=on slows down mem-path=/dev/shm
+  MEM_PATH=/dev/shm MEM_PREALLOC=off
+  MEM_BACKEND="-object memory-backend-file,id=mem,size=$RUNCVM_MEM_SIZE,mem-path=$MEM_PATH,share=on,prealloc=$MEM_PREALLOC"
+else
+  # Fastest performance: +15% CPU/net intensive; 3.5x disk intensive.
+  MEM_BACKEND="-object memory-backend-memfd,id=mem,size=$RUNCVM_MEM_SIZE,share=on,prealloc=on,hugetlb=on"
+fi
+
 ARGS=(
   "${OPTS[@]}"
   -no-user-config
   -nodefaults
   -serial $SERIAL
   -m "$RUNCVM_MEM_SIZE"
   -chardev socket,id=char0,path=$QEMU_VIRTIOFSD_SOCKET
-  -device vhost-user-fs-pci,queue-size=1024,chardev=char0,tag=myfs,ats=on
+  -device vhost-user-fs-pci,queue-size=$VIRTIOFS_QUEUE_SIZE,chardev=char0,tag=myfs,ats=on
   -kernel $RUNCVM_KERNEL_PATH
   -initrd $RUNCVM_KERNEL_INITRAMFS_PATH
   -append "$RUNCVM_KERNEL_ROOT $INIT rw ${APPEND[*]} $PANIC $RUNCVM_KERNEL_APPEND $BREAK"
-  -object memory-backend-file,id=mem,size="$RUNCVM_MEM_SIZE",mem-path=/dev/shm,share=on
+  $MEM_BACKEND
   -numa node,memdev=mem
   -smp $RUNCVM_CPUS
   # Configure host/container tap device with PXE roms disabled

runcvm-scripts/runcvm-runtime

@@ -427,7 +427,13 @@ if [ "$COMMAND" = "create" ]; then
   fi
   log "RUNCVM_MEM_SIZE=$RUNCVM_MEM_SIZE"
   set_config_env "RUNCVM_MEM_SIZE" "$RUNCVM_MEM_SIZE"
-  jq_set "$CFG" --arg size "$RUNCVM_MEM_SIZE" '( .mounts[] | select(.destination == "/dev/shm") ) = {"destination": "/dev/shm","type": "tmpfs","source": "shm","options": ["nosuid","noexec","nodev","mode=1777","size=" + $size]}'
+
+  RUNCVM_HUGETLB=$(get_config_env "RUNCVM_HUGETLB")
+  if [ "$RUNCVM_HUGETLB" != "1" ]; then
+    jq_set "$CFG" --arg size "$RUNCVM_MEM_SIZE" '( .mounts[] | select(.destination == "/dev/shm") ) = {"destination": "/dev/shm","type": "tmpfs","source": "shm","options": ["nosuid","noexec","nodev","mode=1777","size=" + $size]}'
+  # else
+    # --shm-size applies; default 64m.
+  fi
 
   # In future, set the container memory limit to something reasonable to support
   # QEMU + virtiofsd + dnsmasq. Perhaps $RUNCVM_MEM_LIMIT+K (or vice-versa, reduce