[PATCH 2/4] host/rootfs: give VMs a disk-backed directory

[Alyssa Ross <hi@alyssa.is>]

This is the first step towards persistent VM data.  For now, these
directories are never reused, and also don't get cleaned up.  Both are
things to work on in future.  Even without persistence, it's good to
not have to store everything a VM might write into its home directory
in RAM.

For AppImage and Flatpak VMs, disk-backed directories are stored on
the partition containing the AppImage or Flatpak, in the new Spectrum/
hierarchy.  This will enable locating them for later reuse.  System
VMs, on the other hand, don't have a natural partition to store data
on — there may not even be a writable partition at the time they're
launched.  Since we don't expect persistence from system VMs, we just
use a directory on a tmpfs for their "disk-backed" directories.  Doing
it this way avoids the need for multiple routes in img/app — it can
always assume it gets a "disk" directory from the host.  For manually
imported VMs, it's up to the user where these directories should be.

Signed-off-by: Alyssa Ross <hi@alyssa.is>
---
 .../using-spectrum/creating-custom-vms.adoc   | 10 ++++++---
 host/rootfs/image/etc/s6-rc/sys-vmms/up       |  4 +++-
 .../image/usr/bin/create-vm-dependencies      |  1 +
 host/rootfs/image/usr/bin/run-appimage        | 22 +++++++++++++++----
 host/rootfs/image/usr/bin/run-flatpak         | 15 ++++++++++++-
 host/rootfs/image/usr/bin/vm-import           | 12 +++++++++-
 img/app/Makefile                              |  2 +-
 img/app/image/etc/fstab                       |  1 -
 img/app/image/etc/s6-rc/app/run               |  7 +++---
 img/app/scripts/start-virtiofsd.elb           |  2 +-
 release/checks/integration/lib.c              |  3 ++-
 release/checks/integration/networking.c       |  2 +-
 release/checks/integration/portal.c           |  2 +-
 13 files changed, 64 insertions(+), 19 deletions(-)

diff --git a/Documentation/using-spectrum/creating-custom-vms.adoc b/Documentation/using-spectrum/creating-custom-vms.adoc
index 229c0140..68213c89 100644
--- a/Documentation/using-spectrum/creating-custom-vms.adoc
+++ b/Documentation/using-spectrum/creating-custom-vms.adoc
@@ -13,9 +13,13 @@ configurations are directories under a dedicated parent directory, and
 the name of each configuration directory determines the name of
 the VM.  After mounting the persistent storage partition, the
 configured VMs can be made available by running `vm-import user
-/media/4e43cdc2-82b2-4d94-8a90-b6c6189312d2/vms`, replacing
-/media/4e43cdc2-82b2-4d94-8a90-b6c6189312d2/vms with the directory
-containing the VM definitions.
+/media/4e43cdc2-82b2-4d94-8a90-b6c6189312d2/vms
+/media/4e43cdc2-82b2-4d94-8a90-b6c6189312d2/Spectrum/data/spectrum/storage`,
+replacing /media/4e43cdc2-82b2-4d94-8a90-b6c6189312d2/vms with the
+directory containing the VM definitions, and
+/media/4e43cdc2-82b2-4d94-8a90-b6c6189312d2/Spectrum/data/spectrum/storage
+with the directory where disk-backed directories for the VMs should be
+created.
 
 The directory can contain the following files:
 
diff --git a/host/rootfs/image/etc/s6-rc/sys-vmms/up b/host/rootfs/image/etc/s6-rc/sys-vmms/up
index e055de93..7d3ac5ef 100644
--- a/host/rootfs/image/etc/s6-rc/sys-vmms/up
+++ b/host/rootfs/image/etc/s6-rc/sys-vmms/up
@@ -1,4 +1,6 @@
 # SPDX-License-Identifier: EUPL-1.2+
 # SPDX-FileCopyrightText: 2024 Alyssa Ross <hi@alyssa.is>
 
-vm-import sys /usr/lib/spectrum/vm
+if { install -do fs -g fs /run/sys-vm-disk }
+
+vm-import sys /usr/lib/spectrum/vm /run/sys-vm-disk
diff --git a/host/rootfs/image/usr/bin/create-vm-dependencies b/host/rootfs/image/usr/bin/create-vm-dependencies
index 6bf12d03..b7545425 100755
--- a/host/rootfs/image/usr/bin/create-vm-dependencies
+++ b/host/rootfs/image/usr/bin/create-vm-dependencies
@@ -6,6 +6,7 @@ if {
   mkdir -p
     /run/doc/${1}/doc
     /run/fs/${1}/config
+    /run/fs/${1}/disk
     /run/fs/${1}/doc
     /run/vm/by-id/${1}/ns
 }
diff --git a/host/rootfs/image/usr/bin/run-appimage b/host/rootfs/image/usr/bin/run-appimage
index b9464f8b..f62844f8 100755
--- a/host/rootfs/image/usr/bin/run-appimage
+++ b/host/rootfs/image/usr/bin/run-appimage
@@ -25,6 +25,18 @@ if {
   if { create-vm-dependencies $id }
 }
 
+backtick diskdir {
+  s6-setuidgid fs
+
+  backtick -E mountpoint {
+    importas -Siu 1
+    findmnt -no TARGET -T $1
+  }
+
+  if { mkdir -p -- ${mountpoint}/Spectrum/data/spectrum/storage }
+  mktemp -d -- ${mountpoint}/Spectrum/data/spectrum/storage/tmp.XXXXXX
+}
+
 if {
   s6-envuidgid fs
   s6-applyuidgid -Uzu 0
@@ -32,15 +44,17 @@ if {
   multisubstitute {
     importas -Siu id
     importas -Siu 1
+    importas -Siu diskdir
   }
 
   nsenter --preserve-credentials -S0
     --mount=/run/vm/by-id/${id}/ns/mnt
     --user=/run/vm/by-id/${id}/ns/user
-  cd /run/fs/${id}/config
-  if { redirfd -w 1 type echo appimage }
-  if { touch run }
-  mount --bind $1 run
+  cd /run/fs/${id}
+  if { redirfd -w 1 config/type echo appimage }
+  if { touch config/run }
+  if { mount --bind $1 config/run }
+  mount --bind -- $diskdir disk
 }
 
 importas -Siu id
diff --git a/host/rootfs/image/usr/bin/run-flatpak b/host/rootfs/image/usr/bin/run-flatpak
index 2d3e7ea0..9a7ffa33 100755
--- a/host/rootfs/image/usr/bin/run-flatpak
+++ b/host/rootfs/image/usr/bin/run-flatpak
@@ -25,21 +25,34 @@ if {
   if { create-vm-dependencies $id }
 }
 
+backtick diskdir {
+  s6-setuidgid fs
+
+  importas -Siu 1
+
+  if { mkdir -p -- ${1}/Spectrum/data/spectrum/storage }
+  mktemp -d -- ${1}/Spectrum/data/spectrum/storage/tmp.XXXXXX
+}
+
 if {
   s6-envuidgid fs
   s6-applyuidgid -Uzu 0
 
   multisubstitute {
     importas -Siu id
+    importas -Siu diskdir
     elgetpositionals
   }
 
   nsenter --preserve-credentials -S0
     --mount=/run/vm/by-id/${id}/ns/mnt
     --user=/run/vm/by-id/${id}/ns/user
+
   cd /run/fs/${id}/config
   if { redirfd -w 1 type echo flatpak }
-  mount-flatpak $@
+  if { mount-flatpak $@ }
+
+  mount --bind -- $diskdir /run/fs/${id}/disk
 }
 
 importas -Siu id
diff --git a/host/rootfs/image/usr/bin/vm-import b/host/rootfs/image/usr/bin/vm-import
index 014eab87..209be787 100755
--- a/host/rootfs/image/usr/bin/vm-import
+++ b/host/rootfs/image/usr/bin/vm-import
@@ -1,4 +1,4 @@
-#!/bin/execlineb -S2
+#!/bin/execlineb -S3
 # SPDX-License-Identifier: EUPL-1.2+
 # SPDX-FileCopyrightText: 2023-2024 Alyssa Ross <hi@alyssa.is>
 
@@ -20,4 +20,14 @@ if { ln -s -- /run/service/vmm/instance/${id} /run/vm/by-id/${id}/service }
 
 if { create-vm-dependencies $id }
 
+if {
+  s6-envuidgid fs
+  s6-applyuidgid -Uzu 0
+  nsenter --preserve-credentials -S0
+    --mount=/run/vm/by-id/${id}/ns/mnt
+    --user=/run/vm/by-id/${id}/ns/user
+  backtick -E diskdir { mktemp -d -- ${3}/tmp.XXXXXX }
+  mount --bind -- ${diskdir} /run/fs/${id}/disk
+}
+
 s6-instance-create -- /run/service/vmm $id
diff --git a/img/app/Makefile b/img/app/Makefile
index 7e3d05b2..2e720a91 100644
--- a/img/app/Makefile
+++ b/img/app/Makefile
@@ -30,7 +30,7 @@ $(imgdir)/appvm/blk/root.img: ../../scripts/make-gpt.sh ../../scripts/sfdisk-fie
 	    build/rootfs.erofs:root:5460386f-2203-4911-8694-91400125c604:root
 	mv $@.tmp $@
 
-DIRS = dev home/user host run mnt proc sys tmp \
+DIRS = dev host run mnt proc sys tmp \
 	etc/s6-linux-init/run-image/pipewire \
 	etc/s6-linux-init/run-image/service \
 	etc/s6-linux-init/run-image/user \
diff --git a/img/app/image/etc/fstab b/img/app/image/etc/fstab
index 5f78ab87..f51eace0 100644
--- a/img/app/image/etc/fstab
+++ b/img/app/image/etc/fstab
@@ -5,4 +5,3 @@ devpts	/dev/pts	devpts	nosuid,noexec,gid=5,mode=620	0	0
 tmpfs	/dev/shm	tmpfs	nosuid,nodev				0	0
 sysfs	/sys		sysfs	nosuid,nodev,noexec			0	0
 tmpfs	/tmp		tmpfs	nosuid,nodev				0	0
-tmpfs	/home/user	tmpfs	nodev,mode=0700,uid=1000,gid=1000	0	0
diff --git a/img/app/image/etc/s6-rc/app/run b/img/app/image/etc/s6-rc/app/run
index f91877d4..f36d153c 100755
--- a/img/app/image/etc/s6-rc/app/run
+++ b/img/app/image/etc/s6-rc/app/run
@@ -4,11 +4,12 @@
 
 export TMPDIR /run
 
-export HOME /home/user
-cd /home/user
-
 if { /etc/mdev/wait virtiofs-host }
 
+if { install -do user -g user /host/disk/home }
+export HOME /host/disk/home
+cd /host/disk/home
+
 foreground {
   redirfd -r 0 /host/config/type
   withstdinas -E type
diff --git a/img/app/scripts/start-virtiofsd.elb b/img/app/scripts/start-virtiofsd.elb
index 9efb436b..d861a22b 100755
--- a/img/app/scripts/start-virtiofsd.elb
+++ b/img/app/scripts/start-virtiofsd.elb
@@ -7,7 +7,7 @@ background {
   if { mkdir -p build/fs }
   unshare -rUm
   if { mount -t tmpfs -o nosuid,nodev fs build/fs }
-  if { mkdir build/fs/config }
+  if { mkdir build/fs/config build/fs/disk }
   if { importas -Si CONFIG mount --rbind -- ${CONFIG}/fs build/fs/config }
   unshare --map-user 1000 --map-group 1000
   importas -SsD virtiofsd VIRTIOFSD
diff --git a/release/checks/integration/lib.c b/release/checks/integration/lib.c
index 51f6bae7..3dcce471 100644
--- a/release/checks/integration/lib.c
+++ b/release/checks/integration/lib.c
@@ -195,6 +195,7 @@ struct vm *start_qemu(struct config c)
 		"-drive", nullptr,
 		"-drive", nullptr,
 		"-smbios", nullptr,
+		"-snapshot",
 		"-m", "4G",
 		"-nodefaults",
 		"-machine", "virtualization=on",
@@ -242,7 +243,7 @@ struct vm *start_qemu(struct config c)
 
 		if (asprintf(efi_arg, "file=%s,format=raw,if=pflash,readonly=true", c.drives.efi) == -1 ||
 		    asprintf(img_arg, "file=%s,format=raw,if=virtio,readonly=true", c.drives.img) == -1 ||
-		    asprintf(user_data_arg, "file=%s,format=raw,if=virtio,readonly=true", c.drives.user_data) == -1 ||
+		    asprintf(user_data_arg, "file=%s,format=raw,if=virtio", c.drives.user_data) == -1 ||
 		    asprintf(console_arg, "type=11,value=io.systemd.stub.kernel-cmdline-extra=%s%s",
 		             c.serial.console ? "console=" : "",
 		             c.serial.console ? c.serial.console : "") == -1) {
diff --git a/release/checks/integration/networking.c b/release/checks/integration/networking.c
index 078e31fc..6450d2d2 100644
--- a/release/checks/integration/networking.c
+++ b/release/checks/integration/networking.c
@@ -151,7 +151,7 @@ void test(struct config c)
 	          "mkdir /run/mnt && "
 	          "mount \"$(findfs UUID=a7834806-2f82-4faf-8ac4-4f8fd8a474ca)\" /run/mnt && "
 	          "s6-rc -bu change vmm-env && "
-	          "vm-import user /run/mnt/vms && "
+	          "vm-import user /run/mnt/vms /run/sys-vm-disk && "
 	          "vm-start \"$(basename \"$(readlink /run/vm/by-name/user.nc)\")\" && "
 	          "tail -Fc +0 /run/log/current /run/*.log &\n",
 	          vm_console_writer(vm)) == EOF) {
diff --git a/release/checks/integration/portal.c b/release/checks/integration/portal.c
index 6ba5654a..97a06738 100644
--- a/release/checks/integration/portal.c
+++ b/release/checks/integration/portal.c
@@ -16,7 +16,7 @@ void test(struct config c)
 	          "mkdir /run/mnt && "
 	          "mount \"$(findfs UUID=a7834806-2f82-4faf-8ac4-4f8fd8a474ca)\" /run/mnt && "
 	          "s6-rc -bu change vmm-env && "
-	          "vm-import user /run/mnt/vms && "
+	          "vm-import user /run/mnt/vms /run/sys-vm-disk && "
 	          "(tail -Fc +0 /run/*.log &) && "
 	          "s6-svc -O /run/vm/by-name/user.portal/service && "
 	          "vm-start \"$(basename \"$(readlink /run/vm/by-name/user.portal)\")\" && "
-- 
2.51.0