[PATCH 3/5] vm: boot sub-VMs through verified roots

[colbyt <colby@colbyt.com>]

Pass the generated initramfs and root hash through the app VM and net VM
boot paths for QEMU, Cloud Hypervisor, crosvm, and start-vmm.

For start-vmm, include the initramfs in the payload config and validate
the root hash before adding roothash= to the kernel command line.

Signed-off-by: colbyt <colby@colbyt.com>
---
 img/app/Makefile                          | 15 +++++++++------
 tools/start-vmm/ch.rs                     |  3 ++-
 tools/start-vmm/lib.rs                    | 15 ++++++++++++---
 tools/start-vmm/tests/vm_command-basic.rs | 18 ++++++++++++++++--
 vm-lib/make-vm.nix                        |  2 ++
 vm/sys/net/Makefile                       | 15 +++++++++------
 6 files changed, 50 insertions(+), 18 deletions(-)

diff --git a/img/app/Makefile b/img/app/Makefile
index 3f60ef8..96ba2aa 100644
--- a/img/app/Makefile
+++ b/img/app/Makefile
@@ -112,10 +112,11 @@ start-virtiofsd: scripts/start-virtiofsd.elb
 	scripts/start-virtiofsd.elb
 .PHONY: start-virtiofsd
 
-run-qemu: $(imgdir)/appvm/blk/root.img start-vhost-user-net start-virtiofsd
+run-qemu: $(imgdir)/appvm/initramfs $(imgdir)/appvm/blk/root.img $(imgdir)/appvm/blk/rootfs.verity.roothash start-vhost-user-net start-virtiofsd
 	@../../scripts/run-qemu.sh -m 256 -cpu max -kernel $(KERNEL) -vga none \
+	    -initrd $(imgdir)/appvm/initramfs \
 	    -drive file=$(imgdir)/appvm/blk/root.img,if=virtio,format=raw,readonly=on \
-	    -append "root=PARTLABEL=root nokaslr" \
+	    -append "ro roothash=$$(< $(imgdir)/appvm/blk/rootfs.verity.roothash) nokaslr" \
 	    -gdb unix:build/gdb.sock,server,nowait \
 	    -chardev socket,id=vhost-user-net,path=build/vhost-user-net.sock \
 	    -netdev vhost-user,id=net0,chardev=vhost-user-net \
@@ -133,7 +134,7 @@ run-qemu: $(imgdir)/appvm/blk/root.img start-vhost-user-net start-virtiofsd
 	    -device virtconsole,chardev=virtiocon0
 .PHONY: run-qemu
 
-run-cloud-hypervisor: $(imgdir)/appvm/blk/root.img start-vhost-user-gpu start-vhost-user-net start-virtiofsd
+run-cloud-hypervisor: $(imgdir)/appvm/initramfs $(imgdir)/appvm/blk/root.img $(imgdir)/appvm/blk/rootfs.verity.roothash start-vhost-user-gpu start-vhost-user-net start-virtiofsd
 	rm -f build/vmm.sock build/vsock.sock
 	@../../scripts/run-cloud-hypervisor.sh \
 	    --api-socket path=build/vmm.sock \
@@ -144,15 +145,17 @@ run-cloud-hypervisor: $(imgdir)/appvm/blk/root.img start-vhost-user-gpu start-vh
 	    --vsock cid=3,socket=build/vsock.sock \
 	    --net mac=02:00:00:00:00:01,vhost_user=on,socket=build/vhost-user-net.sock \
 	    --kernel $(KERNEL) \
-	    --cmdline "root=PARTLABEL=root" \
+	    --initramfs $(imgdir)/appvm/initramfs \
+	    --cmdline "ro roothash=$$(< $(imgdir)/appvm/blk/rootfs.verity.roothash)" \
 	    --console tty \
 	    --serial file=build/serial.log
 .PHONY: run-cloud-hypervisor
 
-run-crosvm: $(imgdir)/appvm/blk/root.img start-vhost-user-gpu start-virtiofsd
+run-crosvm: $(imgdir)/appvm/initramfs $(imgdir)/appvm/blk/root.img $(imgdir)/appvm/blk/rootfs.verity.roothash start-vhost-user-gpu start-virtiofsd
 	../../scripts/with-taps.elb $(CROSVM_RUN) \
 	    -b path=$(imgdir)/appvm/blk/root.img,ro=true \
-	    -p "console=ttyS0 root=PARTLABEL=root" \
+	    --initrd $(imgdir)/appvm/initramfs \
+	    -p "console=ttyS0 ro roothash=$$(< $(imgdir)/appvm/blk/rootfs.verity.roothash)" \
 	    --net tap-name=tap0,mac=02:00:00:00:00:01 \
 	    --vhost-user fs,socket=build/virtiofsd.sock \
 	    --vhost-user gpu,socket=build/vhost-user-gpu.sock \
diff --git a/tools/start-vmm/ch.rs b/tools/start-vmm/ch.rs
index 239d08e..714c929 100644
--- a/tools/start-vmm/ch.rs
+++ b/tools/start-vmm/ch.rs
@@ -56,7 +56,8 @@ pub struct MemoryConfig {
 #[derive(Serialize)]
 pub struct PayloadConfig {
     pub kernel: String,
-    pub cmdline: &'static str,
+    pub initramfs: String,
+    pub cmdline: String,
 }
 
 #[derive(Serialize)]
diff --git a/tools/start-vmm/lib.rs b/tools/start-vmm/lib.rs
index 0fa6352..99a6a43 100644
--- a/tools/start-vmm/lib.rs
+++ b/tools/start-vmm/lib.rs
@@ -9,7 +9,7 @@ mod s6;
 use std::borrow::Cow;
 use std::env::args_os;
 use std::ffi::OsStr;
-use std::fs::File;
+use std::fs::{File, read_to_string};
 use std::hash::{Hash, Hasher};
 use std::io::ErrorKind;
 use std::path::Path;
@@ -43,8 +43,16 @@ pub fn vm_config(vm_dir: &Path) -> Result<VmConfig, String> {
 
     let config_dir = vm_dir.join("config");
     let blk_dir = config_dir.join("blk");
+    let initramfs_path = config_dir.join("initramfs");
     let kernel_path = config_dir.join("vmlinux");
     let net_providers_dir = config_dir.join("providers/net");
+    let roothash_path = blk_dir.join("rootfs.verity.roothash");
+    let roothash = read_to_string(&roothash_path)
+        .map_err(|e| format!("reading {roothash_path:?}: {e}"))?;
+    let roothash = roothash.trim();
+    if roothash.len() != 64 || !roothash.bytes().all(|b| b.is_ascii_hexdigit()) {
+        return Err(format!("invalid dm-verity roothash in {roothash_path:?}"));
+    }
 
     Ok(VmConfig {
         console: ConsoleConfig {
@@ -155,10 +163,11 @@ pub fn vm_config(vm_dir: &Path) -> Result<VmConfig, String> {
         },
         payload: PayloadConfig {
             kernel: kernel_path.to_str().unwrap().to_string(),
+            initramfs: initramfs_path.to_str().unwrap().to_string(),
             #[cfg(target_arch = "x86_64")]
-            cmdline: "console=ttyS0 root=PARTLABEL=root",
+            cmdline: format!("console=ttyS0 ro roothash={roothash}"),
             #[cfg(not(target_arch = "x86_64"))]
-            cmdline: "root=PARTLABEL=root",
+            cmdline: format!("ro roothash={roothash}"),
         },
         serial: ConsoleConfig {
             mode: "File",
diff --git a/tools/start-vmm/tests/vm_command-basic.rs b/tools/start-vmm/tests/vm_command-basic.rs
index 2e9ad0c..dd4e0c9 100644
--- a/tools/start-vmm/tests/vm_command-basic.rs
+++ b/tools/start-vmm/tests/vm_command-basic.rs
@@ -11,13 +11,20 @@ fn main() -> std::io::Result<()> {
     let tmp_dir = TempDir::new()?;
 
     let vm_dir = tmp_dir.path().join("testvm");
+    let initramfs_path = vm_dir.join("config/initramfs");
     let kernel_path = vm_dir.join("config/vmlinux");
     let image_path = vm_dir.join("config/blk/root.img");
+    let roothash_path = vm_dir.join("config/blk/rootfs.verity.roothash");
 
     create_dir_all(kernel_path.parent().unwrap())?;
     create_dir_all(image_path.parent().unwrap())?;
+    File::create(&initramfs_path)?;
     File::create(&kernel_path)?;
     File::create(&image_path)?;
+    std::fs::write(
+        &roothash_path,
+        "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\n",
+    )?;
 
     let mut config = vm_config(&vm_dir).unwrap();
 
@@ -33,10 +40,17 @@ fn main() -> std::io::Result<()> {
         "/run/service/vm-services/instance/testvm/data/service/vhost-user-fs/env/virtiofsd.sock";
     assert_eq!(fs1.socket, expected);
     assert_eq!(PathBuf::from(config.payload.kernel), kernel_path);
+    assert_eq!(PathBuf::from(config.payload.initramfs), initramfs_path);
     #[cfg(target_arch = "x86_64")]
-    assert_eq!(config.payload.cmdline, "console=ttyS0 root=PARTLABEL=root");
+    assert_eq!(
+        config.payload.cmdline,
+        "console=ttyS0 ro roothash=0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"
+    );
     #[cfg(not(target_arch = "x86_64"))]
-    assert_eq!(config.payload.cmdline, "root=PARTLABEL=root");
+    assert_eq!(
+        config.payload.cmdline,
+        "ro roothash=0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"
+    );
     assert_eq!(config.memory.size, 0x40000000);
     assert!(config.memory.shared);
     assert_eq!(config.serial.mode, "File");
diff --git a/vm-lib/make-vm.nix b/vm-lib/make-vm.nix
index 7847883..3a2d941 100644
--- a/vm-lib/make-vm.nix
+++ b/vm-lib/make-vm.nix
@@ -54,6 +54,8 @@ runCommand "spectrum-vm" {
   popd
 
   ln -s /usr/lib/spectrum/img/appvm/blk/root.img "$out/blk"
+  ln -s /usr/lib/spectrum/img/appvm/blk/rootfs.verity.roothash "$out/blk"
+  ln -s /usr/lib/spectrum/img/appvm/initramfs "$out"
   ln -s /usr/lib/spectrum/img/appvm/vmlinux "$out"
 ''
 ) {}
diff --git a/vm/sys/net/Makefile b/vm/sys/net/Makefile
index d01d766..1e07850 100644
--- a/vm/sys/net/Makefile
+++ b/vm/sys/net/Makefile
@@ -86,10 +86,11 @@ start-vhost-user-net:
 	mkdir -p build
 	../../../scripts/start-passt.elb
 
-run-qemu: $(vmdir)/netvm/blk/root.img
+run-qemu: $(vmdir)/netvm/initramfs $(vmdir)/netvm/blk/root.img $(vmdir)/netvm/blk/rootfs.verity.roothash
 	@../../../scripts/run-qemu.sh -m 256 -cpu max -kernel $(KERNEL) -vga none \
+	    -initrd $(vmdir)/netvm/initramfs \
 	    -drive file=$(vmdir)/netvm/blk/root.img,if=virtio,format=raw,readonly=on \
-	    -append "root=PARTLABEL=root nokaslr" \
+	    -append "ro roothash=$$(< $(vmdir)/netvm/blk/rootfs.verity.roothash) nokaslr" \
 	    -gdb unix:build/gdb.sock,server,nowait \
 	    -netdev user,id=net0 \
 	    -device e1000e,netdev=net0 \
@@ -101,7 +102,7 @@ run-qemu: $(vmdir)/netvm/blk/root.img
 	    -device virtconsole,chardev=virtiocon0
 .PHONY: run-qemu
 
-run-cloud-hypervisor: $(vmdir)/netvm/blk/root.img start-vhost-user-net
+run-cloud-hypervisor: $(vmdir)/netvm/initramfs $(vmdir)/netvm/blk/root.img $(vmdir)/netvm/blk/rootfs.verity.roothash start-vhost-user-net
 	rm -f build/vmm.sock
 	@../../../scripts/with-taps.elb \
 	    ../../../scripts/run-cloud-hypervisor.sh \
@@ -110,15 +111,17 @@ run-cloud-hypervisor: $(vmdir)/netvm/blk/root.img start-vhost-user-net
 	    --disk path=$(vmdir)/netvm/blk/root.img,readonly=on \
 	    --net vhost_user=on,socket=build/vhost-user-net.sock tap=tap1,mac=02:01:00:00:00:01 \
 	    --kernel $(KERNEL) \
-	    --cmdline "root=PARTLABEL=root" \
+	    --initramfs $(vmdir)/netvm/initramfs \
+	    --cmdline "ro roothash=$$(< $(vmdir)/netvm/blk/rootfs.verity.roothash)" \
 	    --console tty \
 	    --serial file=build/serial.log
 .PHONY: run-cloud-hypervisor
 
-run-crosvm: $(vmdir)/netvm/blk/root.img
+run-crosvm: $(vmdir)/netvm/initramfs $(vmdir)/netvm/blk/root.img $(vmdir)/netvm/blk/rootfs.verity.roothash
 	../../../scripts/with-taps.elb $(CROSVM_RUN) \
 	    -b path=$(vmdir)/netvm/blk/root.img,ro=true \
-	    -p "console=ttyS0 root=PARTLABEL=root" \
+	    --initrd $(vmdir)/netvm/initramfs \
+	    -p "console=ttyS0 ro roothash=$$(< $(vmdir)/netvm/blk/rootfs.verity.roothash)" \
 	    --net tap-name=tap0 \
 	    --net tap-name=tap1,mac=02:01:00:00:00:01 \
 	    --serial type=file,hardware=serial,path=build/serial.log \
-- 
2.54.0