[PATCH 15/22] host/start-vm: resolve VM symlinks with /ext root

[Alyssa Ross <hi@alyssa.is>]

Nix profile generations are always symlinks to store paths, i.e. paths
starting with /nix.  On the Spectrum host, these paths would resolve
to the host's /nix, which isn't going to contain the user's VM
definitions.

So to enable managing VM definitions with Nix profiles, the host needs
to resolve VM definition symlinks specially.  It needs to consider the
root directory to be somewhere on the user data partition, so that the
user can place their Nix store at /nix/store.

This special chroot-like symlink resolution is only done to open the
VM definition directory.  Configuration within VM definition
directories is opened as normal.

It would probably be better to use a subdirectory of /ext
(e.g. /ext/vm-root) as the root directory for resolving symlinks,
to keep the root directory of the user data partition tidy, and to
make it clearer that the Nix store used for VM definitions is not a
global Nix store for all things on the user data partition.  I haven't
done that here yet, because it would require handling the additional
case of the root directory not existing.

Signed-off-by: Alyssa Ross <hi@alyssa.is>
---
 host/start-vm/fs.c                            | 17 +++++
 host/start-vm/fs.rs                           | 68 +++++++++++++++++++
 host/start-vm/lib.rs                          | 32 +++++----
 host/start-vm/meson.build                     |  2 +-
 host/start-vm/start-vm.rs                     | 15 ++--
 host/start-vm/tests/meson.build               |  2 +
 host/start-vm/tests/vm_command-basic.rs       |  4 +-
 .../tests/vm_command-config-symlink.rs        | 30 ++++++++
 host/start-vm/tests/vm_command-shared-dir.rs  |  4 +-
 9 files changed, 152 insertions(+), 22 deletions(-)
 create mode 100644 host/start-vm/fs.c
 create mode 100644 host/start-vm/fs.rs
 create mode 100644 host/start-vm/tests/vm_command-config-symlink.rs

diff --git a/host/start-vm/fs.c b/host/start-vm/fs.c
new file mode 100644
index 0000000..d8ec1a1
--- /dev/null
+++ b/host/start-vm/fs.c
@@ -0,0 +1,17 @@
+// SPDX-License-Identifier: EUPL-1.2+
+// SPDX-FileCopyrightText: 2022 Alyssa Ross <hi@alyssa.is>
+
+#include <fcntl.h>
+#include <unistd.h>
+
+#include <sys/syscall.h>
+
+#include <linux/openat2.h>
+
+int openat_path_resolve_in_root(int dirfd, const char *pathname)
+{
+	struct open_how how = { 0 };
+	how.flags = O_PATH | O_CLOEXEC;
+	how.resolve = RESOLVE_IN_ROOT | RESOLVE_NO_MAGICLINKS | RESOLVE_NO_XDEV;
+	return syscall(SYS_openat2, dirfd, pathname, &how, sizeof how);
+}
diff --git a/host/start-vm/fs.rs b/host/start-vm/fs.rs
new file mode 100644
index 0000000..4bc6ada
--- /dev/null
+++ b/host/start-vm/fs.rs
@@ -0,0 +1,68 @@
+// SPDX-License-Identifier: EUPL-1.2+
+// SPDX-FileCopyrightText: 2022 Alyssa Ross <hi@alyssa.is>
+
+use std::ffi::CString;
+use std::fs::{read_link, File};
+use std::io::{Error, Result};
+use std::os::raw::c_char;
+use std::os::unix::prelude::*;
+use std::path::{Path, PathBuf};
+
+extern "C" {
+    fn openat_path_resolve_in_root(dirfd: BorrowedFd, pathname: *const c_char) -> Option<OwnedFd>;
+}
+
+// We can't make a const Path yet.
+// https://github.com/rust-lang/rust/issues/102619
+const PROC_FD_PATH: &str = "/proc/self/fd";
+
+pub trait OwnedFdExt: AsFd + AsRawFd + Sized {
+    /// Returns the path in proc(5) that corresponds to this file descriptor.
+    fn fd_path(&self) -> PathBuf {
+        let mut path = PathBuf::from(PROC_FD_PATH);
+        path.push(self.as_raw_fd().to_string());
+        path
+    }
+
+    /// Returns the path to this file, according to the magic link in proc(5).
+    fn path(&self) -> Result<PathBuf> {
+        read_link(self.fd_path())
+    }
+}
+
+impl OwnedFdExt for OwnedFd {}
+
+/// An `O_PATH` file descriptor.
+#[derive(Debug)]
+#[repr(transparent)]
+pub struct Root(OwnedFd);
+
+fn c_string(path: impl Into<PathBuf>) -> CString {
+    let bytes = path.into().into_os_string().into_vec();
+    CString::new(bytes).unwrap()
+}
+
+impl Root {
+    /// Panics if `path` contains NUL bytes.
+    pub fn open(path: impl AsRef<Path>) -> Result<Self> {
+        Ok(Self(OwnedFd::from(File::open(path)?)))
+    }
+
+    /// Resolves `path` as if the directory represented by `self` was the root directory
+    /// (including for symlink resolution).  The returned file descriptor will not have
+    /// the `O_CLOEXEC` flag set.
+    ///
+    /// Panics if `path` contains NUL bytes.
+    pub fn resolve_no_cloexec(&self, path: impl Into<PathBuf>) -> Result<OwnedFd> {
+        let c_path = c_string(path);
+
+        unsafe { openat_path_resolve_in_root(self.as_fd(), c_path.as_ptr()) }
+            .ok_or_else(Error::last_os_error)
+    }
+}
+
+impl AsFd for Root {
+    fn as_fd(&self) -> BorrowedFd {
+        self.0.as_fd()
+    }
+}
diff --git a/host/start-vm/lib.rs b/host/start-vm/lib.rs
index 3959566..f167efe 100644
--- a/host/start-vm/lib.rs
+++ b/host/start-vm/lib.rs
@@ -2,16 +2,19 @@
 // SPDX-FileCopyrightText: 2022 Alyssa Ross <hi@alyssa.is>
 
 mod ch;
+pub mod fs;
 mod net;
 
 use std::borrow::Cow;
 use std::env::args_os;
 use std::ffi::{CString, OsStr, OsString};
+use std::fs::read_dir;
 use std::io::{self, ErrorKind};
 use std::os::unix::prelude::*;
 use std::path::{Path, PathBuf};
 use std::process::Command;
 
+use fs::{OwnedFdExt, Root};
 use net::{format_mac, net_setup, NetConfig};
 
 pub fn prog_name() -> String {
@@ -25,7 +28,7 @@ pub fn prog_name() -> String {
         .into_owned()
 }
 
-pub fn vm_command(dir: PathBuf, config_root: &Path) -> Result<Command, String> {
+pub fn vm_command(dir: PathBuf, root: &Root) -> Result<Command, String> {
     let dir = dir.into_os_string().into_vec();
     let dir = PathBuf::from(OsString::from_vec(dir));
 
@@ -37,7 +40,10 @@ pub fn vm_command(dir: PathBuf, config_root: &Path) -> Result<Command, String> {
         return Err(format!("VM name may not contain a comma: {:?}", vm_name));
     }
 
-    let config_dir = config_root.join(vm_name);
+    let config = root
+        .resolve_no_cloexec(Path::new("svc/data").join(vm_name))
+        .and_then(|fd| fd.path())
+        .map_err(|e| format!("resolving configuration directory for {:?}: {}", vm_name, e))?;
 
     let mut command = Command::new("s6-notifyoncheck");
     command.args(&["-dc", "test -S env/cloud-hypervisor.sock"]);
@@ -46,11 +52,11 @@ pub fn vm_command(dir: PathBuf, config_root: &Path) -> Result<Command, String> {
     command.args(&["--cmdline", "console=ttyS0 root=PARTLABEL=root"]);
     command.args(&["--memory", "size=128M,shared=on"]);
     command.args(&["--console", "pty"]);
-    command.arg("--kernel");
-    command.arg(config_dir.join("vmlinux"));
 
-    let net_providers_dir = config_dir.join("providers/net");
-    match net_providers_dir.read_dir() {
+    command.arg("--kernel");
+    command.arg(config.join("vmlinux"));
+
+    match read_dir(config.join("providers/net")) {
         Ok(entries) => {
             for r in entries {
                 let entry = r
@@ -78,13 +84,12 @@ pub fn vm_command(dir: PathBuf, config_root: &Path) -> Result<Command, String> {
             }
         }
         Err(e) if e.kind() == ErrorKind::NotFound => {}
-        Err(e) => return Err(format!("reading directory {:?}: {}", net_providers_dir, e)),
+        Err(e) => return Err(format!("reading providers/net: {}", e)),
     }
 
     command.arg("--disk");
 
-    let blk_dir = config_dir.join("blk");
-    match blk_dir.read_dir() {
+    match read_dir(config.join("blk")) {
         Ok(entries) => {
             for result in entries {
                 let entry = result
@@ -105,15 +110,14 @@ pub fn vm_command(dir: PathBuf, config_root: &Path) -> Result<Command, String> {
                 command.arg(arg);
             }
         }
-        Err(e) => return Err(format!("reading directory {:?}: {}", blk_dir, e)),
+        Err(e) => return Err(format!("reading blk: {}", e)),
     }
 
     if command.get_args().last() == Some(OsStr::new("--disk")) {
         return Err("no block devices specified".to_string());
     }
 
-    let shared_dirs_dir = config_dir.join("shared-dirs");
-    match shared_dirs_dir.read_dir().map(Iterator::peekable) {
+    match read_dir(config.join("shared-dirs")).map(Iterator::peekable) {
         Ok(mut entries) => {
             if entries.peek().is_some() {
                 command.arg("--fs");
@@ -135,7 +139,7 @@ pub fn vm_command(dir: PathBuf, config_root: &Path) -> Result<Command, String> {
             }
         }
         Err(e) if e.kind() == ErrorKind::NotFound => {}
-        Err(e) => return Err(format!("reading directory {:?}: {}", shared_dirs_dir, e)),
+        Err(e) => return Err(format!("reading shared-dirs: {}", e)),
     }
 
     command.arg("--serial").arg({
@@ -154,7 +158,7 @@ mod tests {
 
     #[test]
     fn test_vm_name_comma() {
-        assert!(vm_command("/v,m".into(), Path::new("/"))
+        assert!(vm_command("/v,m".into(), &Root::open("/").unwrap())
             .unwrap_err()
             .contains("comma"));
     }
diff --git a/host/start-vm/meson.build b/host/start-vm/meson.build
index 74b88f0..de1defc 100644
--- a/host/start-vm/meson.build
+++ b/host/start-vm/meson.build
@@ -7,7 +7,7 @@ project('start-vm', 'rust', 'c',
 add_project_arguments('-D_GNU_SOURCE', language : 'c')
 add_project_arguments('-C', 'panic=abort', language : 'rust')
 
-c_lib = static_library('start-vm', 'net.c', 'net-util.c')
+c_lib = static_library('start-vm', 'fs.c', 'net.c', 'net-util.c')
 rust_lib = static_library('start_vm', 'lib.rs', link_with : c_lib)
 
 executable('start-vm', 'start-vm.rs', link_with : rust_lib, install : true)
diff --git a/host/start-vm/start-vm.rs b/host/start-vm/start-vm.rs
index 4790841..92b34b6 100644
--- a/host/start-vm/start-vm.rs
+++ b/host/start-vm/start-vm.rs
@@ -3,20 +3,25 @@
 
 use std::env::current_dir;
 use std::os::unix::prelude::*;
-use std::path::Path;
 use std::process::exit;
 
+use start_vm::fs::Root;
 use start_vm::{prog_name, vm_command};
 
-const CONFIG_ROOT: &str = "/ext/svc/data";
+const ROOT: &str = "/ext";
 
 fn run() -> String {
-    let dir = match current_dir().map_err(|e| format!("getting current directory: {}", e)) {
+    let dir = match current_dir() {
         Ok(dir) => dir,
-        Err(e) => return e,
+        Err(e) => return format!("getting current directory: {}", e),
     };
 
-    match vm_command(dir, Path::new(CONFIG_ROOT)) {
+    let config_root = match Root::open(ROOT) {
+        Ok(fd) => fd,
+        Err(e) => return format!("opening {}: {}", ROOT, e),
+    };
+
+    match vm_command(dir, &config_root) {
         Ok(mut command) => format!("failed to exec: {}", command.exec()),
         Err(e) => e,
     }
diff --git a/host/start-vm/tests/meson.build b/host/start-vm/tests/meson.build
index 857414b..efd78bc 100644
--- a/host/start-vm/tests/meson.build
+++ b/host/start-vm/tests/meson.build
@@ -31,5 +31,7 @@ test('tap_open (name too long)', executable('tap_open-name-too-long',
 
 test('vm_command-basic', executable('vm_command-basic',
   'vm_command-basic.rs', link_with : [rust_lib, rust_helper]))
+test('vm_command-config-symlink', executable('vm_command-config-symlink',
+  'vm_command-config-symlink.rs', link_with : [rust_lib, rust_helper]))
 test('vm_command-shared-dir', executable('vm_command-shared-dir',
   'vm_command-shared-dir.rs', link_with : [rust_lib, rust_helper]))
diff --git a/host/start-vm/tests/vm_command-basic.rs b/host/start-vm/tests/vm_command-basic.rs
index 4145b94..234f6e3 100644
--- a/host/start-vm/tests/vm_command-basic.rs
+++ b/host/start-vm/tests/vm_command-basic.rs
@@ -4,11 +4,13 @@
 use std::ffi::{OsStr, OsString};
 use std::fs::{create_dir, create_dir_all, File};
 
+use start_vm::fs::Root;
 use start_vm::vm_command;
 use test_helper::TempDir;
 
 fn main() -> std::io::Result<()> {
     let tmp_dir = TempDir::new()?;
+    let root = Root::open(tmp_dir.path())?;
 
     let service_dir = tmp_dir.path().join("testvm");
     create_dir(&service_dir)?;
@@ -21,7 +23,7 @@ fn main() -> std::io::Result<()> {
     File::create(&kernel_path)?;
     File::create(&image_path)?;
 
-    let command = vm_command(service_dir, &tmp_dir.path().join("svc/data")).unwrap();
+    let command = vm_command(service_dir, &root).unwrap();
     assert_eq!(command.get_program(), "s6-notifyoncheck");
 
     let mut expected_disk_arg = OsString::from("path=");
diff --git a/host/start-vm/tests/vm_command-config-symlink.rs b/host/start-vm/tests/vm_command-config-symlink.rs
new file mode 100644
index 0000000..e1e7112
--- /dev/null
+++ b/host/start-vm/tests/vm_command-config-symlink.rs
@@ -0,0 +1,30 @@
+// SPDX-License-Identifier: EUPL-1.2+
+// SPDX-FileCopyrightText: 2022 Alyssa Ross <hi@alyssa.is>
+
+use std::fs::{create_dir, create_dir_all, File};
+use std::os::unix::fs::symlink;
+
+use start_vm::fs::Root;
+use start_vm::vm_command;
+use test_helper::TempDir;
+
+fn main() -> std::io::Result<()> {
+    let tmp_dir = TempDir::new().unwrap();
+    let root = Root::open(tmp_dir.path()).unwrap();
+
+    let service_dir = tmp_dir.path().join("testvm");
+    create_dir(&service_dir).unwrap();
+
+    create_dir(tmp_dir.path().join("svc"))?;
+    create_dir(tmp_dir.path().join("svc/data"))?;
+    symlink("/config-1/testvm", tmp_dir.path().join("svc/data/testvm"))?;
+
+    let vm_config = tmp_dir.path().join("config-1/testvm");
+    create_dir_all(&vm_config)?;
+    File::create(vm_config.join("vmlinux"))?;
+    create_dir(vm_config.join("blk"))?;
+    symlink("/dev/null", vm_config.join("blk/root.img"))?;
+
+    vm_command(service_dir, &root).unwrap();
+    Ok(())
+}
diff --git a/host/start-vm/tests/vm_command-shared-dir.rs b/host/start-vm/tests/vm_command-shared-dir.rs
index 2b13663..092d55c 100644
--- a/host/start-vm/tests/vm_command-shared-dir.rs
+++ b/host/start-vm/tests/vm_command-shared-dir.rs
@@ -4,6 +4,7 @@
 use std::fs::{create_dir, create_dir_all, File};
 use std::os::unix::fs::symlink;
 
+use start_vm::fs::Root;
 use start_vm::vm_command;
 use test_helper::TempDir;
 
@@ -15,6 +16,7 @@ fn contains_seq<H: PartialEq<N>, N>(haystack: &[H], needle: &[N]) -> bool {
 
 fn main() -> std::io::Result<()> {
     let tmp_dir = TempDir::new()?;
+    let root = Root::open(tmp_dir.path())?;
 
     let service_dir = tmp_dir.path().join("testvm");
     let vm_config = tmp_dir.path().join("svc/data/testvm");
@@ -34,7 +36,7 @@ fn main() -> std::io::Result<()> {
         "tag=root,socket=../testvm-fs-root/env/virtiofsd.sock",
     ];
 
-    let command = vm_command(service_dir, &tmp_dir.path().join("svc/data")).unwrap();
+    let command = vm_command(service_dir, &root).unwrap();
     let args: Box<[_]> = command.get_args().collect();
     assert!(contains_seq(&args, &expected_args));
     Ok(())
-- 
2.37.1