[PATCH v4 07/10] host: integrate router

Yureka Lilian
2025-11-29T19:47:34Z

This removes the old host bridge + taps glue, and instead connects the
apps to their net provider's router instance.

Signed-off-by: Yureka Lilian <yureka@cyberchaos.dev>
---
host/rootfs/default.nix | 4 +-
host/rootfs/file-list.mk | 2 +
.../data/service/spectrum-router/down | 0
.../template/data/service/spectrum-router/run | 9 +++
host/rootfs/image/usr/bin/run-vmm | 21 +++---
host/rootfs/image/usr/bin/vm-import | 13 ----
tools/start-vmm/ch.rs | 66 +----------------
tools/start-vmm/lib.rs | 73 ++++++++++++-------
tools/start-vmm/meson.build | 9 +--
tools/start-vmm/net-util.c | 39 ----------
tools/start-vmm/net-util.h | 6 --
tools/start-vmm/net.c | 55 --------------
tools/start-vmm/net.rs | 11 ---
tools/start-vmm/tests/meson.build | 5 --
.../start-vmm/tests/tap_open-name-too-long.c | 20 -----
tools/start-vmm/tests/tap_open.c | 28 -------
16 files changed, 77 insertions(+), 284 deletions(-)
create mode 100644 host/rootfs/image/etc/s6-linux-init/run-image/service/vm-services/template/data/service/spectrum-router/down
create mode 100755 host/rootfs/image/etc/s6-linux-init/run-image/service/vm-services/template/data/service/spectrum-router/run
delete mode 100644 tools/start-vmm/net-util.c
delete mode 100644 tools/start-vmm/net-util.h
delete mode 100644 tools/start-vmm/net.c
delete mode 100644 tools/start-vmm/tests/tap_open-name-too-long.c
delete mode 100644 tools/start-vmm/tests/tap_open.c

diff --git a/host/rootfs/default.nix b/host/rootfs/default.nix
index b441a51..a759568 100644
--- a/host/rootfs/default.nix
+++ b/host/rootfs/default.nix
@@ -8,7 +8,7 @@ import ../../lib/call-package.nix (
}:
pkgsMusl.callPackage (

-{ spectrum-host-tools
+{ spectrum-host-tools, spectrum-router
, lib, stdenvNoCC, nixos, runCommand, writeClosure, erofs-utils, s6-rc
, busybox, cloud-hypervisor, cosmic-files, crosvm, cryptsetup
, dejavu_fonts, dbus, execline, foot, fuse3, iproute2, inotify-tools
@@ -27,7 +27,7 @@ let
packages = [
btrfs-progs cloud-hypervisor cosmic-files crosvm cryptsetup dbus
execline fuse3 inotify-tools iproute2 jq kmod mdevd s6 s6-linux-init
- s6-rc socat spectrum-host-tools util-linuxMinimal virtiofsd
+ s6-rc socat spectrum-host-tools spectrum-router util-linuxMinimal virtiofsd
xdg-desktop-portal-spectrum-host

(foot.override { allowPgo = false; })
diff --git a/host/rootfs/file-list.mk b/host/rootfs/file-list.mk
index 56f693e..bfe3940 100644
--- a/host/rootfs/file-list.mk
+++ b/host/rootfs/file-list.mk
@@ -27,6 +27,8 @@ FILES = \
image/etc/s6-linux-init/run-image/service/vm-services/run \
image/etc/s6-linux-init/run-image/service/vm-services/template/data/service/dbus/notification-fd \
image/etc/s6-linux-init/run-image/service/vm-services/template/data/service/dbus/run \
+ image/etc/s6-linux-init/run-image/service/vm-services/template/data/service/spectrum-router/down \
+ image/etc/s6-linux-init/run-image/service/vm-services/template/data/service/spectrum-router/run \
image/etc/s6-linux-init/run-image/service/vm-services/template/data/service/vhost-user-fs/notification-fd \
image/etc/s6-linux-init/run-image/service/vm-services/template/data/service/vhost-user-fs/run \
image/etc/s6-linux-init/run-image/service/vm-services/template/data/service/vhost-user-gpu/notification-fd \
diff --git a/host/rootfs/image/etc/s6-linux-init/run-image/service/vm-services/template/data/service/spectrum-router/down b/host/rootfs/image/etc/s6-linux-init/run-image/service/vm-services/template/data/service/spectrum-router/down
new file mode 100644
index 0000000..e69de29
diff --git a/host/rootfs/image/etc/s6-linux-init/run-image/service/vm-services/template/data/service/spectrum-router/run b/host/rootfs/image/etc/s6-linux-init/run-image/service/vm-services/template/data/service/spectrum-router/run
new file mode 100755
index 0000000..7b3e3db
--- /dev/null
+++ b/host/rootfs/image/etc/s6-linux-init/run-image/service/vm-services/template/data/service/spectrum-router/run
@@ -0,0 +1,9 @@
+#!/bin/execlineb -P
+# SPDX-License-Identifier: EUPL-1.2+
+# SPDX-FileCopyrightText: 2025 Yureka Lilian <yureka@cyberchaos.dev>
+
+importas -i VM VM
+
+export RUST_LOG spectrum-router=debug,info
+spectrum-router --app-listen-path ${VM}/router-app.sock --driver-listen-path ${VM}/router-driver.sock
+
diff --git a/host/rootfs/image/usr/bin/run-vmm b/host/rootfs/image/usr/bin/run-vmm
index 5649674..ba8b59c 100755
--- a/host/rootfs/image/usr/bin/run-vmm
+++ b/host/rootfs/image/usr/bin/run-vmm
@@ -20,6 +20,7 @@ background -d {
assign-devices
}

+ # Find any net-provider relationships this VM is involved in
cd /run/vm/by-id
elglob -0 providers */config/providers/net/*
forx -pE path { $providers }
@@ -31,23 +32,21 @@ background -d {
backtick -E link_path { readlink /run/vm/by-name/${router} }
basename -- $link_path
}
-
+ # This VM may be either the driver or the client
if -n {
if { test $client_id != $1 }
test $router_id != $1
}

- backtick -E mac {
- pipeline { ip -j link show client-${client_id} }
- pipeline { jq -r ".[].ifindex" }
- awk "{
- printf \"02:01:%02X:%02X:%02X:%02X\", $0 / 256 ^ 3 % 256,
- $0 / 256 ^ 2 % 256, $0 / 256 % 256, $0 % 256
- }"
+ if {
+ s6-svc -U /run/service/vm-services/instance/${router_id}/data/service/spectrum-router
}
-
- ch-remote --api-socket /run/vm/by-id/${router_id}/vmm add-net
- id=router-${client_id},tap=router-${client_id},mac=${mac}
+ if {
+ s6-svwait -U /run/service/vmm/instance/${router_id}
+ }
+ # Adding the interface is re-entrant and may be called multiple times. Thus, accept failures.
+ redirfd -w 2 /dev/null
+ ch-remote --api-socket /run/vm/by-id/${router_id}/vmm add-net id=router,vhost_user=on,socket=/run/vm/by-id/${router_id}/router-driver.sock,mac=02:01:00:00:00:01
}
unexport !
fdmove -c 3 0
diff --git a/host/rootfs/image/usr/bin/vm-import b/host/rootfs/image/usr/bin/vm-import
index de88f08..c1d1bbc 100755
--- a/host/rootfs/image/usr/bin/vm-import
+++ b/host/rootfs/image/usr/bin/vm-import
@@ -14,19 +14,6 @@ if { ln -s -- ${dir} /run/vm/by-name/${1}.${name} }
if { ln -s -- ${2}/${name} ${dir}/config }
if { ln -s -- /run/service/vmm/instance/${id} ${dir}/service }

-if {
- if -t { elglob -0d " " providers ${name}/providers/net test -n $providers }
-
- if { ip link add br-${id} type bridge }
- if { ip link set br-${id} up }
-
- if { ip tuntap add client-${id} mode tap }
- if { ip link set client-${id} master br-${id} up }
-
- if { ip tuntap add router-${id} mode tap }
- ip link set router-${id} master br-${id} up
-}
-
if { create-vm-dependencies $id }

s6-instance-create -- /run/service/vmm $id
diff --git a/tools/start-vmm/ch.rs b/tools/start-vmm/ch.rs
index abe1742..4764dd5 100644
--- a/tools/start-vmm/ch.rs
+++ b/tools/start-vmm/ch.rs
@@ -1,26 +1,19 @@
// SPDX-License-Identifier: EUPL-1.2+
// SPDX-FileCopyrightText: 2022-2024 Alyssa Ross <hi@alyssa.is>
+// SPDX-FileCopyrightText: 2025 Yureka Lilian <yureka@cyberchaos.dev>

-use std::convert::TryFrom;
use std::ffi::OsStr;
use std::fs::File;
use std::io::Write;
-use std::mem::take;
-use std::num::NonZeroI32;
use std::os::unix::prelude::*;
use std::path::Path;
use std::process::{Command, Stdio};
-use std::string::FromUtf8Error;

use miniserde::{Serialize, json};

use crate::net::MacAddress;
use crate::s6::notify_readiness;

-// Trivially safe.
-const EPERM: NonZeroI32 = NonZeroI32::new(1).unwrap();
-const EPROTO: NonZeroI32 = NonZeroI32::new(71).unwrap();
-
#[derive(Serialize)]
pub struct ConsoleConfig {
pub mode: &'static str,
@@ -46,7 +39,8 @@ pub struct GpuConfig {

#[derive(Serialize)]
pub struct NetConfig {
- pub fd: RawFd,
+ pub vhost_user: bool,
+ pub vhost_socket: String,
pub id: String,
pub mac: MacAddress,
}
@@ -99,11 +93,7 @@ fn command(vm_dir: &Path, s: impl AsRef<OsStr>) -> Command {
command
}

-pub fn create_vm(vm_dir: &Path, ready_fd: File, mut config: VmConfig) -> Result<(), String> {
- // Net devices can't be created from file descriptors in vm.create.
- // https://github.com/cloud-hypervisor/cloud-hypervisor/issues/5523
- let nets = take(&mut config.net);
-
+pub fn create_vm(vm_dir: &Path, ready_fd: File, config: VmConfig) -> Result<(), String> {
let mut ch_remote = command(vm_dir, "create")
.args(["--", "-"])
.stdin(Stdio::piped())
@@ -128,53 +118,5 @@ pub fn create_vm(vm_dir: &Path, ready_fd: File, mut config: VmConfig) -> Result<

notify_readiness(ready_fd)?;

- for net in nets {
- add_net(vm_dir, &net).map_err(|e| format!("failed to add net: {e}"))?;
- }
-
Ok(())
}
-
-pub fn add_net(vm_dir: &Path, net: &NetConfig) -> Result<(), NonZeroI32> {
- let mut ch_remote = command(vm_dir, "add-net")
- .arg(format!("fd={},id={},mac={}", net.fd, net.id, net.mac))
- .stdout(Stdio::piped())
- .spawn()
- .or(Err(EPERM))?;
-
- if let Ok(ch_remote_status) = ch_remote.wait()
- && ch_remote_status.success()
- {
- return Ok(());
- }
-
- Err(EPROTO)
-}
-
-#[repr(C)]
-pub struct NetConfigC {
- pub fd: RawFd,
- pub id: [u8; 18],
- pub mac: MacAddress,
-}
-
-impl<'a> TryFrom<&'a NetConfigC> for NetConfig {
- type Error = FromUtf8Error;
-
- fn try_from(c: &'a NetConfigC) -> Result<NetConfig, Self::Error> {
- let nul_index = c.id.iter().position(|&c| c == 0).unwrap_or(c.id.len());
- Ok(NetConfig {
- fd: c.fd,
- id: String::from_utf8(c.id[..nul_index].to_vec())?,
- mac: c.mac,
- })
- }
-}
-
-impl TryFrom<NetConfigC> for NetConfig {
- type Error = FromUtf8Error;
-
- fn try_from(c: NetConfigC) -> Result<NetConfig, Self::Error> {
- Self::try_from(&c)
- }
-}
diff --git a/tools/start-vmm/lib.rs b/tools/start-vmm/lib.rs
index 0422d85..b44e037 100644
--- a/tools/start-vmm/lib.rs
+++ b/tools/start-vmm/lib.rs
@@ -1,23 +1,24 @@
// SPDX-License-Identifier: EUPL-1.2+
// SPDX-FileCopyrightText: 2022-2024 Alyssa Ross <hi@alyssa.is>
+// SPDX-FileCopyrightText: 2025 Yureka Lilian <yureka@cyberchaos.dev>

mod ch;
mod net;
mod s6;

use std::borrow::Cow;
-use std::convert::TryInto;
use std::env::args_os;
use std::ffi::OsStr;
use std::fs::File;
-use std::io::{self, ErrorKind};
+use std::hash::{Hash, Hasher};
+use std::io::ErrorKind;
use std::path::Path;

use ch::{
- ConsoleConfig, DiskConfig, FsConfig, GpuConfig, LandlockConfig, MemoryConfig, PayloadConfig,
- VmConfig, VsockConfig,
+ ConsoleConfig, DiskConfig, FsConfig, GpuConfig, LandlockConfig, MemoryConfig, NetConfig,
+ PayloadConfig, VmConfig, VsockConfig,
};
-use net::net_setup;
+use net::MacAddress;

pub fn prog_name() -> String {
args_os()
@@ -40,8 +41,6 @@ pub fn vm_config(vm_dir: &Path) -> Result<VmConfig, String> {
return Err(format!("VM name may not contain a colon: {vm_name:?}"));
}

- let name_bytes = vm_name.as_bytes();
-
let config_dir = vm_dir.join("config");
let blk_dir = config_dir.join("blk");
let kernel_path = config_dir.join("vmlinux");
@@ -97,24 +96,48 @@ pub fn vm_config(vm_dir: &Path) -> Result<VmConfig, String> {
shared: true,
},
net: match net_providers_dir.read_dir() {
- Ok(_) => {
- // SAFETY: we check the result.
- let net = unsafe {
- net_setup(
- name_bytes.as_ptr().cast(),
- name_bytes
- .len()
- .try_into()
- .map_err(|e| format!("VM name too long: {e}"))?,
- )
- };
- if net.fd == -1 {
- let e = io::Error::last_os_error();
- return Err(format!("setting up networking failed: {e}"));
- }
-
- vec![net.try_into().unwrap()]
- }
+ Ok(entries) => entries
+ .into_iter()
+ .map(|result| {
+ Ok(result
+ .map_err(|e| format!("examining directory entry: {e}"))?
+ .path())
+ })
+ .map(|result: Result<_, String>| {
+ let provider_name = result?
+ .file_name()
+ .ok_or("unable to get net provider name".to_string())?
+ .to_str()
+ .unwrap()
+ .to_string();
+
+ if provider_name.contains(',') {
+ return Err(format!(
+ "illegal ',' character in net provider name {provider_name:?}"
+ ));
+ }
+
+ let mut hasher = std::hash::DefaultHasher::new();
+ vm_name.hash(&mut hasher);
+ let id_hashed = hasher.finish();
+
+ let mac = MacAddress::new([
+ 0x02, // IEEE 802c administratively assigned
+ 0x00, // Spectrum client
+ (id_hashed >> 24) as u8,
+ (id_hashed >> 16) as u8,
+ (id_hashed >> 8) as u8,
+ id_hashed as u8,
+ ]);
+
+ Ok(NetConfig {
+ vhost_user: true,
+ vhost_socket: format!("/run/vm/by-name/{provider_name}/router-app.sock"),
+ id: provider_name,
+ mac,
+ })
+ })
+ .collect::<Result<_, _>>()?,
Err(e) if e.kind() == ErrorKind::NotFound => Default::default(),
Err(e) => return Err(format!("reading directory {net_providers_dir:?}: {e}")),
},
diff --git a/tools/start-vmm/meson.build b/tools/start-vmm/meson.build
index d07c5a0..4c79f5d 100644
--- a/tools/start-vmm/meson.build
+++ b/tools/start-vmm/meson.build
@@ -1,12 +1,8 @@
# SPDX-License-Identifier: EUPL-1.2+
# SPDX-FileCopyrightText: 2022-2024 Alyssa Ross <hi@alyssa.is>

-c_lib = static_library('start-vmm', 'net.c', 'net-util.c',
- c_args : '-D_GNU_SOURCE')
-
rust_lib = static_library('start_vmm', 'lib.rs',
- dependencies : miniserde_dep,
- link_with : c_lib)
+ dependencies : miniserde_dep)

rust_lib_dep = declare_dependency(
dependencies : miniserde_dep,
@@ -20,8 +16,7 @@ executable('start-vmm', 'start-vmm.rs',
if get_option('tests')
test_exe = executable('start-vmm-test', 'lib.rs',
dependencies : miniserde_dep,
- rust_args : ['--test'],
- link_with : c_lib)
+ rust_args : ['--test'])
test('Rust unit tests', test_exe, protocol : 'rust')

subdir('tests')
diff --git a/tools/start-vmm/net-util.c b/tools/start-vmm/net-util.c
deleted file mode 100644
index 49003e9..0000000
--- a/tools/start-vmm/net-util.c
+++ /dev/null
@@ -1,39 +0,0 @@
-// SPDX-License-Identifier: EUPL-1.2+
-// SPDX-FileCopyrightText: 2022, 2024 Alyssa Ross <hi@alyssa.is>
-
-#include "net-util.h"
-
-#include <errno.h>
-#include <fcntl.h>
-#include <string.h>
-#include <unistd.h>
-
-#include <sys/ioctl.h>
-
-#include <linux/if_tun.h>
-
-int tap_open(char name[static IFNAMSIZ], int flags)
-{
- struct ifreq ifr;
- int fd, e;
-
- if (strnlen(name, IFNAMSIZ) == IFNAMSIZ) {
- errno = ENAMETOOLONG;
- return -1;
- }
-
- strncpy(ifr.ifr_name, name, IFNAMSIZ - 1);
- ifr.ifr_flags = IFF_TAP|flags;
-
- if ((fd = open("/dev/net/tun", O_RDWR)) == -1)
- return -1;
- if (ioctl(fd, TUNSETIFF, &ifr) == -1) {
- e = errno;
- close(fd);
- errno = e;
- return -1;
- }
-
- strncpy(name, ifr.ifr_name, IFNAMSIZ);
- return fd;
-}
diff --git a/tools/start-vmm/net-util.h b/tools/start-vmm/net-util.h
deleted file mode 100644
index 8f55206..0000000
--- a/tools/start-vmm/net-util.h
+++ /dev/null
@@ -1,6 +0,0 @@
-// SPDX-License-Identifier: EUPL-1.2+
-// SPDX-FileCopyrightText: 2022 Alyssa Ross <hi@alyssa.is>
-
-#include <net/if.h>
-
-int tap_open(char name[static IFNAMSIZ], int flags);
diff --git a/tools/start-vmm/net.c b/tools/start-vmm/net.c
deleted file mode 100644
index 78fe7f6..0000000
--- a/tools/start-vmm/net.c
+++ /dev/null
@@ -1,55 +0,0 @@
-// SPDX-License-Identifier: EUPL-1.2+
-// SPDX-FileCopyrightText: 2022-2024 Alyssa Ross <hi@alyssa.is>
-
-#include "ch.h"
-#include "net-util.h"
-
-#include <assert.h>
-#include <errno.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <unistd.h>
-
-#include <arpa/inet.h>
-
-#include <linux/if_tun.h>
-
-static int get_tap_name(char tap_name[static IFNAMSIZ],
- const char tap_prefix[static 1],
- const char name[static 1], int name_len)
-{
- int r = snprintf(tap_name, IFNAMSIZ, "%s-%*s", tap_prefix, name_len, name);
- if (r >= IFNAMSIZ)
- errno = ENAMETOOLONG;
- return r < 0 || r >= IFNAMSIZ ? -1 : 0;
-}
-
-struct net_config net_setup(const char name[static 1], int name_len)
-{
- int e;
- unsigned int client_index;
- struct net_config r = { .fd = -1, .mac = { 0 } };
-
- if ((get_tap_name(r.id, "client", name, name_len)) == -1)
- return r;
-
- if (!(client_index = htonl(if_nametoindex(r.id))))
- return r;
-
- if ((r.fd = tap_open(r.id, IFF_NO_PI|IFF_VNET_HDR)) == -1)
- goto fail_close;
-
- r.mac[0] = 0x02; // IEEE 802c administratively assigned
- r.mac[1] = 0x00; // Spectrum client
- memcpy(&r.mac[2], &client_index, 4);
-
- return r;
-
-fail_close:
- e = errno;
- close(r.fd);
- errno = e;
- r.fd = -1;
- return r;
-}
diff --git a/tools/start-vmm/net.rs b/tools/start-vmm/net.rs
index ebfef7a..7b237df 100644
--- a/tools/start-vmm/net.rs
+++ b/tools/start-vmm/net.rs
@@ -2,14 +2,11 @@
// SPDX-FileCopyrightText: 2022-2025 Alyssa Ross <hi@alyssa.is>

use std::borrow::Cow;
-use std::ffi::{c_char, c_int};
use std::fmt::{self, Display, Formatter};

use miniserde::Serialize;
use miniserde::ser::Fragment;

-use crate::ch::NetConfigC;
-
#[repr(transparent)]
#[derive(Copy, Clone)]
pub struct MacAddress([u8; 6]);
@@ -36,14 +33,6 @@ impl Serialize for MacAddress {
}
}

-// SAFETY: declaration is compatible with C.
-unsafe extern "C" {
- /// # Safety
- ///
- /// The rest of the result is only valid if the returned fd is not -1.
- pub fn net_setup(name: *const c_char, len: c_int) -> NetConfigC;
-}
-
#[cfg(test)]
mod tests {
use super::*;
diff --git a/tools/start-vmm/tests/meson.build b/tools/start-vmm/tests/meson.build
index bfdfc46..5538822 100644
--- a/tools/start-vmm/tests/meson.build
+++ b/tools/start-vmm/tests/meson.build
@@ -4,11 +4,6 @@
rust_helper = static_library('test_helper', 'helper.rs',
dependencies : rust_lib_dep)

-test('tap_open', executable('tap_open', 'tap_open.c', '../net-util.c',
- c_args : '-D_GNU_SOURCE'))
-test('tap_open (name too long)', executable('tap_open-name-too-long',
- 'tap_open-name-too-long.c', '../net-util.c', c_args : '-D_GNU_SOURCE'))
-
test('vm_command-basic', executable('vm_command-basic',
'vm_command-basic.rs',
dependencies : rust_lib_dep,
diff --git a/tools/start-vmm/tests/tap_open-name-too-long.c b/tools/start-vmm/tests/tap_open-name-too-long.c
deleted file mode 100644
index ba4ebd6..0000000
--- a/tools/start-vmm/tests/tap_open-name-too-long.c
+++ /dev/null
@@ -1,20 +0,0 @@
-// SPDX-License-Identifier: EUPL-1.2+
-// SPDX-FileCopyrightText: 2022 Alyssa Ross <hi@alyssa.is>
-
-#include "../net-util.h"
-
-#include <assert.h>
-#include <errno.h>
-#include <net/if.h>
-#include <string.h>
-
-int main(void)
-{
- char name[IFNAMSIZ];
- int fd;
-
- memset(name, 'a', sizeof name);
- fd = tap_open(name, 0);
- assert(fd == -1);
- assert(errno == ENAMETOOLONG);
-}
diff --git a/tools/start-vmm/tests/tap_open.c b/tools/start-vmm/tests/tap_open.c
deleted file mode 100644
index bf5d00c..0000000
--- a/tools/start-vmm/tests/tap_open.c
+++ /dev/null
@@ -1,28 +0,0 @@
-// SPDX-License-Identifier: EUPL-1.2+
-// SPDX-FileCopyrightText: 2022 Alyssa Ross <hi@alyssa.is>
-
-#include "../net-util.h"
-
-#include <assert.h>
-#include <errno.h>
-#include <sched.h>
-#include <string.h>
-
-#include <sys/ioctl.h>
-
-#include <linux/if_tun.h>
-
-int main(void)
-{
- char name[IFNAMSIZ] = "tap%d";
- struct ifreq ifr;
- int fd;
-
- unshare(CLONE_NEWUSER|CLONE_NEWNET);
-
- fd = tap_open(name, 0);
- if (fd == -1 && (errno == EPERM || errno == ENOENT))
- return 77;
- assert(!ioctl(fd, (unsigned)TUNGETIFF, &ifr));
- assert(!strcmp(name, ifr.ifr_name));
-}
--
2.51.2