[PATCH v1 RFC 4/4] host: integrate router

[Yureka Lilian <yureka@cyberchaos.dev>]

This removes the old host bridge + taps glue, and instead connects the
apps to their net provider's router instance.

Signed-off-by: Yureka Lilian <yureka@cyberchaos.dev>
---
 host/rootfs/default.nix                       |  4 +-
 host/rootfs/file-list.mk                      |  2 +
 .../data/service/spectrum-router/down         |  0
 .../template/data/service/spectrum-router/run | 13 ++++
 host/rootfs/image/usr/bin/run-vmm             | 12 ----
 host/rootfs/image/usr/bin/vm-import           | 13 ----
 pkgs/overlay.nix                              |  1 +
 tools/start-vmm/ch.rs                         | 40 ++---------
 tools/start-vmm/lib.rs                        | 68 ++++++++++++-------
 tools/start-vmm/meson.build                   |  2 +-
 tools/start-vmm/net-util.c                    | 39 -----------
 tools/start-vmm/net-util.h                    |  6 --
 tools/start-vmm/net.c                         | 55 ---------------
 tools/start-vmm/net.rs                        | 10 ---
 tools/start-vmm/tests/meson.build             |  5 --
 .../start-vmm/tests/tap_open-name-too-long.c  | 20 ------
 tools/start-vmm/tests/tap_open.c              | 28 --------
 vm/sys/net/default.nix                        |  4 +-
 18 files changed, 68 insertions(+), 254 deletions(-)
 create mode 100644 host/rootfs/image/etc/s6-linux-init/run-image/service/vm-services/template/data/service/spectrum-router/down
 create mode 100644 host/rootfs/image/etc/s6-linux-init/run-image/service/vm-services/template/data/service/spectrum-router/run
 delete mode 100644 tools/start-vmm/net-util.c
 delete mode 100644 tools/start-vmm/net-util.h
 delete mode 100644 tools/start-vmm/net.c
 delete mode 100644 tools/start-vmm/tests/tap_open-name-too-long.c
 delete mode 100644 tools/start-vmm/tests/tap_open.c

diff --git a/host/rootfs/default.nix b/host/rootfs/default.nix
index 0ac70c7..ba0d24a 100644
--- a/host/rootfs/default.nix
+++ b/host/rootfs/default.nix
@@ -8,7 +8,7 @@ import ../../lib/call-package.nix (
 }:
 pkgsStatic.callPackage (
 
-{ spectrum-host-tools
+{ spectrum-host-tools, spectrum-router
 , lib, stdenvNoCC, nixos, runCommand, writeClosure, erofs-utils, s6-rc
 , busybox, cloud-hypervisor, cryptsetup, dbus, execline, inkscape
 , iproute2, inotify-tools, jq, mdevd, s6, s6-linux-init, socat
@@ -34,7 +34,7 @@ let
 
   packages = [
     cloud-hypervisor cryptsetup dbus execline inotify-tools iproute2
-    jq mdevd s6 s6-linux-init s6-rc socat spectrum-host-tools
+    jq mdevd s6 s6-linux-init s6-rc socat spectrum-host-tools spectrum-router
     virtiofsd xdg-desktop-portal-spectrum-host
 
     (busybox.override {
diff --git a/host/rootfs/file-list.mk b/host/rootfs/file-list.mk
index ff6fd1b..3a595b7 100644
--- a/host/rootfs/file-list.mk
+++ b/host/rootfs/file-list.mk
@@ -25,6 +25,8 @@ FILES = \
 	image/etc/s6-linux-init/run-image/service/vm-services/run \
 	image/etc/s6-linux-init/run-image/service/vm-services/template/data/service/dbus/notification-fd \
 	image/etc/s6-linux-init/run-image/service/vm-services/template/data/service/dbus/run \
+	image/etc/s6-linux-init/run-image/service/vm-services/template/data/service/spectrum-router/down \
+	image/etc/s6-linux-init/run-image/service/vm-services/template/data/service/spectrum-router/run \
 	image/etc/s6-linux-init/run-image/service/vm-services/template/data/service/vhost-user-fs/notification-fd \
 	image/etc/s6-linux-init/run-image/service/vm-services/template/data/service/vhost-user-fs/run \
 	image/etc/s6-linux-init/run-image/service/vm-services/template/data/service/vhost-user-gpu/data/check \
diff --git a/host/rootfs/image/etc/s6-linux-init/run-image/service/vm-services/template/data/service/spectrum-router/down b/host/rootfs/image/etc/s6-linux-init/run-image/service/vm-services/template/data/service/spectrum-router/down
new file mode 100644
index 0000000..e69de29
diff --git a/host/rootfs/image/etc/s6-linux-init/run-image/service/vm-services/template/data/service/spectrum-router/run b/host/rootfs/image/etc/s6-linux-init/run-image/service/vm-services/template/data/service/spectrum-router/run
new file mode 100644
index 0000000..b8f831e
--- /dev/null
+++ b/host/rootfs/image/etc/s6-linux-init/run-image/service/vm-services/template/data/service/spectrum-router/run
@@ -0,0 +1,13 @@
+#!/bin/execlineb -P
+# SPDX-License-Identifier: EUPL-1.2+
+# SPDX-FileCopyrightText: 2025 Yureka Lilian <yureka@cyberchaos.dev>
+
+importas -i VM VM
+
+background {
+  ch-remote --api-socket ${VM}/vmm add-net id=router,vhost_user=on,socket=${VM}/router-driver.sock,mac=02:01:00:00:00:01
+}
+
+export RUST_LOG debug
+spectrum-router --app-listen-path ${VM}/router-app.sock --driver-listen-path ${VM}/router-driver.sock --upstream-interface 2
+
diff --git a/host/rootfs/image/usr/bin/run-vmm b/host/rootfs/image/usr/bin/run-vmm
index 5649674..919a69d 100755
--- a/host/rootfs/image/usr/bin/run-vmm
+++ b/host/rootfs/image/usr/bin/run-vmm
@@ -36,18 +36,6 @@ background -d {
     if { test $client_id != $1 }
     test $router_id != $1
   }
-
-  backtick -E mac {
-    pipeline { ip -j link show client-${client_id} }
-    pipeline { jq -r ".[].ifindex" }
-    awk "{
-      printf \"02:01:%02X:%02X:%02X:%02X\", $0 / 256 ^ 3 % 256,
-	$0 / 256 ^ 2 % 256, $0 / 256 % 256, $0 % 256
-    }"
-  }
-
-  ch-remote --api-socket /run/vm/by-id/${router_id}/vmm add-net
-    id=router-${client_id},tap=router-${client_id},mac=${mac}
 }
 unexport !
 fdmove -c 3 0
diff --git a/host/rootfs/image/usr/bin/vm-import b/host/rootfs/image/usr/bin/vm-import
index de88f08..c1d1bbc 100755
--- a/host/rootfs/image/usr/bin/vm-import
+++ b/host/rootfs/image/usr/bin/vm-import
@@ -14,19 +14,6 @@ if { ln -s -- ${dir} /run/vm/by-name/${1}.${name} }
 if { ln -s -- ${2}/${name} ${dir}/config }
 if { ln -s -- /run/service/vmm/instance/${id} ${dir}/service }
 
-if {
-  if -t { elglob -0d " " providers ${name}/providers/net test -n $providers }
-
-  if { ip link add br-${id} type bridge }
-  if { ip link set br-${id} up }
-
-  if { ip tuntap add client-${id} mode tap }
-  if { ip link set client-${id} master br-${id} up }
-
-  if { ip tuntap add router-${id} mode tap }
-  ip link set router-${id} master br-${id} up
-}
-
 if { create-vm-dependencies $id }
 
 s6-instance-create -- /run/service/vmm $id
diff --git a/pkgs/overlay.nix b/pkgs/overlay.nix
index 55cb00c..d2c8331 100644
--- a/pkgs/overlay.nix
+++ b/pkgs/overlay.nix
@@ -3,4 +3,5 @@
 
 (final: super: {
   cloud-hypervisor = import ./cloud-hypervisor { inherit final super; };
+  mailutils = super.mailutils.overrideAttrs (_: { doCheck = false; });
 })
diff --git a/tools/start-vmm/ch.rs b/tools/start-vmm/ch.rs
index 35519cd..5aca1dc 100644
--- a/tools/start-vmm/ch.rs
+++ b/tools/start-vmm/ch.rs
@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: EUPL-1.2+
 // SPDX-FileCopyrightText: 2022-2024 Alyssa Ross <hi@alyssa.is>
+// SPDX-FileCopyrightText: 2025 Yureka Lilian <yureka@cyberchaos.dev>
 
-use std::convert::TryFrom;
 use std::ffi::OsStr;
 use std::fs::File;
 use std::io::Write;
@@ -10,7 +10,6 @@ use std::num::NonZeroI32;
 use std::os::unix::prelude::*;
 use std::path::Path;
 use std::process::{Command, Stdio};
-use std::string::FromUtf8Error;
 
 use miniserde::{json, Serialize};
 
@@ -46,7 +45,7 @@ pub struct GpuConfig {
 
 #[derive(Serialize)]
 pub struct NetConfig {
-    pub fd: RawFd,
+    pub vhost_user_sock: String,
     pub id: String,
     pub mac: MacAddress,
 }
@@ -136,13 +135,10 @@ pub fn create_vm(vm_dir: &Path, ready_fd: File, mut config: VmConfig) -> Result<
 }
 
 pub fn add_net(vm_dir: &Path, net: &NetConfig) -> Result<(), NonZeroI32> {
-    // TODO: re-enable offloading once
-    // https://lore.kernel.org/regressions/87y0ota32b.fsf@alyssa.is/
-    // is fixed.
     let mut ch_remote = command(vm_dir, "add-net")
         .arg(format!(
-            "fd={},id={},mac={},offload_tso=false,offload_ufo=false,offload_csum=false",
-            net.fd, net.id, net.mac
+            "vhost_user=on,socket={},id={},mac={}",
+            net.vhost_user_sock, net.id, net.mac
         ))
         .stdout(Stdio::piped())
         .spawn()
@@ -156,31 +152,3 @@ pub fn add_net(vm_dir: &Path, net: &NetConfig) -> Result<(), NonZeroI32> {
 
     Err(EPROTO)
 }
-
-#[repr(C)]
-pub struct NetConfigC {
-    pub fd: RawFd,
-    pub id: [u8; 18],
-    pub mac: MacAddress,
-}
-
-impl<'a> TryFrom<&'a NetConfigC> for NetConfig {
-    type Error = FromUtf8Error;
-
-    fn try_from(c: &'a NetConfigC) -> Result<NetConfig, Self::Error> {
-        let nul_index = c.id.iter().position(|&c| c == 0).unwrap_or(c.id.len());
-        Ok(NetConfig {
-            fd: c.fd,
-            id: String::from_utf8(c.id[..nul_index].to_vec())?,
-            mac: c.mac,
-        })
-    }
-}
-
-impl TryFrom<NetConfigC> for NetConfig {
-    type Error = FromUtf8Error;
-
-    fn try_from(c: NetConfigC) -> Result<NetConfig, Self::Error> {
-        Self::try_from(&c)
-    }
-}
diff --git a/tools/start-vmm/lib.rs b/tools/start-vmm/lib.rs
index 4586094..112c56c 100644
--- a/tools/start-vmm/lib.rs
+++ b/tools/start-vmm/lib.rs
@@ -1,23 +1,23 @@
 // SPDX-License-Identifier: EUPL-1.2+
 // SPDX-FileCopyrightText: 2022-2024 Alyssa Ross <hi@alyssa.is>
+// SPDX-FileCopyrightText: 2025 Yureka Lilian <yureka@cyberchaos.dev>
 
 mod ch;
 mod net;
 mod s6;
 
 use std::borrow::Cow;
-use std::convert::TryInto;
 use std::env::args_os;
 use std::ffi::OsStr;
 use std::fs::File;
-use std::io::{self, ErrorKind};
+use std::io::ErrorKind;
 use std::path::Path;
 
 use ch::{
-    ConsoleConfig, DiskConfig, FsConfig, GpuConfig, LandlockConfig, MemoryConfig, PayloadConfig,
-    VmConfig, VsockConfig,
+    ConsoleConfig, DiskConfig, FsConfig, GpuConfig, LandlockConfig, MemoryConfig, NetConfig,
+    PayloadConfig, VmConfig, VsockConfig,
 };
-use net::net_setup;
+use net::MacAddress;
 
 pub fn prog_name() -> String {
     args_os()
@@ -40,8 +40,6 @@ pub fn vm_config(vm_dir: &Path) -> Result<VmConfig, String> {
         return Err(format!("VM name may not contain a colon: {vm_name:?}"));
     }
 
-    let name_bytes = vm_name.as_bytes();
-
     let config_dir = vm_dir.join("config");
     let blk_dir = config_dir.join("blk");
     let kernel_path = config_dir.join("vmlinux");
@@ -93,24 +91,44 @@ pub fn vm_config(vm_dir: &Path) -> Result<VmConfig, String> {
             shared: true,
         },
         net: match net_providers_dir.read_dir() {
-            Ok(_) => {
-                // SAFETY: we check the result.
-                let net = unsafe {
-                    net_setup(
-                        name_bytes.as_ptr().cast(),
-                        name_bytes
-                            .len()
-                            .try_into()
-                            .map_err(|e| format!("VM name too long: {e}"))?,
-                    )
-                };
-                if net.fd == -1 {
-                    let e = io::Error::last_os_error();
-                    return Err(format!("setting up networking failed: {e}"));
-                }
-
-                vec![net.try_into().unwrap()]
-            }
+            Ok(entries) => entries
+                .into_iter()
+                .map(|result| {
+                    Ok(result
+                        .map_err(|e| format!("examining directory entry: {e}"))?
+                        .path())
+                })
+                .map(|result: Result<_, String>| {
+                    let provider_name = result?.file_name().ok_or("unable to get net provider name".to_string())?.to_str().unwrap().to_string();
+
+                    if provider_name.contains(',') {
+                        return Err(format!("illegal ',' character in net provider name {provider_name:?}"));
+                    }
+
+                    let mac = MacAddress::new([
+                        0x02, // IEEE 802c administratively assigned
+                        0x00, // Spectrum client
+                        0x00, 0x00, 0x00, 0x01 // TODO
+                    ]);
+
+                    let provider_id = std::fs::read_link(format!("/run/vm/by-name/{provider_name}")).map_err(|e| format!("unable to get net provider id: {e}"))?.file_name().ok_or("unable to get net provider id".to_string())?.to_str().unwrap().to_string();
+
+                    let svc_dir = format!("/run/service/vm-services/instance/{provider_id}/data/service/spectrum-router");
+                    let svc_status = std::process::Command::new("s6-svc")
+                        .args(["-U", &svc_dir])
+                        .status()
+                        .expect("setting up the upstream router via s6-svc failed");
+                    if !svc_status.success() {
+                        return Err(format!("setting up the upstream router via s6-svc failed with exit code {svc_status}"));
+                    }
+
+                    Ok(NetConfig {
+                        vhost_user_sock: format!("/run/vm/by-name/{provider_name}/router-app.sock").into(),
+                        id: provider_name.into(),
+                        mac,
+                    })
+                })
+                .collect::<Result<_, _>>()?,
             Err(e) if e.kind() == ErrorKind::NotFound => Default::default(),
             Err(e) => return Err(format!("reading directory {net_providers_dir:?}: {e}")),
         },
diff --git a/tools/start-vmm/meson.build b/tools/start-vmm/meson.build
index d07c5a0..aa9f6f3 100644
--- a/tools/start-vmm/meson.build
+++ b/tools/start-vmm/meson.build
@@ -1,7 +1,7 @@
 # SPDX-License-Identifier: EUPL-1.2+
 # SPDX-FileCopyrightText: 2022-2024 Alyssa Ross <hi@alyssa.is>
 
-c_lib = static_library('start-vmm', 'net.c', 'net-util.c',
+c_lib = static_library('start-vmm',
   c_args : '-D_GNU_SOURCE')
 
 rust_lib = static_library('start_vmm', 'lib.rs',
diff --git a/tools/start-vmm/net-util.c b/tools/start-vmm/net-util.c
deleted file mode 100644
index 49003e9..0000000
--- a/tools/start-vmm/net-util.c
+++ /dev/null
@@ -1,39 +0,0 @@
-// SPDX-License-Identifier: EUPL-1.2+
-// SPDX-FileCopyrightText: 2022, 2024 Alyssa Ross <hi@alyssa.is>
-
-#include "net-util.h"
-
-#include <errno.h>
-#include <fcntl.h>
-#include <string.h>
-#include <unistd.h>
-
-#include <sys/ioctl.h>
-
-#include <linux/if_tun.h>
-
-int tap_open(char name[static IFNAMSIZ], int flags)
-{
-	struct ifreq ifr;
-	int fd, e;
-
-	if (strnlen(name, IFNAMSIZ) == IFNAMSIZ) {
-		errno = ENAMETOOLONG;
-		return -1;
-	}
-
-	strncpy(ifr.ifr_name, name, IFNAMSIZ - 1);
-	ifr.ifr_flags = IFF_TAP|flags;
-
-	if ((fd = open("/dev/net/tun", O_RDWR)) == -1)
-		return -1;
-	if (ioctl(fd, TUNSETIFF, &ifr) == -1) {
-		e = errno;
-		close(fd);
-		errno = e;
-		return -1;
-	}
-
-	strncpy(name, ifr.ifr_name, IFNAMSIZ);
-	return fd;
-}
diff --git a/tools/start-vmm/net-util.h b/tools/start-vmm/net-util.h
deleted file mode 100644
index 8f55206..0000000
--- a/tools/start-vmm/net-util.h
+++ /dev/null
@@ -1,6 +0,0 @@
-// SPDX-License-Identifier: EUPL-1.2+
-// SPDX-FileCopyrightText: 2022 Alyssa Ross <hi@alyssa.is>
-
-#include <net/if.h>
-
-int tap_open(char name[static IFNAMSIZ], int flags);
diff --git a/tools/start-vmm/net.c b/tools/start-vmm/net.c
deleted file mode 100644
index 78fe7f6..0000000
--- a/tools/start-vmm/net.c
+++ /dev/null
@@ -1,55 +0,0 @@
-// SPDX-License-Identifier: EUPL-1.2+
-// SPDX-FileCopyrightText: 2022-2024 Alyssa Ross <hi@alyssa.is>
-
-#include "ch.h"
-#include "net-util.h"
-
-#include <assert.h>
-#include <errno.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <unistd.h>
-
-#include <arpa/inet.h>
-
-#include <linux/if_tun.h>
-
-static int get_tap_name(char tap_name[static IFNAMSIZ],
-                        const char tap_prefix[static 1],
-                        const char name[static 1], int name_len)
-{
-	int r = snprintf(tap_name, IFNAMSIZ, "%s-%*s", tap_prefix, name_len, name);
-	if (r >= IFNAMSIZ)
-		errno = ENAMETOOLONG;
-	return r < 0 || r >= IFNAMSIZ ? -1 : 0;
-}
-
-struct net_config net_setup(const char name[static 1], int name_len)
-{
-	int e;
-	unsigned int client_index;
-	struct net_config r = { .fd = -1, .mac = { 0 } };
-
-	if ((get_tap_name(r.id, "client", name, name_len)) == -1)
-		return r;
-
-	if (!(client_index = htonl(if_nametoindex(r.id))))
-		return r;
-
-	if ((r.fd = tap_open(r.id, IFF_NO_PI|IFF_VNET_HDR)) == -1)
-		goto fail_close;
-
-	r.mac[0] = 0x02; // IEEE 802c administratively assigned
-	r.mac[1] = 0x00; // Spectrum client
-	memcpy(&r.mac[2], &client_index, 4);
-
-	return r;
-
-fail_close:
-	e = errno;
-	close(r.fd);
-	errno = e;
-	r.fd = -1;
-	return r;
-}
diff --git a/tools/start-vmm/net.rs b/tools/start-vmm/net.rs
index c94bca7..64f6014 100644
--- a/tools/start-vmm/net.rs
+++ b/tools/start-vmm/net.rs
@@ -2,14 +2,11 @@
 // SPDX-FileCopyrightText: 2022-2024 Alyssa Ross <hi@alyssa.is>
 
 use std::borrow::Cow;
-use std::ffi::{c_char, c_int};
 use std::fmt::{self, Display, Formatter};
 
 use miniserde::ser::Fragment;
 use miniserde::Serialize;
 
-use crate::ch::NetConfigC;
-
 #[repr(transparent)]
 #[derive(Copy, Clone)]
 pub struct MacAddress([u8; 6]);
@@ -36,13 +33,6 @@ impl Serialize for MacAddress {
     }
 }
 
-extern "C" {
-    /// # Safety
-    ///
-    /// The rest of the result is only valid if the returned fd is not -1.
-    pub fn net_setup(name: *const c_char, len: c_int) -> NetConfigC;
-}
-
 #[cfg(test)]
 mod tests {
     use super::*;
diff --git a/tools/start-vmm/tests/meson.build b/tools/start-vmm/tests/meson.build
index bfdfc46..5538822 100644
--- a/tools/start-vmm/tests/meson.build
+++ b/tools/start-vmm/tests/meson.build
@@ -4,11 +4,6 @@
 rust_helper = static_library('test_helper', 'helper.rs',
   dependencies : rust_lib_dep)
 
-test('tap_open', executable('tap_open', 'tap_open.c', '../net-util.c',
-  c_args : '-D_GNU_SOURCE'))
-test('tap_open (name too long)', executable('tap_open-name-too-long',
-  'tap_open-name-too-long.c', '../net-util.c', c_args : '-D_GNU_SOURCE'))
-
 test('vm_command-basic', executable('vm_command-basic',
   'vm_command-basic.rs',
   dependencies : rust_lib_dep,
diff --git a/tools/start-vmm/tests/tap_open-name-too-long.c b/tools/start-vmm/tests/tap_open-name-too-long.c
deleted file mode 100644
index ba4ebd6..0000000
--- a/tools/start-vmm/tests/tap_open-name-too-long.c
+++ /dev/null
@@ -1,20 +0,0 @@
-// SPDX-License-Identifier: EUPL-1.2+
-// SPDX-FileCopyrightText: 2022 Alyssa Ross <hi@alyssa.is>
-
-#include "../net-util.h"
-
-#include <assert.h>
-#include <errno.h>
-#include <net/if.h>
-#include <string.h>
-
-int main(void)
-{
-	char name[IFNAMSIZ];
-	int fd;
-
-	memset(name, 'a', sizeof name);
-	fd = tap_open(name, 0);
-	assert(fd == -1);
-	assert(errno == ENAMETOOLONG);
-}
diff --git a/tools/start-vmm/tests/tap_open.c b/tools/start-vmm/tests/tap_open.c
deleted file mode 100644
index bf5d00c..0000000
--- a/tools/start-vmm/tests/tap_open.c
+++ /dev/null
@@ -1,28 +0,0 @@
-// SPDX-License-Identifier: EUPL-1.2+
-// SPDX-FileCopyrightText: 2022 Alyssa Ross <hi@alyssa.is>
-
-#include "../net-util.h"
-
-#include <assert.h>
-#include <errno.h>
-#include <sched.h>
-#include <string.h>
-
-#include <sys/ioctl.h>
-
-#include <linux/if_tun.h>
-
-int main(void)
-{
-	char name[IFNAMSIZ] = "tap%d";
-	struct ifreq ifr;
-	int fd;
-
-	unshare(CLONE_NEWUSER|CLONE_NEWNET);
-
-	fd = tap_open(name, 0);
-	if (fd == -1 && (errno == EPERM || errno == ENOENT))
-		return 77;
-	assert(!ioctl(fd, (unsigned)TUNGETIFF, &ifr));
-	assert(!strcmp(name, ifr.ifr_name));
-}
diff --git a/vm/sys/net/default.nix b/vm/sys/net/default.nix
index fd5bf08..79ca0ad 100644
--- a/vm/sys/net/default.nix
+++ b/vm/sys/net/default.nix
@@ -7,7 +7,7 @@ pkgsMusl.callPackage (
 
 { lib, stdenvNoCC, nixos, runCommand, writeClosure
 , erofs-utils, jq, s6-rc, util-linux, xorg
-, busybox, execline, kmod, linux_latest, mdevd, nftables, xdp-tools
+, busybox, execline, kmod, linux_latest, mdevd, nftables, xdp-tools, wpa_supplicant
 , s6, s6-linux-init
 }:
 
@@ -52,7 +52,7 @@ let
   });
 
   packages = [
-    execline kmod mdevd s6 s6-linux-init s6-rc xdp-tools
+    execline kmod mdevd s6 s6-linux-init s6-rc xdp-tools wpa_supplicant
 
     (busybox.override {
       extraConfig = ''
-- 
2.51.2