From mboxrd@z Thu Jan 1 00:00:00 1970 Received: from atuin.qyliss.net (localhost [IPv6:::1]) by atuin.qyliss.net (Postfix) with ESMTP id D4BB5B94F; Fri, 31 Jul 2026 22:00:32 +0000 (UTC) Received: by atuin.qyliss.net (Postfix, from userid 993) id 689FEB877; Fri, 31 Jul 2026 22:00:27 +0000 (UTC) X-Spam-Checker-Version: SpamAssassin 4.0.1 (2024-03-26) on atuin.qyliss.net X-Spam-Level: X-Spam-Status: No, score=-0.1 required=3.0 tests=DKIM_SIGNED,DKIM_VALID, DKIM_VALID_AU,DMARC_PASS,FREEMAIL_FROM,RCVD_IN_DNSWL_NONE, SPF_HELO_NONE autolearn=unavailable autolearn_force=no version=4.0.1 Received: from mail-yw1-x112b.google.com (mail-yw1-x112b.google.com [IPv6:2607:f8b0:4864:20::112b]) by atuin.qyliss.net (Postfix) with ESMTPS id 52999B86F for ; Fri, 31 Jul 2026 22:00:25 +0000 (UTC) Received: by mail-yw1-x112b.google.com with SMTP id 00721157ae682-81e8fa1b8d6so20831107b3.1 for ; Fri, 31 Jul 2026 15:00:24 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=20251104; t=1785535224; x=1786140024; darn=spectrum-os.org; h=cc:to:in-reply-to:references:message-id:content-transfer-encoding :content-type:mime-version:subject:date:from:from:to:cc:subject:date :message-id:reply-to:content-type; bh=vuuuREu455cA8eBlwFSKIPU6RKDoqfAEW3hEKPRQeP4=; b=c9SHvZ4jam+81x7/DiJutX4UuV4tAYDczMMVO3v+ab1GLn+qGWL5mjzyoe/+ib/FV5 0iOHw8Mlmqf9YDiPbf8CTe21zTDmOuUtmE9MYTeTKgNmkvbxutJTkNLLrNCI8IY7E3mm gZXd0g/ZGesf6yoezr4o+aiOq9zCHCsMMGn7lE3xctBzXf7MjGgiBGz6Pv7TxruKwPUr ulW/kw5BH0jrvMmgvMaQ2OhnkeCHRkSMMubdcg9Fo2Ei+yqA09zF+oOjU8+7lNLzWIWr tFHqBO3Z8CxZjzFdgKZFJ8kutxf38svJiMvf+stRzq7Ph1y77Q97tGnC5f48uZzFkDKV li2A== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20251104; t=1785535224; x=1786140024; h=cc:to:in-reply-to:references:message-id:content-transfer-encoding :content-type:mime-version:subject:date:from:x-gm-gg :x-gm-message-state:from:to:cc:subject:date:message-id:reply-to :content-type; bh=vuuuREu455cA8eBlwFSKIPU6RKDoqfAEW3hEKPRQeP4=; b=tUl75z9xM6fViejU1LiLjrw+BM1zgKKp6neEufKvZN/ZDkl2Xqrbzx7dUK7FlYzAcH C8NniAx+SQUhluCeRpNmXY/0N7myuwqfvVhePrfjF89BlQLM0suFcNpzitX/q+2Y7DRy AR1Ds5Vhv+FNw1csPHqNN/3dvqc6/YBrgyJ0xmmUpoOv/JXtgF9WWxyEwGq4oIZEjnzV azV/rJrshJV8TGA23ttSjReVv3mmOXhfWyX0Xg/dfzpQY+PgJ0yddabULvC9QWlvZ/qD enFrPwe0H1mdE2x+cSj6wKipoUyrgpJoJZ/n7H1/qztAqAnDXPVKzZVwEDD8sbwAmRtj PBJQ== X-Gm-Message-State: AOJu0Yy02RbNnc826RiVGW5kUdmNI6q7/sidbOW8c9qIkNtIdG4uWOZm r2N8FOB3+1FGDCuh0E3IFiVuycEocvlcRBFO7yZdXO1chlrxjHaWWhalM/HjCQ== X-Gm-Gg: AR+sD11jX00ycoJFDOIbDzOiorjBK4tPnl5pbWVuvNkF6VXMYfx6i/hI6p0QnYYxZZt zrj3LGLj+kvoWD4aLGnlFSMf3UNlPZC4uswyM7rUnWHM2AaGzuQWkHL5iYzdRcaF0cSvMz/MtE9 wAXGHEyWIr4B4SNNmto4ODfhtZbTepfuO1WjqpjyAVrPMxnrgyY0h4uHmPWi5AedeOBRuzg8VB0 zw1qneeKznp7DKTqgJ0XyIFCxPRd5T+TagI1psDXhh+VEr4HQLX2zQGYdSf2rUtBJfua1K0Qft3 uPHIcJ9b+OMR9qD4N1HLKIZflQi2n6lglxaeRmRUgGVgKEEGKaEGy0iiqtyP9vFZPVyKglsrheH Qf6zcz5OcLV62KKFuQXY5Zb9z/94pKRnyfrr1i04wtWl2ueBd2ZbERppLt344GHP8i6aUoQ4nXe +9v2oawlV679e+R3nTNfRWtZzZp6XAPCc9Q3W+8wU6ye8FxZ5d/qKZettqX5XWGUpkjRTBz+5tF Q== X-Received: by 2002:a05:690c:4c0c:b0:81c:b865:2221 with SMTP id 00721157ae682-81fd4bf7221mr19254257b3.33.1785535223403; Fri, 31 Jul 2026 15:00:23 -0700 (PDT) Received: from localhost.localdomain ([185.98.168.14]) by smtp.gmail.com with UTF8SMTPSA id 00721157ae682-81fcd14658csm14852927b3.45.2026.07.31.15.00.22 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Fri, 31 Jul 2026 15:00:22 -0700 (PDT) From: Demi Marie Obenour Date: Fri, 31 Jul 2026 17:54:42 -0400 Subject: [PATCH v5 02/19] tools: Add control group manager MIME-Version: 1.0 Content-Type: text/plain; charset="utf-8" Content-Transfer-Encoding: 7bit Message-Id: <20260731-cgroups-v5-2-b325bac9d34f@gmail.com> References: <20260731-cgroups-v5-0-b325bac9d34f@gmail.com> In-Reply-To: <20260731-cgroups-v5-0-b325bac9d34f@gmail.com> To: Spectrum OS Development X-Mailer: b4 0.15.2 X-Developer-Signature: v=1; a=ed25519-sha256; t=1785534880; l=27547; i=demiobenour@gmail.com; s=20250729; h=from:subject:message-id; bh=lUp157LwxT+kO2OM730B10fMET0V2mlzhrV3fnONWO4=; b=JKG3FD3ylwWUNOhaNlnmR5A5ABRaFXAsWxECGVdv6VngvtrPTtyy5jTGx8imqTOkIB4qg/PgG txtS1E44jbTB+Perl3/FTmOV5596+uQD78d9yzv3Z79G25GJpusr4Pr X-Developer-Key: i=demiobenour@gmail.com; a=ed25519; pk=X57Q4/YQDj9t4SBeKaDwvXYKB6quZJVx/DE2Ly2out0= Message-ID-Hash: AKHWH7XIHEI7FUN7BNWJMCSHSBU4EW7S X-Message-ID-Hash: AKHWH7XIHEI7FUN7BNWJMCSHSBU4EW7S X-MailFrom: demiobenour@gmail.com X-Mailman-Rule-Hits: member-moderation X-Mailman-Rule-Misses: dmarc-mitigation; no-senders; approved; loop; banned-address; header-match-devel.spectrum-os.org-0; header-match-devel.spectrum-os.org-1; header-match-devel.spectrum-os.org-2; header-match-devel.spectrum-os.org-3; header-match-devel.spectrum-os.org-4; emergency CC: Demi Marie Obenour , Alyssa Ross X-Mailman-Version: 3.3.10 Precedence: list List-Id: Patches and low-level development discussion Archived-At: List-Archive: List-Help: List-Owner: List-Post: List-Subscribe: List-Unsubscribe: The cgroup-setup Rust program can create and purge cgroups. It can also wait for one to become empty, spawn a program in a cgroup, and more. In the future, it will also support cgroup-based resource control. Locking is used to ensure that concurrent invocations are safe. Signed-off-by: Demi Marie Obenour --- .codespellrc | 2 +- host/rootfs/default.nix | 6 +- pkgs/default.nix | 1 + tools/cgroup-setup/Cargo.lock | 67 ++++++++ tools/cgroup-setup/Cargo.lock.license | 2 + tools/cgroup-setup/Cargo.toml | 10 ++ tools/cgroup-setup/default.nix | 22 +++ tools/cgroup-setup/src/cgroup.rs | 308 ++++++++++++++++++++++++++++++++++ tools/cgroup-setup/src/main.rs | 186 ++++++++++++++++++++ 9 files changed, 600 insertions(+), 4 deletions(-) diff --git a/.codespellrc b/.codespellrc index d8023afc64ec44e98a88c5e397c8d5c681dde063..ae20da8309530759ac729689825a4afc683afa09 100644 --- a/.codespellrc +++ b/.codespellrc @@ -2,4 +2,4 @@ # SPDX-License-Identifier: CC0-1.0 [codespell] -ignore-words-list = crate,passt,rouge,ser +ignore-words-list = crate,passt,rouge,ser,WRONLY diff --git a/host/rootfs/default.nix b/host/rootfs/default.nix index 6bfeefbe0a5f76c1538ccb40e5eb8f291f5d3592..ccf626e2ec0f4bf96573dc5edf058c9375bb65f6 100644 --- a/host/rootfs/default.nix +++ b/host/rootfs/default.nix @@ -8,7 +8,7 @@ import ../../lib/call-package.nix ( }: pkgsMusl.callPackage ( -{ spectrum-host-tools, spectrum-router +{ spectrum-host-tools, spectrum-router, spectrum-cgroup-setup , lib, stdenvNoCC, nixos, runCommand, writeClosure, erofs-utils, s6-rc , btrfs-progs, bubblewrap, busybox, cloud-hypervisor, cosmic-files , crosvm, cryptsetup, dejavu_fonts, dbus, execline, foot, fuse3 @@ -27,8 +27,8 @@ let packages = [ btrfs-progs bubblewrap cloud-hypervisor cosmic-files crosvm cryptsetup dbus execline fuse3 inotify-tools iproute2 jq kmod mdevd mount-flatpak s6 - s6-linux-init s6-rc shadow socat spectrum-host-tools spectrum-router - virtiofsd xdg-desktop-portal-spectrum-host + s6-linux-init s6-rc shadow socat spectrum-cgroup-setup spectrum-host-tools + spectrum-router virtiofsd xdg-desktop-portal-spectrum-host (foot.override { allowPgo = false; }) diff --git a/pkgs/default.nix b/pkgs/default.nix index 44f7b5ff78cb6b9e755292a6a417d0b627ed3fb0..0a13393164ad5d7f752e630763f3f97166479af5 100644 --- a/pkgs/default.nix +++ b/pkgs/default.nix @@ -51,6 +51,7 @@ let driverSupport = true; }; spectrum-router = self.callSpectrumPackage ../tools/router {}; + spectrum-cgroup-setup = self.callSpectrumPackage ../tools/cgroup-setup {}; xdg-desktop-portal-spectrum-host = self.callSpectrumPackage ../tools/xdg-desktop-portal-spectrum-host {}; diff --git a/tools/cgroup-setup/Cargo.lock b/tools/cgroup-setup/Cargo.lock new file mode 100644 index 0000000000000000000000000000000000000000..fe967b3aa02c296c87b6b36ac59253dbe0a32de9 --- /dev/null +++ b/tools/cgroup-setup/Cargo.lock @@ -0,0 +1,67 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "bitflags" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3" + +[[package]] +name = "cgroup-setup" +version = "0.0.0" +dependencies = [ + "libc", + "rustix", +] + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "libc" +version = "0.2.186" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] diff --git a/tools/cgroup-setup/Cargo.lock.license b/tools/cgroup-setup/Cargo.lock.license new file mode 100644 index 0000000000000000000000000000000000000000..62497aa72873adb3e62fb4f186bd19b579b36734 --- /dev/null +++ b/tools/cgroup-setup/Cargo.lock.license @@ -0,0 +1,2 @@ +SPDX-License-Identifier: CC0-1.0 +SPDX-FileCopyrightText: 2026 Demi Marie Obenour diff --git a/tools/cgroup-setup/Cargo.toml b/tools/cgroup-setup/Cargo.toml new file mode 100644 index 0000000000000000000000000000000000000000..bba3f55bcc6dda049f2808efd11c397131742500 --- /dev/null +++ b/tools/cgroup-setup/Cargo.toml @@ -0,0 +1,10 @@ +# SPDX-License-Identifier: CC0-1.0 +# SPDX-FileCopyrightText: 2026 Demi Marie Obenour + +[package] +name = "cgroup-setup" +edition = "2024" + +[dependencies] +libc = "0.2.177" +rustix = { version = "1.1.2", features = ["fs"] } diff --git a/tools/cgroup-setup/default.nix b/tools/cgroup-setup/default.nix new file mode 100644 index 0000000000000000000000000000000000000000..fe3a8bcd4e6118f3cbf49c2fc0e3cda01570c484 --- /dev/null +++ b/tools/cgroup-setup/default.nix @@ -0,0 +1,22 @@ +# SPDX-FileCopyrightText: 2024 Alyssa Ross +# SPDX-FileCopyrightText: 2025 Yureka Lilian +# SPDX-License-Identifier: MIT + +import ../../lib/call-package.nix ( +{ src, lib, rustPlatform }: + +rustPlatform.buildRustPackage { + name = "spectrum-cgroup-setup"; + + src = lib.fileset.toSource { + root = ../..; + fileset = lib.fileset.intersection src ./.; + }; + sourceRoot = "source/tools/cgroup-setup"; + + cargoLock.lockFile = ./Cargo.lock; + + postInstall = '' + ln -s -- cgroup-setup "$out/bin/cgroup-purge" + ''; +}) (_: {}) diff --git a/tools/cgroup-setup/src/cgroup.rs b/tools/cgroup-setup/src/cgroup.rs new file mode 100644 index 0000000000000000000000000000000000000000..4ea89b5787d379e9d0c40405810ce5701167c639 --- /dev/null +++ b/tools/cgroup-setup/src/cgroup.rs @@ -0,0 +1,308 @@ +// SPDX-License-Identifier: EUPL-1.2+ +// SPDX-FileCopyrightText: 2026 Demi Marie Obenour + +use std::cell::RefCell; +use std::ffi::OsStr; +use std::fs::File; +use std::io::{Read as _, Seek as _, Write as _}; +use std::os::unix::prelude::*; + +use std::path::{Component, Path, PathBuf}; +use std::rc::Rc; + +use rustix::fs::{AtFlags, CWD, Dir, FlockOperation}; +use rustix::{ + fs::{Mode, OFlags, ResolveFlags}, + io::Errno, +}; + +#[derive(Debug)] +pub(crate) struct Cgroup { + fd: Vec, +} + +impl AsFd for Cgroup { + fn as_fd(&self) -> BorrowedFd<'_> { + self.fd.last().unwrap().as_fd() + } +} + +fn assert_single_component(component: &Path) { + match component.as_os_str().as_bytes() { + b"" | b"." | b".." => panic!("bad component"), + c if c.contains(&b'\0') => panic!("NUL in component"), + c if c.contains(&b'/') => panic!("/ in component"), + _ => {} + } +} + +// Wrapper around openat2() with better defaults. +pub fn openat2_simple(fd: &dyn AsFd, path: &Path, flags: OFlags) -> Result { + rustix::fs::openat2( + fd.as_fd(), + path, + OFlags::CLOEXEC | OFlags::NOCTTY | flags, + Mode::empty(), + ResolveFlags::NO_SYMLINKS | ResolveFlags::NO_MAGICLINKS | ResolveFlags::NO_XDEV, + ) +} + +pub const DEFAULT_LEAF: &str = "$inner.service"; + +pub fn check_path(path: &Path) -> Result<(), String> { + let bytes = path.as_os_str().as_bytes(); + // Path::components() skips ., so use string manipulation instead. + for component in bytes[path.is_absolute() as usize..].split(|&b| b == b'/') { + if matches!(component, b"" | b"." | b"..") { + return Err(format!("cgroup path {path:?} isn't canonical")); + } + } + Ok(()) +} + +fn push_child_fds(fds: &mut Vec<(Rc>, PathBuf)>, fd: OwnedFd) { + // The rustix source code shows that Dir::new() never fails. + let child_fd = Rc::new(RefCell::new(Dir::new(fd).unwrap())); + while let Some(element) = child_fd.borrow_mut().next() { + let element = element.expect("Iterating through a cgroup directory failed?"); + if element.file_type() != rustix::fs::FileType::Directory { + continue; + } + match element.file_name().to_bytes() { + b"." | b".." => {} + other => { + let other = Path::new(OsStr::from_bytes(other)).to_owned(); + assert_single_component(&other); + fds.push((child_fd.clone(), other)); + } + } + } +} + +// Remove all subdirectories of the given directory recursively, +// but not the directory itself. The directory file descriptor +// is closed. +// +// This isn't the most efficient possible algorithm, but +// simplicity is more important than performance in this +// case. Also, it keeps open more file descriptors than +// strictly necessary, but Spectrum runs with a very high +// limit for the number of open file descriptors, and it +// uses shallow control group hierarchies. +fn remove_child_directories(dirfd: OwnedFd) -> Result<(), Errno> { + let mut fds = Vec::new(); + // Push the children of this directory onto the stack. + push_child_fds(&mut fds, dirfd); + while let Some((d, path)) = fds.pop() { + assert_single_component(&path); + // Try to delete the directory. If that fails because there are child + // directories, push the child directories onto the stack, then push + // this directory again. + match rustix::fs::unlinkat( + // The rustix source code shows that Dir::fd() never fails. + d.borrow().fd().unwrap(), + &path, + AtFlags::REMOVEDIR, + ) { + Err(Errno::NOTEMPTY) => {} + Ok(()) => continue, + Err(bad) => return Err(bad), + } + let fd = openat2_simple( + // The rustix source code shows that Dir::fd() never fails. + &d.borrow().fd().unwrap(), + &path, + OFlags::DIRECTORY | OFlags::RDONLY, + )?; + // Process child directories first, then attempt to delete the + // directory again. + fds.push((d, path)); + push_child_fds(&mut fds, fd); + } + Ok(()) +} + +// If the path is absolute, make it relative. +// Otherwise, read the current cgroup from /proc/thread-self/cgroup +// and prepend it to the path. +fn prepend_current_cgroup_if_needed(path: &Path) -> PathBuf { + if let Ok(suffix) = path.strip_prefix("/") { + suffix.to_owned() + } else { + // /proc/thread-self is the same as /proc/self, except for the current + // thread instead of the initial thread. In this case, the two are + // identical, but using /proc/thread-self is better practice as it is + // correct in more cases. Reading /proc/thread-self/cgroup should + // never fail unless the system is seriously broken. + let current_cgroup = std::fs::read("/proc/thread-self/cgroup") + .expect("cannot read /proc/thread-self/cgroup"); + // Using this on a system without cgroups v2 mounted is user error + // and not supported. + let current_cgroup = current_cgroup + .strip_prefix(b"0::/") + .and_then(|e| e.strip_suffix(b"\n")) + .expect("you don't have cgroups v2 mounted"); + let mut current_cgroup = PathBuf::from(OsStr::from_bytes(current_cgroup)); + // Strip the implied $inner.service suffix. + // This is used to satisfy the "no internal processes" rule. + if current_cgroup.ends_with(Path::new(DEFAULT_LEAF)) { + assert!(current_cgroup.pop()); + } + current_cgroup.push(path); + current_cgroup + } +} + +pub(crate) fn write_value(fd: &dyn AsFd, name: &Path, value: &[u8]) -> Result<(), String> { + let path = Path::new(name); + let fd = openat2_simple(fd, path, OFlags::WRONLY) + .map_err(|e| format!("Cannot open {name:?}: {e}"))?; + File::from(fd).write_all(value).map_err(|e| { + format!( + "Cannot write {:?} to {name:?}: {e}", + OsStr::from_bytes(value) + ) + }) +} + +impl Cgroup { + pub fn open_beneath(&self, path: &Path, flags: OFlags) -> Result { + openat2_simple(self, path, flags) + } + + pub fn new(path: &Path) -> Result { + let cgroup_root = rustix::fs::openat2( + CWD, + Path::new("/sys/fs/cgroup"), + OFlags::CLOEXEC | OFlags::DIRECTORY | OFlags::RDONLY, + Mode::empty(), + ResolveFlags::NO_SYMLINKS | ResolveFlags::NO_MAGICLINKS, + ) + .map_err(|e| format!("Cannot open /sys/fs/cgroup: {e}"))?; + let mut cgroup = Self { + fd: vec![(cgroup_root)], + }; + + let path = prepend_current_cgroup_if_needed(path); + for component in path.components() { + let component = match component { + Component::Normal(component) => component, + _ => unreachable!(), + }; + let sub_fd = cgroup + .open_beneath(Path::new(component), OFlags::RDONLY | OFlags::DIRECTORY) + .map_err(|e| format!("Cannot open sub-cgroup {component:?}: {e}"))?; + // Take a shared lock on the *previous* file descriptor. + rustix::fs::flock(&cgroup, FlockOperation::LockShared) + .map_err(|e| format!("Cannot lock sub-cgroup {component:?}: {e}"))?; + cgroup.fd.push(sub_fd); + } + // Take an exclusive lock on the final file descriptor. + rustix::fs::flock(&cgroup, FlockOperation::LockExclusive) + .map_err(|e| format!("Cannot lock {path:?}: {e}"))?; + Ok(cgroup) + } + + pub fn wait_for_empty(fd: &dyn AsFd) -> std::io::Result<()> { + let wait_file = openat2_simple(fd, Path::new("cgroup.events"), OFlags::RDONLY)?; + let poll_fd = wait_file.as_raw_fd(); + let mut wait_fd = File::from(wait_file); + let mut fds = libc::pollfd { + fd: poll_fd, + events: libc::POLLPRI | libc::POLLERR, + revents: 0, + }; + let mut v = vec![]; + loop { + v.clear(); + wait_fd + .seek(std::io::SeekFrom::Start(0)) + .expect("Seek on control group file should succeed"); + wait_fd + .read_to_end(&mut v) + .expect("reading from control group should work"); + // Check that the cgroup isn't already empty. If it was, + // the kernel would not send an event and poll() would wait + // forever. + if v.split(|&c| c == b'\n').any(|line| line == b"populated 0") { + break; + } + // SAFETY: FFI call, valid arguments, fds contains 1 element + if unsafe { libc::poll(&raw mut fds, 1, -1) } != 1 { + panic!("poll failed"); + } + } + Ok(()) + } + + pub fn purge_child(&mut self, path: &Path) -> Result<(), String> { + assert_single_component(path); + // See if we can just delete the child directly. + match rustix::fs::unlinkat(&self, Path::new(path), AtFlags::REMOVEDIR) { + // If the cgroup was successfully deleted, or if it + // has already been deleted, we are done. + Ok(()) | Err(Errno::NOENT) => return Ok(()), + // If this cgroup is in use, keep going. + Err(Errno::BUSY) => {} + Err(e) => return Err(format!("Cannot purge {path:?}: {e}")), + } + + let sub_fd = match self.open_beneath(path, OFlags::RDONLY | OFlags::DIRECTORY) { + Ok(sub_fd) => sub_fd, + Err(Errno::NOENT) => return Ok(()), + Err(e) => { + return Err(format!("Cannot open sub-cgroup {path:?}: {e}",)); + } + }; + + // Take an exclusive lock on the cgroup that is about to be + // removed. This avoids concurrent executions of this program + // operating on deleted sub-cgroups. + rustix::fs::flock(&sub_fd, FlockOperation::LockExclusive) + .map_err(|e| format!("Cannot lock sub-cgroup: {e}"))?; + + // Drop the exclusive lock on the original cgroup, + // This avoids blocking concurrent operations on other + // child cgroups while the cgroup is being purged, + // or while waiting for programs to exit. + rustix::fs::flock(&self, FlockOperation::LockShared) + .map_err(|e| format!("Cannot relock: {e}"))?; + + // Kill all processes in the child cgroup. + write_value(&sub_fd, Path::new("cgroup.kill"), b"1")?; + + // Wait for the child cgroup to become empty. + Self::wait_for_empty(&sub_fd) + .map_err(|e| format!("Cannot wait for cgroup to become empty: {e}")) + .inspect_err(|_| { + self.fd.pop().unwrap(); + })?; + + // Remove the child cgroup and its contents recursively. + remove_child_directories(sub_fd).map_err(|e| format!("Cannot remove: {e}"))?; + // Re-take an exclusive lock on the parent of the cgroup being purged. + // Otherwise, a concurrent instance of cgroup-setup might create a cgroup + // only for this one to delete it. The other instance could then try to + // create a sub-cgroup of a deleted cgroup, which would fail. Waiting + // until nobody is using the parent cgroup ensures these problems can't + // happen. + // + // This must happen *after* the lock on the cgroup being purged is released. + // Another instance of the program might have a shared lock on the parent + // and be waiting for an exclusive lock on the child. Trying to take an + // exclusive lock on the parent while a lock is held on the child would + // result in an ABBA deadlock. + rustix::fs::flock(&self, FlockOperation::LockExclusive) + .map_err(|e| format!("Cannot re-lock exclusively: {e}"))?; + // Delete the cgroup. If it's been re-created in the meantime + // and is currently in use, this is not an error. Another + // process deleting the cgroup is also not an error. Both of + // these can happen because of the time period between + // remove_child_directories() closing the file descriptor + // (releasing its lock) and the above call to flock(). + match rustix::fs::unlinkat(&self, path, AtFlags::REMOVEDIR) { + Ok(()) | Err(Errno::BUSY) | Err(Errno::NOENT) => Ok(()), + Err(e) => Err(format!("Cannot delete: {e}")), + } + } +} diff --git a/tools/cgroup-setup/src/main.rs b/tools/cgroup-setup/src/main.rs new file mode 100644 index 0000000000000000000000000000000000000000..e8d9e9d7c2111857cc25b36433d8f33ddb28c27b --- /dev/null +++ b/tools/cgroup-setup/src/main.rs @@ -0,0 +1,186 @@ +// SPDX-License-Identifier: EUPL-1.2+ +// SPDX-FileCopyrightText: 2026 Demi Marie Obenour + +mod cgroup; + +use cgroup::{Cgroup, openat2_simple, write_value}; +use rustix::{ + fs::{FlockOperation, Mode, OFlags, XattrFlags}, + io::Errno, +}; +use std::{ + env::ArgsOs, + ffi::OsStr, + fs::File, + io::Read as _, + os::unix::prelude::*, + path::{Path, PathBuf}, +}; + +// Check that the path is canonical, +// then split it into basename and filename. +fn split_path(path: &Path) -> Result<(&Path, &Path), String> { + cgroup::check_path(path) + .map(|()| (path.parent().unwrap(), Path::new(path.file_name().unwrap()))) +} + +fn read_control_file(fd: &dyn AsFd, p: &Path) -> Result, String> { + let mut buf = Vec::new(); + File::from( + openat2_simple(&fd, Path::new(p), OFlags::RDONLY) + .map_err(|e| format!("Cannot open {p:?}: {e}"))?, + ) + .read_to_end(&mut buf) + .map_err(|e| format!("Cannot read {p:?}: {e}"))?; + Ok(buf) +} + +fn enable_subtree_control(fd: &dyn AsFd) -> Result<(), String> { + let p = Path::new("cgroup.controllers"); + let buf = read_control_file(fd, p)?; + let mut subtree = vec![]; + for controller in buf.split(|&b| b == b' ').filter(|e| !e.is_empty()) { + if !subtree.is_empty() { + subtree.push(b' '); + } + subtree.push(b'+'); + subtree.extend_from_slice(controller); + } + if !subtree.is_empty() { + write_value(&fd, Path::new("cgroup.subtree_control"), &subtree)?; + } + Ok(()) +} + +fn cgroup_setup(mut args: ArgsOs) -> Result<(), String> { + let mut leaf = false; + let mut cgroup_path; + let mut systemd_compat = false; + let mut wait = true; + loop { + cgroup_path = args.next(); + let Some(ref arg_) = cgroup_path else { + break; + }; + let arg_ = arg_.as_bytes(); + if arg_ == b"--" { + cgroup_path = args.next(); + break; + } + if !arg_.starts_with(b"-") { + break; + } + + if !arg_.starts_with(b"--") { + return Err("takes no short options".to_owned()); + } + + match &arg_[2..] { + b"leaf" => leaf = true, + b"wait" => wait = true, + b"no-wait" => wait = false, + b"systemd-compat" => systemd_compat = true, + arg => return Err(format!("unknown long option {:?}", OsStr::from_bytes(arg))), + } + } + let Some(cgroup_path) = cgroup_path.map(PathBuf::from) else { + return Err("have no positional arguments, expected at least 1".to_owned()); + }; + + let (parent_cgroup_path, child_cgroup_path) = split_path(&cgroup_path)?; + let cgroup = Cgroup::new(parent_cgroup_path)?; + match rustix::fs::mkdirat(&cgroup, child_cgroup_path, Mode::from_raw_mode(0o755)) { + Ok(()) | Err(Errno::EXIST) => {} + Err(e) => return Err(format!("Cannot make child cgroup: {e}")), + } + let child = cgroup + .open_beneath(child_cgroup_path, OFlags::RDONLY | OFlags::DIRECTORY) + .map_err(|e| format!("Cannot make child cgroup: {e}"))?; + if wait { + // While waiting, only hold an exclusive lock on the child, not the parent. + rustix::fs::flock(&child, FlockOperation::LockExclusive) + .map_err(|e| format!("Cannot take an exclusive lock on child cgroup: {e}"))?; + rustix::fs::flock(&cgroup, FlockOperation::LockShared) + .map_err(|e| format!("Cannot downgrade lock on cgroup to a shared lock: {e}"))?; + Cgroup::wait_for_empty(&child) + .map_err(|e| format!("Cannot wait for {parent_cgroup_path:?} to be empty: {e}"))?; + } + let pid = std::process::id().to_string(); + if leaf { + if args.len() != 0 { + // If we aren't delegating any cgroups, don't create a sub-cgroup. + write_value(&child, Path::new("cgroup.procs"), pid.as_bytes()) + .map_err(|e| format!("Cannot move process to child cgroup: {e}"))?; + } + } else { + // If the child process will need to manage cgroups itself, it will need + // to set up a sub-cgroup due to the "no internal processes" rule. It's + // simplest to just do it automatically. If the cgroup already exists, + // that isn't an error. + match rustix::fs::mkdirat(&child, cgroup::DEFAULT_LEAF, Mode::from_raw_mode(0o755)) { + Ok(()) | Err(Errno::EXIST) => {} + Err(e) => return Err(format!("Cannot make child cgroup: {e}")), + } + if args.len() != 0 { + let child_proc_path = Path::new(cgroup::DEFAULT_LEAF).join(Path::new("cgroup.procs")); + write_value(&child, &child_proc_path, pid.as_bytes()) + .map_err(|e| format!("Cannot move process to child cgroup: {e}"))?; + } + if systemd_compat { + // systemd-aware programs expect to have user.delegate=1 + // and to set cgroup.subtree_control themselves + rustix::fs::fsetxattr(&child, c"user.delegate", b"1", XattrFlags::empty()).map_err( + |e| format!("Cannot enable cgroup delegation in {parent_cgroup_path:?}: {e}"), + )? + } else { + // Spectrum's programs do not check for user.delegate=1 + // and expect the caller to set cgroup.subtree_control. + enable_subtree_control(&child)?; + } + } + let Some(program_name) = args.next() else { + return Ok(()); + }; + let e = std::process::Command::new(&program_name).args(args).exec(); + Err(format!("Cannot spawn child {:?}: {}", program_name, e)) +} + +fn cgroup_purge(mut args: ArgsOs) -> Result<(), String> { + if args.len() != 1 { + return Err("usage: cgroup-purge CGROUP_TO_PURGE".to_owned()); + } + let arg = args.next().unwrap(); + let (parent, child) = split_path(Path::new(&arg))?; + Cgroup::new(parent)?.purge_child(child) +} + +fn run(prog_name: &OsStr, args: ArgsOs) -> Result<(), String> { + match prog_name + .as_bytes() + .split(|&b| b == b'/') + .next_back() + .unwrap() + { + b"cgroup-setup" => cgroup_setup(args), + b"cgroup-purge" => cgroup_purge(args), + _ => Err(format!( + "must be invoked as \"cgroup-setup\" or \ + \"cgroup-purge\", got {prog_name:?}", + )), + } +} + +fn main() { + let mut args = std::env::args_os(); + let Some(prog_name) = args.next() else { + eprintln!("No command line arguments (argv[0] is NULL)"); + std::process::exit(1); + }; + match run(&prog_name, args) { + Ok(()) => {} + Err(e) => { + eprintln!("{prog_name:?}: {}", e); + std::process::exit(1); + } + } +} -- 2.55.0