Merge pull request #206 from keszybz/system

Allow "set!var = program" at top level that makes var = eval(system(program))

Author: keszybz

man/index.txt

@@ -3,6 +3,7 @@ zram-generator.conf(5)  zram-generator.conf.5.ronn
 
 modprobe(8)             https://man7.org/linux/man-pages/man8/modprobe.8.html
 proc(5)                 https://man7.org/linux/man-pages/man5/proc.5.html
+system(3)               https://man7.org/linux/man-pages/man3/system.3.html
 
 systemd-detect-virt(1)  https://freedesktop.org/software/systemd/man/systemd-detect-virt.html
 systemd.generator(7)    https://freedesktop.org/software/systemd/man/systemd.generator.html

man/zram-generator.conf.md

@@ -42,7 +42,7 @@ This option thus has higher priority than the configuration files.
 
 ## OPTIONS
 
-Each device is configured independently in its `[zramN]` section, where N is a nonnegative integer. Other sections are ignored.
+Each device is configured independently in its `[zramN]` section, where N is a nonnegative integer. The global section may contain [DIRECTIVES]. Other sections are ignored.
 
 Devices with the final size of *0* will be discarded.
 
@@ -57,6 +57,7 @@ Devices with the final size of *0* will be discarded.
 * `zram-size`=
 
   Sets the size of the zram device as a function of *MemTotal*, available as the `ram` variable.
+  Additional variables may be provided by [DIRECTIVES].
 
   Arithmetic operators (^%/\*-+), e, π, SI suffixes, log(), int(), ceil(), floor(), round(), abs(), min(), max(), and trigonometric functions are supported.
 
@@ -66,7 +67,7 @@ Devices with the final size of *0* will be discarded.
 
   Sets the maximum resident memory limit of the zram device (or *0* for no limit) as a function of *MemTotal*, available as the `ram` variable.
 
-  Same format as *zram-size*. Defaults to *0*.
+  Same format as `zram-size`. Defaults to *0*.
 
 * `compression-algorithm`=
 
@@ -117,6 +118,17 @@ Devices with the final size of *0* will be discarded.
 
   Defaults to *discard*.
 
+## DIRECTIVES
+
+The global section (before any section header) may contain directives in the following form:
+
+* `set!`*variable*=*program*
+
+  *program* is executed by the shell as-if by system(3),
+  its standard output stream parsed as an arithmetic expression (like `zram-size`/`zram-resident-limit`),
+  then the result is remembered into *variable*,
+  usable in later `set!`s and `zram-size`s/`zram-resident-limit`s.
+
 ## ENVIRONMENT VARIABLES
 
 Setting `ZRAM_GENERATOR_ROOT` during parsing will cause */proc/meminfo* to be read from *$ZRAM_GENERATOR_ROOT/proc/meminfo* instead,

src/config.rs

@@ -10,7 +10,9 @@ use std::ffi::OsString;
 use std::fmt;
 use std::fs;
 use std::io::{prelude::*, BufReader};
+use std::os::unix::process::ExitStatusExt;
 use std::path::{Component, Path, PathBuf};
+use std::process::{Command, Stdio};
 
 const DEFAULT_ZRAM_SIZE: &str = "min(ram / 2, 4096)";
 const DEFAULT_RESIDENT_LIMIT: &str = "0";
@@ -97,14 +99,14 @@ impl Device {
     fn process_size(
         &self,
         zram_option: &Option<(String, fasteval::ExpressionI, fasteval::Slab)>,
-        memtotal_mb: f64,
+        ctx: &mut EvalContext,
         default_size: f64,
         label: &str,
     ) -> Result<u64> {
         Ok((match zram_option {
             Some(zs) => {
                 zs.1.from(&zs.2.ps)
-                    .eval(&zs.2, &mut RamNs(memtotal_mb))
+                    .eval(&zs.2, ctx)
                     .with_context(|| format!("{} {}", self.name, label))
                     .and_then(|f| {
                         if f >= 0. {
@@ -119,29 +121,29 @@ impl Device {
             * 1024.0) as u64)
     }
 
-    fn set_disksize_if_enabled(&mut self, memtotal_mb: u64) -> Result<()> {
-        if !self.is_enabled(memtotal_mb) {
+    fn set_disksize_if_enabled(&mut self, ctx: &mut EvalContext) -> Result<()> {
+        if !self.is_enabled(ctx.memtotal_mb) {
             return Ok(());
         }
 
         if self.zram_fraction.is_some() || self.max_zram_size_mb.is_some() {
             // deprecated path
             let max_mb = self.max_zram_size_mb.unwrap_or(None).unwrap_or(u64::MAX);
-            self.disksize = ((self.zram_fraction.unwrap_or(0.5) * memtotal_mb as f64) as u64)
+            self.disksize = ((self.zram_fraction.unwrap_or(0.5) * ctx.memtotal_mb as f64) as u64)
                 .min(max_mb)
                 * (1024 * 1024);
         } else {
             self.disksize = self.process_size(
                 &self.zram_size,
-                memtotal_mb as f64,
-                (memtotal_mb as f64 / 2.).min(4096.), // DEFAULT_ZRAM_SIZE
+                ctx,
+                (ctx.memtotal_mb as f64 / 2.).min(4096.), // DEFAULT_ZRAM_SIZE
                 "zram-size",
             )?;
         }
 
         self.mem_limit = self.process_size(
             &self.zram_resident_limit,
-            memtotal_mb as f64,
+            ctx,
             0., // DEFAULT_RESIDENT_LIMIT
             "zram-resident-limit",
         )?;
@@ -225,13 +227,19 @@ impl fmt::Display for Algorithms {
     }
 }
 
-struct RamNs(f64);
-impl fasteval::EvalNamespace for RamNs {
+struct EvalContext {
+    memtotal_mb: u64,
+    additional: BTreeMap<String, f64>,
+}
+
+impl fasteval::EvalNamespace for EvalContext {
     fn lookup(&mut self, name: &str, args: Vec<f64>, _: &mut String) -> Option<f64> {
-        if name == "ram" && args.is_empty() {
-            Some(self.0)
-        } else {
+        if !args.is_empty() {
             None
+        } else if name == "ram" {
+            Some(self.memtotal_mb as f64)
+        } else {
+            self.additional.get(name).copied()
         }
     }
 }
@@ -252,6 +260,57 @@ pub fn read_all_devices(root: &Path, kernel_override: bool) -> Result<Vec<Device
         .collect())
 }
 
+fn toplevel_line(
+    path: &Path,
+    k: &str,
+    val: &str,
+    slab: &mut fasteval::Slab,
+    ctx: &mut EvalContext,
+) -> Result<()> {
+    let (op, arg) = if let Some(colon) = k.find('!') {
+        k.split_at(colon + 1)
+    } else {
+        warn!(
+            "{}: invalid outside-of-section key {}, ignoring.",
+            path.display(),
+            k
+        );
+        return Ok(());
+    };
+
+    match op {
+        "set!" => {
+            let out = Command::new("/bin/sh")
+                .args(["-c", "--", val])
+                .stdin(Stdio::null())
+                .stderr(Stdio::inherit())
+                .output()
+                .with_context(|| format!("{}: {}: {}", path.display(), k, val))?;
+            let exit = out
+                .status
+                .code()
+                .unwrap_or_else(|| 128 + out.status.signal().unwrap());
+            if exit != 0 {
+                warn!("{}: {} exited {}", k, val, exit);
+            }
+
+            let expr = String::from_utf8(out.stdout)
+                .with_context(|| format!("{}: {}: {}", path.display(), k, val))?;
+            let evalled = fasteval::Parser::new()
+                .parse(&expr, &mut slab.ps)
+                .and_then(|p| p.from(&slab.ps).eval(slab, ctx))
+                .with_context(|| format!("{}: {}: {}: {}", path.display(), k, val, expr))?;
+            ctx.additional.insert(arg.to_string(), evalled);
+        }
+        _ => warn!(
+            "{}: unknown outside-of-section operation {}, ignoring.",
+            path.display(),
+            op
+        ),
+    }
+    Ok(())
+}
+
 fn read_devices(
     root: &Path,
     kernel_override: bool,
@@ -264,18 +323,21 @@ fn read_devices(
     }
 
     let mut devices: HashMap<String, Device> = HashMap::new();
+    let mut slab = fasteval::Slab::new();
+    let mut ctx = EvalContext {
+        memtotal_mb,
+        additional: BTreeMap::new(),
+    };
 
     for (_, path) in fragments {
         let ini = Ini::load_from_file(&path)?;
 
         for (sname, props) in ini.iter() {
             let sname = match sname {
                 None => {
-                    warn!(
-                        "{}: ignoring settings outside of section: {:?}",
-                        path.display(),
-                        props
-                    );
+                    for (k, v) in props.iter() {
+                        toplevel_line(&path, k, v, &mut slab, &mut ctx)?;
+                    }
                     continue;
                 }
                 Some(sname) if sname.starts_with("zram") && sname[4..].parse::<u64>().is_ok() => {
@@ -304,7 +366,7 @@ fn read_devices(
     }
 
     for dev in devices.values_mut() {
-        dev.set_disksize_if_enabled(memtotal_mb)?;
+        dev.set_disksize_if_enabled(&mut ctx)?;
     }
 
     Ok(devices)
@@ -624,7 +686,11 @@ foo=0
             parse_line(&mut dev, "zram-size", val).unwrap();
         }
         assert!(dev.is_enabled(memtotal_mb));
-        dev.set_disksize_if_enabled(memtotal_mb).unwrap();
+        dev.set_disksize_if_enabled(&mut EvalContext {
+            memtotal_mb,
+            additional: vec![("two".to_string(), 2.)].into_iter().collect(),
+        })
+        .unwrap();
         dev.disksize
     }
 
@@ -636,6 +702,14 @@ foo=0
         );
     }
 
+    #[test]
+    fn test_eval_size_expression_with_additional() {
+        assert_eq!(
+            dev_with_zram_size_size(Some("0.5 * ram * two"), 100),
+            50 * 2 * 1024 * 1024
+        );
+    }
+
     #[test]
     fn test_eval_size_expression_500() {
         assert_eq!(

tests/02-zstd/etc/systemd/zram-generator.conf

@@ -1,4 +1,9 @@
+set!top = echo 3
+set!bottom = echo 4
+set!ratio = ! echo top / bottom
+
 [zram0]
 compression-algorithm = zstd
 host-memory-limit = 2050
-zram-size = ram * 0.75
+zram-resident-limit = 9999
+zram-size = ram * ratio

tests/10-example/bin/xenstore-read

@@ -0,0 +1,2 @@
+#!/bin/sh -x
+echo '8 * 1024'  # MB

tests/test_cases.rs

@@ -4,8 +4,11 @@ use zram_generator::{config, generator};
 
 use anyhow::Result;
 use fs_extra::dir::{copy, CopyOptions};
+use std::env;
+use std::ffi::OsString;
 use std::fs;
 use std::io::{self, Write};
+use std::os::unix::ffi::OsStringExt;
 use std::path::Path;
 use std::process::{exit, Command};
 use tempfile::TempDir;
@@ -43,6 +46,15 @@ fn unshorn() {
         .unwrap();
     fs::create_dir("/proc/self").unwrap();
     symlink("zram-generator", "/proc/self/exe").unwrap();
+
+    let mut path = env::var_os("PATH")
+        .map(|p| p.to_os_string().into_vec())
+        .unwrap_or(b"/usr/bin:/bin".to_vec()); // _PATH_DEFPATH
+    path.insert(0, b':');
+    for &b in "tests/10-example/bin".as_bytes().into_iter().rev() {
+        path.insert(0, b);
+    }
+    env::set_var("PATH", OsString::from_vec(path));
 }
 
 fn prepare_directory(srcroot: &Path) -> Result<TempDir> {
@@ -114,6 +126,9 @@ fn test_01_basic() {
     assert_eq!(d.host_memory_limit_mb, None);
     assert_eq!(d.zram_size.as_ref().map(z_s_name), None);
     assert_eq!(d.options, "discard");
+
+    assert_eq!(d.disksize, 391 * 1024 * 1024);
+    assert_eq!(d.mem_limit, 0);
 }
 
 #[test]
@@ -123,7 +138,7 @@ fn test_02_zstd() {
     let d = &devices[0];
     assert!(d.is_swap());
     assert_eq!(d.host_memory_limit_mb, Some(2050));
-    assert_eq!(d.zram_size.as_ref().map(z_s_name), Some("ram * 0.75"));
+    assert_eq!(d.zram_size.as_ref().map(z_s_name), Some("ram * ratio"));
     assert_eq!(
         d.compression_algorithms,
         config::Algorithms {
@@ -132,6 +147,9 @@ fn test_02_zstd() {
         }
     );
     assert_eq!(d.options, "discard");
+
+    assert_eq!(d.disksize, 782 * 1024 * 1024 * 3 / 4);
+    assert_eq!(d.mem_limit, 9999 * 1024 * 1024);
 }
 
 #[test]
@@ -153,11 +171,17 @@ fn test_04_dropins() {
                 assert_eq!(d.host_memory_limit_mb, Some(1235));
                 assert_eq!(d.zram_size.as_ref().map(z_s_name), None);
                 assert_eq!(d.options, "discard");
+
+                assert_eq!(d.disksize, 782 * 1024 * 1024 / 2);
+                assert_eq!(d.mem_limit, 0);
             }
             "zram2" => {
                 assert_eq!(d.host_memory_limit_mb, None);
                 assert_eq!(d.zram_size.as_ref().map(z_s_name), Some("ram*0.8"));
                 assert_eq!(d.options, "");
+
+                assert_eq!(d.disksize, 782 * 1024 * 1024 * 8 / 10);
+                assert_eq!(d.mem_limit, 0);
             }
             _ => panic!("Unexpected device {}", d),
         }
@@ -306,12 +330,26 @@ fn test_10_example() {
                     }
                 );
                 assert_eq!(d.options, "");
+
+                assert_eq!(
+                    d.zram_resident_limit.as_ref().map(z_s_name),
+                    Some("maxhotplug * 3/4")
+                );
+
+                assert_eq!(d.disksize, 782 * 1024 * 1024 / 10);
+                // This is the combination of tests/10-example/bin/xenstore-read and
+                // zram-resident-limit= in tests/10-example/etc/systemd/zram-generator.conf.
+                assert_eq!(d.mem_limit, 8 * 1024 * 1024 * 1024 * 3 / 4);
             }
+
             "zram1" => {
                 assert_eq!(d.fs_type.as_ref().unwrap(), "ext2");
                 assert_eq!(d.effective_fs_type(), "ext2");
                 assert_eq!(d.zram_size.as_ref().map(z_s_name), Some("ram / 10"));
                 assert_eq!(d.options, "discard");
+
+                assert_eq!(d.disksize, 782 * 1024 * 1024 / 10);
+                assert_eq!(d.mem_limit, 0);
             }
             _ => panic!("Unexpected device {}", d),
         }

zram-generator.conf.example

@@ -1,6 +1,12 @@
 # This file is part of the zram-generator project
 # https://github.com/systemd/zram-generator
 
+# At the top level, a set!variable = program
+# directive executes /bin/sh -c program,
+# parses the output as an expression, and remembers it in variable,
+# usable in later set! and zram-size/zram-resident-limit.
+set!maxhotplug = xenstore-read /local/domain/$(xenstore-read domid)/memory/hotplug-max
+
 [zram0]
 # This section describes the settings for /dev/zram0.
 #
@@ -25,7 +31,7 @@ zram-size = min(ram / 10, 2048)
 # then this device will not consume more than 128 MiB.
 #
 # 0 means no limit; this is the default.
-zram-resident-limit = 0
+zram-resident-limit = maxhotplug * 3/4
 
 # The compression algorithm to use for the zram device,
 # or leave unspecified to keep the kernel default.