Add HW 6

homework/06-membership/tests/src/common.rs

@@ -0,0 +1,44 @@
+use serde::{Deserialize, Serialize};
+use sugars::boxed;
+
+use anysystem::python::PyProcessFactory;
+use anysystem::System;
+
+#[derive(Serialize)]
+pub struct JoinMessage<'a> {
+    pub seed: &'a str,
+}
+
+#[derive(Serialize)]
+pub struct LeaveMessage {}
+
+#[derive(Serialize)]
+pub struct GetMembersMessage {}
+
+#[derive(Deserialize)]
+pub struct MembersMessage {
+    pub members: Vec<String>,
+}
+
+#[derive(Clone, Copy)]
+pub struct TestConfig<'a> {
+    pub process_factory: &'a PyProcessFactory,
+    pub process_count: u32,
+    pub seed: u64,
+}
+
+pub fn build_system(config: &TestConfig) -> System {
+    crate::runner::progress(format!("creating simulation: N={}", config.process_count));
+    let mut sys = System::new(config.seed);
+    sys.network().set_delays(0.01, 0.1);
+    for n in 0..config.process_count {
+        // process and node on which it runs have the same name
+        let name = n.to_string();
+        sys.add_node(&name);
+        let clock_skew = sys.gen_range(0.0..10.0);
+        sys.set_node_clock_skew(&name, clock_skew);
+        let process = config.process_factory.build((&name,), config.seed);
+        sys.add_process(&name, boxed!(process), &name);
+    }
+    sys
+}

homework/06-membership/tests/src/library_policy.rs

@@ -0,0 +1,36 @@
+use std::ffi::CString;
+
+use pyo3::prelude::*;
+use pyo3::types::PyModule;
+
+const POLICY_CODE: &str = include_str!("../library_policy.py");
+
+fn validate_solution(path: &str) -> Result<Vec<String>, String> {
+    Python::attach(|py| {
+        let checker =
+            crate::shared_state::validator_module(py).map_err(|error| error.to_string())?;
+        let code = CString::new(POLICY_CODE).expect("library policy contains a null byte");
+        let policy = PyModule::from_code(py, &code, c"library_policy.py", c"library_policy")
+            .map_err(|error| error.to_string())?;
+        policy
+            .getattr("validate_solution")
+            .and_then(|function| function.call1((path, checker)))
+            .and_then(|result| result.extract::<Vec<String>>())
+            .map_err(|error| error.to_string())
+    })
+}
+
+pub fn validate_or_report(path: &str) -> bool {
+    match validate_solution(path) {
+        Ok(violations) if violations.is_empty() => true,
+        Ok(violations) => {
+            println!("ERROR: HW6 library policy violation");
+            for violation in violations {
+                println!("{violation}");
+            }
+            println!("\nSCORE: 0\n");
+            false
+        }
+        Err(error) => panic!("failed to validate solution libraries: {error}"),
+    }
+}

homework/06-membership/tests/src/main.rs

@@ -0,0 +1,462 @@
+mod common;
+mod library_policy;
+mod python_memory;
+mod resources;
+mod runner;
+mod shared_state;
+mod tests;
+mod tests_mc;
+
+use indexmap::IndexMap;
+use std::collections::HashSet;
+
+use clap::Parser;
+use runner::Outcome;
+
+/// Membership Homework Tests
+#[derive(Parser, Debug)]
+#[clap(about, long_about = None)]
+struct Args {
+    /// Path to Python file with solution
+    #[clap(long = "impl", short = 'i', default_value = "solution/member.py")]
+    solution_path: String,
+
+    /// Test to run (optional)
+    #[clap(long = "test", short)]
+    test: Option<String>,
+
+    /// Print execution trace
+    #[clap(long, short)]
+    debug: bool,
+
+    /// Print test parameters and chaos monkey failures without the execution trace
+    #[clap(long, short = 'v')]
+    verbose: bool,
+
+    /// Random seed used in tests
+    #[clap(long, short, default_value = "123")]
+    seed: u64,
+
+    /// Number of processes
+    #[clap(long, short, default_value = "10")]
+    process_count: u32,
+
+    /// Number of chaos monkey runs
+    #[clap(long, short, default_value = "100")]
+    monkeys: u32,
+
+    /// Do not run model checking tests
+    #[clap(long)]
+    disable_mc_tests: bool,
+
+    /// Wall-clock limit in seconds per ordinary test and per chaos run
+    #[clap(long, default_value = "15", value_parser = clap::value_parser!(u64).range(1..=300))]
+    test_timeout: u64,
+
+    /// Shared wall-clock budget in seconds for all chaos runs
+    #[clap(long, default_value = "300", value_parser = clap::value_parser!(u64).range(1..=540))]
+    chaos_timeout: u64,
+
+    /// Wall-clock limit in seconds per scalability test
+    #[clap(long, default_value = "60", value_parser = clap::value_parser!(u64).range(1..=300))]
+    scalability_timeout: u64,
+
+    /// Wall-clock limit in seconds for both model-checking phases
+    #[clap(long, default_value = "300", value_parser = clap::value_parser!(u64).range(1..=300))]
+    mc_timeout: u64,
+
+    /// Total wall-clock budget in seconds (the Docker watchdog is 600s)
+    #[clap(long, default_value = "540", value_parser = clap::value_parser!(u64).range(1..=540))]
+    suite_timeout: u64,
+
+    /// Fixed address-space limit in MiB for ordinary, chaos and scalability workers
+    #[clap(long, default_value = "4096", value_parser = clap::value_parser!(u64).range(128..=65536))]
+    memory_limit: u64,
+
+    /// Fixed address-space limit in MiB for the model-checking worker
+    #[clap(long, default_value = "14336", value_parser = clap::value_parser!(u64).range(128..=65536))]
+    mc_memory_limit: u64,
+
+    #[clap(long, hide = true, requires = "test-result")]
+    test_worker: Option<String>,
+
+    #[clap(long, hide = true)]
+    test_result: Option<String>,
+
+    #[clap(long, hide = true)]
+    test_progress: Option<String>,
+
+    #[clap(long, hide = true, default_value = "1")]
+    chaos_start: u32,
+}
+
+fn main() {
+    let args = Args::parse();
+    if args.test_worker.is_some() {
+        runner::worker(&args);
+        return;
+    }
+    if !library_policy::validate_or_report(&args.solution_path) {
+        return;
+    }
+    if !shared_state::validate_or_report(&args.solution_path, &["GroupMember"]) {
+        return;
+    }
+    let results = match runner::run(&args) {
+        Ok(results) => results,
+        Err(error) => {
+            eprintln!("ERROR: test environment: {error}");
+            std::process::exit(2);
+        }
+    };
+    if args.test.is_none() {
+        let score = score(results, args.monkeys, args.disable_mc_tests);
+        println!("\nSCORE: {score}\n");
+    }
+}
+
+macro_rules! string_set {
+    ($($x:expr),+ $(,)?) => (
+        HashSet::from([$($x.to_string()),+])
+    );
+}
+
+fn score(results: IndexMap<String, Outcome>, monkeys: u32, disable_mc_tests: bool) -> f32 {
+    let mut basic_group = string_set! {
+        "SIMPLE",
+        "GET MEMBERS SEMANTICS",
+        "RANDOM SEED",
+        "PROCESS JOIN",
+        "PROCESS LEAVE",
+        "MODEL CHECKING",
+    };
+    let process_crash_group = string_set! {
+        "PROCESS CRASH",
+        "SEED PROCESS CRASH",
+        "PROCESS CRASH RECOVER",
+    };
+    let network_groups = [
+        (
+            "Network partitions tests",
+            string_set! {
+                "PROCESS OFFLINE",
+                "SEED PROCESS OFFLINE",
+                "PROCESS OFFLINE RECOVER",
+                "NETWORK PARTITION",
+                "NETWORK PARTITION RECOVER",
+                "NETWORK PARTITION UNEQUAL",
+                "NETWORK PARTITION UNEQUAL RECOVER",
+            },
+        ),
+        (
+            "Network connectivity tests",
+            string_set! {
+                "PROCESS CANNOT RECEIVE",
+                "PROCESS JOIN CANNOT RECEIVE",
+                "PROCESS CANNOT SEND",
+                "TWO PROCESSES CANNOT COMMUNICATE",
+                "TWO PROCESSES CANNOT COMMUNICATE IN TRIANGLE",
+            },
+        ),
+        (
+            "Network delays and losses tests",
+            string_set! {
+                "SLOW NETWORK",
+                "FLAKY NETWORK",
+                "FLAKY NETWORK ON START",
+                "FLAKY NETWORK AND CRASH",
+            },
+        ),
+        (
+            "Chaos monkey tests",
+            (1..=monkeys)
+                .map(|run| format!("CHAOS MONKEY (run {run})"))
+                .collect(),
+        ),
+    ];
+    let scalability_group = string_set! {"SCALABILITY NORMAL", "SCALABILITY CRASH"};
+    // Explicit local opt-out changes the selected basic tests. An MC skipped
+    // because the suite ran out of time remains required during grading.
+    if disable_mc_tests {
+        basic_group.remove("MODEL CHECKING");
+    }
+    let mut all_groups = &(&basic_group | &process_crash_group) | &scalability_group;
+    let mut test_count = basic_group.len() + process_crash_group.len() + scalability_group.len();
+    for (_, group) in &network_groups {
+        all_groups.extend(group.iter().cloned());
+        test_count += group.len();
+    }
+    assert_eq!(all_groups.len(), test_count, "overlapping scoring groups");
+    assert_eq!(all_groups, results.keys().cloned().collect());
+
+    let scalability_skipped = scalability_group
+        .iter()
+        .all(|name| matches!(results.get(name), Some(Outcome::SkippedPrerequisites(_))));
+    let mut passed = HashSet::new();
+    let mut model_checking_ok = false;
+    for (test, result) in results {
+        if test == "MODEL CHECKING" && matches!(result, Outcome::Passed) {
+            model_checking_ok = true;
+        }
+        if result.satisfies_functional_requirement(&test) {
+            passed.insert(test);
+        }
+    }
+
+    let basic_group_ok = basic_group.is_subset(&passed);
+    let mut score = 0.;
+    print!("Basic tests: ");
+    if basic_group_ok {
+        score += 3.;
+        println!("passed");
+    } else {
+        let mut failed: Vec<&String> = basic_group.difference(&passed).collect();
+        failed.sort();
+        println!("not passed {failed:?}");
+    }
+    print!("Process crashes tests: ");
+    if process_crash_group.is_subset(&passed) {
+        score += 3.;
+        println!("passed");
+    } else {
+        let mut failed: Vec<&String> = process_crash_group.difference(&passed).collect();
+        failed.sort();
+        println!("not passed {failed:?}");
+    }
+    for (name, group) in &network_groups {
+        print!("{name}: ");
+        if group.is_subset(&passed) {
+            score += 1.;
+            println!("passed");
+        } else {
+            let mut failed: Vec<&String> = group.difference(&passed).collect();
+            failed.sort();
+            println!("not passed {failed:?}");
+        }
+    }
+    // The basic-group eligibility already includes the resource-limit and
+    // explicit local opt-out exceptions for MC.
+    let functional_tests_ok = basic_group_ok
+        && process_crash_group.is_subset(&passed)
+        && network_groups
+            .iter()
+            .all(|(_, group)| group.is_subset(&passed));
+    print!("Scalability tests: ");
+    if scalability_skipped {
+        println!("skipped (functional tests not passed)");
+    } else if scalability_group.is_subset(&passed) {
+        if functional_tests_ok {
+            score += 4.;
+            println!("passed");
+        } else {
+            println!("passed (no points: functional tests not passed)");
+        }
+    } else {
+        let mut failed: Vec<&String> = scalability_group.difference(&passed).collect();
+        failed.sort();
+        println!("not passed {failed:?}");
+    }
+    print!("Model checking test: ");
+    if model_checking_ok && basic_group_ok {
+        score += 1.;
+        println!("passed");
+    } else if model_checking_ok {
+        println!("passed (no bonus: basic tests not passed)");
+    } else {
+        println!("not passed");
+    }
+    score
+}
+
+#[cfg(test)]
+mod score_tests {
+    use super::*;
+
+    #[test]
+    fn functional_groups_keep_their_points_after_resource_failures() {
+        assert_eq!(score(runner::passed_results(2), 2, false), 15.);
+        for (name, expected) in [
+            ("SCALABILITY NORMAL", 11.),
+            ("PROCESS CRASH", 8.),
+            ("CHAOS MONKEY (run 2)", 10.),
+        ] {
+            for failure in [
+                Outcome::TimeLimit("test budget".into()),
+                Outcome::MemoryLimit("256 MiB".into()),
+                Outcome::WorkerError("SIGKILL".into()),
+                Outcome::Skipped("not started".into()),
+            ] {
+                let mut results = runner::passed_results(2);
+                results.insert(name.into(), failure);
+                assert_eq!(score(results, 2, false), expected, "{name}");
+            }
+        }
+    }
+
+    #[test]
+    fn each_network_test_only_blocks_its_subgroup_and_scalability() {
+        for name in [
+            "PROCESS OFFLINE",
+            "SEED PROCESS OFFLINE",
+            "PROCESS OFFLINE RECOVER",
+            "NETWORK PARTITION",
+            "NETWORK PARTITION RECOVER",
+            "NETWORK PARTITION UNEQUAL",
+            "NETWORK PARTITION UNEQUAL RECOVER",
+            "PROCESS CANNOT RECEIVE",
+            "PROCESS JOIN CANNOT RECEIVE",
+            "PROCESS CANNOT SEND",
+            "TWO PROCESSES CANNOT COMMUNICATE",
+            "TWO PROCESSES CANNOT COMMUNICATE IN TRIANGLE",
+            "SLOW NETWORK",
+            "FLAKY NETWORK",
+            "FLAKY NETWORK ON START",
+            "FLAKY NETWORK AND CRASH",
+            "CHAOS MONKEY (run 1)",
+            "CHAOS MONKEY (run 2)",
+        ] {
+            for failure in [
+                Outcome::Failed("wrong members".into()),
+                Outcome::TimeLimit("test budget".into()),
+                Outcome::MemoryLimit("4096 MiB".into()),
+                Outcome::WorkerError("SIGKILL".into()),
+                Outcome::Skipped("not started".into()),
+            ] {
+                let mut results = runner::passed_results(2);
+                results.insert(name.into(), failure.clone());
+                assert_eq!(score(results, 2, false), 10., "{name}: {failure:?}");
+            }
+        }
+    }
+
+    #[test]
+    fn multiple_failures_in_one_network_subgroup_have_the_same_penalty() {
+        for names in [
+            vec!["PROCESS OFFLINE", "NETWORK PARTITION RECOVER"],
+            vec!["PROCESS CANNOT RECEIVE", "TWO PROCESSES CANNOT COMMUNICATE"],
+            vec!["SLOW NETWORK", "FLAKY NETWORK AND CRASH"],
+            vec!["CHAOS MONKEY (run 1)", "CHAOS MONKEY (run 2)"],
+        ] {
+            let mut results = runner::passed_results(2);
+            for name in &names {
+                results.insert((*name).into(), Outcome::Failed("wrong members".into()));
+            }
+            assert_eq!(score(results, 2, false), 10., "{names:?}");
+        }
+    }
+
+    #[test]
+    fn unsuccessful_ordinary_basic_tests_block_mc_bonus_and_scalability() {
+        for name in [
+            "SIMPLE",
+            "GET MEMBERS SEMANTICS",
+            "RANDOM SEED",
+            "PROCESS JOIN",
+            "PROCESS LEAVE",
+        ] {
+            for failure in [
+                Outcome::Failed("wrong members".into()),
+                Outcome::TimeLimit("test budget".into()),
+                Outcome::MemoryLimit("4096 MiB".into()),
+                Outcome::WorkerError("SIGKILL".into()),
+                Outcome::Skipped("not started".into()),
+            ] {
+                let mut results = runner::passed_results(2);
+                results.insert(name.into(), failure.clone());
+                assert_eq!(score(results, 2, false), 7., "{name}: {failure:?}");
+            }
+        }
+    }
+
+    #[test]
+    fn mc_outcomes_match_the_scoring_table_for_all_other_group_combinations() {
+        // outcome (None = explicit opt-out), basic eligibility, bonus if all basic tests pass.
+        let cases = [
+            (Some(Outcome::Passed), true, 1.),
+            (None, true, 0.),
+            (Some(Outcome::Failed("counterexample".into())), false, 0.),
+            (
+                Some(Outcome::WorkerError("Python exception".into())),
+                false,
+                0.,
+            ),
+            (Some(Outcome::WorkerError("SIGKILL".into())), false, 0.),
+            (Some(Outcome::TimeLimit("MC budget".into())), true, 0.),
+            (
+                Some(Outcome::TimeLimit("suite budget after MC started".into())),
+                true,
+                0.,
+            ),
+            (Some(Outcome::MemoryLimit("14336 MiB".into())), true, 0.),
+            (
+                Some(Outcome::Skipped("suite budget before MC started".into())),
+                false,
+                0.,
+            ),
+        ];
+        let basic = [
+            "SIMPLE",
+            "GET MEMBERS SEMANTICS",
+            "RANDOM SEED",
+            "PROCESS JOIN",
+            "PROCESS LEAVE",
+        ];
+        let other = [
+            ("PROCESS CRASH", 3.),
+            ("PROCESS OFFLINE", 1.),
+            ("PROCESS CANNOT RECEIVE", 1.),
+            ("SLOW NETWORK", 1.),
+            ("CHAOS MONKEY (run 2)", 1.),
+            ("SCALABILITY NORMAL", 4.),
+        ];
+        for (mc, basic_eligible, bonus) in cases {
+            // All 32 combinations of the five ordinary basic tests, and all
+            // 64 combinations of crash, four network subgroups, and scalability.
+            for basic_failures in 0..32 {
+                for other_failures in 0..64 {
+                    let mut results = runner::passed_results(2);
+                    if let Some(outcome) = &mc {
+                        results.insert("MODEL CHECKING".into(), outcome.clone());
+                    } else {
+                        results.shift_remove("MODEL CHECKING");
+                    }
+                    for (index, name) in basic.iter().enumerate() {
+                        if basic_failures & (1 << index) != 0 {
+                            results.insert((*name).into(), Outcome::Failed("wrong members".into()));
+                        }
+                    }
+                    let mut expected = 0.;
+                    if basic_eligible && basic_failures == 0 {
+                        expected += 3. + bonus;
+                    }
+                    for (index, (name, points)) in other.iter().enumerate() {
+                        if other_failures & (1 << index) != 0 {
+                            results.insert((*name).into(), Outcome::Failed("wrong members".into()));
+                        } else if *name != "SCALABILITY NORMAL" {
+                            expected += points;
+                        }
+                    }
+                    // Scalability needs both its own tests and every functional group.
+                    if basic_eligible && basic_failures == 0 && other_failures == 0 {
+                        expected += 4.;
+                    }
+                    let blockers = runner::scalability_blockers(&results, 2, mc.is_none());
+                    let eligible =
+                        basic_eligible && basic_failures == 0 && other_failures & 31 == 0;
+                    assert_eq!(blockers.is_empty(), eligible,
+                        "MC={mc:?}, basic failures={basic_failures:05b}, other failures={other_failures:06b}");
+                    if !eligible {
+                        for name in ["SCALABILITY NORMAL", "SCALABILITY CRASH"] {
+                            results.insert(
+                                name.into(),
+                                Outcome::SkippedPrerequisites(blockers.clone()),
+                            );
+                        }
+                    }
+                    assert_eq!(score(results, 2, mc.is_none()), expected,
+                        "MC={mc:?}, basic failures={basic_failures:05b}, other failures={other_failures:06b}");
+                }
+            }
+        }
+    }
+}

homework/06-membership/tests/src/python_memory.rs

@@ -0,0 +1,83 @@
+//! Observe actual CPython allocation failures before AnySystem replaces a
+//! Python exception with a generic panic. Student stdout is not a result channel.
+use std::ffi::c_void;
+use std::mem::MaybeUninit;
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::Once;
+
+use pyo3::ffi::{self, PyMemAllocatorDomain, PyMemAllocatorEx};
+use pyo3::Python;
+
+static INSTALLED: Once = Once::new();
+static FAILED: AtomicBool = AtomicBool::new(false);
+
+fn observe(pointer: *mut c_void, nonzero: bool) -> *mut c_void {
+    if nonzero && pointer.is_null() {
+        FAILED.store(true, Ordering::Relaxed);
+    }
+    pointer
+}
+
+extern "C" fn malloc(ctx: *mut c_void, size: usize) -> *mut c_void {
+    // SAFETY: ctx points to an immutable original allocator, leaked for the
+    // interpreter's entire lifetime. Calls retain CPython's allocator contract.
+    let allocator = unsafe { &*ctx.cast::<PyMemAllocatorEx>() };
+    observe((allocator.malloc.unwrap())(allocator.ctx, size), size != 0)
+}
+extern "C" fn calloc(ctx: *mut c_void, count: usize, size: usize) -> *mut c_void {
+    // SAFETY: same lifetime and callback contract as malloc above.
+    let allocator = unsafe { &*ctx.cast::<PyMemAllocatorEx>() };
+    observe(
+        (allocator.calloc.unwrap())(allocator.ctx, count, size),
+        count != 0 && size != 0,
+    )
+}
+extern "C" fn realloc(ctx: *mut c_void, pointer: *mut c_void, size: usize) -> *mut c_void {
+    // SAFETY: same lifetime and callback contract as malloc above.
+    let allocator = unsafe { &*ctx.cast::<PyMemAllocatorEx>() };
+    observe(
+        (allocator.realloc.unwrap())(allocator.ctx, pointer, size),
+        size != 0,
+    )
+}
+extern "C" fn free(ctx: *mut c_void, pointer: *mut c_void) {
+    // SAFETY: same lifetime and callback contract as malloc above.
+    let allocator = unsafe { &*ctx.cast::<PyMemAllocatorEx>() };
+    (allocator.free.unwrap())(allocator.ctx, pointer);
+}
+
+pub fn install() {
+    Python::attach(|_| {
+        INSTALLED.call_once(|| {
+            for domain in [
+                PyMemAllocatorDomain::PYMEM_DOMAIN_RAW,
+                PyMemAllocatorDomain::PYMEM_DOMAIN_MEM,
+                PyMemAllocatorDomain::PYMEM_DOMAIN_OBJ,
+            ] {
+                // SAFETY: installation holds the GIL before any student code runs.
+                // CPython requires post-initialization hooks to wrap the previous
+                // allocator. Its four callbacks are supplied by the interpreter.
+                unsafe {
+                    let mut original = MaybeUninit::uninit();
+                    ffi::PyMem_GetAllocator(domain, original.as_mut_ptr());
+                    let original = Box::into_raw(Box::new(original.assume_init()));
+                    let mut wrapper = PyMemAllocatorEx {
+                        ctx: original.cast(),
+                        malloc: Some(malloc),
+                        calloc: Some(calloc),
+                        realloc: Some(realloc),
+                        free: Some(free),
+                    };
+                    ffi::PyMem_SetAllocator(domain, &mut wrapper);
+                }
+            }
+        })
+    });
+}
+
+pub fn reset() {
+    FAILED.store(false, Ordering::Relaxed);
+}
+pub fn failed() -> bool {
+    FAILED.load(Ordering::Relaxed)
+}

homework/06-membership/tests/src/resources.rs

@@ -0,0 +1,165 @@
+//! Hard address-space limits are applied before exec, including Python imports.
+//! Unlike an RSS polling loop, RLIMIT_AS also stops a single oversized allocation.
+use std::fs::File;
+use std::process::{Child, Command, ExitStatus};
+
+#[cfg(target_os = "linux")]
+const MIB: u64 = 1024 * 1024;
+
+#[cfg(target_os = "linux")]
+mod linux {
+    use std::fs;
+    use std::os::fd::OwnedFd;
+    use std::os::unix::net::UnixStream;
+    use std::os::unix::process::CommandExt;
+    use std::process::Stdio;
+
+    use super::*;
+
+    fn number(path: &str) -> Option<u64> {
+        fs::read_to_string(path).ok()?.trim().parse().ok()
+    }
+
+    fn memory_capacity() -> Option<u64> {
+        let meminfo = fs::read_to_string("/proc/meminfo").ok()?;
+        let host = meminfo.lines().find_map(|line| {
+            line.strip_prefix("MemTotal:")?
+                .split_whitespace()
+                .next()?
+                .parse::<u64>()
+                .ok()
+        })? * 1024;
+        // Both namespaced cgroup v2 and the conventional v1 Docker mount.
+        let container = [
+            "/sys/fs/cgroup/memory.max",
+            "/sys/fs/cgroup/memory/memory.limit_in_bytes",
+        ]
+        .into_iter()
+        .filter_map(number)
+        .min();
+        Some(container.map_or(host, |container| host.min(container)))
+    }
+
+    pub fn limit(requested_mib: u64) -> Result<u64, String> {
+        // Check static capacity, never current free memory or cgroup usage.
+        // Insufficient capacity is an environment error, not a lower grading limit.
+        let capacity = memory_capacity().ok_or("cannot determine memory capacity")? / MIB;
+        if requested_mib + 256 > capacity {
+            return Err(format!("fixed worker limit is {requested_mib} MiB, but environment capacity is {capacity} MiB; provide at least {} MiB including the supervisor reserve, or explicitly select a smaller local limit", requested_mib + 256));
+        }
+        let mut inherited = libc::rlimit {
+            rlim_cur: 0,
+            rlim_max: 0,
+        };
+        // SAFETY: getrlimit writes into an initialized rlimit structure.
+        if unsafe { libc::getrlimit(libc::RLIMIT_AS, &mut inherited) } != 0 {
+            return Err(std::io::Error::last_os_error().to_string());
+        }
+        if inherited.rlim_max / MIB < requested_mib {
+            return Err(format!("fixed worker limit is {requested_mib} MiB, but inherited address-space hard limit is {} MiB", inherited.rlim_max / MIB));
+        }
+        Ok(requested_mib)
+    }
+
+    pub fn capture(command: &mut Command) -> std::io::Result<File> {
+        let (reader, writer) = UnixStream::pair()?;
+        command.stdout(Stdio::from(OwnedFd::from(writer.try_clone()?)));
+        command.stderr(Stdio::from(OwnedFd::from(writer)));
+        Ok(File::from(OwnedFd::from(reader)))
+    }
+
+    pub fn configure(command: &mut Command, mib: u64) {
+        let supervisor = std::process::id() as libc::pid_t;
+        // SAFETY: only async-signal-safe libc calls run between fork and exec.
+        unsafe {
+            command.pre_exec(move || {
+                // Interrupting the supervisor must not leave a hanging worker.
+                if libc::prctl(libc::PR_SET_PDEATHSIG, libc::SIGKILL) != 0 {
+                    return Err(std::io::Error::last_os_error());
+                }
+                if libc::getppid() != supervisor {
+                    return Err(std::io::Error::from_raw_os_error(libc::ECHILD));
+                }
+                if libc::setsid() < 0 {
+                    return Err(std::io::Error::last_os_error());
+                }
+                let memory = libc::rlimit {
+                    rlim_cur: mib * MIB,
+                    rlim_max: mib * MIB,
+                };
+                let core = libc::rlimit {
+                    rlim_cur: 0,
+                    rlim_max: 0,
+                };
+                if libc::setrlimit(libc::RLIMIT_AS, &memory) != 0
+                    || libc::setrlimit(libc::RLIMIT_CORE, &core) != 0
+                {
+                    return Err(std::io::Error::last_os_error());
+                }
+                Ok(())
+            });
+        }
+    }
+
+    pub fn terminate(child: &mut Child) {
+        // The child starts a new session: clean up its descendants as well.
+        // SAFETY: the negative pid targets this worker's process group.
+        unsafe {
+            libc::kill(-(child.id() as i32), libc::SIGKILL);
+        }
+        let _ = child.kill();
+        let _ = child.wait();
+    }
+
+    pub fn memory_usage(child: &Child) -> Option<u64> {
+        fs::read_to_string(format!("/proc/{}/status", child.id()))
+            .ok()?
+            .lines()
+            .find_map(|line| {
+                line.strip_prefix("VmSize:")?
+                    .split_whitespace()
+                    .next()?
+                    .parse()
+                    .ok()
+            })
+            .map(|kib: u64| kib / 1024)
+    }
+}
+
+#[cfg(target_os = "linux")]
+pub use linux::{capture, configure, limit, memory_usage, terminate};
+
+#[cfg(not(target_os = "linux"))]
+pub fn limit(_: u64) -> Result<u64, String> {
+    Err("resource-limited workers require Linux; use the homework Docker image".into())
+}
+#[cfg(not(target_os = "linux"))]
+pub fn capture(_: &mut Command) -> std::io::Result<File> {
+    Err(std::io::Error::other("use the homework Docker image"))
+}
+#[cfg(not(target_os = "linux"))]
+pub fn configure(_: &mut Command, _: u64) {}
+#[cfg(not(target_os = "linux"))]
+pub fn terminate(child: &mut Child) {
+    let _ = child.kill();
+    let _ = child.wait();
+}
+#[cfg(not(target_os = "linux"))]
+pub fn memory_usage(_: &Child) -> Option<u64> {
+    None
+}
+
+pub fn allocation_crash(status: Option<ExitStatus>, output: &str) -> bool {
+    #[cfg(target_os = "linux")]
+    {
+        use std::os::unix::process::ExitStatusExt;
+        status.is_some_and(|status| status.signal() == Some(libc::SIGABRT))
+            && output.contains("memory allocation of ")
+            && output.contains(" bytes failed")
+    }
+    #[cfg(not(target_os = "linux"))]
+    {
+        let _ = (status, output);
+        false
+    }
+}

homework/06-membership/tests/src/runner.rs

@@ -0,0 +1,878 @@
+//! Based on HW4's killable-process runner. Only workers execute solution code.
+//! Chaos runs share a worker, with an append-only result journal after every run.
+use std::cell::OnceCell;
+use std::fs::{self, File, OpenOptions};
+use std::io::{BufRead, BufReader, Read, Write};
+use std::path::PathBuf;
+use std::process::Command;
+use std::sync::OnceLock;
+use std::thread;
+use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
+
+use anysystem::python::PyProcessFactory;
+use anysystem::test::TestResult;
+use env_logger::Builder;
+use indexmap::IndexMap;
+use log::{info, LevelFilter};
+use pyo3::prelude::*;
+use rand::{RngCore, SeedableRng};
+use rand_pcg::Pcg64;
+use serde::{Deserialize, Serialize};
+
+use crate::common::TestConfig;
+use crate::tests::*;
+use crate::tests_mc::test_mc_group;
+use crate::Args;
+use crate::{python_memory, resources};
+
+type TestFn = fn(&TestConfig) -> TestResult;
+const ORDINARY: &[(&str, TestFn)] = &[
+    ("SIMPLE", test_simple),
+    ("GET MEMBERS SEMANTICS", test_get_members_semantics),
+    ("RANDOM SEED", test_random_seed),
+    ("PROCESS JOIN", test_process_join),
+    ("PROCESS LEAVE", test_process_leave),
+    ("PROCESS CRASH", test_process_crash),
+    ("SEED PROCESS CRASH", test_seed_process_crash),
+    ("PROCESS CRASH RECOVER", test_process_crash_recover),
+    ("PROCESS OFFLINE", test_process_offline),
+    ("SEED PROCESS OFFLINE", test_seed_process_offline),
+    ("PROCESS OFFLINE RECOVER", test_process_offline_recover),
+    ("PROCESS CANNOT RECEIVE", test_process_cannot_receive),
+    (
+        "PROCESS JOIN CANNOT RECEIVE",
+        test_process_join_cannot_receive,
+    ),
+    ("PROCESS CANNOT SEND", test_process_cannot_send),
+    ("NETWORK PARTITION", test_network_partition),
+    ("NETWORK PARTITION RECOVER", test_network_partition_recover),
+    ("NETWORK PARTITION UNEQUAL", test_network_partition_unequal),
+    (
+        "NETWORK PARTITION UNEQUAL RECOVER",
+        test_network_partition_unequal_recover,
+    ),
+    (
+        "TWO PROCESSES CANNOT COMMUNICATE",
+        test_two_processes_cannot_communicate,
+    ),
+    (
+        "TWO PROCESSES CANNOT COMMUNICATE IN TRIANGLE",
+        test_two_processes_cannot_communicate_in_triangle,
+    ),
+    ("SLOW NETWORK", test_slow_network),
+    ("FLAKY NETWORK", test_flaky_network),
+    ("FLAKY NETWORK ON START", test_flaky_network_on_start),
+    ("FLAKY NETWORK AND CRASH", test_flaky_network_and_crash),
+];
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub enum Outcome {
+    Passed,
+    Failed(String),
+    TimeLimit(String),
+    MemoryLimit(String),
+    WorkerError(String),
+    Skipped(String),
+    SkippedPrerequisites(Vec<String>),
+}
+
+impl Outcome {
+    pub fn resource_limited(&self) -> bool {
+        matches!(self, Self::TimeLimit(_) | Self::MemoryLimit(_))
+    }
+
+    // A resource failure before the Started record is not an attempted test.
+    // In particular it must not grant the MC exemption for the basic group.
+    fn require_started(self, started: bool) -> Self {
+        if !started && self.resource_limited() {
+            Self::Skipped(format!("test did not start; {}", self.description()))
+        } else {
+            self
+        }
+    }
+
+    pub fn satisfies_functional_requirement(&self, name: &str) -> bool {
+        matches!(self, Self::Passed) || (name == "MODEL CHECKING" && self.resource_limited())
+    }
+
+    fn description(&self) -> String {
+        match self {
+            Self::Passed => "passed".into(),
+            Self::Failed(message) => message.clone(),
+            Self::TimeLimit(message) => format!("time limit exceeded: {message}"),
+            Self::MemoryLimit(message) => format!("memory limit exceeded: {message}"),
+            Self::WorkerError(message) => format!("test worker error: {message}"),
+            Self::Skipped(message) => format!("not checked: {message}"),
+            Self::SkippedPrerequisites(tests) => {
+                let mut names = tests.iter().take(8).cloned().collect::<Vec<_>>().join(", ");
+                if tests.len() > 8 {
+                    names.push_str(&format!(" (and {} more)", tests.len() - 8));
+                }
+                format!("required functional tests not passed: {names}")
+            }
+        }
+    }
+}
+
+#[derive(Serialize, Deserialize)]
+enum Event {
+    Started(String),
+    Finished(String, Outcome),
+}
+
+struct Test {
+    name: String,
+    run: TestFn,
+    seed: u64,
+    chaos_run: Option<u32>,
+}
+
+fn tests(args: &Args) -> Vec<Test> {
+    let mut tests: Vec<_> = ORDINARY
+        .iter()
+        .map(|&(name, run)| Test {
+            name: name.into(),
+            run,
+            seed: args.seed,
+            chaos_run: None,
+        })
+        .collect();
+    let mut rand = Pcg64::seed_from_u64(args.seed);
+    for run in 1..=args.monkeys {
+        tests.push(Test {
+            name: format!("CHAOS MONKEY (run {run})"),
+            run: test_chaos_monkey,
+            seed: rand.next_u64(),
+            chaos_run: Some(run),
+        });
+    }
+    for (name, run) in [
+        ("MODEL CHECKING", test_mc_group as TestFn),
+        ("SCALABILITY NORMAL", test_scalability_normal as TestFn),
+        ("SCALABILITY CRASH", test_scalability_crash as TestFn),
+    ] {
+        if name != "MODEL CHECKING" || !args.disable_mc_tests {
+            tests.push(Test {
+                name: name.into(),
+                run,
+                seed: args.seed,
+                chaos_run: None,
+            });
+        }
+    }
+    tests
+}
+
+/// Missing, failed and unstarted functional tests block scalability. Only an
+/// explicitly disabled MC or a resource limit after it started is exempt.
+pub fn scalability_blockers(
+    results: &IndexMap<String, Outcome>,
+    monkeys: u32,
+    disable_mc_tests: bool,
+) -> Vec<String> {
+    ORDINARY
+        .iter()
+        .map(|(name, _)| (*name).to_string())
+        .chain((1..=monkeys).map(|run| format!("CHAOS MONKEY (run {run})")))
+        .chain((!disable_mc_tests).then(|| "MODEL CHECKING".to_string()))
+        .filter(|name| {
+            !results
+                .get(name)
+                .is_some_and(|outcome| outcome.satisfies_functional_requirement(name))
+        })
+        .collect()
+}
+
+static PROGRESS: OnceLock<PathBuf> = OnceLock::new();
+
+/// Bounded, replaceable diagnostics; never mixed with the structured result stream.
+pub fn progress(message: impl AsRef<str>) {
+    if let Some(path) = PROGRESS.get() {
+        let temporary = path.with_extension("next");
+        if fs::write(&temporary, message.as_ref()).is_ok() {
+            let _ = fs::rename(&temporary, path);
+        }
+    }
+}
+
+fn event(file: &mut File, event: Event) {
+    serde_json::to_writer(&mut *file, &event).unwrap();
+    writeln!(file).unwrap();
+    file.flush().unwrap();
+}
+
+fn report(name: &str, outcome: &Outcome) {
+    match outcome {
+        Outcome::Passed => println!("\nPASSED\n"),
+        Outcome::Skipped(_) | Outcome::SkippedPrerequisites(_) => {
+            println!("\nSKIPPED: {}\n", outcome.description())
+        }
+        _ => println!("\nFAILED: {}\n", outcome.description()),
+    }
+    if name == "MODEL CHECKING" && outcome.resource_limited() {
+        println!("INCOMPLETE: model checking reached its resource limit; no MC bonus, basic tests remain eligible\n");
+    }
+    std::io::stdout().flush().unwrap();
+}
+
+pub fn worker(args: &Args) {
+    if let Some(path) = &args.test_progress {
+        let _ = PROGRESS.set(PathBuf::from(path));
+    }
+    if args.debug || args.verbose {
+        let mut logger = Builder::new();
+        logger.filter(Some("distsys_membership"), LevelFilter::Info);
+        if args.debug {
+            logger.filter(Some("anysystem"), LevelFilter::Debug);
+        }
+        logger
+            .format(|buf, record| writeln!(buf, "{}", record.args()))
+            .init();
+    }
+    info!(
+        "Test parameters: seed={}, processes={}",
+        args.seed, args.process_count
+    );
+    let mut journal = OpenOptions::new()
+        .append(true)
+        .open(args.test_result.as_ref().unwrap())
+        .unwrap();
+    let factory = OnceCell::new();
+    for test in tests(args).into_iter().filter(|test| {
+        if args.test_worker.as_deref() == Some("CHAOS MONKEY") {
+            test.chaos_run.is_some_and(|run| run >= args.chaos_start)
+        } else {
+            args.test_worker.as_deref() == Some(test.name.as_str())
+        }
+    }) {
+        progress("loading Python implementation");
+        event(&mut journal, Event::Started(test.name.clone()));
+        println!("\n--- {} ---\n", test.name);
+        std::io::stdout().flush().unwrap();
+        python_memory::reset();
+        let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
+            python_memory::install();
+            let factory = factory.get_or_init(|| {
+                // Match the preflight's local-import root when --impl is outside cwd.
+                let solution = fs::canonicalize(&args.solution_path).unwrap();
+                let directory = solution.parent().unwrap().to_string_lossy();
+                Python::attach(|py| {
+                    py.import("sys")
+                        .and_then(|sys| sys.getattr("path"))
+                        .and_then(|path| path.call_method1("insert", (0, directory.as_ref())))
+                        .unwrap();
+                });
+                PyProcessFactory::new(&args.solution_path, "GroupMember")
+            });
+            progress("initializing simulation");
+            let config = TestConfig {
+                process_factory: factory,
+                process_count: if test.name == "MODEL CHECKING" {
+                    3
+                } else {
+                    args.process_count
+                },
+                seed: test.seed,
+            };
+            (test.run)(&config)
+        }));
+        let mut outcome = match result {
+            Ok(Ok(_)) => Outcome::Passed,
+            Ok(Err(message))
+                if message == "time limit of 300s exceeded" && test.name == "MODEL CHECKING" =>
+            {
+                Outcome::TimeLimit(message)
+            }
+            Ok(Err(message)) => Outcome::Failed(message),
+            Err(payload) => {
+                let message = payload
+                    .downcast_ref::<String>()
+                    .map(String::as_str)
+                    .or_else(|| payload.downcast_ref::<&str>().copied())
+                    .unwrap_or("test panicked")
+                    .to_owned();
+                if python_memory::failed() {
+                    Outcome::MemoryLimit(message)
+                } else {
+                    Outcome::WorkerError(message)
+                }
+            }
+        };
+        if let Outcome::MemoryLimit(message) = &outcome {
+            let phase = PROGRESS
+                .get()
+                .and_then(|path| fs::read_to_string(path).ok())
+                .unwrap_or_default();
+            outcome = Outcome::MemoryLimit(format!(
+                "worker address-space limit {} MiB; {phase}; {message}",
+                memory_limit(args, &test)
+            ));
+        }
+        // Bound IPC diagnostics independently of the worker's memory budget.
+        match &mut outcome {
+            Outcome::Failed(message)
+            | Outcome::TimeLimit(message)
+            | Outcome::MemoryLimit(message)
+            | Outcome::WorkerError(message)
+            | Outcome::Skipped(message)
+                if message.len() > 16384 =>
+            {
+                let mut end = 16384;
+                while !message.is_char_boundary(end) {
+                    end -= 1;
+                }
+                message.truncate(end);
+                message.push_str(" [diagnostic truncated]");
+            }
+            _ => {}
+        }
+        event(
+            &mut journal,
+            Event::Finished(test.name.clone(), outcome.clone()),
+        );
+        // Report panics after draining their diagnostics from the output pipe.
+        if !matches!(outcome, Outcome::WorkerError(_)) {
+            report(&test.name, &outcome);
+        }
+        // A panic can leave the embedded interpreter or simulator in an unknown state.
+        if matches!(outcome, Outcome::WorkerError(_) | Outcome::MemoryLimit(_)) {
+            break;
+        }
+    }
+}
+
+struct Files(PathBuf);
+impl Files {
+    fn new() -> std::io::Result<Self> {
+        let nonce = SystemTime::now()
+            .duration_since(UNIX_EPOCH)
+            .unwrap()
+            .as_nanos();
+        let path =
+            std::env::temp_dir().join(format!("membership-test-{}-{nonce}", std::process::id()));
+        fs::create_dir(&path)?;
+        let files = Self(path);
+        File::create(files.journal())?;
+        Ok(files)
+    }
+    fn journal(&self) -> PathBuf {
+        self.0.join("results.jsonl")
+    }
+    fn progress(&self) -> PathBuf {
+        self.0.join("progress.txt")
+    }
+}
+impl Drop for Files {
+    fn drop(&mut self) {
+        for name in ["results.jsonl", "progress.txt", "progress.next"] {
+            let _ = fs::remove_file(self.0.join(name));
+        }
+        let _ = fs::remove_dir(&self.0);
+    }
+}
+
+// Retain a bounded diagnostic tail for native allocator failures. Python
+// allocation failures are tracked separately; output alone is not an outcome.
+fn output_reader(mut pipe: impl Read + Send + 'static) -> thread::JoinHandle<String> {
+    thread::spawn(move || {
+        let mut tail = Vec::new();
+        let mut chunk = [0; 4096];
+        while let Ok(count) = pipe.read(&mut chunk) {
+            if count == 0 {
+                break;
+            }
+            let _ = std::io::stdout().lock().write_all(&chunk[..count]);
+            tail.extend_from_slice(&chunk[..count]);
+            if tail.len() > 65536 {
+                tail.drain(..tail.len() - 65536);
+            }
+        }
+        String::from_utf8_lossy(&tail).into_owned()
+    })
+}
+
+struct Journal {
+    reader: BufReader<File>,
+    pending: String,
+    next: usize,
+    started: Instant,
+    announced: bool,
+}
+impl Journal {
+    fn read(
+        &mut self,
+        batch: &[Test],
+        results: &mut IndexMap<String, Outcome>,
+    ) -> Result<(), String> {
+        loop {
+            let remaining = 65537_usize.saturating_sub(self.pending.len());
+            let count = (&mut self.reader)
+                .take(remaining as u64)
+                .read_line(&mut self.pending)
+                .map_err(|error| error.to_string())?;
+            if self.pending.len() > 65536 {
+                return Err("worker result exceeds 64 KiB".into());
+            }
+            if !self.pending.ends_with('\n') {
+                return Ok(());
+            }
+            let event: Event = serde_json::from_str(&self.pending)
+                .map_err(|error| format!("invalid worker result: {error}"))?;
+            self.pending.clear();
+            let name = match &event {
+                Event::Started(name) | Event::Finished(name, _) => name,
+            };
+            if batch.get(self.next).map(|test| &test.name) != Some(name) {
+                return Err(format!("unexpected worker result for {name}"));
+            }
+            match event {
+                Event::Started(_) => {
+                    self.started = Instant::now();
+                    self.announced = true;
+                }
+                Event::Finished(name, outcome) => {
+                    results.insert(name, outcome);
+                    self.next += 1;
+                    self.started = Instant::now();
+                    self.announced = false;
+                }
+            }
+            if count == 0 {
+                return Ok(());
+            }
+        }
+    }
+}
+
+fn memory_limit(args: &Args, test: &Test) -> u64 {
+    if test.name == "MODEL CHECKING" {
+        args.mc_memory_limit
+    } else {
+        args.memory_limit
+    }
+}
+
+fn seconds(args: &Args, test: &Test) -> u64 {
+    if test.name == "MODEL CHECKING" {
+        args.mc_timeout
+    } else if test.name.starts_with("SCALABILITY") {
+        args.scalability_timeout
+    } else {
+        args.test_timeout
+    }
+}
+
+fn run_worker(
+    args: &Args,
+    batch: &[Test],
+    results: &mut IndexMap<String, Outcome>,
+    suite: Instant,
+    phase: Instant,
+) -> usize {
+    let setup = || -> Result<_, String> {
+        let files = Files::new().map_err(|error| error.to_string())?;
+        let limit = resources::limit(memory_limit(args, &batch[0]))?;
+        let mut command = Command::new(std::env::current_exe().map_err(|error| error.to_string())?);
+        command
+            .arg("--test-worker")
+            .arg(if batch.len() > 1 {
+                "CHAOS MONKEY"
+            } else {
+                &batch[0].name
+            })
+            .args([
+                "-i",
+                &args.solution_path,
+                "-s",
+                &args.seed.to_string(),
+                "-p",
+                &args.process_count.to_string(),
+                "-m",
+                &args.monkeys.to_string(),
+                "--chaos-start",
+                &batch[0].chaos_run.unwrap_or(1).to_string(),
+                "--memory-limit",
+                &args.memory_limit.to_string(),
+                "--mc-memory-limit",
+                &args.mc_memory_limit.to_string(),
+                "--test-result",
+            ])
+            .arg(files.journal())
+            .arg("--test-progress")
+            .arg(files.progress())
+            .env("PYTHONHASHSEED", (args.seed as u32).to_string())
+            .env("PYTHONUNBUFFERED", "1");
+        if args.debug {
+            command.arg("-d");
+        }
+        if args.verbose {
+            command.arg("-v");
+        }
+        if args.disable_mc_tests {
+            command.arg("--disable-mc-tests");
+        }
+        resources::configure(&mut command, limit);
+        let reader =
+            BufReader::new(File::open(files.journal()).map_err(|error| error.to_string())?);
+        let output = resources::capture(&mut command).map_err(|error| error.to_string())?;
+        let child = command.spawn().map_err(|error| error.to_string())?;
+        Ok((files, limit, reader, output, child))
+    };
+    let (files, limit, reader, output, mut child) = match setup() {
+        Ok(setup) => setup,
+        Err(error) => {
+            let outcome = Outcome::WorkerError(error);
+            println!("\n--- {} ---\n", batch[0].name);
+            report(&batch[0].name, &outcome);
+            results.insert(batch[0].name.clone(), outcome);
+            return 1;
+        }
+    };
+    let stderr = output_reader(output);
+    let mut journal = Journal {
+        reader,
+        pending: String::new(),
+        next: 0,
+        started: Instant::now(),
+        announced: false,
+    };
+    let mut last_progress = Instant::now();
+    let mut detail = "starting worker".to_owned();
+    let mut stopped = None;
+    let status = loop {
+        if let Err(error) = journal.read(batch, results) {
+            stopped = Some(Outcome::WorkerError(error));
+            break None;
+        }
+        match child.try_wait() {
+            Ok(Some(status)) => break Some(status),
+            Err(error) => {
+                stopped = Some(Outcome::WorkerError(error.to_string()));
+                break None;
+            }
+            Ok(None) => {}
+        }
+        if let Ok(message) = fs::read_to_string(files.progress()) {
+            detail = message;
+        }
+        let now = Instant::now();
+        let current = &batch[journal.next.min(batch.len() - 1)];
+        let limit_seconds = seconds(args, current);
+        if now >= suite
+            || now >= phase
+            || now.duration_since(journal.started).as_secs() >= limit_seconds
+        {
+            let budget = if now >= suite {
+                format!("suite budget of {}s", args.suite_timeout)
+            } else if now >= phase {
+                format!("chaos budget of {}s", args.chaos_timeout)
+            } else {
+                format!("test budget of {limit_seconds}s")
+            };
+            stopped = Some(Outcome::TimeLimit(format!("{budget}; {detail}")));
+            break None;
+        }
+        if last_progress.elapsed() >= Duration::from_secs(5) {
+            let memory = resources::memory_usage(&child).map_or(String::new(), |mib| {
+                format!("; address space {mib}/{limit} MiB")
+            });
+            eprintln!(
+                "RUNNING: {}; {}; elapsed {:.0}s{}",
+                current.name,
+                detail,
+                journal.started.elapsed().as_secs_f64(),
+                memory
+            );
+            last_progress = now;
+        }
+        thread::sleep(Duration::from_millis(10));
+    };
+    resources::terminate(&mut child);
+    let stderr = stderr.join().unwrap_or_default();
+    // The worker may have committed another result between the last poll and exit.
+    if let Err(error) = journal.read(batch, results) {
+        stopped = Some(Outcome::WorkerError(error));
+    }
+    if journal.next > 0 {
+        let finished = &batch[journal.next - 1];
+        if let Some(Outcome::WorkerError(message)) = results.get(&finished.name) {
+            let phase = fs::read_to_string(files.progress()).unwrap_or_default();
+            let outcome = Outcome::WorkerError(format!("{message}; {phase}"));
+            report(&finished.name, &outcome);
+            results.insert(finished.name.clone(), outcome);
+        }
+    }
+    if journal.next == batch.len() {
+        return journal.next;
+    }
+    // A handled panic/allocation failure ends the batch cleanly. Restart the
+    // remaining runs without inventing a failure for a test that never started.
+    if stopped.is_none() && journal.next > 0 && !journal.announced {
+        return journal.next;
+    }
+    if let Ok(message) = fs::read_to_string(files.progress()) {
+        detail = message;
+    }
+    let test = &batch[journal.next];
+    let outcome = stopped.unwrap_or_else(|| {
+        if resources::allocation_crash(status, &stderr) {
+            Outcome::MemoryLimit(format!("worker address-space limit {limit} MiB; {detail}"))
+        } else {
+            Outcome::WorkerError(format!(
+                "{}; {detail}",
+                status.map_or("worker stopped".into(), |status| format!(
+                    "exited with {status} without a result"
+                ))
+            ))
+        }
+    });
+    // Read the final journal first: Started may have arrived after the timeout
+    // poll. Only an actually started test is eligible for a resource exemption.
+    let outcome = outcome.require_started(journal.announced);
+    if !journal.announced {
+        println!("\n--- {} ---\n", test.name);
+    }
+    report(&test.name, &outcome);
+    println!(
+        "REPRODUCE: -t {:?} -s {} -p {} -m {} (effective run seed: {})\n",
+        test.name, args.seed, args.process_count, args.monkeys, test.seed
+    );
+    results.insert(test.name.clone(), outcome);
+    journal.next + 1
+}
+
+pub fn run(args: &Args) -> Result<IndexMap<String, Outcome>, String> {
+    let selected: Vec<_> = tests(args)
+        .into_iter()
+        .filter(|test| args.test.as_ref().is_none_or(|name| name == &test.name))
+        .collect();
+    if selected.is_empty() {
+        println!("FAILED: unknown test {:?}", args.test);
+        return Ok(IndexMap::new());
+    }
+    let maximum_memory = selected
+        .iter()
+        .map(|test| memory_limit(args, test))
+        .max()
+        .unwrap();
+    resources::limit(maximum_memory)?;
+    let suite = Instant::now() + Duration::from_secs(args.suite_timeout);
+    let mut results: IndexMap<_, _> = selected
+        .iter()
+        .map(|test| {
+            (
+                test.name.clone(),
+                Outcome::Skipped(format!(
+                    "suite budget of {}s exhausted before this test started",
+                    args.suite_timeout
+                )),
+            )
+        })
+        .collect();
+    let mut next = 0;
+    while next < selected.len() && Instant::now() < suite {
+        // A single explicitly selected test remains independently runnable.
+        if args.test.is_none() && selected[next].name.starts_with("SCALABILITY") {
+            let blockers = scalability_blockers(&results, args.monkeys, args.disable_mc_tests);
+            if !blockers.is_empty() {
+                for test in &selected[next..] {
+                    let outcome = Outcome::SkippedPrerequisites(blockers.clone());
+                    println!("\n--- {} ---\n", test.name);
+                    report(&test.name, &outcome);
+                    results.insert(test.name.clone(), outcome);
+                }
+                break;
+            }
+        }
+        let mut end = next + 1;
+        let chaos = selected[next].chaos_run.is_some();
+        if chaos {
+            while end < selected.len() && selected[end].chaos_run.is_some() {
+                end += 1;
+            }
+        }
+        let phase = if chaos {
+            Instant::now() + Duration::from_secs(args.chaos_timeout)
+        } else {
+            suite
+        };
+        while next < end && Instant::now() < suite && Instant::now() < phase {
+            next += run_worker(args, &selected[next..end], &mut results, suite, phase);
+        }
+        if next < end {
+            if Instant::now() < suite {
+                for test in &selected[next..end] {
+                    results.insert(
+                        test.name.clone(),
+                        Outcome::Skipped(format!(
+                            "chaos budget of {}s exhausted before this run started",
+                            args.chaos_timeout
+                        )),
+                    );
+                }
+            }
+            next = end;
+        }
+    }
+    if args.test.is_none() {
+        let passed = results
+            .values()
+            .filter(|result| matches!(result, Outcome::Passed))
+            .count();
+        println!("-------------------------------------------------------------------------------\n\nPassed {passed} from {} tests\n", selected.len());
+        for (name, result) in &results {
+            if !matches!(result, Outcome::Passed) {
+                println!("- {name}: {}", result.description());
+            }
+        }
+        println!();
+    }
+    Ok(results)
+}
+
+#[cfg(test)]
+pub fn passed_results(monkeys: u32) -> IndexMap<String, Outcome> {
+    let args = <Args as clap::Parser>::parse_from(["membership", "-m", &monkeys.to_string()]);
+    tests(&args)
+        .into_iter()
+        .map(|test| (test.name, Outcome::Passed))
+        .collect()
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn resource_failure_requires_a_started_record_even_at_the_deadline() {
+        // Model the final journal drain on either side of the startup boundary.
+        // With no Started record, the result must remain ineligible for MC credit.
+        for resource in [
+            Outcome::TimeLimit("suite budget exhausted during startup".into()),
+            Outcome::MemoryLimit("allocation failed during startup".into()),
+        ] {
+            let files = Files::new().unwrap();
+            let args = <Args as clap::Parser>::parse_from(["membership", "-m", "0"]);
+            let registry = tests(&args);
+            let mc_index = registry
+                .iter()
+                .position(|test| test.name == "MODEL CHECKING")
+                .unwrap();
+            let batch = &registry[mc_index..mc_index + 1];
+            let mut journal = Journal {
+                reader: BufReader::new(File::open(files.journal()).unwrap()),
+                pending: String::new(),
+                next: 0,
+                started: Instant::now(),
+                announced: false,
+            };
+            let mut results = IndexMap::new();
+            journal.read(batch, &mut results).unwrap();
+            let outcome = resource.clone().require_started(journal.announced);
+            assert!(matches!(outcome, Outcome::Skipped(_)));
+            assert!(!outcome.resource_limited());
+
+            // Started committed just before termination is picked up by the
+            // final drain, and makes a genuine interrupted run resource-limited.
+            let mut writer = OpenOptions::new()
+                .append(true)
+                .open(files.journal())
+                .unwrap();
+            event(&mut writer, Event::Started(batch[0].name.clone()));
+            journal.read(batch, &mut results).unwrap();
+            assert!(resource
+                .require_started(journal.announced)
+                .resource_limited());
+        }
+    }
+
+    #[test]
+    fn default_memory_limits_are_separate_for_mc_and_simulation() {
+        let args = <Args as clap::Parser>::parse_from(["membership"]);
+        for test in tests(&args) {
+            assert_eq!(
+                memory_limit(&args, &test),
+                if test.name == "MODEL CHECKING" {
+                    14336
+                } else {
+                    4096
+                }
+            );
+        }
+    }
+
+    #[test]
+    fn registry_preserves_all_tests_and_chaos_seeds() {
+        let args = <Args as clap::Parser>::parse_from(["membership", "-m", "1000"]);
+        let registry = tests(&args);
+        assert_eq!(registry.len(), 1027);
+        let ordinary = ORDINARY.len();
+        assert_eq!(registry[0].name, "SIMPLE");
+        assert_eq!(registry[ordinary].name, "CHAOS MONKEY (run 1)");
+        assert_eq!(registry[ordinary + 999].name, "CHAOS MONKEY (run 1000)");
+        assert_eq!(registry[ordinary + 1000].name, "MODEL CHECKING");
+        assert_eq!(registry[ordinary + 1001].name, "SCALABILITY NORMAL");
+        assert_eq!(registry[ordinary + 1002].name, "SCALABILITY CRASH");
+        let mut rand = Pcg64::seed_from_u64(123);
+        for test in &registry[ordinary..ordinary + 1000] {
+            assert_eq!(test.seed, rand.next_u64());
+        }
+    }
+
+    #[test]
+    fn oversized_ipc_records_do_not_exhaust_supervisor_memory() {
+        let files = Files::new().unwrap();
+        fs::write(files.journal(), vec![b'x'; 100_000]).unwrap();
+        let args = <Args as clap::Parser>::parse_from(["membership"]);
+        let mut journal = Journal {
+            reader: BufReader::new(File::open(files.journal()).unwrap()),
+            pending: String::new(),
+            next: 0,
+            started: Instant::now(),
+            announced: false,
+        };
+        let error = journal
+            .read(&tests(&args), &mut IndexMap::new())
+            .unwrap_err();
+        assert_eq!(error, "worker result exceeds 64 KiB");
+        assert!(journal.pending.len() <= 65537);
+    }
+
+    #[test]
+    fn journal_keeps_completed_results_and_waits_for_partial_records() {
+        let files = Files::new().unwrap();
+        let args = <Args as clap::Parser>::parse_from(["membership", "-m", "2"]);
+        let registry = tests(&args);
+        let batch = &registry[ORDINARY.len()..ORDINARY.len() + 2];
+        let mut writer = OpenOptions::new()
+            .append(true)
+            .open(files.journal())
+            .unwrap();
+        event(&mut writer, Event::Started(batch[0].name.clone()));
+        event(
+            &mut writer,
+            Event::Finished(batch[0].name.clone(), Outcome::Passed),
+        );
+        let mut journal = Journal {
+            reader: BufReader::new(File::open(files.journal()).unwrap()),
+            pending: String::new(),
+            next: 0,
+            started: Instant::now(),
+            announced: false,
+        };
+        let mut results = IndexMap::new();
+        journal.read(batch, &mut results).unwrap();
+        assert_eq!(journal.next, 1);
+        assert!(matches!(results[&batch[0].name], Outcome::Passed));
+        let record = serde_json::to_string(&Event::Started(batch[1].name.clone())).unwrap();
+        writer.write_all(record.as_bytes()).unwrap();
+        writer.flush().unwrap();
+        journal.read(batch, &mut results).unwrap();
+        assert!(!journal.announced);
+        writeln!(writer).unwrap();
+        writer.flush().unwrap();
+        journal.read(batch, &mut results).unwrap();
+        assert!(journal.announced);
+        assert_eq!(results.len(), 1);
+    }
+}

homework/06-membership/tests/src/shared_state.rs

@@ -0,0 +1,78 @@
+use std::ffi::CString;
+
+use pyo3::prelude::*;
+use pyo3::types::{PyList, PyModule};
+
+const VALIDATOR_CODE: &str = include_str!("../shared_state_validator.py");
+
+pub(crate) fn validator_module<'py>(py: Python<'py>) -> PyResult<Bound<'py, PyModule>> {
+    let code = CString::new(VALIDATOR_CODE).expect("validator source contains a null byte");
+    PyModule::from_code(
+        py,
+        &code,
+        c"shared_state_validator.py",
+        c"shared_state_validator",
+    )
+}
+
+#[cfg(test)]
+fn validate_source(
+    source: &str,
+    filename: &str,
+    class_names: &[&str],
+) -> Result<Vec<String>, String> {
+    Python::attach(|py| {
+        let module = validator_module(py).map_err(|error| error.to_string())?;
+        let class_names = PyList::new(py, class_names).map_err(|error| error.to_string())?;
+        module
+            .getattr("validate_source")
+            .and_then(|function| function.call1((source, filename, class_names)))
+            .and_then(|result| result.extract::<Vec<String>>())
+            .map_err(|error| error.to_string())
+    })
+}
+
+fn validate_solution(path: &str, class_names: &[&str]) -> Result<Vec<String>, String> {
+    Python::attach(|py| {
+        let module = validator_module(py).map_err(|error| error.to_string())?;
+        let class_names = PyList::new(py, class_names).map_err(|error| error.to_string())?;
+        module
+            .getattr("validate_solution")
+            .and_then(|function| function.call1((path, class_names)))
+            .and_then(|result| result.extract::<Vec<String>>())
+            .map_err(|error| error.to_string())
+    })
+}
+
+pub fn validate_or_report(path: &str, class_names: &[&str]) -> bool {
+    match validate_solution(path, class_names) {
+        Ok(violations) if violations.is_empty() => true,
+        Ok(violations) => {
+            println!("ERROR: shared state between AnySystem processes is forbidden");
+            for violation in violations {
+                println!("{violation}");
+            }
+            println!("\nSCORE: 0\n");
+            false
+        }
+        Err(error) => panic!("failed to validate solution for shared state: {error}"),
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::validate_source;
+
+    #[test]
+    fn embeds_validator_and_rejects_shared_state() {
+        let violations = validate_source(
+            "class ProcessImpl:\n    shared = {}\n",
+            "solution.py",
+            &["ProcessImpl"],
+        )
+        .unwrap();
+
+        assert_eq!(violations.len(), 1);
+        assert!(violations[0].contains("ProcessImpl.shared"));
+    }
+}

homework/06-membership/tests/src/tests.rs

@@ -0,0 +1,813 @@
+use std::collections::{BTreeSet, HashMap};
+
+use assertables::{assume, assume_eq};
+use log::info;
+use rand::prelude::*;
+use rand_pcg::Pcg64;
+use sugars::boxed;
+
+use anysystem::test::TestResult;
+use anysystem::{Message, System};
+
+use crate::common::*;
+
+pub fn test_simple(config: &TestConfig) -> TestResult {
+    let mut sys = build_system(config);
+    let group = sys.process_names();
+    let seed = &group[0];
+    initialize_group(&mut sys, &group, seed)
+}
+
+pub fn test_get_members_semantics(config: &TestConfig) -> TestResult {
+    let mut sys = build_system(config);
+    let group = sys.process_names();
+    let seed = &group[0];
+    initialize_group(&mut sys, &group, seed)?;
+    for proc in sys.process_names() {
+        get_members(&mut sys, &proc)?;
+    }
+    Ok(true)
+}
+
+pub fn test_random_seed(config: &TestConfig) -> TestResult {
+    let mut rand = Pcg64::seed_from_u64(config.seed);
+    let mut sys = build_system(config);
+    let mut group = Vec::new();
+    for proc in sys.process_names() {
+        let seed = match group.len() {
+            0 => &proc,
+            _ => group.choose(&mut rand).unwrap(),
+        };
+        sys.send_local_message(&proc, Message::json("JOIN", &JoinMessage { seed }));
+        group.push(proc);
+    }
+    step_until_stabilized(&mut sys, group.into_iter().collect())
+}
+
+pub fn test_process_join(config: &TestConfig) -> TestResult {
+    let mut rand = Pcg64::seed_from_u64(config.seed);
+    let mut sys = build_system(config);
+    let mut group = sys.process_names();
+    group.shuffle(&mut rand);
+    let new_proc = group.remove(rand.gen_range(0..group.len()));
+    let seed = &group[0];
+    initialize_group(&mut sys, &group, seed)?;
+
+    // process joins the system
+    sys.send_local_message(&new_proc, Message::json("JOIN", &JoinMessage { seed }));
+    group.push(new_proc);
+    step_until_stabilized(&mut sys, group.into_iter().collect())
+}
+
+pub fn test_process_leave(config: &TestConfig) -> TestResult {
+    let mut rand = Pcg64::seed_from_u64(config.seed);
+    let mut sys = build_system(config);
+    let mut group = sys.process_names();
+    group.shuffle(&mut rand);
+    let seed = &group[0];
+    initialize_group(&mut sys, &group, seed)?;
+
+    // process leaves the system
+    let left_proc = group.remove(rand.gen_range(0..group.len()));
+    sys.send_local_message(&left_proc, Message::json("LEAVE", &LeaveMessage {}));
+    step_until_stabilized(&mut sys, group.into_iter().collect())
+}
+
+pub fn test_process_crash(config: &TestConfig) -> TestResult {
+    let mut rand = Pcg64::seed_from_u64(config.seed);
+    let mut sys = build_system(config);
+    let mut group = sys.process_names();
+    group.shuffle(&mut rand);
+    let seed = &group[0];
+    initialize_group(&mut sys, &group, seed)?;
+
+    // process crashes
+    let crashed = group.remove(rand.gen_range(0..group.len()));
+    crash_process(&crashed, &mut sys);
+    step_until_stabilized(&mut sys, group.into_iter().collect())
+}
+
+pub fn test_seed_process_crash(config: &TestConfig) -> TestResult {
+    let mut rand = Pcg64::seed_from_u64(config.seed);
+    let mut sys = build_system(config);
+    let mut group = sys.process_names();
+    group.shuffle(&mut rand);
+    let seed = &group[0].clone();
+    initialize_group(&mut sys, &group, seed)?;
+
+    // seed process crashes
+    group.remove(0);
+    crash_process(seed, &mut sys);
+    step_until_stabilized(&mut sys, group.into_iter().collect())
+}
+
+pub fn test_process_crash_recover(config: &TestConfig) -> TestResult {
+    let mut rand = Pcg64::seed_from_u64(config.seed);
+    let mut sys = build_system(config);
+    let mut group = sys.process_names();
+    group.shuffle(&mut rand);
+    let seed = &group[0].clone();
+    initialize_group(&mut sys, &group, seed)?;
+
+    // process crashes
+    let crashed = group.remove(rand.gen_range(0..group.len()));
+    crash_process(&crashed, &mut sys);
+    step_until_stabilized(&mut sys, group.clone().into_iter().collect())?;
+
+    // process recovers
+    recover_process(&crashed, &mut sys, config);
+    sys.send_local_message(&crashed, Message::json("JOIN", &JoinMessage { seed }));
+
+    group.push(crashed);
+    step_until_membership_stable(&mut sys, group.into_iter().collect(), "process recovery")
+}
+
+pub fn test_process_offline(config: &TestConfig) -> TestResult {
+    let mut rand = Pcg64::seed_from_u64(config.seed);
+    let mut sys = build_system(config);
+    let mut group = sys.process_names();
+    group.shuffle(&mut rand);
+    let seed = &group[0];
+    initialize_group(&mut sys, &group, seed)?;
+
+    // process goes offline
+    let offline_proc = group.remove(rand.gen_range(0..group.len()));
+    sys.network().disconnect_node(&offline_proc);
+    step_until_stabilized(&mut sys, group.into_iter().collect())?;
+    step_until_stabilized_at(&mut sys, BTreeSet::from([offline_proc]), "isolated process")
+}
+
+pub fn test_seed_process_offline(config: &TestConfig) -> TestResult {
+    let mut rand = Pcg64::seed_from_u64(config.seed);
+    let mut sys = build_system(config);
+    let mut group = sys.process_names();
+    group.shuffle(&mut rand);
+    let seed = &group[0].clone();
+    initialize_group(&mut sys, &group, seed)?;
+
+    // seed process goes offline
+    group.remove(0);
+    sys.network().disconnect_node(seed);
+    step_until_stabilized(&mut sys, group.into_iter().collect())?;
+    step_until_stabilized_at(&mut sys, BTreeSet::from([seed.clone()]), "isolated process")
+}
+
+pub fn test_process_offline_recover(config: &TestConfig) -> TestResult {
+    let mut rand = Pcg64::seed_from_u64(config.seed);
+    let mut sys = build_system(config);
+    let mut group = sys.process_names();
+    group.shuffle(&mut rand);
+    let seed = &group[0];
+    initialize_group(&mut sys, &group, seed)?;
+
+    // process goes offline
+    let offline_proc = group.remove(rand.gen_range(0..group.len()));
+    sys.network().disconnect_node(&offline_proc);
+    step_until_stabilized(&mut sys, group.clone().into_iter().collect())?;
+    step_until_stabilized_at(
+        &mut sys,
+        BTreeSet::from([offline_proc.clone()]),
+        "isolated process",
+    )?;
+
+    // process goes back online
+    sys.network().connect_node(&offline_proc);
+    group.push(offline_proc);
+    step_until_membership_stable(
+        &mut sys,
+        group.into_iter().collect(),
+        "network reconnection",
+    )
+}
+
+pub fn test_network_partition(config: &TestConfig) -> TestResult {
+    run_network_partition(config, false, false)
+}
+
+pub fn test_network_partition_recover(config: &TestConfig) -> TestResult {
+    run_network_partition(config, false, true)
+}
+
+pub fn test_network_partition_unequal(config: &TestConfig) -> TestResult {
+    run_network_partition(config, true, false)
+}
+
+pub fn test_network_partition_unequal_recover(config: &TestConfig) -> TestResult {
+    run_network_partition(config, true, true)
+}
+
+fn run_network_partition(config: &TestConfig, unequal: bool, recover: bool) -> TestResult {
+    if unequal {
+        assume!(
+            config.process_count >= 3,
+            "Unequal network partition requires at least 3 processes"
+        )?;
+    }
+    let mut rand = Pcg64::seed_from_u64(config.seed);
+    let mut sys = build_system(config);
+    let mut group = sys.process_names();
+    group.shuffle(&mut rand);
+    let seed = &group[0];
+    initialize_group(&mut sys, &group, seed)?;
+
+    let (group1, group2): (Vec<_>, Vec<_>) = if unequal {
+        // Always a nonempty strict minority, including odd process counts.
+        let (minority, majority) = group.split_at((group.len() - 1) / 2);
+        (
+            minority.iter().map(String::as_str).collect(),
+            majority.iter().map(String::as_str).collect(),
+        )
+    } else {
+        // Preserve the existing random partition and its seeds.
+        group
+            .iter()
+            .map(String::as_str)
+            .partition(|_| rand.gen_range(0.0..1.0) > 0.6)
+    };
+    info!(
+        "Network partition: [{}] ({} processes) | [{}] ({} processes)",
+        group1.join(", "),
+        group1.len(),
+        group2.join(", "),
+        group2.len()
+    );
+    sys.network().make_partition(&group1, &group2);
+    step_until_stabilized_at(
+        &mut sys,
+        group1.into_iter().map(String::from).collect(),
+        "network partition: first subgroup",
+    )?;
+    step_until_stabilized_at(
+        &mut sys,
+        group2.into_iter().map(String::from).collect(),
+        "network partition: second subgroup",
+    )?;
+
+    if recover {
+        sys.network().reset();
+        step_until_membership_stable(&mut sys, group.into_iter().collect(), "network recovery")?;
+    }
+    Ok(true)
+}
+
+pub fn test_process_cannot_receive(config: &TestConfig) -> TestResult {
+    let mut rand = Pcg64::seed_from_u64(config.seed);
+    let mut sys = build_system(config);
+    let mut group = sys.process_names();
+    group.shuffle(&mut rand);
+    let seed = &group[0];
+    initialize_group(&mut sys, &group, seed)?;
+
+    // process goes partially offline (cannot receive incoming messages)
+    let blocked_proc = group.remove(rand.gen_range(0..group.len()));
+    sys.network().drop_incoming(&blocked_proc);
+    step_until_stabilized(&mut sys, group.into_iter().collect())?;
+    step_until_stabilized_at(&mut sys, BTreeSet::from([blocked_proc]), "isolated process")
+}
+
+pub fn test_process_join_cannot_receive(config: &TestConfig) -> TestResult {
+    let mut rand = Pcg64::seed_from_u64(config.seed);
+    let mut sys = build_system(config);
+    let mut group = sys.process_names();
+    group.shuffle(&mut rand);
+    let new_proc = group.remove(rand.gen_range(0..group.len()));
+    let seed = &group[0];
+    initialize_group(&mut sys, &group, seed)?;
+
+    // process tries to join the system but cannot receive messages
+    // it must not be kept in the group
+    sys.network().drop_incoming(&new_proc);
+    sys.send_local_message(&new_proc, Message::json("JOIN", &JoinMessage { seed }));
+    step_until_stabilized(&mut sys, group.into_iter().collect())
+}
+
+pub fn test_process_cannot_send(config: &TestConfig) -> TestResult {
+    let mut rand = Pcg64::seed_from_u64(config.seed);
+    let mut sys = build_system(config);
+    let mut group = sys.process_names();
+    group.shuffle(&mut rand);
+    let seed = &group[0];
+    initialize_group(&mut sys, &group, seed)?;
+
+    // process goes partially offline (cannot send outgoing messages)
+    let blocked_proc = group.remove(rand.gen_range(0..group.len()));
+    sys.network().drop_outgoing(&blocked_proc);
+    step_until_stabilized(&mut sys, group.into_iter().collect())?;
+    step_until_stabilized_at(&mut sys, BTreeSet::from([blocked_proc]), "isolated process")
+}
+
+pub fn test_two_processes_cannot_communicate(config: &TestConfig) -> TestResult {
+    run_pairwise_link_failure(config, false)
+}
+
+pub fn test_two_processes_cannot_communicate_in_triangle(config: &TestConfig) -> TestResult {
+    let config = TestConfig {
+        process_count: 3,
+        ..*config
+    };
+    run_pairwise_link_failure(&config, true)
+}
+
+fn run_pairwise_link_failure(config: &TestConfig, triangle: bool) -> TestResult {
+    assume!(
+        config.process_count >= 3,
+        "Pairwise link failure with a connected group requires at least 3 processes"
+    )?;
+    let mut rand = Pcg64::seed_from_u64(config.seed);
+    let mut sys = build_system(config);
+    let mut group = sys.process_names();
+    if triangle {
+        group.sort();
+    } else {
+        group.shuffle(&mut rand);
+    }
+    let seed = &group[0];
+    initialize_group(&mut sys, &group, seed)?;
+
+    // In a triangle, the seed remains the bridge between the other two nodes.
+    let proc1 = if triangle { &group[1] } else { seed };
+    let proc2 = if triangle {
+        &group[2]
+    } else {
+        group.get(rand.gen_range(1..group.len())).unwrap()
+    };
+    info!(
+        "Network links disabled: {proc1} <-> {proc2}; group remains connected (N={})",
+        group.len()
+    );
+    sys.network().disable_link(proc1, proc2);
+    sys.network().disable_link(proc2, proc1);
+    // Preserve the original test's execution before checking convergence.
+    sys.steps(1000);
+    step_until_membership_stable(
+        &mut sys,
+        group.into_iter().collect(),
+        "pairwise link failure",
+    )
+}
+
+/// Once membership is correct, observe every event without advancing time in GET_MEMBERS.
+/// A transient mismatch restarts the stability window, allowing eventual convergence.
+fn step_until_membership_stable(
+    sys: &mut System,
+    group: BTreeSet<String>,
+    stage: &str,
+) -> TestResult {
+    const STABLE_WINDOW: f64 = 30.0;
+    const CONVERGENCE_LIMIT: f64 = 300.0;
+    let deadline = sys.time() + CONVERGENCE_LIMIT;
+    let mut stable_since = None;
+    let mut checked_events = 0_u64;
+    let mut search_exhausted = false;
+    let mut last_mismatch = String::from("no correct membership snapshot");
+    loop {
+        let now = sys.time();
+        // The next event may lie beyond the observed window. Its effects must
+        // not invalidate the unchanged state inside the completed window.
+        if stable_since
+            .is_some_and(|since| now > since + STABLE_WINDOW && since + STABLE_WINDOW <= deadline)
+        {
+            break;
+        }
+        if now > deadline {
+            return Err(format!(
+                "Group members lists are not stable for {STABLE_WINDOW}s within {CONVERGENCE_LIMIT}s; {last_mismatch}"
+            ));
+        }
+        if checked_events.is_multiple_of(100) {
+            crate::runner::progress(format!(
+                "{stage}: membership stability: N={}, simulation time={now:.1}, stable for {:.1}/{STABLE_WINDOW}s, checked events={checked_events}",
+                group.len(), stable_since.map_or(0.0, |since| now - since)
+            ));
+        }
+        let mut correct = true;
+        for proc in &group {
+            let data = get_members(sys, proc)?;
+            let members: BTreeSet<_> = data.members.into_iter().collect();
+            if members != group {
+                correct = false;
+                last_mismatch = format!(
+                    "process {proc} at simulation time {now:.3}: expected {group:?}, got {members:?}"
+                );
+            }
+        }
+        if correct {
+            let since = *stable_since.get_or_insert(now);
+            if now - since >= STABLE_WINDOW {
+                break;
+            }
+        } else {
+            stable_since = None;
+        }
+        if !correct && !search_exhausted && now < deadline {
+            // Search for the next candidate snapshot at the usual 5s cadence.
+            // Only a correct candidate starts event-by-event observation. This
+            // avoids filling AnySystem's retained trace with failed queries.
+            search_exhausted = !sys.step_for_duration(5.0_f64.min(deadline - now));
+            continue;
+        }
+        if !correct {
+            return Err(format!(
+                "Group members lists are not stable for {STABLE_WINDOW}s within {CONVERGENCE_LIMIT}s; {last_mismatch}"
+            ));
+        }
+        if !sys.step() {
+            // With no pending events, a correct membership cannot change.
+            break;
+        }
+        checked_events += 1;
+    }
+    info!(
+        "Membership remained stable for {STABLE_WINDOW}s ({stage}; checked after {checked_events} simulation events)"
+    );
+    Ok(true)
+}
+
+pub fn test_slow_network(config: &TestConfig) -> TestResult {
+    let mut rand = Pcg64::seed_from_u64(config.seed);
+    let mut sys = build_system(config);
+    let mut group = sys.process_names();
+    group.shuffle(&mut rand);
+    let seed = &group[0];
+    initialize_group(&mut sys, &group, seed)?;
+
+    // slow down network for a while
+    sys.network().set_delays(0.1, 1.0);
+    sys.steps(200);
+    sys.network().set_delays(0.01, 0.1);
+    step_until_stabilized(&mut sys, group.into_iter().collect())
+}
+
+pub fn test_flaky_network(config: &TestConfig) -> TestResult {
+    let mut rand = Pcg64::seed_from_u64(config.seed);
+    let mut sys = build_system(config);
+    let mut group = sys.process_names();
+    group.shuffle(&mut rand);
+    let seed = &group[0];
+    initialize_group(&mut sys, &group, seed)?;
+
+    // make network unreliable for a while
+    sys.network().set_drop_rate(0.5);
+    sys.steps(1000);
+    sys.network().set_drop_rate(0.0);
+    step_until_stabilized(&mut sys, group.into_iter().collect())
+}
+
+pub fn test_flaky_network_on_start(config: &TestConfig) -> TestResult {
+    let mut rand = Pcg64::seed_from_u64(config.seed);
+    let mut sys = build_system(config);
+    let mut group = sys.process_names();
+    group.shuffle(&mut rand);
+    let seed = &group[0];
+
+    // make network unreliable from the start
+    sys.network().set_drop_rate(0.2);
+    for proc in &group {
+        sys.send_local_message(proc, Message::json("JOIN", &JoinMessage { seed }));
+    }
+    sys.steps(1000);
+    sys.network().set_drop_rate(0.0);
+    step_until_stabilized(&mut sys, group.into_iter().collect())
+}
+
+pub fn test_flaky_network_and_crash(config: &TestConfig) -> TestResult {
+    let mut rand = Pcg64::seed_from_u64(config.seed);
+    let mut sys = build_system(config);
+    let mut group = sys.process_names();
+    group.shuffle(&mut rand);
+    let seed = &group[0];
+    initialize_group(&mut sys, &group, seed)?;
+
+    // make network unreliable for a while + crash process
+    sys.network().set_drop_rate(0.5);
+    let crashed = group.remove(rand.gen_range(0..group.len()));
+    crash_process(&crashed, &mut sys);
+    sys.steps(1000);
+    sys.network().set_drop_rate(0.0);
+    step_until_stabilized(&mut sys, group.into_iter().collect())
+}
+
+pub fn test_chaos_monkey(config: &TestConfig) -> TestResult {
+    let mut rand = Pcg64::seed_from_u64(config.seed);
+    let mut sys = build_system(config);
+    let mut group = sys.process_names();
+    group.shuffle(&mut rand);
+    let seed = &group[0];
+    initialize_group(&mut sys, &group, seed)?;
+
+    info!(
+        "CHAOS MONKEY run seed: {}; join order: [{}]; seed process: {seed}",
+        config.seed,
+        group.join(", ")
+    );
+    // Keep at least two participants; pairwise link failures must not isolate
+    // a participant still expected in the control group.
+    let failure_count = 6.min(group.len().saturating_sub(2));
+    for step in 1..=failure_count {
+        let p = rand.gen_range(0.0..1.0);
+        // do some nasty things
+        match p {
+            p if p < 0.25 => {
+                // crash process
+                let victim = group.remove(rand.gen_range(0..group.len()));
+                info!(
+                    "- Step {step} at {:.2}: process {victim} crashes",
+                    sys.time()
+                );
+                crash_process(&victim, &mut sys);
+            }
+            p if p < 0.5 => {
+                // disconnect process
+                let victim = group.remove(rand.gen_range(0..group.len()));
+                info!(
+                    "- Step {step} at {:.2}: process {victim} disconnects",
+                    sys.time()
+                );
+                sys.network().disconnect_node(&victim);
+            }
+            p if p < 0.75 => {
+                // partially disconnect process (cannot receive)
+                let victim = group.remove(rand.gen_range(0..group.len()));
+                info!(
+                    "- Step {step} at {:.2}: process {victim} cannot receive messages",
+                    sys.time()
+                );
+                sys.network().drop_incoming(&victim);
+            }
+            _ => {
+                // two processes cannot communicate with each other
+                let proc1 = group.get(rand.gen_range(0..group.len())).unwrap();
+                let mut proc2 = group.get(rand.gen_range(0..group.len())).unwrap();
+                while proc1 == proc2 {
+                    proc2 = group.get(rand.gen_range(0..group.len())).unwrap();
+                }
+                info!(
+                    "- Step {step} at {:.2}: processes {proc1} and {proc2} cannot communicate",
+                    sys.time()
+                );
+                sys.network().disable_link(proc1, proc2);
+                sys.network().disable_link(proc2, proc1);
+            }
+        }
+        info!(
+            "Control group after step {step}: [{}] ({} processes)",
+            group.join(", "),
+            group.len()
+        );
+        step_until_stabilized(&mut sys, group.clone().into_iter().collect())?;
+    }
+    Ok(true)
+}
+
+pub fn test_scalability_normal(config: &TestConfig) -> TestResult {
+    let sys_sizes = [
+        config.process_count,
+        config.process_count * 2,
+        config.process_count * 5,
+        config.process_count * 10,
+    ];
+    let mut measurements = Vec::new();
+    for size in sys_sizes {
+        let mut run_config = *config;
+        run_config.process_count = size;
+        let mut rand = Pcg64::seed_from_u64(config.seed);
+        let mut sys = build_system(&run_config);
+        let mut group = sys.process_names();
+        group.shuffle(&mut rand);
+        let seed = &group[0];
+        initialize_group(&mut sys, &group, seed)?;
+
+        let init_time = sys.time();
+        let init_net_traffic = sys.network().traffic();
+        let init_msg_count = sys.network().network_message_count();
+        let mut init_loads = HashMap::new();
+        for proc in sys.process_names() {
+            init_loads.insert(
+                proc.clone(),
+                sys.sent_message_count(&proc) + sys.received_message_count(&proc),
+            );
+        }
+
+        crate::runner::progress(format!(
+            "steady-state measurement: N={size}, simulation time={:.1}",
+            sys.time()
+        ));
+        sys.step_for_duration(10.0);
+
+        let mut loads = Vec::new();
+        for proc in sys.process_names() {
+            let load = sys.sent_message_count(&proc) + sys.received_message_count(&proc);
+            loads.push(load - init_loads.get(&proc).unwrap());
+        }
+        let min_load = *loads.iter().min().unwrap();
+        let max_load = *loads.iter().max().unwrap();
+        let duration = sys.time() - init_time;
+        let traffic = sys.network().traffic();
+        let message_count = sys.network().network_message_count();
+        measurements.push((
+            duration,
+            (traffic - init_net_traffic) as f64 / duration,
+            (message_count - init_msg_count) as f64 / duration,
+            max_load as f64 / duration,
+            max_load as f64 / min_load as f64,
+        ));
+    }
+    let mut scaling_ok = true;
+    let mut load_ratio_ok = true;
+    for i in 0..sys_sizes.len() {
+        let (time, traffic, message_count, max_load, load_ratio) = measurements[i];
+        println!(
+            "- N = {}: time - {:.2}, traffic/s - {:.2}, messages/s - {:.2}, max load - {:.2}, max/min load - {:.2}",
+            sys_sizes[i], time, traffic, message_count, max_load, load_ratio
+        );
+        if load_ratio > 5.0 {
+            load_ratio_ok = false;
+        }
+        if i > 0 {
+            let size_ratio = sys_sizes[i] as f64 / sys_sizes[i - 1] as f64;
+            let traffic_ratio = traffic / measurements[i - 1].1;
+            let messages_ratio = message_count / measurements[i - 1].2;
+            if traffic_ratio > 2.0 * size_ratio || messages_ratio > 2.0 * size_ratio {
+                scaling_ok = false;
+            }
+        }
+    }
+    assume!(scaling_ok, "Bad network load scaling")?;
+    assume!(load_ratio_ok, "Bad max/min process load")?;
+    Ok(true)
+}
+
+pub fn test_scalability_crash(config: &TestConfig) -> TestResult {
+    let sys_sizes = [
+        config.process_count,
+        config.process_count * 2,
+        config.process_count * 5,
+        config.process_count * 10,
+    ];
+    let mut measurements = Vec::new();
+    for size in sys_sizes {
+        let mut run_config = *config;
+        run_config.process_count = size;
+        let mut rand = Pcg64::seed_from_u64(config.seed);
+        let mut sys = build_system(&run_config);
+        let mut group = sys.process_names();
+        group.shuffle(&mut rand);
+        let seed = &group[0];
+        initialize_group(&mut sys, &group, seed)?;
+
+        let init_time = sys.time();
+        let init_net_traffic = sys.network().traffic();
+        let init_msg_count = sys.network().network_message_count();
+        let mut init_loads = HashMap::new();
+        for proc in sys.process_names() {
+            init_loads.insert(
+                proc.clone(),
+                sys.sent_message_count(&proc) + sys.received_message_count(&proc),
+            );
+        }
+
+        let crashed = group.remove(rand.gen_range(0..group.len()));
+        crash_process(&crashed, &mut sys);
+        step_until_stabilized_at(
+            &mut sys,
+            group.clone().into_iter().collect(),
+            "stabilization after crash",
+        )?;
+
+        let mut loads = Vec::new();
+        for proc in sys.process_names() {
+            if proc != crashed {
+                let load = sys.sent_message_count(&proc) + sys.received_message_count(&proc);
+                loads.push(load - init_loads.get(&proc).unwrap());
+            }
+        }
+        let min_load = *loads.iter().min().unwrap();
+        let max_load = *loads.iter().max().unwrap();
+        let duration = sys.time() - init_time;
+        let traffic = sys.network().traffic();
+        let message_count = sys.network().network_message_count();
+        measurements.push((
+            duration,
+            (traffic - init_net_traffic) as f64 / duration,
+            (message_count - init_msg_count) as f64 / duration,
+            max_load as f64 / duration,
+            max_load as f64 / min_load as f64,
+        ));
+    }
+    let mut scaling_ok = true;
+    let mut load_ratio_ok = true;
+    for i in 0..sys_sizes.len() {
+        let (time, traffic, message_count, max_load, load_ratio) = measurements[i];
+        println!(
+            "- N = {}: time - {:.2}, traffic/s - {:.2}, messages/s - {:.2}, max load - {:.2}, max/min load - {:.2}",
+            sys_sizes[i], time, traffic, message_count, max_load, load_ratio
+        );
+        if load_ratio > 5.0 {
+            load_ratio_ok = false;
+        }
+        if i > 0 {
+            let size_ratio = sys_sizes[i] as f64 / sys_sizes[i - 1] as f64;
+            let traffic_ratio = traffic / measurements[i - 1].1;
+            let messages_ratio = message_count / measurements[i - 1].2;
+            if traffic_ratio > 2.0 * size_ratio || messages_ratio > 2.0 * size_ratio {
+                scaling_ok = false;
+            }
+        }
+    }
+    assume!(scaling_ok, "Bad network load scaling")?;
+    assume!(load_ratio_ok, "Bad max/min process load")?;
+    Ok(true)
+}
+
+fn initialize_group(sys: &mut System, group: &Vec<String>, seed: &str) -> TestResult {
+    for proc in group {
+        sys.send_local_message(proc, Message::json("JOIN", &JoinMessage { seed }));
+    }
+    step_until_stabilized_at(
+        sys,
+        group.clone().into_iter().collect(),
+        "initial group assembly",
+    )
+}
+
+fn crash_process(name: &str, sys: &mut System) {
+    // we just crash the node on which the process is running
+    sys.crash_node(name);
+}
+
+fn recover_process(name: &str, sys: &mut System, config: &TestConfig) {
+    sys.recover_node(name);
+    let clock_skew = sys.gen_range(0.0..10.0);
+    // A reboot may change the local clock. Apply it before on_start is called.
+    sys.set_node_clock_skew(name, clock_skew);
+    info!(
+        "Process {name} recovers at simulation time {:.6} with clock skew {clock_skew:.6}",
+        sys.time()
+    );
+    let process = config.process_factory.build((name,), config.seed);
+    sys.add_process(name, boxed!(process), name);
+}
+
+fn get_members(sys: &mut System, proc: &str) -> Result<MembersMessage, String> {
+    let sent_before = sys.sent_message_count(proc);
+    sys.send_local_message(proc, Message::json("GET_MEMBERS", &GetMembersMessage {}));
+    assume_eq!(
+        sys.sent_message_count(proc),
+        sent_before,
+        format!("GET_MEMBERS must not send network messages: process {proc}")
+    )?;
+    let messages = sys
+        .step_until_local_message_max_steps(proc, 0)
+        .map_err(|_| {
+            format!("GET_MEMBERS must return immediately without simulation steps: process {proc}")
+        })?;
+    assume_eq!(messages.len(), 1, "expected exactly one MEMBERS message")?;
+    assume_eq!(messages[0].tip, "MEMBERS", "Wrong message type")?;
+    serde_json::from_str(&messages[0].data)
+        .map_err(|error| format!("Invalid MEMBERS message from {proc}: {error}"))
+}
+
+fn step_until_stabilized(sys: &mut System, group: BTreeSet<String>) -> TestResult {
+    step_until_stabilized_at(sys, group, "membership stabilization")
+}
+
+fn step_until_stabilized_at(sys: &mut System, group: BTreeSet<String>, stage: &str) -> TestResult {
+    let max_time = sys.time() + 300.; // timeout is 5 minutes
+    let mut stabilized = BTreeSet::new();
+    let mut memberlists = HashMap::new();
+
+    while stabilized.len() < group.len() && sys.time() < max_time {
+        crate::runner::progress(format!("{stage}: N={}, correct members lists={}/{}, simulation time={:.1}, network messages={}",
+            group.len(), stabilized.len(), group.len(), sys.time(), sys.network().network_message_count()));
+        let cont = sys.step_for_duration(5.);
+        stabilized.clear();
+        // Query processes in a stable order without advancing simulation time.
+        for proc in group.iter() {
+            let data = get_members(sys, proc)?;
+            let members: BTreeSet<String> = data.members.clone().into_iter().collect();
+            if members.eq(&group) {
+                stabilized.insert(proc.clone());
+            }
+            memberlists.insert(proc.clone(), data.members);
+        }
+        if !cont {
+            break;
+        }
+    }
+
+    if stabilized != group && group.len() <= 10 {
+        println!("Members lists:");
+        for proc in sys.process_names() {
+            if group.contains(&proc) {
+                let members = memberlists.get_mut(&proc).unwrap();
+                members.sort();
+                println!("- [{}] {}", proc, members.join(", "));
+            }
+        }
+        let mut expected = group.clone().into_iter().collect::<Vec<_>>();
+        expected.sort();
+        println!("Expected group: {}", expected.join(", "));
+    }
+    assume_eq!(stabilized, group, "Group members lists are not stabilized")?;
+    Ok(true)
+}

homework/06-membership/tests/src/tests_mc.rs

@@ -0,0 +1,159 @@
+use std::collections::HashSet;
+use std::time::Duration;
+
+use rand::prelude::*;
+use rand_pcg::Pcg64;
+use sugars::boxed;
+
+use anysystem::logger::{LogEntry, LogEntry::McMessageReceived};
+use anysystem::mc::{
+    predicates::{collects, goals, invariants, prunes},
+    strategies::Bfs,
+    CollectFn, EventOrderingMode, InvariantFn, McState, ModelChecker, StrategyConfig,
+};
+use anysystem::test::TestResult;
+use anysystem::{Message, System};
+
+use crate::common::*;
+
+pub fn test_mc_group(config: &TestConfig) -> TestResult {
+    let mut rand = Pcg64::seed_from_u64(config.seed);
+    let mut sys = build_system(config);
+    let group = sys.process_names();
+    let seed = group.choose(&mut rand).unwrap();
+
+    crate::runner::progress("MC: exploring group assembly, N=3");
+    let collected_states = mc_explore_after_joins(&mut sys, seed.to_string())?;
+    if collected_states.is_empty() {
+        return Err("no states collected during explore stage".to_string());
+    }
+    crate::runner::progress(format!(
+        "MC: checking membership in {} collected states, N=3",
+        collected_states.len()
+    ));
+    mc_check_members(&mut sys, collected_states)
+}
+
+fn mc_explore_after_joins(sys: &mut System, seed_proc: String) -> Result<HashSet<McState>, String> {
+    let procs = sys.process_names();
+    let mut mc = ModelChecker::new(sys);
+
+    let strategy_config = StrategyConfig::default()
+        // Explore only states with up to 2 timer firings per process
+        // (we expect each process to communicate with others at most 3 times:
+        // 1 time on a local message and 2 times on a timer).
+        .prune(prunes::events_limit_per_proc(
+            |entry: &LogEntry, proc_name: &String| match entry {
+                LogEntry::McTimerFired { proc, .. } => proc_name == proc,
+                _ => false,
+            },
+            procs.clone(),
+            2,
+        ))
+        // Stop when no events left or reached depth 20 (steps of simulation).
+        .goal(goals::any_goal(vec![
+            goals::no_events(),
+            goals::depth_reached(20),
+        ]))
+        // Inner exploration safeguard; the supervisor also bounds both MC phases.
+        .invariant(invariants::time_limit(Duration::from_secs(300)))
+        // Collect states in which the group should be stabilized, namely:
+        // either no events left, or every process received at least 3 messages.
+        //
+        // Considering a system with 3 processes and a stable network, we can show this is enough:
+        // * seed process (A) should get information from both other processes (B & C)
+        // * because C joins later than B, it will know the full group as well
+        // * B is able to get information about the full group either from A or C, depending on implementation
+        //
+        // Technically, there can be execution A <-> B (x3), A <-> C (x3) where B's information becomes outdated,
+        // but we consider solution wrong if it allows such execution.
+        .collect(collects::any_collect(vec![
+            collects::no_events(),
+            collects::all_collects(
+                procs
+                    .clone()
+                    .into_iter()
+                    .map(|proc_name| {
+                        collects::event_happened_n_times_current_run(
+                            move |log_entry| match log_entry {
+                                McMessageReceived { dst: proc, .. } => proc == &proc_name,
+                                _ => false,
+                            },
+                            3,
+                        )
+                    })
+                    .collect::<Vec<CollectFn>>(),
+            ),
+        ]));
+
+    let res = mc.run_with_change::<Bfs>(strategy_config, |sys| {
+        // Use event ordering mode which prioritizes messages over timers
+        // (this emulates perfect network where timeouts do not occur).
+        sys.set_event_ordering_mode(EventOrderingMode::MessagesFirst);
+        for proc in &procs {
+            sys.send_local_message(
+                proc.clone(),
+                proc.clone(),
+                Message::json("JOIN", &JoinMessage { seed: &seed_proc }),
+            );
+        }
+    });
+    match res {
+        Err(e) => {
+            e.print_trace();
+            Err(e.message())
+        }
+        Ok(stats) => {
+            // println!("collected {} states", stats.collected_states.len());
+            Ok(stats.collected_states)
+        }
+    }
+}
+
+fn mc_check_members(sys: &mut System, collected: HashSet<McState>) -> TestResult {
+    let procs = sys.process_names();
+    let mut mc = ModelChecker::new(sys);
+    let strategy_config = StrategyConfig::default()
+        .invariant(mc_invariant_check_stabilized(procs.clone()))
+        .goal(goals::always_ok());
+
+    let res = mc.run_from_states_with_change::<Bfs>(strategy_config, collected, |sys| {
+        for node in procs.iter() {
+            sys.send_local_message(
+                node,
+                node,
+                Message::json("GET_MEMBERS", &GetMembersMessage {}),
+            );
+        }
+    });
+    if let Err(e) = res {
+        e.print_trace();
+        Err(e.message())
+    } else {
+        Ok(true)
+    }
+}
+
+fn mc_invariant_check_stabilized(group: Vec<String>) -> InvariantFn {
+    boxed!(move |state| {
+        let group = group.clone().into_iter().collect::<HashSet<String>>();
+        for node in state.node_states.keys() {
+            if let Some(msg) = state.node_states[node].proc_states[node]
+                .local_outbox
+                .first()
+            {
+                if msg.tip != "MEMBERS" {
+                    return Err("wrong message type".to_owned());
+                }
+                let data: MembersMessage = serde_json::from_str(&msg.data).unwrap();
+                let members = HashSet::from_iter(data.members.into_iter());
+                if !members.eq(&group) {
+                    return Err(format!(
+                        "expected a stabilized group {group:?} but got {members:?}"
+                    ));
+                }
+            }
+        }
+        Ok(())
+    })
+}