syakovlev/hse-2026
1
Fork 0
forked from course/hse-2026
Code Pull Requests Activity

Add homework 01 guarantees assignment

Browse Source
This commit is contained in:
osukhoroslov
2026-09-10 13:22:31 +03:00
parent 16a7a0e469
commit e29c7eb39e
17 changed files with 5613 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
homework/01-guarantees/tests/src/common.rs
+285
View File
@@ -0,0 +1,285 @@
use std::collections::HashMap;
use assertables::{assume, assume_eq};
use sugars::boxed;
use anysystem::python::PyProcessFactory;
use anysystem::test::TestResult;
use anysystem::{Message, System};
#[derive(Copy, Clone)]
pub struct TestConfig<'a> {
pub impl_path: &'a str,
pub sender_class: &'a str,
pub receiver_class: &'a str,
pub seed: u64,
pub monkeys: u32,
pub reliable: bool,
pub once: bool,
pub ordered: bool,
}
pub fn build_system(config: &TestConfig, measure_max_size: bool) -> System {
let mut sys = System::new(config.seed);
sys.add_node("sender-node");
sys.add_node("receiver-node");
let sender_f = PyProcessFactory::new(config.impl_path, config.sender_class);
let mut sender = sender_f.build(("sender", "receiver"), config.seed);
if measure_max_size {
sender.set_max_size_freq(100);
}
sys.add_process("sender", boxed!(sender), "sender-node");
let receiver_f = PyProcessFactory::new(config.impl_path, config.receiver_class);
let mut receiver = receiver_f.build(("receiver",), config.seed);
if measure_max_size {
receiver.set_max_size_freq(100);
}
sys.add_process("receiver", boxed!(receiver), "receiver-node");
sys
}
pub fn generate_message_texts(sys: &mut System, message_count: usize) -> Vec<String> {
if message_count == 5 {
["distributed", "systems", "need", "some", "guarantees"]
.map(String::from)
.to_vec()
} else {
let mut messages = Vec::new();
for _i in 0..message_count {
let msg = if message_count == 10 {
format!("{}C", sys.gen_range(20..30))
} else {
sys.random_string(100)
};
messages.push(msg);
}
messages
}
}
pub fn send_messages(sys: &mut System, message_count: usize) -> Vec<Message> {
let texts = generate_message_texts(sys, message_count);
let mut messages = Vec::new();
for text in texts {
let msg = Message::new("MESSAGE", &format!(r#"{{"text": "{text}"}}"#));
sys.send_local_message("sender", msg.clone());
if message_count <= 50 {
let steps = sys.gen_range(0..2);
if steps > 0 {
sys.steps(steps);
}
} else {
let duration = sys.gen_range(0.0..2.0);
sys.step_for_duration(duration);
};
messages.push(msg);
}
messages
}
pub fn check_delivered_messages(
delivered: &[Message],
expected_msg_count: &HashMap<String, i32>,
expected_tip: &String,
) -> Result<HashMap<String, i32>, String> {
assert!(!expected_msg_count.is_empty());
let mut delivered_msg_count = HashMap::default();
for msg in delivered.iter() {
// assuming all messages have the same type
assume_eq!(msg.tip, *expected_tip, format!("Wrong message type {}", msg.tip))?;
assume!(
expected_msg_count.contains_key(&msg.data),
format!("Wrong message data: {}", msg.data)
)?;
*delivered_msg_count.entry(msg.data.clone()).or_insert(0) += 1;
}
Ok(delivered_msg_count)
}
pub fn check_message_delivery_reliable(
delivered_msg_count: &HashMap<String, i32>,
expected_msg_count: &HashMap<String, i32>,
) -> TestResult {
for (data, expected_count) in expected_msg_count {
let delivered_count = delivered_msg_count.get(data).unwrap_or(&0);
assume!(
delivered_count >= expected_count,
format!(
"Message {} is not delivered (observed count {} < expected count {})",
data, delivered_count, expected_count
)
)?;
}
Ok(true)
}
pub fn check_message_delivery_once(
delivered_msg_count: &HashMap<String, i32>,
expected_msg_count: &HashMap<String, i32>,
) -> TestResult {
for (data, delivered_count) in delivered_msg_count {
if expected_msg_count.contains_key(data) {
let expected_count = expected_msg_count[data];
assume!(
*delivered_count <= expected_count,
format!(
"Message {} is delivered more than once (observed count {} > expected count {})",
data, delivered_count, expected_count
)
)?;
}
}
Ok(true)
}
pub fn check_message_delivery_ordered(delivered: &[Message], sent: &[Message]) -> TestResult {
let mut next_idx = 0;
for i in 0..delivered.len() {
let msg = &delivered[i];
let mut matched = false;
while !matched && next_idx < sent.len() {
if msg.data == sent[next_idx].data {
matched = true;
} else {
next_idx += 1;
}
}
assume!(
matched,
format!("Order violation: {} after {}", msg.data, &delivered[i - 1].data)
)?;
}
Ok(true)
}
pub fn check_guarantees(sys: &mut System, sent: &[Message], config: &TestConfig) -> TestResult {
let mut expected_msg_count = HashMap::new();
for msg in sent {
*expected_msg_count.entry(msg.data.clone()).or_insert(0) += 1;
}
let delivered = sys.read_local_messages("receiver");
// check that delivered messages have expected type and data
let delivered_msg_count = check_delivered_messages(&delivered, &expected_msg_count, &sent[0].tip)?;
// check delivered message count according to expected guarantees
if config.reliable {
check_message_delivery_reliable(&delivered_msg_count, &expected_msg_count)?;
}
if config.once {
check_message_delivery_once(&delivered_msg_count, &expected_msg_count)?;
}
if config.ordered {
check_message_delivery_ordered(&delivered, sent)?;
}
Ok(true)
}
#[allow(clippy::too_many_arguments)]
pub fn check_overhead(
guarantee: &str,
faulty: bool,
message_count: usize,
sender_mem: u64,
receiver_mem: u64,
net_message_count: u64,
net_traffic: u64,
throughput: f64,
) -> TestResult {
let (sender_mem_limit, receiver_mem_limit, net_message_count_limit, net_traffic_limit, throughput_limit) =
match guarantee {
"AMO" => match message_count {
100 => {
if !faulty {
(800, 1500, 100, 20000, 0.6)
} else {
(800, 3500, 100, 20000, 0.6)
}
}
1000 => {
if !faulty {
(800, 1500, 1000, 200000, 0.6)
} else {
(800, 30000, 1000, 200000, 0.6)
}
}
_ => (u64::MAX, u64::MAX, u64::MAX, u64::MAX, 0.),
},
"ALO" => match message_count {
100 => {
if !faulty {
(1700, 600, 200, 20000, 0.6)
} else {
(5000, 600, 500, 40000, 0.6)
}
}
1000 => {
if !faulty {
(2200, 600, 2000, 200000, 0.6)
} else {
(6000, 600, 5000, 400000, 0.6)
}
}
_ => (u64::MAX, u64::MAX, u64::MAX, u64::MAX, 0.),
},
"EO" => match message_count {
100 => {
if !faulty {
(1700, 1500, 200, 20000, 0.6)
} else {
(5000, 2200, 500, 40000, 0.6)
}
}
1000 => {
if !faulty {
(2200, 1500, 2000, 200000, 0.6)
} else {
(6000, 2200, 5000, 400000, 0.6)
}
}
_ => (u64::MAX, u64::MAX, u64::MAX, u64::MAX, 0.),
},
"EOO" => match message_count {
100 => {
if !faulty {
(2900, 1200, 200, 25000, 0.4)
} else {
(9000, 2500, 500, 45000, 0.4)
}
}
1000 => {
if !faulty {
(3400, 1200, 2000, 250000, 0.4)
} else {
(55000, 4000, 5000, 450000, 0.4)
}
}
_ => (u64::MAX, u64::MAX, u64::MAX, u64::MAX, 0.),
},
_ => (u64::MAX, u64::MAX, u64::MAX, u64::MAX, 0.),
};
assume!(
sender_mem <= sender_mem_limit,
format!("Sender memory > {}", sender_mem_limit)
)?;
assume!(
receiver_mem <= receiver_mem_limit,
format!("Receiver memory > {}", receiver_mem_limit)
)?;
assume!(
net_message_count <= net_message_count_limit,
format!("Message count > {}", net_message_count_limit)
)?;
assume!(
net_traffic <= net_traffic_limit,
format!("Traffic > {}", net_traffic_limit)
)?;
assume!(
throughput >= throughput_limit,
format!("Throughput < {}", throughput_limit)
)?;
Ok(true)
}
homework/01-guarantees/tests/src/main.rs
+310
View File
@@ -0,0 +1,310 @@
mod common;
mod shared_state;
mod tests;
mod tests_mc;
use indexmap::IndexMap;
use std::collections::HashSet;
use std::env;
use std::io::Write;
use clap::Parser;
use env_logger::Builder;
use log::LevelFilter;
use anysystem::test::{TestResult, TestSuite};
use crate::common::TestConfig;
use crate::tests::*;
use crate::tests_mc::*;
/// Guarantees Homework Tests
#[derive(Parser, Debug)]
#[clap(about, long_about = None)]
struct Args {
/// Path to Python file with solution
#[clap(long = "impl", short = 'i', default_value = "solution/guarantees.py")]
solution_path: String,
/// Test to run (optional)
#[clap(long = "test", short)]
test: Option<String>,
/// Print execution trace
#[clap(long, short)]
debug: bool,
/// Guarantee to check
#[clap(long, short, possible_values = ["AMO", "ALO", "EO", "EOO"])]
guarantee: Option<String>,
/// Random seed used in tests
#[clap(long, short, default_value = "123")]
seed: u64,
/// Number of chaos monkey runs
#[clap(long, short, default_value = "0")]
monkeys: u32,
/// Run overhead tests
#[clap(long, short)]
overhead: bool,
/// Run model checking tests
#[clap(long, short = 'c')]
model_checking: bool,
}
fn main() {
let args = Args::parse();
if !shared_state::validate_or_report(
&args.solution_path,
&[
"AtMostOnceSender",
"AtMostOnceReceiver",
"AtLeastOnceSender",
"AtLeastOnceReceiver",
"ExactlyOnceSender",
"ExactlyOnceReceiver",
"ExactlyOnceOrderedSender",
"ExactlyOnceOrderedReceiver",
],
) {
return;
}
if args.debug {
Builder::new()
.filter(Some("anysystem"), LevelFilter::Debug)
.format(|buf, record| writeln!(buf, "{}", record.args()))
.init();
}
let guarantee = args.guarantee.as_deref();
env::set_var("PYTHONHASHSEED", args.seed.to_string());
let mut config = TestConfig {
impl_path: &args.solution_path,
sender_class: "",
receiver_class: "",
seed: args.seed,
monkeys: args.monkeys,
reliable: false,
once: false,
ordered: false,
};
let mut tests = TestSuite::new();
// At most once
if guarantee.is_none() || guarantee == Some("AMO") {
config.sender_class = "AtMostOnceSender";
config.receiver_class = "AtMostOnceReceiver";
config.once = true;
// without drops should be reliable
config.reliable = true;
tests.add("[AT MOST ONCE] NORMAL", test_normal, config);
tests.add("[AT MOST ONCE] NORMAL NON-UNIQUE", test_normal_non_unique, config);
tests.add("[AT MOST ONCE] DELAYED", test_delayed, config);
tests.add("[AT MOST ONCE] DUPLICATED", test_duplicated, config);
tests.add("[AT MOST ONCE] DELAYED+DUPLICATED", test_delayed_duplicated, config);
tests.add("[AT MOST ONCE] OLD DUPLICATE", test_old_duplicate, config);
// with drops is not reliable
config.reliable = false;
tests.add("[AT MOST ONCE] DROPPED", test_dropped, config);
if args.monkeys > 0 {
tests.add("[AT MOST ONCE] CHAOS MONKEY", test_chaos_monkey, config);
}
if args.overhead {
config.reliable = true;
tests.add(
"[AT MOST ONCE] OVERHEAD NORMAL",
|x| test_overhead(x, "AMO", false),
config,
);
config.reliable = false;
tests.add(
"[AT MOST ONCE] OVERHEAD FAULTY",
|x| test_overhead(x, "AMO", true),
config,
);
}
if args.model_checking {
tests.add("[AT MOST ONCE] MODEL CHECKING", test_mc_reliable_network, config);
tests.add(
"[AT MOST ONCE] MODEL CHECKING MESSAGE DROPS",
test_mc_message_drops,
config,
);
tests.add(
"[AT MOST ONCE] MODEL CHECKING UNSTABLE NETWORK",
test_mc_unstable_network,
config,
);
}
}
// At least once
if guarantee.is_none() || guarantee == Some("ALO") {
config.sender_class = "AtLeastOnceSender";
config.receiver_class = "AtLeastOnceReceiver";
config.reliable = true;
config.once = false;
tests.add("[AT LEAST ONCE] NORMAL", test_normal, config);
tests.add("[AT LEAST ONCE] NORMAL NON-UNIQUE", test_normal_non_unique, config);
tests.add("[AT LEAST ONCE] DELAYED", test_delayed, config);
tests.add("[AT LEAST ONCE] DUPLICATED", test_duplicated, config);
tests.add("[AT LEAST ONCE] DELAYED+DUPLICATED", test_delayed_duplicated, config);
tests.add("[AT LEAST ONCE] DROPPED", test_dropped, config);
if args.monkeys > 0 {
tests.add("[AT LEAST ONCE] CHAOS MONKEY", test_chaos_monkey, config);
}
if args.overhead {
tests.add(
"[AT LEAST ONCE] OVERHEAD NORMAL",
|x| test_overhead(x, "ALO", false),
config,
);
tests.add(
"[AT LEAST ONCE] OVERHEAD FAULTY",
|x| test_overhead(x, "ALO", true),
config,
);
}
if args.model_checking {
tests.add("[AT LEAST ONCE] MODEL CHECKING", test_mc_reliable_network, config);
tests.add(
"[AT LEAST ONCE] MODEL CHECKING MESSAGE DROPS",
test_mc_message_drops,
config,
);
tests.add(
"[AT LEAST ONCE] MODEL CHECKING UNSTABLE NETWORK",
test_mc_unstable_network,
config,
);
}
}
// Exactly once
if guarantee.is_none() || guarantee == Some("EO") {
config.sender_class = "ExactlyOnceSender";
config.receiver_class = "ExactlyOnceReceiver";
config.reliable = true;
config.once = true;
tests.add("[EXACTLY ONCE] NORMAL", test_normal, config);
tests.add("[EXACTLY ONCE] NORMAL NON-UNIQUE", test_normal_non_unique, config);
tests.add("[EXACTLY ONCE] DELAYED", test_delayed, config);
tests.add("[EXACTLY ONCE] DUPLICATED", test_duplicated, config);
tests.add("[EXACTLY ONCE] DELAYED+DUPLICATED", test_delayed_duplicated, config);
tests.add("[EXACTLY ONCE] DROPPED", test_dropped, config);
if args.monkeys > 0 {
tests.add("[EXACTLY ONCE] CHAOS MONKEY", test_chaos_monkey, config);
}
if args.overhead {
tests.add(
"[EXACTLY ONCE] OVERHEAD NORMAL",
|x| test_overhead(x, "EO", false),
config,
);
tests.add(
"[EXACTLY ONCE] OVERHEAD FAULTY",
|x| test_overhead(x, "EO", true),
config,
);
}
if args.model_checking {
tests.add("[EXACTLY ONCE] MODEL CHECKING", test_mc_reliable_network, config);
tests.add(
"[EXACTLY ONCE] MODEL CHECKING MESSAGE DROPS",
test_mc_message_drops,
config,
);
tests.add(
"[EXACTLY ONCE] MODEL CHECKING UNSTABLE NETWORK",
test_mc_unstable_network,
config,
);
}
}
// EXACTLY ONCE ORDERED
if guarantee.is_none() || guarantee == Some("EOO") {
config.sender_class = "ExactlyOnceOrderedSender";
config.receiver_class = "ExactlyOnceOrderedReceiver";
config.reliable = true;
config.once = true;
config.ordered = true;
tests.add("[EXACTLY ONCE ORDERED] NORMAL", test_normal, config);
tests.add(
"[EXACTLY ONCE ORDERED] NORMAL NON-UNIQUE",
test_normal_non_unique,
config,
);
tests.add("[EXACTLY ONCE ORDERED] DELAYED", test_delayed, config);
tests.add("[EXACTLY ONCE ORDERED] DUPLICATED", test_duplicated, config);
tests.add(
"[EXACTLY ONCE ORDERED] DELAYED+DUPLICATED",
test_delayed_duplicated,
config,
);
tests.add("[EXACTLY ONCE ORDERED] DROPPED", test_dropped, config);
if args.monkeys > 0 {
tests.add("[EXACTLY ONCE ORDERED] CHAOS MONKEY", test_chaos_monkey, config);
}
if args.overhead {
tests.add(
"[EXACTLY ONCE ORDERED] OVERHEAD NORMAL",
|x| test_overhead(x, "EOO", false),
config,
);
tests.add(
"[EXACTLY ONCE ORDERED] OVERHEAD FAULTY",
|x| test_overhead(x, "EOO", true),
config,
);
}
if args.model_checking {
tests.add(
"[EXACTLY ONCE ORDERED] MODEL CHECKING",
test_mc_reliable_network,
config,
);
tests.add(
"[EXACTLY ONCE ORDERED] MODEL CHECKING MESSAGE DROPS",
test_mc_message_drops,
config,
);
tests.add(
"[EXACTLY ONCE ORDERED] MODEL CHECKING UNSTABLE NETWORK",
test_mc_unstable_network,
config,
);
}
}
if let Some(test) = &args.test {
tests.run_test(test);
} else {
let (_, results) = tests.run();
let score = score(results);
println!("SCORE: {score}\n");
}
}
fn score(results: IndexMap<String, TestResult>) -> f32 {
let guarantees = HashSet::from(["AT MOST ONCE", "AT LEAST ONCE", "EXACTLY ONCE", "EXACTLY ONCE ORDERED"]);
let mut failed_guarantees: HashSet<&str> = HashSet::new();
let mut failed_overheads: HashSet<&str> = HashSet::new();
for (test, result) in results {
if result.is_err() {
for guarantee in guarantees.iter() {
if test.contains(format!("[{guarantee}]").as_str()) {
if test.contains("OVERHEAD") {
failed_overheads.insert(guarantee);
} else {
failed_guarantees.insert(guarantee);
}
}
}
}
}
9. - failed_guarantees.len() as f32 * 2. - f32::from(!failed_overheads.is_empty())
}
homework/01-guarantees/tests/src/shared_state.rs
+636
View File
@@ -0,0 +1,636 @@
use std::ffi::CString;
use pyo3::prelude::*;
use pyo3::types::{PyList, PyModule};
const VALIDATOR_CODE: &str = include_str!("../shared_state_validator.py");
fn validator_module<'py>(py: Python<'py>) -> PyResult<Bound<'py, PyModule>> {
let code = CString::new(VALIDATOR_CODE).expect("validator source contains a null byte");
PyModule::from_code(py, &code, c"shared_state_validator.py", c"shared_state_validator")
}
#[cfg(test)]
fn validate_source(source: &str, filename: &str, class_names: &[&str]) -> Result<Vec<String>, String> {
Python::attach(|py| {
let module = validator_module(py).map_err(|error| error.to_string())?;
let class_names = PyList::new(py, class_names).map_err(|error| error.to_string())?;
module
.getattr("validate_source")
.and_then(|function| function.call1((source, filename, class_names)))
.and_then(|result| result.extract::<Vec<String>>())
.map_err(|error| error.to_string())
})
}
fn validate_solution(path: &str, class_names: &[&str]) -> Result<Vec<String>, String> {
Python::attach(|py| {
let module = validator_module(py).map_err(|error| error.to_string())?;
let class_names = PyList::new(py, class_names).map_err(|error| error.to_string())?;
module
.getattr("validate_solution")
.and_then(|function| function.call1((path, class_names)))
.and_then(|result| result.extract::<Vec<String>>())
.map_err(|error| error.to_string())
})
}
pub fn validate_or_report(path: &str, class_names: &[&str]) -> bool {
match validate_solution(path, class_names) {
Ok(violations) if violations.is_empty() => true,
Ok(violations) => {
println!("ERROR: shared state between AnySystem processes is forbidden");
for violation in violations {
println!("{violation}");
}
println!("\nSCORE: 0\n");
false
}
Err(error) => panic!("failed to validate solution for shared state: {error}"),
}
}
#[cfg(test)]
mod tests {
use super::{validate_solution, validate_source};
const PROCESS: &[&str] = &["ProcessImpl"];
fn violations(source: &str) -> Vec<String> {
validate_source(source, "solution.py", PROCESS).unwrap()
}
#[test]
fn accepts_imports_helpers_constants_and_instance_state() {
let source = r#"
import random
from dataclasses import dataclass, field
from enum import Enum, auto
from types import MappingProxyType
LIMIT = 2 ** 10
NAMES = ("a", "b")
LOOKUP = MappingProxyType({"a": 1, "b": 2})
@dataclass
class Helper:
values: list[str] = field(default_factory=list)
@dataclass(frozen=True)
class Status:
ok: bool
class Kind(Enum):
READY = auto()
class ProcessImpl:
RETRIES = 3
READY = Status(True)
def __init__(self):
self.values = []
self.helper = Helper()
def choose(self):
return random.choice(NAMES)
"#;
assert_eq!(violations(source), Vec::<String>::new());
}
#[test]
fn checks_local_imports_without_executing_them() {
let suffix = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_nanos();
let directory = std::env::temp_dir().join(format!(
"distsys-shared-state-validator-{}-{suffix}",
std::process::id()
));
std::fs::create_dir_all(&directory).unwrap();
let entrypoint = directory.join("solution.py");
std::fs::write(&entrypoint, "import helper\n\nclass ProcessImpl:\n pass\n").unwrap();
std::fs::write(
directory.join("helper.py"),
"class Shared:\n data = {}\n\nraise RuntimeError('must not execute')\n",
)
.unwrap();
let result = validate_solution(entrypoint.to_str().unwrap(), PROCESS);
std::fs::remove_dir_all(directory).unwrap();
let violations = result.unwrap();
assert_eq!(violations.len(), 1);
assert!(violations[0].contains("Shared.data"));
}
#[test]
fn rejects_shared_state_hidden_behind_helper_class() {
let source = r#"
from enum import Enum
class Shared:
data = {}
@classmethod
def data_ref(cls):
return cls.data
class UnsafeKind(Enum):
VALUES = []
class ProcessImpl:
def write(self):
Shared.data_ref()["key"] = "value"
"#;
let violations = violations(source);
assert_eq!(violations.len(), 2);
assert!(violations.iter().any(|item| item.contains("Shared.data")));
assert!(violations.iter().any(|item| item.contains("UnsafeKind.VALUES")));
}
#[test]
fn rejects_module_and_process_class_state() {
let source = r#"
CACHE = {}
from dataclasses import dataclass, field
@dataclass(frozen=True)
class FrozenButMutable:
values: list[int] = field(default_factory=list)
BROKEN = FrozenButMutable()
class Base:
pending = set()
class ProcessImpl(Base):
values = []
"#;
let violations = violations(source);
assert_eq!(violations.len(), 4);
assert!(violations.iter().any(|message| message.contains("CACHE")));
assert!(violations.iter().any(|message| message.contains("BROKEN")));
assert!(violations.iter().any(|message| message.contains("Base.pending")));
assert!(violations.iter().any(|message| message.contains("ProcessImpl.values")));
}
#[test]
fn rejects_module_level_dynamic_shared_state() {
let source = r#"
def helper():
pass
class ProcessImpl:
COUNT = 0
alias = ProcessImpl
alias.cache = {}
helper.cache = []
setattr(ProcessImpl, "pending", set())
"#;
let violations = violations(source);
assert_eq!(violations.len(), 3);
assert!(violations
.iter()
.all(|message| message.contains("module-level shared state")));
}
#[test]
fn rejects_mutable_defaults_and_scope_state() {
let source = r#"
class ProcessImpl:
def on_message(self, message, cache={}):
global counter
def next_value():
nonlocal message
return message
"#;
let violations = violations(source);
assert_eq!(violations.len(), 2);
}
#[test]
fn rejects_shared_but_accepts_local_closure_state() {
let source = r#"
def make_counter():
count = 0
def next_value():
nonlocal count
count += 1
return count
return next_value
SHARED_COUNTER = make_counter()
class ProcessImpl:
def local_counter(self):
count = 0
def next_value():
nonlocal count
count += 1
return count
return next_value()
"#;
let violations = violations(source);
assert_eq!(violations.len(), 2);
assert!(violations.iter().any(|message| message.contains("shared closure")));
}
#[test]
fn rejects_getattr_eval_and_shared_cache_decorator() {
let source = r#"
from functools import lru_cache
@lru_cache
def cached_value(key):
return key
class ProcessImpl:
cache = ()
def on_message(self, message):
getattr(type(self), "cache").clear()
eval("globals()")
"#;
let violations = violations(source);
assert_eq!(violations.len(), 3);
assert!(violations.iter().any(|message| message.contains("lru_cache")));
assert!(violations.iter().any(|message| message.contains("clear")));
assert!(violations.iter().any(|message| message.contains("eval")));
}
#[test]
fn rejects_direct_aliased_and_reflective_mutation() {
let source = r#"
class ProcessImpl:
COUNT = 0
def on_message(self, message):
cls = type(self)
state = cls.__dict__
cls.COUNT += 1
state.update({"x": 1})
setattr(cls, "other", [])
globals()["hidden"] = {}
exec("hidden = {}")
"#;
let violations = violations(source);
assert!(violations.len() >= 5);
assert!(violations.iter().any(|message| message.contains("assignment mutates")));
assert!(violations.iter().any(|message| message.contains("setattr")));
assert!(violations.iter().any(|message| message.contains("globals")));
assert!(violations.iter().any(|message| message.contains("exec")));
}
#[test]
fn rejects_spoofed_safe_names_and_aliased_cache_decorator() {
let source = r#"
def tuple():
return []
def MappingProxyType(value):
return value
def dataclass(*args, **kwargs):
def wrap(cls):
return cls
return wrap
SHARED_TUPLE = tuple()
SHARED_MAPPING = MappingProxyType({})
@dataclass(frozen=True)
class Mutable:
pass
SHARED_OBJECT = Mutable()
from functools import lru_cache as memo
@memo
def cached(value):
return value
class ProcessImpl:
pass
"#;
let violations = violations(source);
assert!(violations.iter().any(|message| message.contains("SHARED_TUPLE")));
assert!(violations.iter().any(|message| message.contains("SHARED_MAPPING")));
assert!(violations.iter().any(|message| message.contains("SHARED_OBJECT")));
assert!(violations.iter().any(|message| message.contains("lru_cache")));
}
#[test]
fn rejects_imported_and_function_object_state() {
let source = r#"
import math
import sys
SHARED = sys.modules
holder = lambda: None
module_alias = math
math.shared = {}
holder.shared = {}
module_alias.other = {}
class ProcessImpl:
def __init__(self):
self.shared = sys.modules
def write(self):
local_alias = math
local_alias.more = {}
self.shared["hidden"] = {}
"#;
let violations = violations(source);
assert!(violations.iter().any(|message| message.contains("SHARED")));
assert!(violations.len() >= 6);
}
#[test]
fn rejects_unknown_constructor_as_mutable_default() {
let source = r#"
class Box:
def __init__(self):
self.values = {}
class ProcessImpl:
def on_message(self, message, box=Box()):
pass
"#;
let violations = violations(source);
assert!(violations
.iter()
.any(|message| message.contains("mutable default argument")));
}
#[test]
fn rejects_dunder_and_class_namespace_reflection() {
let source = r#"
from dataclasses import dataclass
@dataclass(frozen=True)
class Frozen:
value: int
SHARED = Frozen(1)
class ProcessImpl:
locals()["shared"] = {}
def on_message(self, message):
type.__setattr__(ProcessImpl, "cache", {})
object.__setattr__(SHARED, "value", 2)
"#;
let violations = violations(source);
assert!(violations
.iter()
.any(|message| message.contains("class-level shared state")));
assert!(violations.iter().any(|message| message.contains("__setattr__")));
assert!(violations.len() >= 3);
}
#[test]
fn rejects_mutating_frozen_dataclass_methods() {
let source = r#"
from dataclasses import dataclass
@dataclass(frozen=True)
class Frozen:
value: int = 1
def __post_init__(self):
object.__setattr__(self, "state", {})
def put(self, key, value):
self.state[key] = value
GLOBAL = Frozen()
class ProcessImpl:
def write(self):
GLOBAL.put("x", 1)
"#;
let violations = violations(source);
assert!(violations.iter().any(|message| message.contains("GLOBAL")));
}
#[test]
fn rejects_dunder_and_operator_item_mutation() {
let source = r#"
import math
import operator
class ProcessImpl:
def write(self):
math.__dict__.__setitem__("shared", {})
operator.setitem(math.__dict__, "other", {})
"#;
let violations = violations(source);
assert!(violations.len() >= 2);
assert!(violations.iter().any(|message| message.contains("__setitem__")));
assert!(violations.iter().any(|message| message.contains("operator")));
}
#[test]
fn rejects_class_local_and_star_cache_decorators() {
let source = r#"
from functools import *
@lru_cache
def module_cached():
return []
class ProcessImpl:
from functools import lru_cache as memo
@staticmethod
@memo
def shared():
return []
"#;
let violations = violations(source);
assert_eq!(
violations
.iter()
.filter(|message| message.contains("lru_cache"))
.count(),
2
);
}
#[test]
fn checks_literal_dynamic_local_imports() {
let suffix = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_nanos();
let directory = std::env::temp_dir().join(format!(
"distsys-dynamic-import-validator-{}-{suffix}",
std::process::id()
));
std::fs::create_dir_all(&directory).unwrap();
let entrypoint = directory.join("solution.py");
std::fs::write(
&entrypoint,
"from importlib import import_module as load\n\nPREFIX = 'hel'\nMODULE = PREFIX + 'per'\n\nclass ProcessImpl:\n def __init__(self):\n self.shared = load(MODULE).CACHE\n",
)
.unwrap();
std::fs::write(directory.join("helper.py"), "CACHE = {}\n").unwrap();
let result = validate_solution(entrypoint.to_str().unwrap(), PROCESS);
std::fs::remove_dir_all(directory).unwrap();
let violations = result.unwrap();
assert_eq!(violations.len(), 1);
assert!(violations[0].contains("CACHE"));
}
#[test]
fn rejects_shared_values_returned_by_wrappers_and_wrapped_cache() {
let source = r#"
import sys
from functools import lru_cache
def get_shared():
return sys.modules
def memo(function):
return lru_cache(function)
@memo
def shared():
return []
class ProcessImpl:
def __init__(self):
self.shared = get_shared()
def write(self):
self.shared["key"] = shared()
"#;
let violations = violations(source);
assert!(violations.iter().any(|message| message.contains("assignment mutates")));
assert!(violations.iter().any(|message| message.contains("memo")));
}
#[test]
fn rejects_aliased_mutator_callables() {
let source = r#"
import math
import operator
class ProcessImpl:
def write(self):
mutate = operator.setitem
mutate(math.__dict__, "first", {})
bound_mutate = math.__dict__.__setitem__
bound_mutate("second", {})
"#;
let violations = violations(source);
assert_eq!(violations.len(), 2);
assert!(violations.iter().any(|message| message.contains("operator")));
assert!(violations.iter().any(|message| message.contains("aliased")));
}
#[test]
fn rejects_unresolved_dynamic_local_imports() {
let suffix = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_nanos();
let directory = std::env::temp_dir().join(format!(
"distsys-unresolved-import-validator-{}-{suffix}",
std::process::id()
));
std::fs::create_dir_all(&directory).unwrap();
let entrypoint = directory.join("solution.py");
std::fs::write(
&entrypoint,
"from importlib import import_module as load\n\nclass ProcessImpl:\n def write(self):\n name = ''.join(['helper'])\n state = load(name).CACHE\n state['key'] = 1\n",
)
.unwrap();
std::fs::write(directory.join("helper.py"), "CACHE = {}\n").unwrap();
let result = validate_solution(entrypoint.to_str().unwrap(), PROCESS);
std::fs::remove_dir_all(directory).unwrap();
let violations = result.unwrap();
assert_eq!(violations.len(), 1);
assert!(violations[0].contains("cannot be resolved statically"));
}
#[test]
fn rejects_stateful_imported_api_but_allows_reviewed_calls() {
let source = r#"
import random
from random import seed as direct_seed
random.seed(1)
class ProcessImpl:
def write(self, value):
direct_seed(value)
reseed = random.seed
reseed(value)
def read(self):
return random.getstate()
def choose(self):
return random.choice((1, 2))
"#;
let violations = violations(source);
assert_eq!(violations.len(), 4);
assert!(violations.iter().any(|message| message.contains("random.seed")));
assert!(violations.iter().any(|message| message.contains("random.getstate")));
assert!(!violations.iter().any(|message| message.contains("random.choice")));
}
#[test]
fn rejects_passthrough_and_lambda_shared_references() {
let source = r#"
import sys
def identity(value):
return value
get_shared = lambda: sys.modules
lambda_identity = lambda value: value
class ProcessImpl:
def __init__(self):
self.first = identity(sys.modules)
self.second = get_shared()
self.third = lambda_identity(sys.modules)
def write(self):
self.first["first"] = {}
self.second["second"] = {}
self.third["third"] = {}
"#;
let violations = violations(source);
assert_eq!(violations.len(), 3);
assert!(violations.iter().all(|message| message.contains("assignment mutates")));
}
#[test]
fn rejects_getattr_hidden_imported_api_and_mutator() {
let source = r#"
import math
import random
class ProcessImpl:
def write(self, value):
getattr(random, "seed")(value)
getattr(math.__dict__, "__setitem__")("shared", {})
"#;
let violations = violations(source);
assert_eq!(violations.len(), 2);
assert!(violations.iter().any(|message| message.contains("random.seed")));
assert!(violations.iter().any(|message| message.contains("__setitem__")));
}
}
homework/01-guarantees/tests/src/tests.rs
+158
View File
@@ -0,0 +1,158 @@
use std::fs;
use assertables::assume;
use rand::prelude::*;
use rand_pcg::Pcg64;
use anysystem::test::TestResult;
use anysystem::Message;
use crate::common::{build_system, check_guarantees, check_overhead, send_messages, TestConfig};
pub fn test_normal(config: &TestConfig) -> TestResult {
let mut sys = build_system(config, false);
let messages = send_messages(&mut sys, 5);
sys.step_until_no_events();
check_guarantees(&mut sys, &messages, config)?;
// We expect no more than 5 messages from sender in normal network conditions
let sent_count = sys.sent_message_count("sender");
assume!(
sent_count <= 5,
format!("Sender sent {} messages, expected at most 5", sent_count)
)
}
pub fn test_normal_non_unique(config: &TestConfig) -> TestResult {
let mut sys = build_system(config, false);
let messages = send_messages(&mut sys, 10);
sys.step_until_no_events();
check_guarantees(&mut sys, &messages, config)?;
// We expect no more than 10 messages from sender in normal network conditions (stable delay, no loss).
// If solution sends multiple messages without or with too small (<RTT) delay, this results in extra redundant
// traffic. We want to avoid wasting network resources, assuming that normal conditions happen most of the time.
let sent_count = sys.sent_message_count("sender");
assume!(
sent_count <= 10,
format!("Sender sent {} messages, expected at most 10", sent_count)
)
}
pub fn test_delayed(config: &TestConfig) -> TestResult {
let mut sys = build_system(config, false);
sys.network().set_delays(1., 3.);
let messages = send_messages(&mut sys, 5);
sys.step_until_no_events();
check_guarantees(&mut sys, &messages, config)
}
pub fn test_duplicated(config: &TestConfig) -> TestResult {
let mut sys = build_system(config, false);
sys.network().set_dupl_rate(0.3);
let messages = send_messages(&mut sys, 5);
sys.step_until_no_events();
check_guarantees(&mut sys, &messages, config)
}
pub fn test_delayed_duplicated(config: &TestConfig) -> TestResult {
let mut sys = build_system(config, false);
sys.network().set_delays(1., 3.);
sys.network().set_dupl_rate(0.3);
let messages = send_messages(&mut sys, 5);
sys.step_until_no_events();
check_guarantees(&mut sys, &messages, config)
}
pub fn test_old_duplicate(config: &TestConfig) -> TestResult {
let mut sys = build_system(config, false);
sys.network().set_delays(1., 3.);
sys.network().set_dupl_rate(1.);
let first = Message::new("MESSAGE", r#"{"text": "first"}"#);
let mut messages = vec![first.clone()];
sys.send_local_message("sender", first);
// Stop after the first copy is delivered.
while sys.local_outbox("receiver").is_empty() {
if !sys.step() {
return Err("The first message was not delivered".to_string());
}
}
// Keep the old duplicate pending while newer messages are delivered first.
// Zero-delay messages are processed before the delayed copy.
sys.network().set_dupl_rate(0.);
sys.network().set_delay(0.);
for i in 0..50 {
let msg = Message::new("MESSAGE", &format!(r#"{{"text": "message-{i}"}}"#));
sys.send_local_message("sender", msg.clone());
messages.push(msg);
}
sys.step_until_no_events();
check_guarantees(&mut sys, &messages, config)
}
pub fn test_dropped(config: &TestConfig) -> TestResult {
let mut sys = build_system(config, false);
sys.network().set_drop_rate(0.3);
let messages = send_messages(&mut sys, 5);
sys.step_until_no_events();
check_guarantees(&mut sys, &messages, config)
}
pub fn test_chaos_monkey(config: &TestConfig) -> TestResult {
let mut rand = Pcg64::seed_from_u64(config.seed);
for i in 1..=config.monkeys {
let mut run_config = *config;
run_config.seed = rand.next_u64();
println!("Run {} (seed: {})", i, run_config.seed);
let mut sys = build_system(&run_config, false);
sys.network().set_delays(1., 3.);
sys.network().set_dupl_rate(0.3);
sys.network().set_drop_rate(0.3);
let messages = send_messages(&mut sys, 50);
sys.step_until_no_events();
let res = check_guarantees(&mut sys, &messages, &run_config);
res.as_ref()?;
}
Ok(true)
}
pub fn test_overhead(config: &TestConfig, guarantee: &str, faulty: bool) -> TestResult {
for message_count in [100, 500, 1000] {
let mut sys = build_system(config, true);
if faulty {
sys.network().set_delays(1., 3.);
sys.network().set_dupl_rate(0.3);
sys.network().set_drop_rate(0.3);
}
let messages = send_messages(&mut sys, message_count);
sys.step_until_no_events();
let res = check_guarantees(&mut sys, &messages, config);
res.as_ref()?;
let sender_mem = sys.max_size("sender");
let receiver_mem = sys.max_size("receiver");
let net_message_count = sys.network().network_message_count();
let net_traffic = sys.network().traffic();
let throughput = message_count as f64 / sys.time();
println!(
"{message_count:<6} Send Mem: {sender_mem:<8} Recv Mem: {receiver_mem:<8} Messages: {net_message_count:<8} Traffic: {net_traffic:<8} Throughput: {throughput:.3}"
);
check_overhead(
guarantee,
faulty,
message_count,
sender_mem,
receiver_mem,
net_message_count,
net_traffic,
throughput,
)?;
}
let impl_code = fs::read_to_string(config.impl_path).unwrap();
assume!(
!impl_code.contains("<<") && !impl_code.contains(">>"),
"Implementation contains bitwise shift operators"
)?;
Ok(true)
}
homework/01-guarantees/tests/src/tests_mc.rs
+148
View File
@@ -0,0 +1,148 @@
use std::collections::HashMap;
use std::time::Duration;
use sugars::boxed;
use anysystem::logger::LogEntry;
use anysystem::mc::{
predicates::{goals, invariants, prunes},
strategies::Bfs,
InvariantFn, ModelChecker, StrategyConfig,
};
use anysystem::test::TestResult;
use anysystem::Message;
use crate::common::{
build_system, check_delivered_messages, check_message_delivery_once, check_message_delivery_ordered,
check_message_delivery_reliable, generate_message_texts, TestConfig,
};
fn mc_invariant_guarantees(messages_expected: Vec<Message>, config: TestConfig) -> InvariantFn {
boxed!(move |state| {
let mut expected_msg_count = HashMap::new();
for msg in &messages_expected {
*expected_msg_count.entry(msg.data.clone()).or_insert(0) += 1;
}
let delivered = &state.node_states["receiver-node"].proc_states["receiver"].local_outbox;
// check that delivered messages have expected type and data
let delivered_msg_count = check_delivered_messages(delivered, &expected_msg_count, &messages_expected[0].tip)?;
// check delivered message count according to expected guarantees
if config.reliable && state.events.is_empty() {
check_message_delivery_reliable(&delivered_msg_count, &expected_msg_count)?;
}
if config.once {
check_message_delivery_once(&delivered_msg_count, &expected_msg_count)?;
}
if config.ordered {
check_message_delivery_ordered(delivered, &messages_expected)?;
}
Ok(())
})
}
pub fn test_mc_reliable_network(config: &TestConfig) -> TestResult {
let mut sys = build_system(config, false);
let messages: Vec<Message> = generate_message_texts(&mut sys, 2)
.into_iter()
.map(|text| Message::new("MESSAGE", &format!(r#"{{"text": "{text}"}}"#)))
.collect();
let strategy_config = StrategyConfig::default()
.prune(prunes::sent_messages_limit(4))
.goal(goals::got_n_local_messages("receiver-node", "receiver", 2))
.invariant(invariants::all_invariants(vec![
invariants::state_depth(20),
mc_invariant_guarantees(messages.clone(), *config),
]));
let mut mc = ModelChecker::new(&sys);
let res = mc.run_with_change::<Bfs>(strategy_config, move |sys| {
for message in messages {
sys.send_local_message("sender-node", "sender", message);
}
});
if let Err(e) = res {
e.print_trace();
Err(e.message())
} else {
Ok(true)
}
}
pub fn test_mc_message_drops(config: &TestConfig) -> TestResult {
let mut sys = build_system(config, false);
sys.network().set_drop_rate(0.1);
let messages: Vec<Message> = generate_message_texts(&mut sys, 2)
.into_iter()
.map(|text| Message::new("MESSAGE", &format!(r#"{{"text": "{text}"}}"#)))
.collect();
let strategy_config = StrategyConfig::default()
.prune(prunes::state_depth(7))
.goal(goals::any_goal(vec![
goals::got_n_local_messages("receiver-node", "receiver", 2),
goals::no_events(),
]))
.invariant(mc_invariant_guarantees(messages.clone(), *config));
let mut mc = ModelChecker::new(&sys);
let res = mc.run_with_change::<Bfs>(strategy_config, move |sys| {
for message in messages {
sys.send_local_message("sender-node", "sender", message);
}
});
if let Err(e) = res {
e.print_trace();
Err(e.message())
} else {
Ok(true)
}
}
pub fn test_mc_unstable_network(config: &TestConfig) -> TestResult {
let mut sys = build_system(config, false);
sys.network().set_drop_rate(0.1);
sys.network().set_dupl_rate(0.1);
let msg_count = if config.ordered { 3 } else { 2 };
let messages: Vec<Message> = generate_message_texts(&mut sys, msg_count)
.into_iter()
.map(|text| Message::new("MESSAGE", &format!(r#"{{"text": "{text}"}}"#)))
.collect();
let num_drops_allowed = 1;
let num_duplication_allowed = 1;
let goal = if config.reliable && config.once {
goals::all_goals(vec![
goals::got_n_local_messages("receiver-node", "receiver", msg_count),
goals::no_events(),
])
} else {
goals::no_events()
};
let mut invariants = vec![
invariants::state_depth(20),
mc_invariant_guarantees(messages.clone(), *config),
];
if config.ordered {
invariants.push(invariants::time_limit(Duration::from_secs(80)))
};
let strategy_config = StrategyConfig::default()
.prune(prunes::any_prune(vec![
prunes::events_limit(LogEntry::is_mc_message_dropped, num_drops_allowed),
prunes::events_limit(LogEntry::is_mc_message_duplicated, num_duplication_allowed),
prunes::events_limit(LogEntry::is_mc_timer_fired, 1),
prunes::events_limit(LogEntry::is_mc_message_received, msg_count + num_drops_allowed),
]))
.goal(goal)
.invariant(invariants::all_invariants(invariants));
let mut mc = ModelChecker::new(&sys);
let res = mc.run_with_change::<Bfs>(strategy_config, |sys| {
for msg in messages {
sys.send_local_message("sender-node", "sender", msg.clone());
}
});
if let Err(e) = res {
e.print_trace();
Err(e.message())
} else {
Ok(true)
}
}