This commit is contained in:
osukhoroslov committed 2026-11-15 10:29:10 +03:00
1 parent a4dc591ca0
commit 3030f979d1
17 files changed
+6982

No files matched your search

@@ -0,0 +1,5 @@
target
**/target
**/__pycache__
**/.pytest_cache
*.pyc
File diff suppressed because it is too large. Load diff
@@ -0,0 +1,21 @@
[package]
name = "distsys-kv-replication"
version = "0.1.0"
edition = "2021"
resolver = "3"
[dependencies]
anysystem = "=0.3.0"
indexmap = "2.12"
assertables = "3.2.2"
byteorder = "1.4.3"
clap = { version = "3.2.17", features = ["cargo", "derive"] }
decorum = "0.3.1"
env_logger = "0.9.0"
log = "0.4.17"
md5 = "0.7.0"
pyo3 = { version = "=0.29.2", features = ["auto-initialize"] }
rand = "0.8.5"
rand_pcg = "0.3.1"
serde = { version = "1.0.142", features = ["derive"] }
serde_json = "1.0.83"
@@ -0,0 +1,19 @@
# syntax=docker/dockerfile:1
FROM rust:1.97.1-bookworm@sha256:0e2bcaef56d041a486784e54104a81aebe0da44bd03019bd70bc0401e42e4a97 AS builder
RUN apt-get update && apt-get install -y build-essential python3-dev
WORKDIR /tests
COPY . .
RUN --mount=type=cache,id=distsys-course-cargo-registry,target=/usr/local/cargo/registry,sharing=locked \
--mount=type=cache,id=distsys-course-cargo-target-rust-1-97-1,target=/tests/target,sharing=locked \
cargo install --locked --path . --root /opt/course
FROM debian:bookworm-slim
RUN apt-get update && apt-get install -y python3-dev && rm -rf /var/lib/apt/lists/*
COPY --from=builder /opt/course/bin/distsys-kv-replication /usr/local/bin/distsys-kv-replication
WORKDIR /solution
ENTRYPOINT ["timeout", "--verbose", "-k", "10", "1260", "distsys-kv-replication", "-i", "node.py"]
@@ -0,0 +1,38 @@
"""Retain exceptions from StorageNode calls for the Rust test runner."""
from functools import wraps
from inspect import getattr_static
def watch(cls):
if not isinstance(cls, type):
raise TypeError("StorageNode must be a class")
missing = sorted(getattr(cls, "__abstractmethods__", ()))
if missing:
raise TypeError(f"{cls.__name__} must implement {', '.join(missing)}")
errors = []
def checked(name, method):
@wraps(method)
def call(*args, **kwargs):
try:
result = method(*args, **kwargs)
except BaseException as exc:
errors[:] = [f"{name}: {type(exc).__name__}: {exc}"]
raise
if errors:
errors.clear() # A nested exception was handled by the solution.
return result
return call
for name in ("__new__", "__init__", "on_start", "on_local_message", "on_message", "on_timer",
"get_state", "set_state", "__deepcopy__"):
# Preserve static and class methods and inherited implementations.
method = getattr_static(cls, name, None)
if name == "__new__" and method is object.__new__:
continue # Keep the default allocation and constructor argument handling.
if isinstance(method, (staticmethod, classmethod)):
setattr(cls, name, type(method)(checked(name, method.__func__)))
elif callable(method):
setattr(cls, name, checked(name, method))
return errors
File diff suppressed because it is too large. Load diff
@@ -0,0 +1,139 @@
use std::cell::OnceCell;
use byteorder::{ByteOrder, LittleEndian};
use rand::distributions::WeightedIndex;
use rand::prelude::*;
use rand_pcg::Pcg64;
use serde::Serialize;
use serde_json::{Map, Value};
use anysystem::python::{PyProcess, PyProcessFactory};
use anysystem::{Message, Process, System};
use crate::count_keys::TestSystem;
#[derive(Serialize)]
pub struct GetReqMessage<'a> {
pub key: &'a str,
pub quorum: u8,
}
pub struct ValueResponse {
pub key: String,
pub value: Option<String>,
}
pub fn parse_value_response(msg: &Message, proc: &str) -> Result<ValueResponse, String> {
let tip = &msg.tip;
let mut data: Map<String, Value> = serde_json::from_str(&msg.data)
.map_err(|err| format!("Invalid {tip} from {proc}: {err}"))?;
let Some(Value::String(key)) = data.remove("key") else {
return Err(format!("Missing or invalid key in {tip} from {proc}"));
};
let value = match data.remove("value") {
Some(Value::String(value)) => Some(value),
Some(Value::Null) => None,
None => {
return Err(format!(
"Missing field 'value' in {tip} from {proc} for key {key}"
))
}
Some(_) => {
return Err(format!(
"Invalid 'value' in {tip} from {proc} for key {key}: expected a string or null"
))
}
};
Ok(ValueResponse { key, value })
}
#[derive(Serialize)]
pub struct PutReqMessage<'a> {
pub key: &'a str,
pub value: &'a str,
pub quorum: u8,
}
#[derive(Serialize)]
pub struct DeleteReqMessage<'a> {
pub key: &'a str,
pub quorum: u8,
}
#[derive(Copy, Clone)]
pub struct TestConfig<'a> {
pub proc_factory: &'a PyProcessFactory,
pub count_keys_error: &'a OnceCell<String>,
pub proc_count: u32,
pub seed: u64,
}
pub const MIN_NETWORK_DELAY: f64 = 0.01;
pub fn build_system<'a>(config: &TestConfig<'a>) -> TestSystem<'a> {
build_system_with(config, |proc| Box::new(proc))
}
pub fn build_system_with<'a>(
config: &TestConfig<'a>,
wrap: impl Fn(PyProcess) -> Box<dyn Process>,
) -> TestSystem<'a> {
let mut sys = System::new(config.seed);
sys.network().set_delays(MIN_NETWORK_DELAY, 0.1);
let mut proc_names = Vec::new();
for n in 0..config.proc_count {
proc_names.push(format!("{n}"));
}
for proc_name in proc_names.iter() {
let proc = config
.proc_factory
.build((proc_name, proc_names.clone()), config.seed);
// process and node on which it runs have the same name
let node_name = proc_name.clone();
sys.add_node(&node_name);
sys.add_process(proc_name, wrap(proc), &node_name);
}
TestSystem::new(sys, config.count_keys_error)
}
pub fn key_replicas(key: &str, sys: &System) -> Vec<String> {
let proc_count = sys.process_names().len();
let mut replicas = Vec::new();
let hash = md5::compute(key);
let hash128 = LittleEndian::read_u128(&hash.0);
let mut replica = (hash128 % proc_count as u128) as usize;
for _ in 0..3 {
replicas.push(replica.to_string());
replica += 1;
if replica == proc_count {
replica = 0;
}
}
replicas
}
pub fn key_non_replicas(key: &str, sys: &System) -> Vec<String> {
let proc_count = sys.process_names().len();
let replicas = key_replicas(key, sys);
let last_primary = replicas.last().unwrap().parse::<usize>().unwrap();
(1..=proc_count - replicas.len())
.map(|offset| ((last_primary + offset) % proc_count).to_string())
.collect()
}
const SYMBOLS: [char; 36] = [
'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's',
't', 'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9',
];
const WEIGHTS: [usize; 36] = [
13, 16, 3, 8, 8, 5, 6, 23, 4, 8, 24, 12, 2, 1, 1, 10, 5, 8, 10, 1, 24, 3, 1, 8, 12, 22, 5, 20,
18, 5, 5, 2, 1, 3, 16, 22,
];
pub fn random_string(length: usize, rand: &mut Pcg64) -> String {
let dist = WeightedIndex::new(WEIGHTS).unwrap();
rand.sample_iter(&dist)
.take(length)
.map(|x| SYMBOLS[x])
.collect()
}
@@ -0,0 +1,80 @@
use std::cell::OnceCell;
use std::collections::HashSet;
use std::ops::{Deref, DerefMut};
use anysystem::logger::LogEntry;
use anysystem::System;
// One test case can build several simulations. Check each finished trace once,
// including callbacks that ran after the client received COUNT_KEYS_RESP.
pub struct TestSystem<'a> {
system: System,
count_keys_error: &'a OnceCell<String>,
}
impl<'a> TestSystem<'a> {
pub fn new(system: System, count_keys_error: &'a OnceCell<String>) -> Self {
Self {
system,
count_keys_error,
}
}
}
impl Deref for TestSystem<'_> {
type Target = System;
fn deref(&self) -> &System {
&self.system
}
}
impl DerefMut for TestSystem<'_> {
fn deref_mut(&mut self) -> &mut System {
&mut self.system
}
}
impl Drop for TestSystem<'_> {
fn drop(&mut self) {
if !std::thread::panicking() && self.count_keys_error.get().is_none() {
if let Some(sender) = count_keys_network_sender(self.system.logger().trace()) {
let _ = self.count_keys_error.set(format!(
"COUNT_KEYS on {sender} sent a message to another node"
));
}
}
}
}
fn count_keys_network_sender(trace: &[LogEntry]) -> Option<&str> {
let mut timers = HashSet::new();
let mut self_messages = HashSet::new();
let mut handling_count = false;
for entry in trace {
match entry {
LogEntry::LocalMessageReceived { msg, .. } => handling_count = msg.tip == "COUNT_KEYS",
LogEntry::TimerFired { timer_id, .. } => handling_count = timers.remove(timer_id),
LogEntry::MessageReceived { msg_id, .. } => {
handling_count = self_messages.remove(msg_id)
}
// AnySystem gives every scheduled timer a new ID, even when the name is reused.
LogEntry::TimerSet { timer_id, .. } if handling_count => {
timers.insert(timer_id);
}
LogEntry::MessageSent {
src_node,
dst_node,
msg_id,
..
} if handling_count => {
if src_node != dst_node {
return Some(src_node);
}
self_messages.insert(msg_id);
}
_ => {}
}
}
None
}
@@ -0,0 +1,171 @@
use indexmap::IndexMap;
use anysystem::test::TestResult;
pub type Results = IndexMap<&'static str, TestResult>;
// Each item is worth half a point. Paired tests check the same skill.
pub const CORE: &[(&str, &[&[&str]])] = &[
(
"Operations and placement",
&[
&["BASIC"],
&["MC BASIC"],
&["REPLICAS CHECK"],
&["DATA PLACEMENT"],
&["CONCURRENT GET PUT", "CONCURRENT GETS"],
&["DELETE ABSENT"],
],
),
(
"Conflicts and repair",
&[
&["CONCURRENT WRITES", "MC CONCURRENT WRITES"],
&["CONCURRENT WRITES TIE"],
&["STALE REPLICA", "DIVERGED REPLICAS"],
&["STALE REPLICA DELETE", "READ REPAIR DELETE"],
&["CONCURRENT DELETES"],
],
),
(
"Quorums and partitions",
&[
&["SLOPPY QUORUM READ", "SLOPPY QUORUM WRITE"],
&["SLOPPY QUORUM TRICKY"],
&["SLOPPY QUORUM DELETE"],
&["PARTITION CLIENTS", "PARTITION MIXED"],
&["HINTED HANDOFF", "MC SLOPPY QUORUM HINTED HANDOFF"],
],
),
];
pub const TRACKS: &[(&str, &[&str])] = &[
(
"A",
&[
"CONCURRENT PUT DELETE TIE",
"READ REPAIR PUT DELETE TIE",
"READ REPAIR LATE",
"READ REPAIR TIE",
"READ REPAIR MULTI KEY",
"DATA PLACEMENT MULTI KEY",
],
),
(
"B",
&[
"SLOPPY QUORUM GET DISTINCT",
"SLOPPY QUORUM PUT DISTINCT",
"SLOPPY QUORUM DELETE DISTINCT",
"SLOPPY QUORUM GET ORDER",
"SLOPPY QUORUM PUT ORDER",
"SLOPPY QUORUM DELETE ORDER",
"SLOPPY QUORUM DELETE ORDER PARTITION",
"SLOPPY QUORUM DELETE PARTITION",
],
),
(
"C",
&[
"HINTED HANDOFF PUT DELETE TIE",
"HINTED HANDOFF MULTI KEY",
"HINTED HANDOFF RETENTION",
"HINTED HANDOFF LWW",
"SLOPPY QUORUM READ REPAIR",
"SLOPPY QUORUM READ REPAIR DELETE",
"SLOPPY QUORUM READ REPAIR MULTI KEY",
],
),
];
pub fn all_passed(results: &Results, names: &[&str]) -> bool {
names
.iter()
.all(|name| results.get(name).is_some_and(Result::is_ok))
}
pub fn core_points(results: &Results, items: &[&[&str]]) -> f32 {
items
.iter()
.filter(|item| all_passed(results, item))
.count() as f32
/ 2.0
}
pub fn score(core: f32, results: &Results) -> f32 {
if core < 8.0 {
return core;
}
let complete = TRACKS
.iter()
.filter(|(_, names)| all_passed(results, names))
.count();
core + complete.min(2) as f32
}
#[cfg(test)]
mod tests {
use super::*;
use std::collections::HashSet;
#[test]
fn rubric_covers_each_case_once() {
let mut names = HashSet::new();
for &(_, items) in CORE {
for item in items {
for &name in *item {
assert!(names.insert(name), "{name}");
}
}
}
assert_eq!(names.len(), 23);
for &(_, track) in TRACKS {
for &name in track {
assert!(names.insert(name), "{name}");
}
}
assert_eq!(names.len(), 44);
assert_eq!(names, crate::CASES.iter().map(|&(name, _)| name).collect());
}
#[test]
fn independent_core_credits_and_advanced_thresholds() {
let reference: Results = crate::CASES
.iter()
.map(|&(name, _)| (name, Ok(true)))
.collect();
let total = |results: &Results| {
score(
CORE.iter()
.map(|(_, items)| core_points(results, items))
.sum(),
results,
)
};
for (i, &(_, items)) in CORE.iter().enumerate() {
let mut results = reference.clone();
for item in items {
// A failure on either half of a pair loses only this item's credit.
for &name in *item {
results.insert(name, Err(String::new()));
assert_eq!(total(&results), 7.5);
results.insert(name, Ok(true));
}
}
for item in items {
results.insert(item[0], Err(String::new()));
}
assert_eq!(total(&results), [5.0, 5.5, 5.5][i]);
}
for mask in 0_u32..8 {
let mut results = reference.clone();
for (i, &(_, names)) in TRACKS.iter().enumerate() {
if mask & (1 << i) == 0 {
results.shift_remove(names[0]);
}
}
assert_eq!(total(&results), 8.0 + mask.count_ones().min(2) as f32);
}
assert_eq!(total(&Results::new()), 0.0);
}
}
@@ -0,0 +1,252 @@
mod common;
mod count_keys;
mod grading;
mod python_errors;
mod shared_state;
mod tests;
mod tests_mc;
use std::collections::HashSet;
use std::env;
use std::io::Write;
use std::path::Path;
use clap::Parser;
use env_logger::Builder;
use log::LevelFilter;
use crate::grading::{core_points, score, Results, CORE, TRACKS};
use anysystem::test::TestSuite;
use crate::python_errors::{run_case, Case, CaseConfig, Solution};
use crate::tests::*;
use crate::tests_mc::*;
/// Replicated KV Store Homework Tests
#[derive(Parser, Debug)]
#[clap(about, long_about = None)]
struct Args {
/// Path to Python file with solution
#[clap(long = "impl", short = 'i', default_value = "solution/node.py")]
solution_path: String,
/// Test to run (optional)
#[clap(long = "test", short)]
test: Option<String>,
/// Run only the 23 core tests
#[clap(long, conflicts_with_all = &["all", "test"])]
core: bool,
/// Run all 44 tests for diagnostics, ignoring track selection and the core prerequisite
#[clap(long, conflicts_with = "test")]
all: bool,
/// Print execution trace
#[clap(long, short)]
debug: bool,
/// Number of nodes used in tests
#[clap(long, short, default_value = "6")]
node_count: u32,
/// Random seed used in tests
#[clap(long, short, default_value = "123")]
seed: u64,
}
fn main() {
let args = Args::parse();
// Initialize CPython with a 32-bit hash seed; the simulator uses the full u64.
env::set_var("PYTHONHASHSEED", (args.seed as u32).to_string());
python_errors::install_panic_hook();
let validation = shared_state::validate_or_report(&args.solution_path, &["StorageNode"]);
if matches!(validation, Ok(false)) {
return;
}
if args.debug {
Builder::new()
.filter(Some("anysystem"), LevelFilter::Debug)
.format(|buf, record| writeln!(buf, "{}", record.args()))
.init();
}
let solution = validation.and_then(|_| Solution::load(&args.solution_path));
let config = CaseConfig {
solution: &solution,
proc_count: args.node_count,
seed: args.seed,
};
if let Some(test) = &args.test {
let mut suite = TestSuite::new();
for &(name, case) in CASES {
suite.add(name, run_case, (&config, case));
}
suite.run_test(test);
return;
}
let mut results = Results::new();
let mut core = 0.0;
for &(block, items) in CORE {
for item in items {
for name in *item {
run_named_case(name, &config, &mut results);
}
}
let points = core_points(&results, items);
println!("{block}: {points}/{}", items.len() as f32 / 2.0);
core += points;
}
println!("Core: {core}/8");
if args.all || (!args.core && core == 8.0) {
let selected = if args.all {
Ok(None)
} else {
selected_tracks(Path::new(&args.solution_path))
};
match selected {
Ok(selected) => {
for &(track, names) in TRACKS {
if selected.as_ref().is_some_and(|ids| !ids.contains(track)) {
println!("Track {track}: skipped (not selected)");
continue;
}
for name in names {
run_named_case(name, &config, &mut results);
}
let passed = names.iter().filter(|name| results[*name].is_ok()).count();
println!("Track {track}: {passed}/{}", names.len());
}
}
Err(error) => eprintln!("Cannot select advanced tracks: {error}. Grading core only."),
}
} else if args.core {
println!("Advanced tracks skipped (--core)");
} else {
println!("Advanced tracks skipped (full core required)");
}
let passed = results.values().filter(|result| result.is_ok()).count();
println!("\nPassed {passed} from {} tests", results.len());
for (name, result) in &results {
if let Err(error) = result {
println!("- {name}: {error}");
}
}
println!("\nSCORE: {}\n", score(core, &results));
}
fn run_named_case(name: &'static str, config: &CaseConfig, results: &mut Results) {
let &(_, case) = CASES
.iter()
.find(|(test, _)| *test == name)
.expect("registered case");
println!("\n--- {name} ---\n");
let result = run_case(&(config, case));
match &result {
Ok(_) => println!("\nPASSED\n"),
Err(error) => println!("\nFAILED: {error}\n"),
}
results.insert(name, result);
}
// A missing file selects all tracks; an empty file selects only the core.
fn selected_tracks(solution: &Path) -> Result<Option<HashSet<String>>, String> {
let path = solution.with_file_name("tracks.txt");
let contents = match std::fs::read_to_string(&path) {
Ok(contents) => contents,
Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None),
Err(error) => return Err(format!("{}: {error}", path.display())),
};
let mut selected = HashSet::new();
for word in contents.trim_start_matches('\u{feff}').split_whitespace() {
let id = word.to_ascii_uppercase();
if !TRACKS.iter().any(|(track, _)| *track == id) {
return Err(format!(
"unknown track {word:?} in tracks.txt (expected A, B or C)"
));
}
selected.insert(id);
}
Ok(Some(selected))
}
const CASES: &[(&str, Case)] = &[
("BASIC", test_basic),
("REPLICAS CHECK", test_replicas_check),
("DATA PLACEMENT", test_data_placement),
("DATA PLACEMENT MULTI KEY", test_data_placement_multi_key),
("CONCURRENT GET PUT", test_concurrent_get_put),
("CONCURRENT GETS", test_concurrent_gets),
("MC BASIC", test_mc_basic),
("CONCURRENT WRITES", test_concurrent_writes),
("CONCURRENT WRITES TIE", test_concurrent_writes_tie),
("CONCURRENT PUT DELETE TIE", test_concurrent_put_delete_tie),
(
"READ REPAIR PUT DELETE TIE",
test_read_repair_put_delete_tie,
),
(
"HINTED HANDOFF PUT DELETE TIE",
test_hinted_handoff_put_delete_tie,
),
("MC CONCURRENT WRITES", test_mc_concurrent_writes),
("STALE REPLICA", test_stale_replica),
("STALE REPLICA DELETE", test_stale_replica_delete),
("DIVERGED REPLICAS", test_diverged_replicas),
("DELETE ABSENT", test_delete_absent),
("CONCURRENT DELETES", test_concurrent_deletes),
("SLOPPY QUORUM DELETE", test_sloppy_quorum_delete),
(
"SLOPPY QUORUM DELETE PARTITION",
test_sloppy_quorum_delete_partition,
),
(
"SLOPPY QUORUM DELETE DISTINCT",
test_sloppy_quorum_delete_distinct,
),
("READ REPAIR LATE", test_read_repair_late),
("READ REPAIR DELETE", test_read_repair_delete),
("READ REPAIR TIE", test_read_repair_tie),
("READ REPAIR MULTI KEY", test_read_repair_multi_key),
("SLOPPY QUORUM READ REPAIR", test_sloppy_quorum_read_repair),
(
"SLOPPY QUORUM READ REPAIR DELETE",
test_sloppy_quorum_read_repair_delete,
),
(
"SLOPPY QUORUM READ REPAIR MULTI KEY",
test_sloppy_quorum_read_repair_multi_key,
),
("SLOPPY QUORUM GET ORDER", test_sloppy_quorum_get_order),
(
"SLOPPY QUORUM GET DISTINCT",
test_sloppy_quorum_get_distinct,
),
("SLOPPY QUORUM PUT ORDER", test_sloppy_quorum_put_order),
(
"SLOPPY QUORUM PUT DISTINCT",
test_sloppy_quorum_put_distinct,
),
(
"SLOPPY QUORUM DELETE ORDER",
test_sloppy_quorum_delete_order,
),
(
"SLOPPY QUORUM DELETE ORDER PARTITION",
test_sloppy_quorum_delete_order_partition,
),
("SLOPPY QUORUM READ", test_sloppy_quorum_read),
("SLOPPY QUORUM WRITE", test_sloppy_quorum_write),
("SLOPPY QUORUM TRICKY", test_sloppy_quorum_tricky),
(
"MC SLOPPY QUORUM HINTED HANDOFF",
test_mc_sloppy_quorum_hinted_handoff,
),
("HINTED HANDOFF", test_hinted_handoff),
("HINTED HANDOFF MULTI KEY", test_hinted_handoff_multi_key),
("HINTED HANDOFF RETENTION", test_hinted_handoff_retention),
("HINTED HANDOFF LWW", test_hinted_handoff_lww),
("PARTITION CLIENTS", test_partition_clients),
("PARTITION MIXED", test_partition_mixed),
];
@@ -0,0 +1,111 @@
use std::cell::OnceCell;
use std::ffi::CString;
use std::panic::{catch_unwind, resume_unwind, AssertUnwindSafe};
use std::path::Path;
use anysystem::python::PyProcessFactory;
use anysystem::test::TestResult;
use pyo3::prelude::*;
use pyo3::types::{PyList, PyModule};
use crate::common::TestConfig;
// AnySystem 0.3 prints the Python traceback and then unwraps the same error.
// Suppress this duplicate diagnostic only at those bridge sites.
pub fn install_panic_hook() {
let default = std::panic::take_hook();
std::panic::set_hook(Box::new(move |info| {
let site = info
.location()
.map(|l| l.file().replace('\\', "/"))
.unwrap_or_default();
let message = info.payload_as_str().unwrap_or("");
let python = site.ends_with("/anysystem-0.3.0/src/python/mod.rs")
&& (message.contains("PyErr {")
|| message.contains("Error when creating Python process"));
let callback = (site.ends_with("/anysystem-0.3.0/src/node.rs")
|| site.ends_with("/anysystem-0.3.0/src/mc/node.rs"))
&& message.contains("Error when calling process");
if !python && !callback {
default(info);
}
}));
}
pub struct Solution {
factory: PyProcessFactory,
errors: Py<PyList>,
}
impl Solution {
pub fn load(path: &str) -> Result<Self, String> {
let path = Path::new(path).canonicalize().map_err(|e| e.to_string())?;
let path = path.to_str().ok_or("Solution path is not UTF-8")?;
// AnySystem unwraps Python errors during import. Preserve unrelated loader panics.
let factory = catch_unwind(|| PyProcessFactory::new(path, "StorageNode"))
.map_err(|panic| format!("Cannot load StorageNode: {}", python_panic(panic)))?;
let errors = Python::attach(|py| -> PyResult<_> {
// AnySystem 0.3 registers the solution under its canonical filename without ".py".
let modules = PyModule::import(py, "sys")?.getattr("modules")?;
let class = modules
.get_item(path.replace(".py", ""))?
.getattr("StorageNode")?;
let code = CString::new(include_str!("../python_errors.py")).unwrap();
let adapter =
PyModule::from_code(py, &code, c"python_errors.py", c"hw8_python_errors")?;
Ok(adapter
.getattr("watch")?
.call1((class,))?
.extract::<Py<PyList>>()?)
})
.map_err(|error| error.to_string())?;
Ok(Self { factory, errors })
}
}
fn python_panic(panic: Box<dyn std::any::Any + Send>) -> String {
if let Some(message) = panic.downcast_ref::<String>() {
if let Some(start) = message.find("PyErr {") {
return message[start..].to_string();
}
}
resume_unwind(panic)
}
pub type Case = fn(&TestConfig) -> TestResult;
pub struct CaseConfig<'a> {
pub solution: &'a Result<Solution, String>,
pub proc_count: u32,
pub seed: u64,
}
pub fn run_case((config, test): &(&CaseConfig, Case)) -> TestResult {
let solution = config.solution.as_ref().map_err(Clone::clone)?;
let count_keys_error = OnceCell::new();
let config = TestConfig {
count_keys_error: &count_keys_error,
proc_factory: &solution.factory,
proc_count: config.proc_count,
seed: config.seed,
};
// Each case owns its simulator. Unwinding drops all nodes from the failed case.
match catch_unwind(AssertUnwindSafe(|| test(&config))) {
Ok(result) => count_keys_error.into_inner().map_or(result, Err),
Err(panic) => {
let error = Python::attach(|py| {
solution
.errors
.bind(py)
.call_method0("pop")
.and_then(|item| item.extract::<String>())
.ok()
});
match error {
Some(error) => Err(format!("Python exception in {error}")),
// PyO3 can also report Python errors outside a callback, for example in deepcopy.
None => Err(format!("Python exception: {}", python_panic(panic))),
}
}
}
}
@@ -0,0 +1,76 @@
use std::ffi::CString;
use pyo3::prelude::*;
use pyo3::types::{PyList, PyModule};
const VALIDATOR_CODE: &str = include_str!("../shared_state_validator.py");
fn validator_module<'py>(py: Python<'py>) -> PyResult<Bound<'py, PyModule>> {
let code = CString::new(VALIDATOR_CODE).expect("validator source contains a null byte");
PyModule::from_code(
py,
&code,
c"shared_state_validator.py",
c"shared_state_validator",
)
}
#[cfg(test)]
fn validate_source(
source: &str,
filename: &str,
class_names: &[&str],
) -> Result<Vec<String>, String> {
Python::attach(|py| {
let module = validator_module(py).map_err(|error| error.to_string())?;
let class_names = PyList::new(py, class_names).map_err(|error| error.to_string())?;
module
.getattr("validate_source")
.and_then(|function| function.call1((source, filename, class_names)))
.and_then(|result| result.extract::<Vec<String>>())
.map_err(|error| error.to_string())
})
}
fn validate_solution(path: &str, class_names: &[&str]) -> Result<Vec<String>, String> {
Python::attach(|py| {
let module = validator_module(py).map_err(|error| error.to_string())?;
let class_names = PyList::new(py, class_names).map_err(|error| error.to_string())?;
module
.getattr("validate_solution")
.and_then(|function| function.call1((path, class_names)))
.and_then(|result| result.extract::<Vec<String>>())
.map_err(|error| error.to_string())
})
}
pub fn validate_or_report(path: &str, class_names: &[&str]) -> Result<bool, String> {
let violations = validate_solution(path, class_names)?;
if violations.is_empty() {
return Ok(true);
}
println!("ERROR: shared state between AnySystem processes is forbidden");
for violation in violations {
println!("{violation}");
}
println!("\nSCORE: 0\n");
Ok(false)
}
#[cfg(test)]
mod tests {
use super::validate_source;
#[test]
fn embeds_validator_and_rejects_shared_state() {
let violations = validate_source(
"class ProcessImpl:\n shared = {}\n",
"solution.py",
&["ProcessImpl"],
)
.unwrap();
assert_eq!(violations.len(), 1);
assert!(violations[0].contains("ProcessImpl.shared"));
}
}
File diff suppressed because it is too large. Load diff
@@ -0,0 +1,100 @@
use anysystem::python::PyProcess;
use anysystem::{Context, Process};
use super::*;
// Hold network deliveries from one coordinator at their destinations. This also
// covers retries and extra protocol rounds without inspecting student messages.
#[derive(Clone)]
struct DeliveryGate {
proc: PyProcess,
held_sender: Option<String>,
pending: Vec<Message>,
}
impl Process for DeliveryGate {
fn on_start(&mut self, ctx: &mut Context) -> Result<(), String> {
self.proc.on_start(ctx)
}
fn on_message(&mut self, msg: Message, from: String, ctx: &mut Context) -> Result<(), String> {
if self.held_sender.as_ref() == Some(&from) {
self.pending.push(msg);
return Ok(());
}
self.proc.on_message(msg, from, ctx)
}
fn on_local_message(&mut self, msg: Message, ctx: &mut Context) -> Result<(), String> {
match msg.tip.as_str() {
"HOLD_FROM" => self.held_sender = Some(msg.data),
"RELEASE" => {
if let Some(from) = self.held_sender.take() {
for msg in self.pending.drain(..) {
self.proc.on_message(msg, from.clone(), ctx)?;
}
}
}
_ => return self.proc.on_local_message(msg, ctx),
}
Ok(())
}
fn on_timer(&mut self, timer: String, ctx: &mut Context) -> Result<(), String> {
self.proc.on_timer(timer, ctx)
}
}
pub(super) fn check_put_before_later_delete(config: &TestConfig) -> TestResult {
// Two non-replica coordinators and three primaries suffice. Any two
// quorums of three intersect, even if a retry reaches a backup.
let config = TestConfig {
proc_count: 5,
..*config
};
for value in ["abc", ""] {
for hold_put in [false, true] {
let mut sys = build_system_with(&config, |proc| {
Box::new(DeliveryGate {
proc,
held_sender: None,
pending: Vec::new(),
})
});
let mut rand = Pcg64::seed_from_u64(config.seed);
let key = random_string(8, &mut rand).to_uppercase();
let others = key_non_replicas(&key, &sys);
let (writer, deleter) = (&others[0], &others[1]);
let held = if hold_put { writer } else { deleter };
let procs = sys.process_names();
for proc in &procs {
sys.send_local_message(proc, Message::new("HOLD_FROM", held));
}
// PUT is older, but both clients submit before the network delivers anything.
send_put(&mut sys, writer, &key, value, 3);
sys.step_for_duration(MIN_NETWORK_DELAY / 2.0);
send_delete(&mut sys, deleter, &key, 3);
let expected = [(!hold_put).then_some(value)];
let put = ExpectedResponse::put(writer, &key, &expected);
let delete = ExpectedResponse::delete(deleter, &key, &expected);
let (first, second, delivery) = if hold_put {
(delete, put, "DELETE then PUT")
} else {
(put, delete, "PUT then DELETE")
};
let order = format!("replicas receive {delivery}, value={value:?}");
println!("Earlier PUT, later DELETE: {order}");
wait_for_responses(&mut sys, &[first], 300).map_err(|err| format!("{order}: {err}"))?;
for proc in &procs {
sys.send_local_message(proc, Message::new("RELEASE", ""));
}
wait_for_responses(&mut sys, &[second], 300)
.map_err(|err| format!("{order}: {err}"))?;
finish_background_work(&mut sys, "Concurrent PUT before later DELETE")?;
check_isolated_reads(&mut sys, &[(&key, None)])?;
}
}
Ok(true)
}
@@ -0,0 +1,483 @@
use std::collections::HashSet;
use rand::prelude::*;
use rand_pcg::Pcg64;
use anysystem::logger::LogEntry;
use anysystem::mc::{
predicates::{collects, goals, prunes},
strategies::bfs::Bfs,
EventOrderingMode, InvariantFn, McState, McStats, McSystem, ModelChecker, StrategyConfig,
};
use anysystem::test::TestResult;
use anysystem::{Message, System};
use crate::common::*;
pub fn test_mc_basic(config: &TestConfig) -> TestResult {
let sys = build_system(config);
let procs = sys.process_names();
let mut rand = Pcg64::seed_from_u64(config.seed);
let mut mc = ModelChecker::new(&sys);
let key = random_string(8, &mut rand).to_uppercase();
let replicas = key_replicas(&key, &sys);
let non_replicas = key_non_replicas(&key, &sys);
println!("Key {key} replicas: {replicas:?}");
println!("Key {key} non-replicas: {non_replicas:?}");
// stage 1: get key from the first node
let stage1_strategy = mc_query_strategy(&procs[0], McQuery::Get(key.clone(), None));
let stage1_msg = Message::json(
"GET",
&GetReqMessage {
key: &key,
quorum: 2,
},
);
let stage1_states = run_mc(
&mut mc,
stage1_strategy,
&procs[0],
stage1_msg,
McNetworkChange::None,
None,
)?
.collected_states;
println!("stage 1: {}", stage1_states.len());
if stage1_states.is_empty() {
return Err("stage 1 - GET response is not received".to_owned());
}
// stage 2: put key to the first replica
let value = random_string(8, &mut rand);
let sys = build_system(config);
mc = ModelChecker::new(&sys);
let stage2_strategy = mc_query_strategy(&replicas[0], McQuery::Put(key.clone(), value.clone()));
let stage2_msg = Message::json(
"PUT",
&PutReqMessage {
key: &key,
value: &value,
quorum: 2,
},
);
let stage2_states = run_mc(
&mut mc,
stage2_strategy,
&replicas[0],
stage2_msg,
McNetworkChange::None,
None,
)?
.collected_states;
println!("stage 2: {}", stage2_states.len());
if stage2_states.is_empty() {
return Err("stage 2 - PUT response is not received".to_owned());
}
// stage 3: get key from the last replica
let stage3_strategy = mc_query_strategy(&replicas[2], McQuery::Get(key.clone(), Some(value)));
let stage3_msg = Message::json(
"GET",
&GetReqMessage {
key: &key,
quorum: 2,
},
);
let stage3_states = run_mc(
&mut mc,
stage3_strategy,
&replicas[2],
stage3_msg,
McNetworkChange::None,
Some(stage2_states),
)?
.collected_states;
println!("stage 3: {}", stage3_states.len());
if stage3_states.is_empty() {
return Err("stage 3 - GET response is not received".to_owned());
}
Ok(true)
}
pub fn test_mc_concurrent_writes(config: &TestConfig) -> TestResult {
let sys = build_system(config);
let mut rand = Pcg64::seed_from_u64(config.seed);
let key = random_string(8, &mut rand).to_uppercase();
let replicas = key_replicas(&key, &sys);
let non_replicas = key_non_replicas(&key, &sys);
println!("Key {key} replicas: {replicas:?}");
println!("Key {key} non-replicas: {non_replicas:?}");
// isolate all processes
for proc in sys.process_names() {
sys.network().disconnect_node(&proc);
}
let mut mc = ModelChecker::new(&sys);
// put (key, value) to the first replica
// and then put (key, value2) to the second replica
let value = random_string(8, &mut rand);
let value2 = random_string(8, &mut rand);
let strategy_config = mc_query_strategy(&replicas[0], McQuery::Put(key.clone(), value.clone()));
let msg1 = Message::json(
"PUT",
&PutReqMessage {
key: &key,
value: &value,
quorum: 1,
},
);
let states = run_mc(
&mut mc,
strategy_config,
&replicas[0],
msg1,
McNetworkChange::None,
None,
)?
.collected_states;
if states.is_empty() {
return Err(format!("put({key}, {value}) response is not received"));
}
println!("put({key}, {value}): {} states collected", states.len());
let strategy_config =
mc_query_strategy(&replicas[1], McQuery::Put(key.clone(), value2.clone()));
let msg2 = Message::json(
"PUT",
&PutReqMessage {
key: &key,
value: &value2,
quorum: 1,
},
);
let states = run_mc(
&mut mc,
strategy_config,
&replicas[1],
msg2,
McNetworkChange::None,
Some(states),
)?
.collected_states;
if states.is_empty() {
return Err(format!("put({key}, {value2}) response is not received"));
}
println!("put({key}, {value2}): {} states collected", states.len());
// now reset the network state and ask the third replica about the key's value
// we expect the value written later (value2) to be returned
let strategy_config = mc_query_strategy(
&replicas[2],
McQuery::Get(key.to_string(), Some(value2.to_string())),
);
let msg = Message::json(
"GET",
&GetReqMessage {
key: &key,
quorum: 3,
},
);
let states = run_mc(
&mut mc,
strategy_config,
&replicas[2],
msg,
McNetworkChange::Reset,
Some(states),
)?
.collected_states;
if states.is_empty() {
return Err(format!("get({key}) response is not received"));
}
println!("get({key}): {} states collected", states.len());
Ok(true)
}
pub fn test_mc_sloppy_quorum_hinted_handoff(config: &TestConfig) -> TestResult {
let mut sys = build_system(config);
let mut rand = Pcg64::seed_from_u64(config.seed);
let mut mc = ModelChecker::new(&sys);
let key = random_string(8, &mut rand).to_uppercase();
let replicas = key_replicas(&key, &sys);
let non_replicas = key_non_replicas(&key, &sys);
println!("Key {key} replicas: {replicas:?}");
println!("Key {key} non-replicas: {non_replicas:?}");
// stage 1: get key from the first replica (during the network partition)
let stage1_strategy = mc_query_strategy(&replicas[0], McQuery::Get(key.clone(), None));
let stage1_msg = Message::json(
"GET",
&GetReqMessage {
key: &key,
quorum: 2,
},
);
let stage1_states = run_mc(
&mut mc,
stage1_strategy,
&replicas[0],
stage1_msg,
McNetworkChange::Partition([
vec![
replicas[0].clone(),
non_replicas[0].clone(),
non_replicas[1].clone(),
non_replicas[2].clone(),
],
vec![replicas[1].clone(), replicas[2].clone()],
]),
None,
)?
.collected_states;
println!("stage 1: {}", stage1_states.len());
if stage1_states.is_empty() {
return Err("stage 1 - GET response is not received".to_owned());
}
// stage 2: put key from the first replica (network partition still exists)
let value = random_string(8, &mut rand);
sys = build_system(config);
mc = ModelChecker::new(&sys);
let stage2_strategy = mc_query_strategy(&replicas[0], McQuery::Put(key.clone(), value.clone()));
let stage2_msg = Message::json(
"PUT",
&PutReqMessage {
key: &key,
value: &value,
quorum: 2,
},
);
let stage2_states = run_mc(
&mut mc,
stage2_strategy,
&replicas[0],
stage2_msg,
McNetworkChange::Partition([
vec![
replicas[0].clone(),
non_replicas[0].clone(),
non_replicas[1].clone(),
non_replicas[2].clone(),
],
vec![replicas[1].clone(), replicas[2].clone()],
]),
None,
)?
.collected_states;
println!("stage 2: {}", stage2_states.len());
if stage2_states.is_empty() {
return Err("stage 2 - PUT response is not received".to_owned());
}
// stage 3: recover network and let data propagate to all replicas
let stage3_states = mc_stabilize(&mut sys, stage2_states)?.collected_states;
println!("stage 3: {}", stage3_states.len());
if stage3_states.is_empty() {
return Err(
"stage 3 - no states found during the exploration phase with recovered network"
.to_owned(),
);
}
// stage 4: get key from the last replica (again during the network partition)
let stage4_strategy = mc_query_strategy(&replicas[2], McQuery::Get(key.clone(), Some(value)));
let stage4_msg = Message::json(
"GET",
&GetReqMessage {
key: &key,
quorum: 2,
},
);
let stage4_states = run_mc(
&mut mc,
stage4_strategy,
&replicas[2],
stage4_msg,
McNetworkChange::Partition([
vec![
replicas[0].clone(),
non_replicas[0].clone(),
non_replicas[1].clone(),
non_replicas[2].clone(),
],
vec![replicas[1].clone(), replicas[2].clone()],
]),
Some(stage3_states),
)?
.collected_states;
println!("stage 4: {}", stage4_states.len());
if stage4_states.is_empty() {
return Err("stage 4 - GET response is not received".to_owned());
}
Ok(true)
}
// UTILS ---------------------------------------------------------------------------------------------------------------
enum McQuery {
Get(String, Option<String>),
Put(String, String),
}
fn mc_query_strategy(proc: &str, query_data: McQuery) -> StrategyConfig {
let proc_name = proc.to_string();
let invariant = match query_data {
McQuery::Get(key, expected) => mc_get_invariant(proc, key, expected),
McQuery::Put(key, value) => mc_put_invariant(proc, key, value),
};
StrategyConfig::default()
.prune(prunes::any_prune(vec![
prunes::event_happened_n_times_current_run(LogEntry::is_mc_timer_fired, 5_usize),
prunes::event_happened_n_times_current_run(LogEntry::is_mc_message_received, 10_usize),
]))
.goal(goals::event_happened_n_times_current_run(
LogEntry::is_mc_local_message_sent,
1,
))
.invariant(invariant)
.collect(collects::event_happened_n_times_current_run(
move |log_entry| match log_entry {
LogEntry::McLocalMessageSent { proc, .. } => proc == &proc_name,
_ => false,
},
1,
))
}
fn mc_get_invariant<S>(proc: S, key: String, expected: Option<String>) -> InvariantFn
where
S: Into<String>,
{
let proc_name = proc.into();
Box::new(move |state: &McState| -> Result<(), String> {
for entry in state.current_run_trace().iter() {
if let LogEntry::McLocalMessageSent { msg, proc } = entry {
if &proc_name != proc {
return Err("local message received on wrong process".to_string());
}
if msg.tip != "GET_RESP" {
return Err(format!("wrong type {}", msg.tip));
}
let data = parse_value_response(msg, proc)?;
if data.key != key {
return Err(format!("wrong key {}", data.key));
}
if data.value != expected {
return Err(format!("wrong value {:?}", data.value));
}
}
}
Ok(())
})
}
fn mc_put_invariant<S>(proc: S, key: String, value: String) -> InvariantFn
where
S: Into<String>,
{
let proc_name = proc.into();
Box::new(move |state: &McState| -> Result<(), String> {
for entry in state.current_run_trace().iter() {
if let LogEntry::McLocalMessageSent { msg, proc } = entry {
if &proc_name != proc {
return Err("local message received on wrong process".to_string());
}
if msg.tip != "PUT_RESP" {
return Err(format!("wrong type {}", msg.tip));
}
let data = parse_value_response(msg, proc)?;
if data.key != key {
return Err(format!("wrong key {}", data.key));
}
if data.value.as_deref() != Some(value.as_str()) {
return Err(format!("wrong value {:?}", data.value));
}
}
}
Ok(())
})
}
enum McNetworkChange {
None,
Reset,
Partition([Vec<String>; 2]),
}
fn run_mc<S>(
mc: &mut ModelChecker,
strategy_config: StrategyConfig,
proc: S,
msg: Message,
network_change: McNetworkChange,
states: Option<HashSet<McState>>,
) -> Result<McStats, String>
where
S: Into<String>,
{
let proc = proc.into();
let callback = |sys: &mut McSystem| {
match &network_change {
McNetworkChange::Partition([part1, part2]) => sys.network().partition(part1, part2),
McNetworkChange::Reset => sys.network().reset(),
McNetworkChange::None => {}
}
sys.set_event_ordering_mode(EventOrderingMode::MessagesFirst);
sys.send_local_message(proc.clone(), proc.clone(), msg.clone());
};
let res = if let Some(states) = states {
mc.run_from_states_with_change::<Bfs>(strategy_config, states, callback)
} else {
mc.run_with_change::<Bfs>(strategy_config, callback)
};
match res {
Err(e) => {
e.print_trace();
Err(e.message())
}
Ok(stats) => Ok(stats),
}
}
fn mc_stabilize(sys: &mut System, states: HashSet<McState>) -> Result<McStats, String> {
let strategy_config = StrategyConfig::default()
.prune(prunes::any_prune(vec![
prunes::event_happened_n_times_current_run(LogEntry::is_mc_timer_fired, 6),
prunes::event_happened_n_times_current_run(LogEntry::is_mc_message_received, 24),
]))
.goal(goals::any_goal(vec![
goals::depth_reached(30),
goals::no_events(),
]))
.collect(collects::any_collect(vec![
collects::no_events(),
collects::event_happened_n_times_current_run(LogEntry::is_mc_timer_fired, 6),
collects::event_happened_n_times_current_run(LogEntry::is_mc_message_received, 24),
]));
let mut mc = ModelChecker::new(sys);
let res = mc.run_from_states_with_change::<Bfs>(strategy_config, states, |sys| {
sys.network().reset();
sys.set_event_ordering_mode(EventOrderingMode::MessagesFirst);
});
match res {
Err(e) => {
e.print_trace();
Err(e.message())
}
Ok(stats) => Ok(stats),
}
}