Add HW 8
This commit is contained in:
1 parent
a4dc591ca0
commit
3030f979d1
17 files changed
+6982
No files matched your search
@@ -0,0 +1,5 @@
|
||||
target
|
||||
**/target
|
||||
**/__pycache__
|
||||
**/.pytest_cache
|
||||
*.pyc
|
||||
+1027
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,21 @@
|
||||
[package]
|
||||
name = "distsys-kv-replication"
|
||||
version = "0.1.0"
|
||||
edition = "2021"
|
||||
resolver = "3"
|
||||
|
||||
[dependencies]
|
||||
anysystem = "=0.3.0"
|
||||
indexmap = "2.12"
|
||||
assertables = "3.2.2"
|
||||
byteorder = "1.4.3"
|
||||
clap = { version = "3.2.17", features = ["cargo", "derive"] }
|
||||
decorum = "0.3.1"
|
||||
env_logger = "0.9.0"
|
||||
log = "0.4.17"
|
||||
md5 = "0.7.0"
|
||||
pyo3 = { version = "=0.29.2", features = ["auto-initialize"] }
|
||||
rand = "0.8.5"
|
||||
rand_pcg = "0.3.1"
|
||||
serde = { version = "1.0.142", features = ["derive"] }
|
||||
serde_json = "1.0.83"
|
||||
@@ -0,0 +1,19 @@
|
||||
# syntax=docker/dockerfile:1
|
||||
FROM rust:1.97.1-bookworm@sha256:0e2bcaef56d041a486784e54104a81aebe0da44bd03019bd70bc0401e42e4a97 AS builder
|
||||
|
||||
RUN apt-get update && apt-get install -y build-essential python3-dev
|
||||
|
||||
WORKDIR /tests
|
||||
COPY . .
|
||||
RUN --mount=type=cache,id=distsys-course-cargo-registry,target=/usr/local/cargo/registry,sharing=locked \
|
||||
--mount=type=cache,id=distsys-course-cargo-target-rust-1-97-1,target=/tests/target,sharing=locked \
|
||||
cargo install --locked --path . --root /opt/course
|
||||
|
||||
FROM debian:bookworm-slim
|
||||
|
||||
RUN apt-get update && apt-get install -y python3-dev && rm -rf /var/lib/apt/lists/*
|
||||
|
||||
COPY --from=builder /opt/course/bin/distsys-kv-replication /usr/local/bin/distsys-kv-replication
|
||||
|
||||
WORKDIR /solution
|
||||
ENTRYPOINT ["timeout", "--verbose", "-k", "10", "1260", "distsys-kv-replication", "-i", "node.py"]
|
||||
@@ -0,0 +1,38 @@
|
||||
"""Retain exceptions from StorageNode calls for the Rust test runner."""
|
||||
from functools import wraps
|
||||
from inspect import getattr_static
|
||||
|
||||
|
||||
def watch(cls):
|
||||
if not isinstance(cls, type):
|
||||
raise TypeError("StorageNode must be a class")
|
||||
missing = sorted(getattr(cls, "__abstractmethods__", ()))
|
||||
if missing:
|
||||
raise TypeError(f"{cls.__name__} must implement {', '.join(missing)}")
|
||||
errors = []
|
||||
|
||||
def checked(name, method):
|
||||
@wraps(method)
|
||||
def call(*args, **kwargs):
|
||||
try:
|
||||
result = method(*args, **kwargs)
|
||||
except BaseException as exc:
|
||||
errors[:] = [f"{name}: {type(exc).__name__}: {exc}"]
|
||||
raise
|
||||
if errors:
|
||||
errors.clear() # A nested exception was handled by the solution.
|
||||
return result
|
||||
|
||||
return call
|
||||
|
||||
for name in ("__new__", "__init__", "on_start", "on_local_message", "on_message", "on_timer",
|
||||
"get_state", "set_state", "__deepcopy__"):
|
||||
# Preserve static and class methods and inherited implementations.
|
||||
method = getattr_static(cls, name, None)
|
||||
if name == "__new__" and method is object.__new__:
|
||||
continue # Keep the default allocation and constructor argument handling.
|
||||
if isinstance(method, (staticmethod, classmethod)):
|
||||
setattr(cls, name, type(method)(checked(name, method.__func__)))
|
||||
elif callable(method):
|
||||
setattr(cls, name, checked(name, method))
|
||||
return errors
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,139 @@
|
||||
use std::cell::OnceCell;
|
||||
|
||||
use byteorder::{ByteOrder, LittleEndian};
|
||||
use rand::distributions::WeightedIndex;
|
||||
use rand::prelude::*;
|
||||
use rand_pcg::Pcg64;
|
||||
use serde::Serialize;
|
||||
use serde_json::{Map, Value};
|
||||
|
||||
use anysystem::python::{PyProcess, PyProcessFactory};
|
||||
use anysystem::{Message, Process, System};
|
||||
|
||||
use crate::count_keys::TestSystem;
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub struct GetReqMessage<'a> {
|
||||
pub key: &'a str,
|
||||
pub quorum: u8,
|
||||
}
|
||||
|
||||
pub struct ValueResponse {
|
||||
pub key: String,
|
||||
pub value: Option<String>,
|
||||
}
|
||||
|
||||
pub fn parse_value_response(msg: &Message, proc: &str) -> Result<ValueResponse, String> {
|
||||
let tip = &msg.tip;
|
||||
let mut data: Map<String, Value> = serde_json::from_str(&msg.data)
|
||||
.map_err(|err| format!("Invalid {tip} from {proc}: {err}"))?;
|
||||
let Some(Value::String(key)) = data.remove("key") else {
|
||||
return Err(format!("Missing or invalid key in {tip} from {proc}"));
|
||||
};
|
||||
let value = match data.remove("value") {
|
||||
Some(Value::String(value)) => Some(value),
|
||||
Some(Value::Null) => None,
|
||||
None => {
|
||||
return Err(format!(
|
||||
"Missing field 'value' in {tip} from {proc} for key {key}"
|
||||
))
|
||||
}
|
||||
Some(_) => {
|
||||
return Err(format!(
|
||||
"Invalid 'value' in {tip} from {proc} for key {key}: expected a string or null"
|
||||
))
|
||||
}
|
||||
};
|
||||
Ok(ValueResponse { key, value })
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub struct PutReqMessage<'a> {
|
||||
pub key: &'a str,
|
||||
pub value: &'a str,
|
||||
pub quorum: u8,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub struct DeleteReqMessage<'a> {
|
||||
pub key: &'a str,
|
||||
pub quorum: u8,
|
||||
}
|
||||
|
||||
#[derive(Copy, Clone)]
|
||||
pub struct TestConfig<'a> {
|
||||
pub proc_factory: &'a PyProcessFactory,
|
||||
pub count_keys_error: &'a OnceCell<String>,
|
||||
pub proc_count: u32,
|
||||
pub seed: u64,
|
||||
}
|
||||
|
||||
pub const MIN_NETWORK_DELAY: f64 = 0.01;
|
||||
|
||||
pub fn build_system<'a>(config: &TestConfig<'a>) -> TestSystem<'a> {
|
||||
build_system_with(config, |proc| Box::new(proc))
|
||||
}
|
||||
|
||||
pub fn build_system_with<'a>(
|
||||
config: &TestConfig<'a>,
|
||||
wrap: impl Fn(PyProcess) -> Box<dyn Process>,
|
||||
) -> TestSystem<'a> {
|
||||
let mut sys = System::new(config.seed);
|
||||
sys.network().set_delays(MIN_NETWORK_DELAY, 0.1);
|
||||
let mut proc_names = Vec::new();
|
||||
for n in 0..config.proc_count {
|
||||
proc_names.push(format!("{n}"));
|
||||
}
|
||||
for proc_name in proc_names.iter() {
|
||||
let proc = config
|
||||
.proc_factory
|
||||
.build((proc_name, proc_names.clone()), config.seed);
|
||||
// process and node on which it runs have the same name
|
||||
let node_name = proc_name.clone();
|
||||
sys.add_node(&node_name);
|
||||
sys.add_process(proc_name, wrap(proc), &node_name);
|
||||
}
|
||||
TestSystem::new(sys, config.count_keys_error)
|
||||
}
|
||||
|
||||
pub fn key_replicas(key: &str, sys: &System) -> Vec<String> {
|
||||
let proc_count = sys.process_names().len();
|
||||
let mut replicas = Vec::new();
|
||||
let hash = md5::compute(key);
|
||||
let hash128 = LittleEndian::read_u128(&hash.0);
|
||||
let mut replica = (hash128 % proc_count as u128) as usize;
|
||||
for _ in 0..3 {
|
||||
replicas.push(replica.to_string());
|
||||
replica += 1;
|
||||
if replica == proc_count {
|
||||
replica = 0;
|
||||
}
|
||||
}
|
||||
replicas
|
||||
}
|
||||
|
||||
pub fn key_non_replicas(key: &str, sys: &System) -> Vec<String> {
|
||||
let proc_count = sys.process_names().len();
|
||||
let replicas = key_replicas(key, sys);
|
||||
let last_primary = replicas.last().unwrap().parse::<usize>().unwrap();
|
||||
(1..=proc_count - replicas.len())
|
||||
.map(|offset| ((last_primary + offset) % proc_count).to_string())
|
||||
.collect()
|
||||
}
|
||||
|
||||
const SYMBOLS: [char; 36] = [
|
||||
'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's',
|
||||
't', 'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9',
|
||||
];
|
||||
const WEIGHTS: [usize; 36] = [
|
||||
13, 16, 3, 8, 8, 5, 6, 23, 4, 8, 24, 12, 2, 1, 1, 10, 5, 8, 10, 1, 24, 3, 1, 8, 12, 22, 5, 20,
|
||||
18, 5, 5, 2, 1, 3, 16, 22,
|
||||
];
|
||||
|
||||
pub fn random_string(length: usize, rand: &mut Pcg64) -> String {
|
||||
let dist = WeightedIndex::new(WEIGHTS).unwrap();
|
||||
rand.sample_iter(&dist)
|
||||
.take(length)
|
||||
.map(|x| SYMBOLS[x])
|
||||
.collect()
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
use std::cell::OnceCell;
|
||||
use std::collections::HashSet;
|
||||
use std::ops::{Deref, DerefMut};
|
||||
|
||||
use anysystem::logger::LogEntry;
|
||||
use anysystem::System;
|
||||
|
||||
// One test case can build several simulations. Check each finished trace once,
|
||||
// including callbacks that ran after the client received COUNT_KEYS_RESP.
|
||||
pub struct TestSystem<'a> {
|
||||
system: System,
|
||||
count_keys_error: &'a OnceCell<String>,
|
||||
}
|
||||
|
||||
impl<'a> TestSystem<'a> {
|
||||
pub fn new(system: System, count_keys_error: &'a OnceCell<String>) -> Self {
|
||||
Self {
|
||||
system,
|
||||
count_keys_error,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Deref for TestSystem<'_> {
|
||||
type Target = System;
|
||||
|
||||
fn deref(&self) -> &System {
|
||||
&self.system
|
||||
}
|
||||
}
|
||||
|
||||
impl DerefMut for TestSystem<'_> {
|
||||
fn deref_mut(&mut self) -> &mut System {
|
||||
&mut self.system
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for TestSystem<'_> {
|
||||
fn drop(&mut self) {
|
||||
if !std::thread::panicking() && self.count_keys_error.get().is_none() {
|
||||
if let Some(sender) = count_keys_network_sender(self.system.logger().trace()) {
|
||||
let _ = self.count_keys_error.set(format!(
|
||||
"COUNT_KEYS on {sender} sent a message to another node"
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn count_keys_network_sender(trace: &[LogEntry]) -> Option<&str> {
|
||||
let mut timers = HashSet::new();
|
||||
let mut self_messages = HashSet::new();
|
||||
let mut handling_count = false;
|
||||
for entry in trace {
|
||||
match entry {
|
||||
LogEntry::LocalMessageReceived { msg, .. } => handling_count = msg.tip == "COUNT_KEYS",
|
||||
LogEntry::TimerFired { timer_id, .. } => handling_count = timers.remove(timer_id),
|
||||
LogEntry::MessageReceived { msg_id, .. } => {
|
||||
handling_count = self_messages.remove(msg_id)
|
||||
}
|
||||
// AnySystem gives every scheduled timer a new ID, even when the name is reused.
|
||||
LogEntry::TimerSet { timer_id, .. } if handling_count => {
|
||||
timers.insert(timer_id);
|
||||
}
|
||||
LogEntry::MessageSent {
|
||||
src_node,
|
||||
dst_node,
|
||||
msg_id,
|
||||
..
|
||||
} if handling_count => {
|
||||
if src_node != dst_node {
|
||||
return Some(src_node);
|
||||
}
|
||||
self_messages.insert(msg_id);
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
@@ -0,0 +1,171 @@
|
||||
use indexmap::IndexMap;
|
||||
|
||||
use anysystem::test::TestResult;
|
||||
|
||||
pub type Results = IndexMap<&'static str, TestResult>;
|
||||
|
||||
// Each item is worth half a point. Paired tests check the same skill.
|
||||
pub const CORE: &[(&str, &[&[&str]])] = &[
|
||||
(
|
||||
"Operations and placement",
|
||||
&[
|
||||
&["BASIC"],
|
||||
&["MC BASIC"],
|
||||
&["REPLICAS CHECK"],
|
||||
&["DATA PLACEMENT"],
|
||||
&["CONCURRENT GET PUT", "CONCURRENT GETS"],
|
||||
&["DELETE ABSENT"],
|
||||
],
|
||||
),
|
||||
(
|
||||
"Conflicts and repair",
|
||||
&[
|
||||
&["CONCURRENT WRITES", "MC CONCURRENT WRITES"],
|
||||
&["CONCURRENT WRITES TIE"],
|
||||
&["STALE REPLICA", "DIVERGED REPLICAS"],
|
||||
&["STALE REPLICA DELETE", "READ REPAIR DELETE"],
|
||||
&["CONCURRENT DELETES"],
|
||||
],
|
||||
),
|
||||
(
|
||||
"Quorums and partitions",
|
||||
&[
|
||||
&["SLOPPY QUORUM READ", "SLOPPY QUORUM WRITE"],
|
||||
&["SLOPPY QUORUM TRICKY"],
|
||||
&["SLOPPY QUORUM DELETE"],
|
||||
&["PARTITION CLIENTS", "PARTITION MIXED"],
|
||||
&["HINTED HANDOFF", "MC SLOPPY QUORUM HINTED HANDOFF"],
|
||||
],
|
||||
),
|
||||
];
|
||||
|
||||
pub const TRACKS: &[(&str, &[&str])] = &[
|
||||
(
|
||||
"A",
|
||||
&[
|
||||
"CONCURRENT PUT DELETE TIE",
|
||||
"READ REPAIR PUT DELETE TIE",
|
||||
"READ REPAIR LATE",
|
||||
"READ REPAIR TIE",
|
||||
"READ REPAIR MULTI KEY",
|
||||
"DATA PLACEMENT MULTI KEY",
|
||||
],
|
||||
),
|
||||
(
|
||||
"B",
|
||||
&[
|
||||
"SLOPPY QUORUM GET DISTINCT",
|
||||
"SLOPPY QUORUM PUT DISTINCT",
|
||||
"SLOPPY QUORUM DELETE DISTINCT",
|
||||
"SLOPPY QUORUM GET ORDER",
|
||||
"SLOPPY QUORUM PUT ORDER",
|
||||
"SLOPPY QUORUM DELETE ORDER",
|
||||
"SLOPPY QUORUM DELETE ORDER PARTITION",
|
||||
"SLOPPY QUORUM DELETE PARTITION",
|
||||
],
|
||||
),
|
||||
(
|
||||
"C",
|
||||
&[
|
||||
"HINTED HANDOFF PUT DELETE TIE",
|
||||
"HINTED HANDOFF MULTI KEY",
|
||||
"HINTED HANDOFF RETENTION",
|
||||
"HINTED HANDOFF LWW",
|
||||
"SLOPPY QUORUM READ REPAIR",
|
||||
"SLOPPY QUORUM READ REPAIR DELETE",
|
||||
"SLOPPY QUORUM READ REPAIR MULTI KEY",
|
||||
],
|
||||
),
|
||||
];
|
||||
|
||||
pub fn all_passed(results: &Results, names: &[&str]) -> bool {
|
||||
names
|
||||
.iter()
|
||||
.all(|name| results.get(name).is_some_and(Result::is_ok))
|
||||
}
|
||||
|
||||
pub fn core_points(results: &Results, items: &[&[&str]]) -> f32 {
|
||||
items
|
||||
.iter()
|
||||
.filter(|item| all_passed(results, item))
|
||||
.count() as f32
|
||||
/ 2.0
|
||||
}
|
||||
|
||||
pub fn score(core: f32, results: &Results) -> f32 {
|
||||
if core < 8.0 {
|
||||
return core;
|
||||
}
|
||||
let complete = TRACKS
|
||||
.iter()
|
||||
.filter(|(_, names)| all_passed(results, names))
|
||||
.count();
|
||||
core + complete.min(2) as f32
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::collections::HashSet;
|
||||
|
||||
#[test]
|
||||
fn rubric_covers_each_case_once() {
|
||||
let mut names = HashSet::new();
|
||||
for &(_, items) in CORE {
|
||||
for item in items {
|
||||
for &name in *item {
|
||||
assert!(names.insert(name), "{name}");
|
||||
}
|
||||
}
|
||||
}
|
||||
assert_eq!(names.len(), 23);
|
||||
for &(_, track) in TRACKS {
|
||||
for &name in track {
|
||||
assert!(names.insert(name), "{name}");
|
||||
}
|
||||
}
|
||||
assert_eq!(names.len(), 44);
|
||||
assert_eq!(names, crate::CASES.iter().map(|&(name, _)| name).collect());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn independent_core_credits_and_advanced_thresholds() {
|
||||
let reference: Results = crate::CASES
|
||||
.iter()
|
||||
.map(|&(name, _)| (name, Ok(true)))
|
||||
.collect();
|
||||
let total = |results: &Results| {
|
||||
score(
|
||||
CORE.iter()
|
||||
.map(|(_, items)| core_points(results, items))
|
||||
.sum(),
|
||||
results,
|
||||
)
|
||||
};
|
||||
for (i, &(_, items)) in CORE.iter().enumerate() {
|
||||
let mut results = reference.clone();
|
||||
for item in items {
|
||||
// A failure on either half of a pair loses only this item's credit.
|
||||
for &name in *item {
|
||||
results.insert(name, Err(String::new()));
|
||||
assert_eq!(total(&results), 7.5);
|
||||
results.insert(name, Ok(true));
|
||||
}
|
||||
}
|
||||
for item in items {
|
||||
results.insert(item[0], Err(String::new()));
|
||||
}
|
||||
assert_eq!(total(&results), [5.0, 5.5, 5.5][i]);
|
||||
}
|
||||
for mask in 0_u32..8 {
|
||||
let mut results = reference.clone();
|
||||
for (i, &(_, names)) in TRACKS.iter().enumerate() {
|
||||
if mask & (1 << i) == 0 {
|
||||
results.shift_remove(names[0]);
|
||||
}
|
||||
}
|
||||
assert_eq!(total(&results), 8.0 + mask.count_ones().min(2) as f32);
|
||||
}
|
||||
assert_eq!(total(&Results::new()), 0.0);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,252 @@
|
||||
mod common;
|
||||
mod count_keys;
|
||||
mod grading;
|
||||
mod python_errors;
|
||||
mod shared_state;
|
||||
mod tests;
|
||||
mod tests_mc;
|
||||
|
||||
use std::collections::HashSet;
|
||||
use std::env;
|
||||
use std::io::Write;
|
||||
use std::path::Path;
|
||||
|
||||
use clap::Parser;
|
||||
use env_logger::Builder;
|
||||
use log::LevelFilter;
|
||||
|
||||
use crate::grading::{core_points, score, Results, CORE, TRACKS};
|
||||
use anysystem::test::TestSuite;
|
||||
|
||||
use crate::python_errors::{run_case, Case, CaseConfig, Solution};
|
||||
use crate::tests::*;
|
||||
use crate::tests_mc::*;
|
||||
|
||||
/// Replicated KV Store Homework Tests
|
||||
#[derive(Parser, Debug)]
|
||||
#[clap(about, long_about = None)]
|
||||
struct Args {
|
||||
/// Path to Python file with solution
|
||||
#[clap(long = "impl", short = 'i', default_value = "solution/node.py")]
|
||||
solution_path: String,
|
||||
|
||||
/// Test to run (optional)
|
||||
#[clap(long = "test", short)]
|
||||
test: Option<String>,
|
||||
|
||||
/// Run only the 23 core tests
|
||||
#[clap(long, conflicts_with_all = &["all", "test"])]
|
||||
core: bool,
|
||||
|
||||
/// Run all 44 tests for diagnostics, ignoring track selection and the core prerequisite
|
||||
#[clap(long, conflicts_with = "test")]
|
||||
all: bool,
|
||||
|
||||
/// Print execution trace
|
||||
#[clap(long, short)]
|
||||
debug: bool,
|
||||
|
||||
/// Number of nodes used in tests
|
||||
#[clap(long, short, default_value = "6")]
|
||||
node_count: u32,
|
||||
|
||||
/// Random seed used in tests
|
||||
#[clap(long, short, default_value = "123")]
|
||||
seed: u64,
|
||||
}
|
||||
|
||||
fn main() {
|
||||
let args = Args::parse();
|
||||
// Initialize CPython with a 32-bit hash seed; the simulator uses the full u64.
|
||||
env::set_var("PYTHONHASHSEED", (args.seed as u32).to_string());
|
||||
python_errors::install_panic_hook();
|
||||
let validation = shared_state::validate_or_report(&args.solution_path, &["StorageNode"]);
|
||||
if matches!(validation, Ok(false)) {
|
||||
return;
|
||||
}
|
||||
if args.debug {
|
||||
Builder::new()
|
||||
.filter(Some("anysystem"), LevelFilter::Debug)
|
||||
.format(|buf, record| writeln!(buf, "{}", record.args()))
|
||||
.init();
|
||||
}
|
||||
let solution = validation.and_then(|_| Solution::load(&args.solution_path));
|
||||
let config = CaseConfig {
|
||||
solution: &solution,
|
||||
proc_count: args.node_count,
|
||||
seed: args.seed,
|
||||
};
|
||||
if let Some(test) = &args.test {
|
||||
let mut suite = TestSuite::new();
|
||||
for &(name, case) in CASES {
|
||||
suite.add(name, run_case, (&config, case));
|
||||
}
|
||||
suite.run_test(test);
|
||||
return;
|
||||
}
|
||||
|
||||
let mut results = Results::new();
|
||||
let mut core = 0.0;
|
||||
for &(block, items) in CORE {
|
||||
for item in items {
|
||||
for name in *item {
|
||||
run_named_case(name, &config, &mut results);
|
||||
}
|
||||
}
|
||||
let points = core_points(&results, items);
|
||||
println!("{block}: {points}/{}", items.len() as f32 / 2.0);
|
||||
core += points;
|
||||
}
|
||||
println!("Core: {core}/8");
|
||||
|
||||
if args.all || (!args.core && core == 8.0) {
|
||||
let selected = if args.all {
|
||||
Ok(None)
|
||||
} else {
|
||||
selected_tracks(Path::new(&args.solution_path))
|
||||
};
|
||||
match selected {
|
||||
Ok(selected) => {
|
||||
for &(track, names) in TRACKS {
|
||||
if selected.as_ref().is_some_and(|ids| !ids.contains(track)) {
|
||||
println!("Track {track}: skipped (not selected)");
|
||||
continue;
|
||||
}
|
||||
for name in names {
|
||||
run_named_case(name, &config, &mut results);
|
||||
}
|
||||
let passed = names.iter().filter(|name| results[*name].is_ok()).count();
|
||||
println!("Track {track}: {passed}/{}", names.len());
|
||||
}
|
||||
}
|
||||
Err(error) => eprintln!("Cannot select advanced tracks: {error}. Grading core only."),
|
||||
}
|
||||
} else if args.core {
|
||||
println!("Advanced tracks skipped (--core)");
|
||||
} else {
|
||||
println!("Advanced tracks skipped (full core required)");
|
||||
}
|
||||
let passed = results.values().filter(|result| result.is_ok()).count();
|
||||
println!("\nPassed {passed} from {} tests", results.len());
|
||||
for (name, result) in &results {
|
||||
if let Err(error) = result {
|
||||
println!("- {name}: {error}");
|
||||
}
|
||||
}
|
||||
println!("\nSCORE: {}\n", score(core, &results));
|
||||
}
|
||||
|
||||
fn run_named_case(name: &'static str, config: &CaseConfig, results: &mut Results) {
|
||||
let &(_, case) = CASES
|
||||
.iter()
|
||||
.find(|(test, _)| *test == name)
|
||||
.expect("registered case");
|
||||
println!("\n--- {name} ---\n");
|
||||
let result = run_case(&(config, case));
|
||||
match &result {
|
||||
Ok(_) => println!("\nPASSED\n"),
|
||||
Err(error) => println!("\nFAILED: {error}\n"),
|
||||
}
|
||||
results.insert(name, result);
|
||||
}
|
||||
|
||||
// A missing file selects all tracks; an empty file selects only the core.
|
||||
fn selected_tracks(solution: &Path) -> Result<Option<HashSet<String>>, String> {
|
||||
let path = solution.with_file_name("tracks.txt");
|
||||
let contents = match std::fs::read_to_string(&path) {
|
||||
Ok(contents) => contents,
|
||||
Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None),
|
||||
Err(error) => return Err(format!("{}: {error}", path.display())),
|
||||
};
|
||||
let mut selected = HashSet::new();
|
||||
for word in contents.trim_start_matches('\u{feff}').split_whitespace() {
|
||||
let id = word.to_ascii_uppercase();
|
||||
if !TRACKS.iter().any(|(track, _)| *track == id) {
|
||||
return Err(format!(
|
||||
"unknown track {word:?} in tracks.txt (expected A, B or C)"
|
||||
));
|
||||
}
|
||||
selected.insert(id);
|
||||
}
|
||||
Ok(Some(selected))
|
||||
}
|
||||
|
||||
const CASES: &[(&str, Case)] = &[
|
||||
("BASIC", test_basic),
|
||||
("REPLICAS CHECK", test_replicas_check),
|
||||
("DATA PLACEMENT", test_data_placement),
|
||||
("DATA PLACEMENT MULTI KEY", test_data_placement_multi_key),
|
||||
("CONCURRENT GET PUT", test_concurrent_get_put),
|
||||
("CONCURRENT GETS", test_concurrent_gets),
|
||||
("MC BASIC", test_mc_basic),
|
||||
("CONCURRENT WRITES", test_concurrent_writes),
|
||||
("CONCURRENT WRITES TIE", test_concurrent_writes_tie),
|
||||
("CONCURRENT PUT DELETE TIE", test_concurrent_put_delete_tie),
|
||||
(
|
||||
"READ REPAIR PUT DELETE TIE",
|
||||
test_read_repair_put_delete_tie,
|
||||
),
|
||||
(
|
||||
"HINTED HANDOFF PUT DELETE TIE",
|
||||
test_hinted_handoff_put_delete_tie,
|
||||
),
|
||||
("MC CONCURRENT WRITES", test_mc_concurrent_writes),
|
||||
("STALE REPLICA", test_stale_replica),
|
||||
("STALE REPLICA DELETE", test_stale_replica_delete),
|
||||
("DIVERGED REPLICAS", test_diverged_replicas),
|
||||
("DELETE ABSENT", test_delete_absent),
|
||||
("CONCURRENT DELETES", test_concurrent_deletes),
|
||||
("SLOPPY QUORUM DELETE", test_sloppy_quorum_delete),
|
||||
(
|
||||
"SLOPPY QUORUM DELETE PARTITION",
|
||||
test_sloppy_quorum_delete_partition,
|
||||
),
|
||||
(
|
||||
"SLOPPY QUORUM DELETE DISTINCT",
|
||||
test_sloppy_quorum_delete_distinct,
|
||||
),
|
||||
("READ REPAIR LATE", test_read_repair_late),
|
||||
("READ REPAIR DELETE", test_read_repair_delete),
|
||||
("READ REPAIR TIE", test_read_repair_tie),
|
||||
("READ REPAIR MULTI KEY", test_read_repair_multi_key),
|
||||
("SLOPPY QUORUM READ REPAIR", test_sloppy_quorum_read_repair),
|
||||
(
|
||||
"SLOPPY QUORUM READ REPAIR DELETE",
|
||||
test_sloppy_quorum_read_repair_delete,
|
||||
),
|
||||
(
|
||||
"SLOPPY QUORUM READ REPAIR MULTI KEY",
|
||||
test_sloppy_quorum_read_repair_multi_key,
|
||||
),
|
||||
("SLOPPY QUORUM GET ORDER", test_sloppy_quorum_get_order),
|
||||
(
|
||||
"SLOPPY QUORUM GET DISTINCT",
|
||||
test_sloppy_quorum_get_distinct,
|
||||
),
|
||||
("SLOPPY QUORUM PUT ORDER", test_sloppy_quorum_put_order),
|
||||
(
|
||||
"SLOPPY QUORUM PUT DISTINCT",
|
||||
test_sloppy_quorum_put_distinct,
|
||||
),
|
||||
(
|
||||
"SLOPPY QUORUM DELETE ORDER",
|
||||
test_sloppy_quorum_delete_order,
|
||||
),
|
||||
(
|
||||
"SLOPPY QUORUM DELETE ORDER PARTITION",
|
||||
test_sloppy_quorum_delete_order_partition,
|
||||
),
|
||||
("SLOPPY QUORUM READ", test_sloppy_quorum_read),
|
||||
("SLOPPY QUORUM WRITE", test_sloppy_quorum_write),
|
||||
("SLOPPY QUORUM TRICKY", test_sloppy_quorum_tricky),
|
||||
(
|
||||
"MC SLOPPY QUORUM HINTED HANDOFF",
|
||||
test_mc_sloppy_quorum_hinted_handoff,
|
||||
),
|
||||
("HINTED HANDOFF", test_hinted_handoff),
|
||||
("HINTED HANDOFF MULTI KEY", test_hinted_handoff_multi_key),
|
||||
("HINTED HANDOFF RETENTION", test_hinted_handoff_retention),
|
||||
("HINTED HANDOFF LWW", test_hinted_handoff_lww),
|
||||
("PARTITION CLIENTS", test_partition_clients),
|
||||
("PARTITION MIXED", test_partition_mixed),
|
||||
];
|
||||
@@ -0,0 +1,111 @@
|
||||
use std::cell::OnceCell;
|
||||
use std::ffi::CString;
|
||||
use std::panic::{catch_unwind, resume_unwind, AssertUnwindSafe};
|
||||
use std::path::Path;
|
||||
|
||||
use anysystem::python::PyProcessFactory;
|
||||
use anysystem::test::TestResult;
|
||||
use pyo3::prelude::*;
|
||||
use pyo3::types::{PyList, PyModule};
|
||||
|
||||
use crate::common::TestConfig;
|
||||
|
||||
// AnySystem 0.3 prints the Python traceback and then unwraps the same error.
|
||||
// Suppress this duplicate diagnostic only at those bridge sites.
|
||||
pub fn install_panic_hook() {
|
||||
let default = std::panic::take_hook();
|
||||
std::panic::set_hook(Box::new(move |info| {
|
||||
let site = info
|
||||
.location()
|
||||
.map(|l| l.file().replace('\\', "/"))
|
||||
.unwrap_or_default();
|
||||
let message = info.payload_as_str().unwrap_or("");
|
||||
let python = site.ends_with("/anysystem-0.3.0/src/python/mod.rs")
|
||||
&& (message.contains("PyErr {")
|
||||
|| message.contains("Error when creating Python process"));
|
||||
let callback = (site.ends_with("/anysystem-0.3.0/src/node.rs")
|
||||
|| site.ends_with("/anysystem-0.3.0/src/mc/node.rs"))
|
||||
&& message.contains("Error when calling process");
|
||||
if !python && !callback {
|
||||
default(info);
|
||||
}
|
||||
}));
|
||||
}
|
||||
|
||||
pub struct Solution {
|
||||
factory: PyProcessFactory,
|
||||
errors: Py<PyList>,
|
||||
}
|
||||
|
||||
impl Solution {
|
||||
pub fn load(path: &str) -> Result<Self, String> {
|
||||
let path = Path::new(path).canonicalize().map_err(|e| e.to_string())?;
|
||||
let path = path.to_str().ok_or("Solution path is not UTF-8")?;
|
||||
// AnySystem unwraps Python errors during import. Preserve unrelated loader panics.
|
||||
let factory = catch_unwind(|| PyProcessFactory::new(path, "StorageNode"))
|
||||
.map_err(|panic| format!("Cannot load StorageNode: {}", python_panic(panic)))?;
|
||||
let errors = Python::attach(|py| -> PyResult<_> {
|
||||
// AnySystem 0.3 registers the solution under its canonical filename without ".py".
|
||||
let modules = PyModule::import(py, "sys")?.getattr("modules")?;
|
||||
let class = modules
|
||||
.get_item(path.replace(".py", ""))?
|
||||
.getattr("StorageNode")?;
|
||||
let code = CString::new(include_str!("../python_errors.py")).unwrap();
|
||||
let adapter =
|
||||
PyModule::from_code(py, &code, c"python_errors.py", c"hw8_python_errors")?;
|
||||
Ok(adapter
|
||||
.getattr("watch")?
|
||||
.call1((class,))?
|
||||
.extract::<Py<PyList>>()?)
|
||||
})
|
||||
.map_err(|error| error.to_string())?;
|
||||
Ok(Self { factory, errors })
|
||||
}
|
||||
}
|
||||
|
||||
fn python_panic(panic: Box<dyn std::any::Any + Send>) -> String {
|
||||
if let Some(message) = panic.downcast_ref::<String>() {
|
||||
if let Some(start) = message.find("PyErr {") {
|
||||
return message[start..].to_string();
|
||||
}
|
||||
}
|
||||
resume_unwind(panic)
|
||||
}
|
||||
|
||||
pub type Case = fn(&TestConfig) -> TestResult;
|
||||
|
||||
pub struct CaseConfig<'a> {
|
||||
pub solution: &'a Result<Solution, String>,
|
||||
pub proc_count: u32,
|
||||
pub seed: u64,
|
||||
}
|
||||
|
||||
pub fn run_case((config, test): &(&CaseConfig, Case)) -> TestResult {
|
||||
let solution = config.solution.as_ref().map_err(Clone::clone)?;
|
||||
let count_keys_error = OnceCell::new();
|
||||
let config = TestConfig {
|
||||
count_keys_error: &count_keys_error,
|
||||
proc_factory: &solution.factory,
|
||||
proc_count: config.proc_count,
|
||||
seed: config.seed,
|
||||
};
|
||||
// Each case owns its simulator. Unwinding drops all nodes from the failed case.
|
||||
match catch_unwind(AssertUnwindSafe(|| test(&config))) {
|
||||
Ok(result) => count_keys_error.into_inner().map_or(result, Err),
|
||||
Err(panic) => {
|
||||
let error = Python::attach(|py| {
|
||||
solution
|
||||
.errors
|
||||
.bind(py)
|
||||
.call_method0("pop")
|
||||
.and_then(|item| item.extract::<String>())
|
||||
.ok()
|
||||
});
|
||||
match error {
|
||||
Some(error) => Err(format!("Python exception in {error}")),
|
||||
// PyO3 can also report Python errors outside a callback, for example in deepcopy.
|
||||
None => Err(format!("Python exception: {}", python_panic(panic))),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
use std::ffi::CString;
|
||||
|
||||
use pyo3::prelude::*;
|
||||
use pyo3::types::{PyList, PyModule};
|
||||
|
||||
const VALIDATOR_CODE: &str = include_str!("../shared_state_validator.py");
|
||||
|
||||
fn validator_module<'py>(py: Python<'py>) -> PyResult<Bound<'py, PyModule>> {
|
||||
let code = CString::new(VALIDATOR_CODE).expect("validator source contains a null byte");
|
||||
PyModule::from_code(
|
||||
py,
|
||||
&code,
|
||||
c"shared_state_validator.py",
|
||||
c"shared_state_validator",
|
||||
)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn validate_source(
|
||||
source: &str,
|
||||
filename: &str,
|
||||
class_names: &[&str],
|
||||
) -> Result<Vec<String>, String> {
|
||||
Python::attach(|py| {
|
||||
let module = validator_module(py).map_err(|error| error.to_string())?;
|
||||
let class_names = PyList::new(py, class_names).map_err(|error| error.to_string())?;
|
||||
module
|
||||
.getattr("validate_source")
|
||||
.and_then(|function| function.call1((source, filename, class_names)))
|
||||
.and_then(|result| result.extract::<Vec<String>>())
|
||||
.map_err(|error| error.to_string())
|
||||
})
|
||||
}
|
||||
|
||||
fn validate_solution(path: &str, class_names: &[&str]) -> Result<Vec<String>, String> {
|
||||
Python::attach(|py| {
|
||||
let module = validator_module(py).map_err(|error| error.to_string())?;
|
||||
let class_names = PyList::new(py, class_names).map_err(|error| error.to_string())?;
|
||||
module
|
||||
.getattr("validate_solution")
|
||||
.and_then(|function| function.call1((path, class_names)))
|
||||
.and_then(|result| result.extract::<Vec<String>>())
|
||||
.map_err(|error| error.to_string())
|
||||
})
|
||||
}
|
||||
|
||||
pub fn validate_or_report(path: &str, class_names: &[&str]) -> Result<bool, String> {
|
||||
let violations = validate_solution(path, class_names)?;
|
||||
if violations.is_empty() {
|
||||
return Ok(true);
|
||||
}
|
||||
println!("ERROR: shared state between AnySystem processes is forbidden");
|
||||
for violation in violations {
|
||||
println!("{violation}");
|
||||
}
|
||||
println!("\nSCORE: 0\n");
|
||||
Ok(false)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::validate_source;
|
||||
|
||||
#[test]
|
||||
fn embeds_validator_and_rejects_shared_state() {
|
||||
let violations = validate_source(
|
||||
"class ProcessImpl:\n shared = {}\n",
|
||||
"solution.py",
|
||||
&["ProcessImpl"],
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(violations.len(), 1);
|
||||
assert!(violations[0].contains("ProcessImpl.shared"));
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,100 @@
|
||||
use anysystem::python::PyProcess;
|
||||
use anysystem::{Context, Process};
|
||||
|
||||
use super::*;
|
||||
|
||||
// Hold network deliveries from one coordinator at their destinations. This also
|
||||
// covers retries and extra protocol rounds without inspecting student messages.
|
||||
#[derive(Clone)]
|
||||
struct DeliveryGate {
|
||||
proc: PyProcess,
|
||||
held_sender: Option<String>,
|
||||
pending: Vec<Message>,
|
||||
}
|
||||
|
||||
impl Process for DeliveryGate {
|
||||
fn on_start(&mut self, ctx: &mut Context) -> Result<(), String> {
|
||||
self.proc.on_start(ctx)
|
||||
}
|
||||
|
||||
fn on_message(&mut self, msg: Message, from: String, ctx: &mut Context) -> Result<(), String> {
|
||||
if self.held_sender.as_ref() == Some(&from) {
|
||||
self.pending.push(msg);
|
||||
return Ok(());
|
||||
}
|
||||
self.proc.on_message(msg, from, ctx)
|
||||
}
|
||||
|
||||
fn on_local_message(&mut self, msg: Message, ctx: &mut Context) -> Result<(), String> {
|
||||
match msg.tip.as_str() {
|
||||
"HOLD_FROM" => self.held_sender = Some(msg.data),
|
||||
"RELEASE" => {
|
||||
if let Some(from) = self.held_sender.take() {
|
||||
for msg in self.pending.drain(..) {
|
||||
self.proc.on_message(msg, from.clone(), ctx)?;
|
||||
}
|
||||
}
|
||||
}
|
||||
_ => return self.proc.on_local_message(msg, ctx),
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn on_timer(&mut self, timer: String, ctx: &mut Context) -> Result<(), String> {
|
||||
self.proc.on_timer(timer, ctx)
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn check_put_before_later_delete(config: &TestConfig) -> TestResult {
|
||||
// Two non-replica coordinators and three primaries suffice. Any two
|
||||
// quorums of three intersect, even if a retry reaches a backup.
|
||||
let config = TestConfig {
|
||||
proc_count: 5,
|
||||
..*config
|
||||
};
|
||||
for value in ["abc", ""] {
|
||||
for hold_put in [false, true] {
|
||||
let mut sys = build_system_with(&config, |proc| {
|
||||
Box::new(DeliveryGate {
|
||||
proc,
|
||||
held_sender: None,
|
||||
pending: Vec::new(),
|
||||
})
|
||||
});
|
||||
let mut rand = Pcg64::seed_from_u64(config.seed);
|
||||
let key = random_string(8, &mut rand).to_uppercase();
|
||||
let others = key_non_replicas(&key, &sys);
|
||||
let (writer, deleter) = (&others[0], &others[1]);
|
||||
let held = if hold_put { writer } else { deleter };
|
||||
let procs = sys.process_names();
|
||||
for proc in &procs {
|
||||
sys.send_local_message(proc, Message::new("HOLD_FROM", held));
|
||||
}
|
||||
|
||||
// PUT is older, but both clients submit before the network delivers anything.
|
||||
send_put(&mut sys, writer, &key, value, 3);
|
||||
sys.step_for_duration(MIN_NETWORK_DELAY / 2.0);
|
||||
send_delete(&mut sys, deleter, &key, 3);
|
||||
|
||||
let expected = [(!hold_put).then_some(value)];
|
||||
let put = ExpectedResponse::put(writer, &key, &expected);
|
||||
let delete = ExpectedResponse::delete(deleter, &key, &expected);
|
||||
let (first, second, delivery) = if hold_put {
|
||||
(delete, put, "DELETE then PUT")
|
||||
} else {
|
||||
(put, delete, "PUT then DELETE")
|
||||
};
|
||||
let order = format!("replicas receive {delivery}, value={value:?}");
|
||||
println!("Earlier PUT, later DELETE: {order}");
|
||||
wait_for_responses(&mut sys, &[first], 300).map_err(|err| format!("{order}: {err}"))?;
|
||||
for proc in &procs {
|
||||
sys.send_local_message(proc, Message::new("RELEASE", ""));
|
||||
}
|
||||
wait_for_responses(&mut sys, &[second], 300)
|
||||
.map_err(|err| format!("{order}: {err}"))?;
|
||||
finish_background_work(&mut sys, "Concurrent PUT before later DELETE")?;
|
||||
check_isolated_reads(&mut sys, &[(&key, None)])?;
|
||||
}
|
||||
}
|
||||
Ok(true)
|
||||
}
|
||||
@@ -0,0 +1,483 @@
|
||||
use std::collections::HashSet;
|
||||
|
||||
use rand::prelude::*;
|
||||
use rand_pcg::Pcg64;
|
||||
|
||||
use anysystem::logger::LogEntry;
|
||||
use anysystem::mc::{
|
||||
predicates::{collects, goals, prunes},
|
||||
strategies::bfs::Bfs,
|
||||
EventOrderingMode, InvariantFn, McState, McStats, McSystem, ModelChecker, StrategyConfig,
|
||||
};
|
||||
use anysystem::test::TestResult;
|
||||
use anysystem::{Message, System};
|
||||
|
||||
use crate::common::*;
|
||||
|
||||
pub fn test_mc_basic(config: &TestConfig) -> TestResult {
|
||||
let sys = build_system(config);
|
||||
let procs = sys.process_names();
|
||||
let mut rand = Pcg64::seed_from_u64(config.seed);
|
||||
|
||||
let mut mc = ModelChecker::new(&sys);
|
||||
|
||||
let key = random_string(8, &mut rand).to_uppercase();
|
||||
let replicas = key_replicas(&key, &sys);
|
||||
let non_replicas = key_non_replicas(&key, &sys);
|
||||
println!("Key {key} replicas: {replicas:?}");
|
||||
println!("Key {key} non-replicas: {non_replicas:?}");
|
||||
|
||||
// stage 1: get key from the first node
|
||||
let stage1_strategy = mc_query_strategy(&procs[0], McQuery::Get(key.clone(), None));
|
||||
let stage1_msg = Message::json(
|
||||
"GET",
|
||||
&GetReqMessage {
|
||||
key: &key,
|
||||
quorum: 2,
|
||||
},
|
||||
);
|
||||
let stage1_states = run_mc(
|
||||
&mut mc,
|
||||
stage1_strategy,
|
||||
&procs[0],
|
||||
stage1_msg,
|
||||
McNetworkChange::None,
|
||||
None,
|
||||
)?
|
||||
.collected_states;
|
||||
println!("stage 1: {}", stage1_states.len());
|
||||
if stage1_states.is_empty() {
|
||||
return Err("stage 1 - GET response is not received".to_owned());
|
||||
}
|
||||
|
||||
// stage 2: put key to the first replica
|
||||
let value = random_string(8, &mut rand);
|
||||
let sys = build_system(config);
|
||||
mc = ModelChecker::new(&sys);
|
||||
|
||||
let stage2_strategy = mc_query_strategy(&replicas[0], McQuery::Put(key.clone(), value.clone()));
|
||||
let stage2_msg = Message::json(
|
||||
"PUT",
|
||||
&PutReqMessage {
|
||||
key: &key,
|
||||
value: &value,
|
||||
quorum: 2,
|
||||
},
|
||||
);
|
||||
let stage2_states = run_mc(
|
||||
&mut mc,
|
||||
stage2_strategy,
|
||||
&replicas[0],
|
||||
stage2_msg,
|
||||
McNetworkChange::None,
|
||||
None,
|
||||
)?
|
||||
.collected_states;
|
||||
println!("stage 2: {}", stage2_states.len());
|
||||
if stage2_states.is_empty() {
|
||||
return Err("stage 2 - PUT response is not received".to_owned());
|
||||
}
|
||||
|
||||
// stage 3: get key from the last replica
|
||||
let stage3_strategy = mc_query_strategy(&replicas[2], McQuery::Get(key.clone(), Some(value)));
|
||||
let stage3_msg = Message::json(
|
||||
"GET",
|
||||
&GetReqMessage {
|
||||
key: &key,
|
||||
quorum: 2,
|
||||
},
|
||||
);
|
||||
let stage3_states = run_mc(
|
||||
&mut mc,
|
||||
stage3_strategy,
|
||||
&replicas[2],
|
||||
stage3_msg,
|
||||
McNetworkChange::None,
|
||||
Some(stage2_states),
|
||||
)?
|
||||
.collected_states;
|
||||
println!("stage 3: {}", stage3_states.len());
|
||||
if stage3_states.is_empty() {
|
||||
return Err("stage 3 - GET response is not received".to_owned());
|
||||
}
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
pub fn test_mc_concurrent_writes(config: &TestConfig) -> TestResult {
|
||||
let sys = build_system(config);
|
||||
let mut rand = Pcg64::seed_from_u64(config.seed);
|
||||
|
||||
let key = random_string(8, &mut rand).to_uppercase();
|
||||
let replicas = key_replicas(&key, &sys);
|
||||
let non_replicas = key_non_replicas(&key, &sys);
|
||||
println!("Key {key} replicas: {replicas:?}");
|
||||
println!("Key {key} non-replicas: {non_replicas:?}");
|
||||
|
||||
// isolate all processes
|
||||
for proc in sys.process_names() {
|
||||
sys.network().disconnect_node(&proc);
|
||||
}
|
||||
let mut mc = ModelChecker::new(&sys);
|
||||
|
||||
// put (key, value) to the first replica
|
||||
// and then put (key, value2) to the second replica
|
||||
let value = random_string(8, &mut rand);
|
||||
let value2 = random_string(8, &mut rand);
|
||||
|
||||
let strategy_config = mc_query_strategy(&replicas[0], McQuery::Put(key.clone(), value.clone()));
|
||||
let msg1 = Message::json(
|
||||
"PUT",
|
||||
&PutReqMessage {
|
||||
key: &key,
|
||||
value: &value,
|
||||
quorum: 1,
|
||||
},
|
||||
);
|
||||
let states = run_mc(
|
||||
&mut mc,
|
||||
strategy_config,
|
||||
&replicas[0],
|
||||
msg1,
|
||||
McNetworkChange::None,
|
||||
None,
|
||||
)?
|
||||
.collected_states;
|
||||
if states.is_empty() {
|
||||
return Err(format!("put({key}, {value}) response is not received"));
|
||||
}
|
||||
println!("put({key}, {value}): {} states collected", states.len());
|
||||
|
||||
let strategy_config =
|
||||
mc_query_strategy(&replicas[1], McQuery::Put(key.clone(), value2.clone()));
|
||||
let msg2 = Message::json(
|
||||
"PUT",
|
||||
&PutReqMessage {
|
||||
key: &key,
|
||||
value: &value2,
|
||||
quorum: 1,
|
||||
},
|
||||
);
|
||||
let states = run_mc(
|
||||
&mut mc,
|
||||
strategy_config,
|
||||
&replicas[1],
|
||||
msg2,
|
||||
McNetworkChange::None,
|
||||
Some(states),
|
||||
)?
|
||||
.collected_states;
|
||||
if states.is_empty() {
|
||||
return Err(format!("put({key}, {value2}) response is not received"));
|
||||
}
|
||||
println!("put({key}, {value2}): {} states collected", states.len());
|
||||
|
||||
// now reset the network state and ask the third replica about the key's value
|
||||
// we expect the value written later (value2) to be returned
|
||||
let strategy_config = mc_query_strategy(
|
||||
&replicas[2],
|
||||
McQuery::Get(key.to_string(), Some(value2.to_string())),
|
||||
);
|
||||
let msg = Message::json(
|
||||
"GET",
|
||||
&GetReqMessage {
|
||||
key: &key,
|
||||
quorum: 3,
|
||||
},
|
||||
);
|
||||
let states = run_mc(
|
||||
&mut mc,
|
||||
strategy_config,
|
||||
&replicas[2],
|
||||
msg,
|
||||
McNetworkChange::Reset,
|
||||
Some(states),
|
||||
)?
|
||||
.collected_states;
|
||||
if states.is_empty() {
|
||||
return Err(format!("get({key}) response is not received"));
|
||||
}
|
||||
println!("get({key}): {} states collected", states.len());
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
pub fn test_mc_sloppy_quorum_hinted_handoff(config: &TestConfig) -> TestResult {
|
||||
let mut sys = build_system(config);
|
||||
let mut rand = Pcg64::seed_from_u64(config.seed);
|
||||
|
||||
let mut mc = ModelChecker::new(&sys);
|
||||
|
||||
let key = random_string(8, &mut rand).to_uppercase();
|
||||
let replicas = key_replicas(&key, &sys);
|
||||
let non_replicas = key_non_replicas(&key, &sys);
|
||||
println!("Key {key} replicas: {replicas:?}");
|
||||
println!("Key {key} non-replicas: {non_replicas:?}");
|
||||
|
||||
// stage 1: get key from the first replica (during the network partition)
|
||||
let stage1_strategy = mc_query_strategy(&replicas[0], McQuery::Get(key.clone(), None));
|
||||
let stage1_msg = Message::json(
|
||||
"GET",
|
||||
&GetReqMessage {
|
||||
key: &key,
|
||||
quorum: 2,
|
||||
},
|
||||
);
|
||||
let stage1_states = run_mc(
|
||||
&mut mc,
|
||||
stage1_strategy,
|
||||
&replicas[0],
|
||||
stage1_msg,
|
||||
McNetworkChange::Partition([
|
||||
vec![
|
||||
replicas[0].clone(),
|
||||
non_replicas[0].clone(),
|
||||
non_replicas[1].clone(),
|
||||
non_replicas[2].clone(),
|
||||
],
|
||||
vec![replicas[1].clone(), replicas[2].clone()],
|
||||
]),
|
||||
None,
|
||||
)?
|
||||
.collected_states;
|
||||
println!("stage 1: {}", stage1_states.len());
|
||||
if stage1_states.is_empty() {
|
||||
return Err("stage 1 - GET response is not received".to_owned());
|
||||
}
|
||||
|
||||
// stage 2: put key from the first replica (network partition still exists)
|
||||
let value = random_string(8, &mut rand);
|
||||
sys = build_system(config);
|
||||
mc = ModelChecker::new(&sys);
|
||||
|
||||
let stage2_strategy = mc_query_strategy(&replicas[0], McQuery::Put(key.clone(), value.clone()));
|
||||
let stage2_msg = Message::json(
|
||||
"PUT",
|
||||
&PutReqMessage {
|
||||
key: &key,
|
||||
value: &value,
|
||||
quorum: 2,
|
||||
},
|
||||
);
|
||||
let stage2_states = run_mc(
|
||||
&mut mc,
|
||||
stage2_strategy,
|
||||
&replicas[0],
|
||||
stage2_msg,
|
||||
McNetworkChange::Partition([
|
||||
vec![
|
||||
replicas[0].clone(),
|
||||
non_replicas[0].clone(),
|
||||
non_replicas[1].clone(),
|
||||
non_replicas[2].clone(),
|
||||
],
|
||||
vec![replicas[1].clone(), replicas[2].clone()],
|
||||
]),
|
||||
None,
|
||||
)?
|
||||
.collected_states;
|
||||
println!("stage 2: {}", stage2_states.len());
|
||||
if stage2_states.is_empty() {
|
||||
return Err("stage 2 - PUT response is not received".to_owned());
|
||||
}
|
||||
|
||||
// stage 3: recover network and let data propagate to all replicas
|
||||
let stage3_states = mc_stabilize(&mut sys, stage2_states)?.collected_states;
|
||||
println!("stage 3: {}", stage3_states.len());
|
||||
if stage3_states.is_empty() {
|
||||
return Err(
|
||||
"stage 3 - no states found during the exploration phase with recovered network"
|
||||
.to_owned(),
|
||||
);
|
||||
}
|
||||
|
||||
// stage 4: get key from the last replica (again during the network partition)
|
||||
let stage4_strategy = mc_query_strategy(&replicas[2], McQuery::Get(key.clone(), Some(value)));
|
||||
let stage4_msg = Message::json(
|
||||
"GET",
|
||||
&GetReqMessage {
|
||||
key: &key,
|
||||
quorum: 2,
|
||||
},
|
||||
);
|
||||
let stage4_states = run_mc(
|
||||
&mut mc,
|
||||
stage4_strategy,
|
||||
&replicas[2],
|
||||
stage4_msg,
|
||||
McNetworkChange::Partition([
|
||||
vec![
|
||||
replicas[0].clone(),
|
||||
non_replicas[0].clone(),
|
||||
non_replicas[1].clone(),
|
||||
non_replicas[2].clone(),
|
||||
],
|
||||
vec![replicas[1].clone(), replicas[2].clone()],
|
||||
]),
|
||||
Some(stage3_states),
|
||||
)?
|
||||
.collected_states;
|
||||
println!("stage 4: {}", stage4_states.len());
|
||||
if stage4_states.is_empty() {
|
||||
return Err("stage 4 - GET response is not received".to_owned());
|
||||
}
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
// UTILS ---------------------------------------------------------------------------------------------------------------
|
||||
|
||||
enum McQuery {
|
||||
Get(String, Option<String>),
|
||||
Put(String, String),
|
||||
}
|
||||
|
||||
fn mc_query_strategy(proc: &str, query_data: McQuery) -> StrategyConfig {
|
||||
let proc_name = proc.to_string();
|
||||
|
||||
let invariant = match query_data {
|
||||
McQuery::Get(key, expected) => mc_get_invariant(proc, key, expected),
|
||||
McQuery::Put(key, value) => mc_put_invariant(proc, key, value),
|
||||
};
|
||||
|
||||
StrategyConfig::default()
|
||||
.prune(prunes::any_prune(vec![
|
||||
prunes::event_happened_n_times_current_run(LogEntry::is_mc_timer_fired, 5_usize),
|
||||
prunes::event_happened_n_times_current_run(LogEntry::is_mc_message_received, 10_usize),
|
||||
]))
|
||||
.goal(goals::event_happened_n_times_current_run(
|
||||
LogEntry::is_mc_local_message_sent,
|
||||
1,
|
||||
))
|
||||
.invariant(invariant)
|
||||
.collect(collects::event_happened_n_times_current_run(
|
||||
move |log_entry| match log_entry {
|
||||
LogEntry::McLocalMessageSent { proc, .. } => proc == &proc_name,
|
||||
_ => false,
|
||||
},
|
||||
1,
|
||||
))
|
||||
}
|
||||
|
||||
fn mc_get_invariant<S>(proc: S, key: String, expected: Option<String>) -> InvariantFn
|
||||
where
|
||||
S: Into<String>,
|
||||
{
|
||||
let proc_name = proc.into();
|
||||
Box::new(move |state: &McState| -> Result<(), String> {
|
||||
for entry in state.current_run_trace().iter() {
|
||||
if let LogEntry::McLocalMessageSent { msg, proc } = entry {
|
||||
if &proc_name != proc {
|
||||
return Err("local message received on wrong process".to_string());
|
||||
}
|
||||
if msg.tip != "GET_RESP" {
|
||||
return Err(format!("wrong type {}", msg.tip));
|
||||
}
|
||||
let data = parse_value_response(msg, proc)?;
|
||||
if data.key != key {
|
||||
return Err(format!("wrong key {}", data.key));
|
||||
}
|
||||
if data.value != expected {
|
||||
return Err(format!("wrong value {:?}", data.value));
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
|
||||
fn mc_put_invariant<S>(proc: S, key: String, value: String) -> InvariantFn
|
||||
where
|
||||
S: Into<String>,
|
||||
{
|
||||
let proc_name = proc.into();
|
||||
Box::new(move |state: &McState| -> Result<(), String> {
|
||||
for entry in state.current_run_trace().iter() {
|
||||
if let LogEntry::McLocalMessageSent { msg, proc } = entry {
|
||||
if &proc_name != proc {
|
||||
return Err("local message received on wrong process".to_string());
|
||||
}
|
||||
if msg.tip != "PUT_RESP" {
|
||||
return Err(format!("wrong type {}", msg.tip));
|
||||
}
|
||||
let data = parse_value_response(msg, proc)?;
|
||||
if data.key != key {
|
||||
return Err(format!("wrong key {}", data.key));
|
||||
}
|
||||
if data.value.as_deref() != Some(value.as_str()) {
|
||||
return Err(format!("wrong value {:?}", data.value));
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
|
||||
enum McNetworkChange {
|
||||
None,
|
||||
Reset,
|
||||
Partition([Vec<String>; 2]),
|
||||
}
|
||||
|
||||
fn run_mc<S>(
|
||||
mc: &mut ModelChecker,
|
||||
strategy_config: StrategyConfig,
|
||||
proc: S,
|
||||
msg: Message,
|
||||
network_change: McNetworkChange,
|
||||
states: Option<HashSet<McState>>,
|
||||
) -> Result<McStats, String>
|
||||
where
|
||||
S: Into<String>,
|
||||
{
|
||||
let proc = proc.into();
|
||||
|
||||
let callback = |sys: &mut McSystem| {
|
||||
match &network_change {
|
||||
McNetworkChange::Partition([part1, part2]) => sys.network().partition(part1, part2),
|
||||
McNetworkChange::Reset => sys.network().reset(),
|
||||
McNetworkChange::None => {}
|
||||
}
|
||||
sys.set_event_ordering_mode(EventOrderingMode::MessagesFirst);
|
||||
sys.send_local_message(proc.clone(), proc.clone(), msg.clone());
|
||||
};
|
||||
|
||||
let res = if let Some(states) = states {
|
||||
mc.run_from_states_with_change::<Bfs>(strategy_config, states, callback)
|
||||
} else {
|
||||
mc.run_with_change::<Bfs>(strategy_config, callback)
|
||||
};
|
||||
match res {
|
||||
Err(e) => {
|
||||
e.print_trace();
|
||||
Err(e.message())
|
||||
}
|
||||
Ok(stats) => Ok(stats),
|
||||
}
|
||||
}
|
||||
|
||||
fn mc_stabilize(sys: &mut System, states: HashSet<McState>) -> Result<McStats, String> {
|
||||
let strategy_config = StrategyConfig::default()
|
||||
.prune(prunes::any_prune(vec![
|
||||
prunes::event_happened_n_times_current_run(LogEntry::is_mc_timer_fired, 6),
|
||||
prunes::event_happened_n_times_current_run(LogEntry::is_mc_message_received, 24),
|
||||
]))
|
||||
.goal(goals::any_goal(vec![
|
||||
goals::depth_reached(30),
|
||||
goals::no_events(),
|
||||
]))
|
||||
.collect(collects::any_collect(vec![
|
||||
collects::no_events(),
|
||||
collects::event_happened_n_times_current_run(LogEntry::is_mc_timer_fired, 6),
|
||||
collects::event_happened_n_times_current_run(LogEntry::is_mc_message_received, 24),
|
||||
]));
|
||||
let mut mc = ModelChecker::new(sys);
|
||||
let res = mc.run_from_states_with_change::<Bfs>(strategy_config, states, |sys| {
|
||||
sys.network().reset();
|
||||
sys.set_event_ordering_mode(EventOrderingMode::MessagesFirst);
|
||||
});
|
||||
match res {
|
||||
Err(e) => {
|
||||
e.print_trace();
|
||||
Err(e.message())
|
||||
}
|
||||
Ok(stats) => Ok(stats),
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user