172 lines
4.9 KiB
Rust
172 lines
4.9 KiB
Rust
use indexmap::IndexMap;
|
|
|
|
use anysystem::test::TestResult;
|
|
|
|
pub type Results = IndexMap<&'static str, TestResult>;
|
|
|
|
// Each item is worth half a point. Paired tests check the same skill.
|
|
pub const CORE: &[(&str, &[&[&str]])] = &[
|
|
(
|
|
"Operations and placement",
|
|
&[
|
|
&["BASIC"],
|
|
&["MC BASIC"],
|
|
&["REPLICAS CHECK"],
|
|
&["DATA PLACEMENT"],
|
|
&["CONCURRENT GET PUT", "CONCURRENT GETS"],
|
|
&["DELETE ABSENT"],
|
|
],
|
|
),
|
|
(
|
|
"Conflicts and repair",
|
|
&[
|
|
&["CONCURRENT WRITES", "MC CONCURRENT WRITES"],
|
|
&["CONCURRENT WRITES TIE"],
|
|
&["STALE REPLICA", "DIVERGED REPLICAS"],
|
|
&["STALE REPLICA DELETE", "READ REPAIR DELETE"],
|
|
&["CONCURRENT DELETES"],
|
|
],
|
|
),
|
|
(
|
|
"Quorums and partitions",
|
|
&[
|
|
&["SLOPPY QUORUM READ", "SLOPPY QUORUM WRITE"],
|
|
&["SLOPPY QUORUM TRICKY"],
|
|
&["SLOPPY QUORUM DELETE"],
|
|
&["PARTITION CLIENTS", "PARTITION MIXED"],
|
|
&["HINTED HANDOFF", "MC SLOPPY QUORUM HINTED HANDOFF"],
|
|
],
|
|
),
|
|
];
|
|
|
|
pub const TRACKS: &[(&str, &[&str])] = &[
|
|
(
|
|
"A",
|
|
&[
|
|
"CONCURRENT PUT DELETE TIE",
|
|
"READ REPAIR PUT DELETE TIE",
|
|
"READ REPAIR LATE",
|
|
"READ REPAIR TIE",
|
|
"READ REPAIR MULTI KEY",
|
|
"DATA PLACEMENT MULTI KEY",
|
|
],
|
|
),
|
|
(
|
|
"B",
|
|
&[
|
|
"SLOPPY QUORUM GET DISTINCT",
|
|
"SLOPPY QUORUM PUT DISTINCT",
|
|
"SLOPPY QUORUM DELETE DISTINCT",
|
|
"SLOPPY QUORUM GET ORDER",
|
|
"SLOPPY QUORUM PUT ORDER",
|
|
"SLOPPY QUORUM DELETE ORDER",
|
|
"SLOPPY QUORUM DELETE ORDER PARTITION",
|
|
"SLOPPY QUORUM DELETE PARTITION",
|
|
],
|
|
),
|
|
(
|
|
"C",
|
|
&[
|
|
"HINTED HANDOFF PUT DELETE TIE",
|
|
"HINTED HANDOFF MULTI KEY",
|
|
"HINTED HANDOFF RETENTION",
|
|
"HINTED HANDOFF LWW",
|
|
"SLOPPY QUORUM READ REPAIR",
|
|
"SLOPPY QUORUM READ REPAIR DELETE",
|
|
"SLOPPY QUORUM READ REPAIR MULTI KEY",
|
|
],
|
|
),
|
|
];
|
|
|
|
pub fn all_passed(results: &Results, names: &[&str]) -> bool {
|
|
names
|
|
.iter()
|
|
.all(|name| results.get(name).is_some_and(Result::is_ok))
|
|
}
|
|
|
|
pub fn core_points(results: &Results, items: &[&[&str]]) -> f32 {
|
|
items
|
|
.iter()
|
|
.filter(|item| all_passed(results, item))
|
|
.count() as f32
|
|
/ 2.0
|
|
}
|
|
|
|
pub fn score(core: f32, results: &Results) -> f32 {
|
|
if core < 8.0 {
|
|
return core;
|
|
}
|
|
let complete = TRACKS
|
|
.iter()
|
|
.filter(|(_, names)| all_passed(results, names))
|
|
.count();
|
|
core + complete.min(2) as f32
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
use std::collections::HashSet;
|
|
|
|
#[test]
|
|
fn rubric_covers_each_case_once() {
|
|
let mut names = HashSet::new();
|
|
for &(_, items) in CORE {
|
|
for item in items {
|
|
for &name in *item {
|
|
assert!(names.insert(name), "{name}");
|
|
}
|
|
}
|
|
}
|
|
assert_eq!(names.len(), 23);
|
|
for &(_, track) in TRACKS {
|
|
for &name in track {
|
|
assert!(names.insert(name), "{name}");
|
|
}
|
|
}
|
|
assert_eq!(names.len(), 44);
|
|
assert_eq!(names, crate::CASES.iter().map(|&(name, _)| name).collect());
|
|
}
|
|
|
|
#[test]
|
|
fn independent_core_credits_and_advanced_thresholds() {
|
|
let reference: Results = crate::CASES
|
|
.iter()
|
|
.map(|&(name, _)| (name, Ok(true)))
|
|
.collect();
|
|
let total = |results: &Results| {
|
|
score(
|
|
CORE.iter()
|
|
.map(|(_, items)| core_points(results, items))
|
|
.sum(),
|
|
results,
|
|
)
|
|
};
|
|
for (i, &(_, items)) in CORE.iter().enumerate() {
|
|
let mut results = reference.clone();
|
|
for item in items {
|
|
// A failure on either half of a pair loses only this item's credit.
|
|
for &name in *item {
|
|
results.insert(name, Err(String::new()));
|
|
assert_eq!(total(&results), 7.5);
|
|
results.insert(name, Ok(true));
|
|
}
|
|
}
|
|
for item in items {
|
|
results.insert(item[0], Err(String::new()));
|
|
}
|
|
assert_eq!(total(&results), [5.0, 5.5, 5.5][i]);
|
|
}
|
|
for mask in 0_u32..8 {
|
|
let mut results = reference.clone();
|
|
for (i, &(_, names)) in TRACKS.iter().enumerate() {
|
|
if mask & (1 << i) == 0 {
|
|
results.shift_remove(names[0]);
|
|
}
|
|
}
|
|
assert_eq!(total(&results), 8.0 + mask.count_ones().min(2) as f32);
|
|
}
|
|
assert_eq!(total(&Results::new()), 0.0);
|
|
}
|
|
}
|