From 3030f979d1aa56292c7bd19720f9f4097affd7dc Mon Sep 17 00:00:00 2001 From: Oleg Sukhoroslov Date: Sun, 15 Nov 2026 10:29:10 +0300 Subject: [PATCH] Add HW 8 --- homework/08-kv-replication/readme.md | 108 + homework/08-kv-replication/solution/node.py | 91 + .../08-kv-replication/tests/.dockerignore | 5 + homework/08-kv-replication/tests/Cargo.lock | 1027 ++++++++ homework/08-kv-replication/tests/Cargo.toml | 21 + homework/08-kv-replication/tests/Dockerfile | 19 + .../08-kv-replication/tests/python_errors.py | 38 + .../tests/shared_state_validator.py | 1938 ++++++++++++++ .../08-kv-replication/tests/src/common.rs | 139 + .../08-kv-replication/tests/src/count_keys.rs | 80 + .../08-kv-replication/tests/src/grading.rs | 171 ++ homework/08-kv-replication/tests/src/main.rs | 252 ++ .../tests/src/python_errors.rs | 111 + .../tests/src/shared_state.rs | 76 + homework/08-kv-replication/tests/src/tests.rs | 2323 +++++++++++++++++ .../tests/src/tests/ordered_delivery.rs | 100 + .../08-kv-replication/tests/src/tests_mc.rs | 483 ++++ 17 files changed, 6982 insertions(+) create mode 100644 homework/08-kv-replication/readme.md create mode 100644 homework/08-kv-replication/solution/node.py create mode 100644 homework/08-kv-replication/tests/.dockerignore create mode 100644 homework/08-kv-replication/tests/Cargo.lock create mode 100644 homework/08-kv-replication/tests/Cargo.toml create mode 100644 homework/08-kv-replication/tests/Dockerfile create mode 100644 homework/08-kv-replication/tests/python_errors.py create mode 100644 homework/08-kv-replication/tests/shared_state_validator.py create mode 100644 homework/08-kv-replication/tests/src/common.rs create mode 100644 homework/08-kv-replication/tests/src/count_keys.rs create mode 100644 homework/08-kv-replication/tests/src/grading.rs create mode 100644 homework/08-kv-replication/tests/src/main.rs create mode 100644 homework/08-kv-replication/tests/src/python_errors.rs create mode 100644 homework/08-kv-replication/tests/src/shared_state.rs create mode 100644 homework/08-kv-replication/tests/src/tests.rs create mode 100644 homework/08-kv-replication/tests/src/tests/ordered_delivery.rs create mode 100644 homework/08-kv-replication/tests/src/tests_mc.rs diff --git a/homework/08-kv-replication/readme.md b/homework/08-kv-replication/readme.md new file mode 100644 index 0000000..e597a23 --- /dev/null +++ b/homework/08-kv-replication/readme.md @@ -0,0 +1,108 @@ +# Распределенное key-value хранилище с репликацией + +В этом задании вам предстоит усовершенствовать key-value хранилище из предыдущего ДЗ, обеспечив отказоустойчивое хранение данных путем их репликации. Теперь каждая запись должна храниться не на одном узле, а на трех. Если все три реплики уже сохранили запись, она переживёт отказ любых двух узлов. При этом остается шардинг данных, то есть в системе из 6 узлов каждый будет хранить примерно половину данных. Чтобы не создавать зависимость от предыдущего ДЗ, будем использовать простейший (и не самый эффективный) способ шардинга данных и обойдемся без перебалансировки шардов. Тем самым, сконцентрируемся теперь на репликации данных и связанных с ней проблемах. + +Пусть в нашей системе есть _N_ узлов с идентификаторами _[0, N)_. Тогда за хранение ключа _K_ будет отвечать узел с идентификатором _P_ = _hash(K) mod N_. Так как мы хотим реплицировать данные на трех узлах, то добавим к этому узлу еще два узла, следующие за ним в порядке их идентификаторов. Например, если _N_=6 и для некоторого ключа _P_=4, то за хранение этого ключа будут отвечать узлы _{4, 5, 0}_. Далее будем называть эти узлы и хранимые ими копии данных _репликами_. В исправной сети после завершения репликации запись должна оставаться только на этих трёх узлах, даже если запрос принял другой узел. + +Для реализации репликации будем использовать подход без лидера, описанный в [статье про Amazon Dynamo](https://www.allthingsdistributed.com/files/amazon-dynamo-sosp2007.pdf). Операции на чтение (GET) и запись (PUT, DELETE) выполняются с _кворумами_, размеры которых указывает клиент при отправке операции. Клиент может отправить локальный запрос любому узлу. Узел, принявший запрос, называется *координатором этого запроса*. Он рассылает операцию по репликам и возвращает результат клиенту, когда собран кворум. В кворуме учитываются разные узлы: повторные ответы одного узла не увеличивают кворум, даже если он заменяет несколько основных реплик. + +Мы хотим сделать наше хранилище высокодоступным и уметь успешно обрабатывать запросы в присутствии отказов узлов и сети. Если доступных основных реплик не хватает для кворума, координатор использует другие доступные узлы как резервные реплики. В Dynamo такой подход называется *sloppy quorum*. В нашем хранилище координатор выбирает резервные узлы по кольцу, начиная с узла, следующего за последней основной репликой. Например, пусть основные реплики ключа - узлы *{2, 3, 4}*, а для операции нужен кворум из трёх узлов. Если одна из реплик недоступна, координатор обращается к узлу *5*. Если он тоже недоступен, обход продолжается по возрастанию числовых идентификаторов. После последнего узла следует узел *0*. + +Если при PUT или DELETE резервная реплика замещает недоступную основную, она временно хранит изменение. Когда связь восстанавливается, резервная реплика передаёт изменение основной, даже если клиент больше не обращается к ключу. Такая передача называется *hinted handoff*. После успешной передачи всем основным репликам, которые она замещала для этого ключа, резервная реплика удаляет временную копию. Ключи, для которых этот узел сам служит основной репликой, остаются на нём. + +Узлы и сеть могут отказывать во время обработки запросов. Обнаруживать отказы можно с помощью таймаутов. Для простоты будем считать, что отказы временные, и в случае обнаружения отказа не требуется делать перебалансировку шардов или переназначать реплики между узлами. Кроме того, вашей реализации не требуется поддерживать добавление и удаление узлов из системы, в тестах состав узлов фиксирован. + +В силу описанного подхода, операции записи в нашей системе могут возвращать клиенту успех до того, как изменения достигнут всех реплик, а чтение может возвращать устаревшее значение. При этом система должна обеспечивать _согласованность в конечном счёте_ (eventual consistency) - изменения должны рано или поздно оказаться на всех живых репликах. Для этого реализуйте *hinted handoff* и *read repair*. Фоновая синхронизация реплик (*anti-entropy*) - по желанию. + +При GET координатор может получить от реплик разные версии одного ключа. Тогда он выбирает актуальную версию по правилам ниже и передаёт её отставшим репликам. Такое восстановление по результатам чтения называется *read repair*. Клиент получает ответ, как только собран кворум. Если более свежая версия пришла позже, координатор учитывает её при восстановлении. В направлении A проверяется восстановление основных реплик по позднему ответу, а в направлении C - восстановление данных на резервных репликах и их последующая передача основным через *hinted handoff*. + +Из-за отказов и конкурентных операций записи на разных репликах могут оказаться разные версии одного ключа. При записи и чтении разрешайте такие конфликты автоматически по правилу *last write wins (LWW)*. Из двух версий с разными метками времени выбирайте более позднюю. Метка времени операции PUT или DELETE - значение `ctx.time()` в момент получения локального запроса. Сохраняйте эту метку при репликации, read repair и hinted handoff. + +При одинаковом времени PUT побеждает DELETE, а из нескольких PUT выбирается лексикографически большее значение. Это правило относится и к пустой строке `""`: она считается сохранённым значением и отличается от отсутствия записи (`null`). Две операции DELETE с одинаковым временем равнозначны. Применяйте эти правила при обработке операций, чтении и синхронизации реплик. + +Чтобы при синхронизации не восстановить старое значение, при удалении ключа реплики сохраняют *маркер удаления* - запись о том, какой ключ удалён и в какой момент. Допустим, реплика пропустила DELETE в момент 20 и вернулась в сеть со значением `"abc"`, записанным в момент 10. Получив маркер, она сравнит метки времени 20 и 10, увидит, что удаление произошло позже записи, и заменит `"abc"` маркером. + +Чтобы метки времени отражали порядок записей на разных узлах, их часы должны быть синхронизированы. В тестах это условие выполняется, поэтому для определения времени записи можно использовать `ctx.time()`. Однако даже при синхронизированных часах LWW сохраняет только одну из конкурирующих версий ключа, поэтому изменения другого клиента могут потеряться. Мы устраним эти ограничения в следующем задании (**Внимание! Следующее ДЗ будет требовать наличия готового решения текущего задания**). + +Если вам плохо понятны некоторые требования, изучите соответствующие тесты - это часть условия задания. + +## Реализация + +Для реализации и тестирования решения используется фреймворк AnySystem (см. материалы первого семинара). В папке `solution` размещена заготовка для решения [node.py](solution/node.py). Вам надо доработать реализацию класса `StorageNode` так, чтобы проходили тесты выбранного уровня. + +При инициализации узлу передается его уникальный id, а также список id всех узлов в системе. + +Узел должен поддерживать обработку следующих локальных сообщений (форматы запросов и ответов описаны в заготовке): +- _GET(key, quorum)_ - вернуть значение записи с ключом `key` или `null`, если запись отсутствует, +- _PUT(key, value, quorum)_ - сохранить строку `value` по ключу `key`. В ответе верните ключ и значение версии, победившей по LWW среди ответов кворума. Если победил конкурентный DELETE, верните `null` в поле `value`, +- _DELETE(key, quorum)_ - удалить запись с ключом `key` и вернуть значение, которое кворум наблюдал до удаления, или `null`, если ключ отсутствовал. + +Для GET, PUT и DELETE клиент указывает размер кворума. Если один узел одновременно принимает два GET одного ключа, клиент должен получить ответ на каждый запрос. + +Для проверки размещения данных нужно поддержать локальный запрос `COUNT_KEYS {}` с ответом `COUNT_KEYS_RESP {"count": N}`. Он возвращает число различных ключей с сохранёнными на узле значениями, включая временные копии и пустые строки. Маркеры удаления и служебные метаданные не учитываются; один ключ считается один раз. Запрос не должен обращаться к другим узлам или изменять состояние хранилища. + +Для вычисления реплик по ключу используйте функцию `get_key_replicas()` из заготовки (использовать другую функцию не следует, так как это влияет на тесты). + +Для взаимодействия между узлами вы можете использовать любые собственные типы сообщений. + +Ваши реализации процессов AnySystem должны соблюдать правило [изоляции процессов](../readme.md#изоляция-процессов-anysystem): изменяемое состояние каждого процесса должно храниться только в `self`, а общая изменяемая память между процессами запрещена. Все взаимодействия между процессами должны происходить через сообщения, как и в реальной распределённой системе. + +## Тестирование + +### Локальное тестирование + +Тесты находятся в папке `tests`. Есть два варианта их запуска. + +Рекомендуем запускать тесты в готовом Docker-образе. Чтобы проверить решение с теми же параметрами, что и в тестирующей системе, выполните команду из папки задания. + +```commandline +docker run --pull always --rm -t -v ./solution:/solution distsys.ru/course/kv-replication:latest +``` + +При обычном запуске тестер выводит название и результат каждого сценария, а в конце - число пройденных тестов и итоговый балл. Чтобы запустить один сценарий и увидеть трассу событий, добавьте в конец команды `-t "REPLICAS CHECK" -d`. Список всех опций можно посмотреть с помощью `-h`. + +Вы также можете запустить тесты, скомпилировав их локально с помощью компилятора Rust. Такой вариант может быть удобен, если вы хотите лучше изучить или доработать тесты. Скомпилируйте тесты с помощью команды `cargo install --locked --path tests`. Для запуска тестов выполните команду: + +```commandline +distsys-kv-replication [ЗДЕСЬ МОЖНО УКАЗАТЬ ОПЦИИ] +``` + +Тестер сначала проверяет основную часть. Если набраны все 8 баллов, он автоматически запускает три повышенных направления. Чтобы выбрать направления для локального запуска и сдачи на сервер, укажите их латинские буквы через пробел в файле `solution/tracks.txt`, например `A C`. Пустой файл оставляет только основную часть. Для локальной проверки есть также флаги `--core` (только основная часть) и `--all` (все 44 теста независимо от набранных баллов и выбора направлений). + +На весь прогон в Docker отводится 21 минута. Если решение зависнет, по истечении этого времени проверка завершится без итоговой сводки. Последний запущенный тест можно найти по его названию в выводе и повторить отдельно с опцией `-t`. + +Бонусы за пробелы в тестах начисляются по [общим правилам](../readme.md#бонусы-за-пробелы-в-тестах). + +### Проверка в тестирующей системе + +Отправьте ваше решение в тестирующую систему следуя [инструкции](../readme.md) и дождитесь результатов. + +## Оценивание + +За основную часть можно получить до 8 баллов. Баллы трёх блоков складываются независимо. В каждой строке таблицы перечислены пункты по 0,5 балла. Если в пункте два теста, нужно пройти оба. + +| Блок | Максимум | Пункты по 0,5 балла | +|---|---:|---| +| Базовые операции и размещение | 3 | `BASIC` · `MC BASIC` · `REPLICAS CHECK` · `DATA PLACEMENT` · `CONCURRENT GET PUT` и `CONCURRENT GETS` · `DELETE ABSENT` | +| Конфликты и восстановление | 2,5 | `CONCURRENT WRITES` и `MC CONCURRENT WRITES` · `CONCURRENT WRITES TIE` · `STALE REPLICA` и `DIVERGED REPLICAS` · `STALE REPLICA DELETE` и `READ REPAIR DELETE` · `CONCURRENT DELETES` | +| Кворумы и разделение сети | 2,5 | `SLOPPY QUORUM READ` и `SLOPPY QUORUM WRITE` · `SLOPPY QUORUM TRICKY` · `SLOPPY QUORUM DELETE` · `PARTITION CLIENTS` и `PARTITION MIXED` · `HINTED HANDOFF` и `MC SLOPPY QUORUM HINTED HANDOFF` | + +Когда пройдены все 23 теста основной части, можно получить **9 баллов** за любое одно полностью пройденное направление или **10 баллов** за любые два. + +| Направление | Ожидаемый результат | Тесты | +|---|---|---| +| A. Конфликты и восстановление нескольких ключей | При одновременных PUT и DELETE одного ключа с одинаковой меткой времени побеждает PUT, в том числе с пустой строкой. Если метка DELETE больше, ответ на PUT зависит от того, успел ли его кворум увидеть удаление. Если более свежая версия обнаружилась уже после ответа на GET, основные реплики в итоге сходятся к этой версии. Обновление или удаление одного ключа сохраняет значения остальных. | `CONCURRENT PUT DELETE TIE`, `READ REPAIR PUT DELETE TIE`, `READ REPAIR LATE`, `READ REPAIR TIE`, `READ REPAIR MULTI KEY`, `DATA PLACEMENT MULTI KEY` | +| B. Кворум при дублях и разделении сети | Недоступные основные реплики замещают первые доступные узлы по числовому порядку кольца. Для ответа клиенту нужен кворум разных узлов даже при дублировании сообщений и участии самого координатора. Например, если все узлы будущего кворума хранят `"abc"`, первый DELETE возвращает `"abc"` даже при повторной доставке внутренних сообщений. Следующий DELETE без промежуточной записи возвращает `null`. Пока доступных узлов недостаточно для кворума, запрос остаётся в ожидании. Когда кворум снова достижим, координатор завершает тот же запрос без повторного обращения клиента. | Тесты с префиксом `SLOPPY QUORUM` и окончаниями `GET DISTINCT`, `PUT DISTINCT`, `DELETE DISTINCT`, `GET ORDER`, `PUT ORDER`, `DELETE ORDER`, `DELETE ORDER PARTITION`, `DELETE PARTITION` | +| C. Восстановление через резервные реплики | Если при GET обнаружилась устаревшая копия на резервной реплике, в итоге на ней оказывается актуальное значение или маркер удаления. После восстановления связи эту версию получают все основные реплики, которые она замещала для данного ключа. Временная копия сохраняется до передачи последнему адресату, затем исчезает. Ключи, для которых этот узел сам служит основной репликой, остаются на нём. | `HINTED HANDOFF PUT DELETE TIE`, `HINTED HANDOFF MULTI KEY`, `HINTED HANDOFF RETENTION`, `HINTED HANDOFF LWW`, `SLOPPY QUORUM READ REPAIR`, `SLOPPY QUORUM READ REPAIR DELETE`, `SLOPPY QUORUM READ REPAIR MULTI KEY` | + +К решению приложите отчёт с описанием реализации в файле `solution/readme.md`. **Без отчёта решение не засчитывается.** + +## Рекомендации + +1. **Репликация в исправной сети.** Отправляйте запрос трём основным репликам и отвечайте клиенту, когда собран кворум. Учитывайте каждый клиентский запрос отдельно. На одном узле могут одновременно выполняться два `GET` одного ключа. В тесте `REPLICAS CHECK` после записи с кворумом 3 каждая реплика должна вернуть значение, даже когда её отключили от сети. +2. **Разрешение конфликтов.** Применяйте одно правило LWW при записи на реплику и при выборе значения из ответов на чтение. В `CONCURRENT WRITES TIE` две записи получают одинаковую метку времени. Проверьте, что в итоге побеждает лексикографически большее значение при любом порядке их доставки. +3. **Read repair.** При чтении сравнивайте версии и отправляйте свежую запись отставшим репликам. В `STALE REPLICA` одна реплика пропускает запись, возвращается в сеть и после кворумного чтения должна выдать новое значение самостоятельно. Клиенту отвечайте сразу по достижении кворума. В направлении A тест `READ REPAIR LATE` проверяет, что поздний ответ реплики тоже учитывается при восстановлении. +4. **Sloppy quorum.** Если по тайм-ауту ответов основных реплик не хватает для кворума, продолжайте обход кольца и обращайтесь к резервным узлам. В `SLOPPY QUORUM READ` чтение с кворумом 2 должно завершиться, хотя доступна только одна основная реплика. +5. **Hinted handoff.** Когда основная реплика снова станет доступна, передайте ей данные с резервной. В тесте `HINTED HANDOFF` после успешной передачи значение должно появиться на вернувшейся реплике, а временная копия - исчезнуть. + +Запустите тестер с `--core`. Когда пройдёте все тесты основной части, выберите повышенные направления в разделе [«Оценивание»](#оценивание). diff --git a/homework/08-kv-replication/solution/node.py b/homework/08-kv-replication/solution/node.py new file mode 100644 index 0000000..d7565d4 --- /dev/null +++ b/homework/08-kv-replication/solution/node.py @@ -0,0 +1,91 @@ +import hashlib +from anysystem import Context, Message, Process +from typing import List + + +class StorageNode(Process): + def __init__(self, node_id: str, nodes: List[str]): + self._id = node_id + self._nodes = nodes + self._data = {} + + def on_local_message(self, msg: Message, ctx: Context): + # Get key value. + # Request: + # GET {"key": "some key", "quorum": 1-3} + # Response: + # GET_RESP {"key": "some key", "value": "value for this key"} + # GET_RESP {"key": "some key", "value": null} - if record for this key is not found + if msg.type == 'GET': + key = msg['key'] + print("[py] Key", key, "replicas:", get_key_replicas(key, len(self._nodes))) + value = self._data.get(key) + resp = Message('GET_RESP', { + 'key': key, + 'value': value + }) + ctx.send_local(resp) + + # Store (key, value) record + # Request: + # PUT {"key": "some key", "value": "some value", "quorum": 1-3} + # Response: + # PUT_RESP {"key": "some key", "value": "some value"} + # PUT_RESP {"key": "some key", "value": null} - if a newer DELETE wins in the quorum + elif msg.type == 'PUT': + key = msg['key'] + value = msg['value'] + self._data[key] = value + resp = Message('PUT_RESP', { + 'key': key, + 'value': value + }) + ctx.send_local(resp) + + # Delete value for the key + # Request: + # DELETE {"key": "some key", "quorum": 1-3} + # Response: + # DELETE_RESP {"key": "some key", "value": "some value"} + # DELETE_RESP {"key": "some key", "value": null} - if the key is absent + elif msg.type == 'DELETE': + key = msg['key'] + value = self._data.pop(key, None) + resp = Message('DELETE_RESP', { + 'key': key, + 'value': value + }) + ctx.send_local(resp) + + # Count distinct keys with locally stored values, including temporary + # copies and empty strings, but excluding tombstones and metadata. + # Request: COUNT_KEYS {} + # Response: COUNT_KEYS_RESP {"count": 100} + elif msg.type == 'COUNT_KEYS': + ctx.send_local(Message('COUNT_KEYS_RESP', { + 'count': sum(value is not None for value in self._data.values()), + })) + + def on_start(self, ctx: Context): + pass + + def on_message(self, msg: Message, sender: str, ctx: Context): + # Implement node-to-node communication using any message types + pass + + def on_timer(self, timer_name: str, ctx: Context): + pass + + +def get_key_replicas(key: str, node_count: int): + replicas = [] + key_hash = int.from_bytes(hashlib.md5(key.encode('utf8')).digest(), 'little', signed=False) + cur = key_hash % node_count + for _ in range(3): + replicas.append(str(cur)) + cur = get_next_replica(cur, node_count) + return replicas + + +def get_next_replica(i, node_count: int): + return (i + 1) % node_count diff --git a/homework/08-kv-replication/tests/.dockerignore b/homework/08-kv-replication/tests/.dockerignore new file mode 100644 index 0000000..d291891 --- /dev/null +++ b/homework/08-kv-replication/tests/.dockerignore @@ -0,0 +1,5 @@ +target +**/target +**/__pycache__ +**/.pytest_cache +*.pyc diff --git a/homework/08-kv-replication/tests/Cargo.lock b/homework/08-kv-replication/tests/Cargo.lock new file mode 100644 index 0000000..d3299ee --- /dev/null +++ b/homework/08-kv-replication/tests/Cargo.lock @@ -0,0 +1,1027 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "aho-corasick" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" +dependencies = [ + "memchr", +] + +[[package]] +name = "anysystem" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e01badd3bc0d042e4399f8f685c394a79a72e7f233b3d3b5eabb3b94734f4dbe" +dependencies = [ + "colored", + "downcast-rs", + "dyn-clone", + "indexmap 2.14.0", + "lazy_static", + "log", + "ordered-float", + "pyo3", + "rand", + "rand_pcg", + "regex", + "rstest", + "serde", + "serde_json", + "simcore", + "sugars", +] + +[[package]] +name = "approx" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0e60b75072ecd4168020818c0107f2857bb6c4e64252d8d3983f6263b40a5c3" +dependencies = [ + "num-traits", +] + +[[package]] +name = "assertables" +version = "3.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d84b91d63c67e6c0b1a74728dad0057c4a6124b74e671ff91b7fe9f8bbd671aa" + +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi", + "libc", + "winapi", +] + +[[package]] +name = "autocfg" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "cfg-if" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2fd1289c04a9ea8cb22300a459a72a385d7c73d3259e2ed7dcb2af674838cfa9" + +[[package]] +name = "clap" +version = "3.2.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ea181bf566f71cb9a5d17a59e1871af638180a18fb0035c92ae62b705207123" +dependencies = [ + "atty", + "bitflags", + "clap_derive", + "clap_lex", + "indexmap 1.9.3", + "once_cell", + "strsim", + "termcolor", + "textwrap", +] + +[[package]] +name = "clap_derive" +version = "3.2.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae6371b8bdc8b7d3959e9cf7b22d4435ef3e79e138688421ec654acf8c81b008" +dependencies = [ + "heck 0.4.1", + "proc-macro-error", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "clap_lex" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2850f2f5a82cbf437dd5af4d49848fbdfc27c157c3d010345776f952765261c5" +dependencies = [ + "os_str_bytes", +] + +[[package]] +name = "colored" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "117725a109d387c937a1533ce01b450cbde6b88abceea8473c4d7a85853cda3c" +dependencies = [ + "lazy_static", + "windows-sys 0.59.0", +] + +[[package]] +name = "decorum" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "281759d3c8a14f5c3f0c49363be56810fcd7f910422f97f2db850c2920fde5cf" +dependencies = [ + "approx", + "num-traits", + "serde", + "serde_derive", +] + +[[package]] +name = "distsys-kv-replication" +version = "0.1.0" +dependencies = [ + "anysystem", + "assertables", + "byteorder", + "clap", + "decorum", + "env_logger", + "indexmap 2.14.0", + "log", + "md5", + "pyo3", + "rand", + "rand_pcg", + "serde", + "serde_json", +] + +[[package]] +name = "downcast-rs" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75b325c5dbd37f80359721ad39aca5a29fb04c89279657cffdda8736d0c0b9d2" + +[[package]] +name = "dyn-clone" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" + +[[package]] +name = "env_logger" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a12e6657c4c97ebab115a42dcee77225f7f482cdd841cf7088c657a42e9e00e7" +dependencies = [ + "atty", + "humantime", + "log", + "regex", + "termcolor", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "erased-serde" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e004d887f51fcb9fef17317a2f3525c887d8aa3f4f50fed920816a688284a5b7" +dependencies = [ + "serde", + "typeid", +] + +[[package]] +name = "futures" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" + +[[package]] +name = "futures-executor" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" + +[[package]] +name = "futures-macro" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.106", +] + +[[package]] +name = "futures-sink" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" + +[[package]] +name = "futures-task" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" + +[[package]] +name = "futures-timer" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f288b0a4f20f9a56b5d1da57e2227c661b7b16168e2f72365f57b63326e29b24" + +[[package]] +name = "futures-util" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "pin-utils", + "slab", +] + +[[package]] +name = "getrandom" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "335ff9f135e4384c8150d6f27c6daed433577f86b4750418338c01a1a2528592" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "glob" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" + +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "hermit-abi" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +dependencies = [ + "libc", +] + +[[package]] +name = "humantime" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424" + +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown 0.12.3", +] + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown 0.17.1", +] + +[[package]] +name = "itoa" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "libc" +version = "0.2.175" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a82ae493e598baaea5209805c49bbf2ea7de956d50d7da0da1164f9c6d28543" + +[[package]] +name = "log" +version = "0.4.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34080505efa8e45a4b816c349525ebe327ceaa8559756f0356cba97ef3bf7432" + +[[package]] +name = "md5" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771" + +[[package]] +name = "memchr" +version = "2.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a282da65faaf38286cf3be983213fcf1d2e2a58700e808f83f4ea9a4804bc0" + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "once_cell" +version = "1.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" + +[[package]] +name = "ordered-float" +version = "4.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7bb71e1b3fa6ca1c61f383464aaf2bb0e2f8e772a1f01d486832464de363b951" +dependencies = [ + "num-traits", + "rand", + "serde", +] + +[[package]] +name = "os_str_bytes" +version = "6.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2355d85b9a3786f481747ced0e0ff2ba35213a1f9bd406ed906554d7af805a1" + +[[package]] +name = "pin-project-lite" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "portable-atomic" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "proc-macro-crate" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edce586971a4dfaa28950c6f18ed55e0406c1ab88bbce2c6f6293a7aaba73d35" +dependencies = [ + "toml_edit", +] + +[[package]] +name = "proc-macro-error" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" +dependencies = [ + "proc-macro-error-attr", + "proc-macro2", + "quote", + "syn 1.0.109", + "version_check", +] + +[[package]] +name = "proc-macro-error-attr" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" +dependencies = [ + "proc-macro2", + "quote", + "version_check", +] + +[[package]] +name = "proc-macro2" +version = "1.0.101" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89ae43fd86e4158d6db51ad8e2b80f313af9cc74f5c0e03ccb87de09998732de" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "pyo3" +version = "0.29.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4688ddedf473e32662b9b067670129a8afb8c18e351482c70d62ba4a88171e8b" +dependencies = [ + "libc", + "once_cell", + "portable-atomic", + "pyo3-build-config", + "pyo3-ffi", + "pyo3-macros", +] + +[[package]] +name = "pyo3-build-config" +version = "0.29.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f41027e41b4bd03f6e60f9f417fe24a6341a6bb744edd62b6f709f2a52ea30e9" +dependencies = [ + "target-lexicon", +] + +[[package]] +name = "pyo3-ffi" +version = "0.29.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e591a95526fead067432c3b3a33fc74770b87b1e04e73671090d9c2055a2b327" +dependencies = [ + "libc", + "pyo3-build-config", +] + +[[package]] +name = "pyo3-macros" +version = "0.29.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73225868fc1cd84eef2c3c230ddb91273bf1de46aeb8a4248da76d32a0924a1c" +dependencies = [ + "proc-macro2", + "pyo3-macros-backend", + "quote", + "syn 2.0.106", +] + +[[package]] +name = "pyo3-macros-backend" +version = "0.29.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "571575aa3749fa6216757dd47d2a3e7ef360f329a40f0666a9fbd14889024952" +dependencies = [ + "heck 0.5.0", + "proc-macro2", + "quote", + "syn 2.0.106", +] + +[[package]] +name = "quote" +version = "1.0.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", + "serde", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom", + "serde", +] + +[[package]] +name = "rand_pcg" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59cad018caf63deb318e5a4586d99a24424a364f40f1e5778c29aca23f4fc73e" +dependencies = [ + "rand_core", +] + +[[package]] +name = "regex" +version = "1.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23d7fd106d8c02486a8d64e778353d1cffe08ce79ac2e82f540c86d0facf6912" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b9458fa0bfeeac22b5ca447c63aaf45f28439a709ccd244698632f9aa6394d6" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caf4aa5b0f434c91fe5c7f1ecb6a5ece2130b02ad2a590589dda5146df959001" + +[[package]] +name = "relative-path" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba39f3699c378cd8970968dcbff9c43159ea4cfbd88d43c00b22f2ef10a435d2" + +[[package]] +name = "rstest" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9afd55a67069d6e434a95161415f5beeada95a01c7b815508a82dcb0e1593682" +dependencies = [ + "futures", + "futures-timer", + "rstest_macros", + "rustc_version", +] + +[[package]] +name = "rstest_macros" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4165dfae59a39dd41d8dec720d3cbfbc71f69744efb480a3920f5d4e0cc6798d" +dependencies = [ + "cfg-if", + "glob", + "proc-macro-crate", + "proc-macro2", + "quote", + "regex", + "relative-path", + "rustc_version", + "syn 2.0.106", + "unicode-ident", +] + +[[package]] +name = "rustc-hash" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" + +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + +[[package]] +name = "ryu" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" + +[[package]] +name = "semver" +version = "1.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56e6fa9c48d24d85fb3de5ad847117517440f6beceb7798af16b4a87d616b8d0" + +[[package]] +name = "serde" +version = "1.0.219" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f0e2c6ed6606019b4e29e69dbaba95b11854410e5347d525002456dbbb786b6" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.219" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.106", +] + +[[package]] +name = "serde_json" +version = "1.0.143" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d401abef1d108fbd9cbaebc3e46611f4b1021f714a0597a71f41ee463f5f4a5a" +dependencies = [ + "indexmap 2.14.0", + "itoa", + "memchr", + "ryu", + "serde", +] + +[[package]] +name = "serde_type_name" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92acc4cd6ae20767c54d6cf1a075624e7f4d9e99d7ebc685398ff243144d8714" +dependencies = [ + "serde", +] + +[[package]] +name = "simcore" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a20dfec85e94e569fa5b810b2d992c68b361f31bfba471447427536488c9c292" +dependencies = [ + "colored", + "downcast-rs", + "dyn-clone", + "erased-serde", + "futures", + "log", + "rand", + "rand_pcg", + "rustc-hash", + "serde", + "serde_json", + "serde_type_name", +] + +[[package]] +name = "slab" +version = "0.4.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a2ae44ef20feb57a68b23d846850f861394c2e02dc425a50098ae8c90267589" + +[[package]] +name = "strsim" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" + +[[package]] +name = "sugars" +version = "3.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc0db74f9ee706e039d031a560bd7d110c7022f016051b3d33eeff9583e3e67a" + +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "2.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ede7c438028d4436d71104916910f5bb611972c5cfd7f89b8300a8186e6fada6" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "target-lexicon" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca" + +[[package]] +name = "termcolor" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "textwrap" +version = "0.16.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c13547615a44dc9c452a8a534638acdf07120d4b6847c8178705da06306a3057" + +[[package]] +name = "toml_datetime" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" + +[[package]] +name = "toml_edit" +version = "0.22.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" +dependencies = [ + "indexmap 2.14.0", + "toml_datetime", + "winnow", +] + +[[package]] +name = "typeid" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc7d623258602320d5c55d1bc22793b57daff0ec7efc270ea7d55ce1d5f5471c" + +[[package]] +name = "unicode-ident" +version = "1.0.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f63a545481291138910575129486daeaf8ac54aee4387fe7906919f7830c7d9d" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.61.0", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-link" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45e46c0661abb7180e7b9c281db115305d49ca1709ab8242adf09666d2173c65" + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.61.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e201184e40b2ede64bc2ea34968b28e33622acdbbf37104f0e4a33f7abe657aa" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "winnow" +version = "0.7.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21a0236b59786fed61e2a80582dd500fe61f18b5dca67a4a067d0bc9039339cf" +dependencies = [ + "memchr", +] + +[[package]] +name = "zerocopy" +version = "0.8.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0894878a5fa3edfd6da3f88c4805f4c8558e2b996227a3d864f47fe11e38282c" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88d2b8d9c68ad2b9e4340d7832716a4d21a22a1154777ad56ea55c51a9cf3831" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.106", +] diff --git a/homework/08-kv-replication/tests/Cargo.toml b/homework/08-kv-replication/tests/Cargo.toml new file mode 100644 index 0000000..ee370a6 --- /dev/null +++ b/homework/08-kv-replication/tests/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "distsys-kv-replication" +version = "0.1.0" +edition = "2021" +resolver = "3" + +[dependencies] +anysystem = "=0.3.0" +indexmap = "2.12" +assertables = "3.2.2" +byteorder = "1.4.3" +clap = { version = "3.2.17", features = ["cargo", "derive"] } +decorum = "0.3.1" +env_logger = "0.9.0" +log = "0.4.17" +md5 = "0.7.0" +pyo3 = { version = "=0.29.2", features = ["auto-initialize"] } +rand = "0.8.5" +rand_pcg = "0.3.1" +serde = { version = "1.0.142", features = ["derive"] } +serde_json = "1.0.83" diff --git a/homework/08-kv-replication/tests/Dockerfile b/homework/08-kv-replication/tests/Dockerfile new file mode 100644 index 0000000..3c7ec56 --- /dev/null +++ b/homework/08-kv-replication/tests/Dockerfile @@ -0,0 +1,19 @@ +# syntax=docker/dockerfile:1 +FROM rust:1.97.1-bookworm@sha256:0e2bcaef56d041a486784e54104a81aebe0da44bd03019bd70bc0401e42e4a97 AS builder + +RUN apt-get update && apt-get install -y build-essential python3-dev + +WORKDIR /tests +COPY . . +RUN --mount=type=cache,id=distsys-course-cargo-registry,target=/usr/local/cargo/registry,sharing=locked \ + --mount=type=cache,id=distsys-course-cargo-target-rust-1-97-1,target=/tests/target,sharing=locked \ + cargo install --locked --path . --root /opt/course + +FROM debian:bookworm-slim + +RUN apt-get update && apt-get install -y python3-dev && rm -rf /var/lib/apt/lists/* + +COPY --from=builder /opt/course/bin/distsys-kv-replication /usr/local/bin/distsys-kv-replication + +WORKDIR /solution +ENTRYPOINT ["timeout", "--verbose", "-k", "10", "1260", "distsys-kv-replication", "-i", "node.py"] diff --git a/homework/08-kv-replication/tests/python_errors.py b/homework/08-kv-replication/tests/python_errors.py new file mode 100644 index 0000000..4bbddf7 --- /dev/null +++ b/homework/08-kv-replication/tests/python_errors.py @@ -0,0 +1,38 @@ +"""Retain exceptions from StorageNode calls for the Rust test runner.""" +from functools import wraps +from inspect import getattr_static + + +def watch(cls): + if not isinstance(cls, type): + raise TypeError("StorageNode must be a class") + missing = sorted(getattr(cls, "__abstractmethods__", ())) + if missing: + raise TypeError(f"{cls.__name__} must implement {', '.join(missing)}") + errors = [] + + def checked(name, method): + @wraps(method) + def call(*args, **kwargs): + try: + result = method(*args, **kwargs) + except BaseException as exc: + errors[:] = [f"{name}: {type(exc).__name__}: {exc}"] + raise + if errors: + errors.clear() # A nested exception was handled by the solution. + return result + + return call + + for name in ("__new__", "__init__", "on_start", "on_local_message", "on_message", "on_timer", + "get_state", "set_state", "__deepcopy__"): + # Preserve static and class methods and inherited implementations. + method = getattr_static(cls, name, None) + if name == "__new__" and method is object.__new__: + continue # Keep the default allocation and constructor argument handling. + if isinstance(method, (staticmethod, classmethod)): + setattr(cls, name, type(method)(checked(name, method.__func__))) + elif callable(method): + setattr(cls, name, checked(name, method)) + return errors diff --git a/homework/08-kv-replication/tests/shared_state_validator.py b/homework/08-kv-replication/tests/shared_state_validator.py new file mode 100644 index 0000000..bf28c31 --- /dev/null +++ b/homework/08-kv-replication/tests/shared_state_validator.py @@ -0,0 +1,1938 @@ +"""Static checks for Python state shared between AnySystem processes. + +The checker parses source files only. It never imports or executes a student's +solution. Local imports which can be resolved next to the entrypoint are checked +recursively. Standard-library calls are allowed unless they access known shared +runtime state; other external calls need an explicitly reviewed API. +""" + +from __future__ import annotations + +import ast +import sys +from pathlib import Path +from typing import Callable, Iterable + + +MUTABLE_NODES = ( + ast.List, + ast.ListComp, + ast.Dict, + ast.DictComp, + ast.Set, + ast.SetComp, + ast.GeneratorExp, +) +MUTABLE_CONSTRUCTORS = { + "builtins.bytearray", + "builtins.dict", + "builtins.list", + "builtins.set", + "collections.ChainMap", + "collections.Counter", + "collections.OrderedDict", + "collections.defaultdict", + "collections.deque", + "weakref.WeakKeyDictionary", + "weakref.WeakValueDictionary", +} +IMMUTABLE_SCALAR_CONSTRUCTORS = { + "builtins.bool", + "builtins.bytes", + "builtins.complex", + "builtins.float", + "builtins.int", + "builtins.range", + "builtins.str", +} +KNOWN_IMMUTABLE_CALLS = { + "os.getcwd", + "os.path.abspath", + "os.path.basename", + "os.path.dirname", + "os.path.join", + "os.path.normpath", + "os.path.realpath", + "random.randint", + "random.randrange", +} +TYPE_SUBSCRIPT_NAMES = { + "builtins.dict", + "builtins.frozenset", + "builtins.list", + "builtins.set", + "builtins.tuple", + "builtins.type", + "collections.abc.Mapping", + "collections.abc.Sequence", + "typing.Annotated", + "typing.Callable", + "typing.ClassVar", + "typing.Dict", + "typing.FrozenSet", + "typing.List", + "typing.Literal", + "typing.Mapping", + "typing.Optional", + "typing.Sequence", + "typing.Set", + "typing.Tuple", + "typing.Type", + "typing.Union", +} +TYPE_FACTORIES = { + "collections.namedtuple", + "typing.NewType", + "typing.ParamSpec", + "typing.TypeVar", +} +ENUM_BASE_NAMES = { + "enum.Enum", + "enum.Flag", + "enum.IntEnum", + "enum.IntFlag", + "enum.StrEnum", +} +MUTATOR_METHODS = frozenset( + "__delitem__ __setitem__ add append clear discard extend insert move_to_end " + "pop popitem remove reverse rotate setdefault sort update".split() +) +UNBOUND_MUTATOR_METHODS = {"__delattr__", "__delitem__", "__setattr__", "__setitem__"} +OPERATOR_MUTATORS = frozenset( + f"operator.{name}" + for name in "delitem iadd iand iconcat imatmul imul ior isub ixor setitem".split() +) +STATEFUL_DECORATORS = {"functools.cache", "functools.lru_cache"} +# Reviewed external APIs and a few interpreter operations used by the course. +# Ordinary stdlib functions do not need to be enumerated here: see +# _imported_call_allowed. Shared-state checks still apply to their arguments. +ALLOWED_IMPORTED_CALLS = frozenset( + """ + anysystem.Message anysystem.Message.from_json anysystem.python.anysystem.Message + array.array ast.literal_eval + bisect.bisect bisect.bisect_left bisect.bisect_right bisect.insort builtins.print + collections.defaultdict collections.deque collections.namedtuple copy.copy copy.deepcopy + dataclasses.asdict dataclasses.field dataclasses.fields dataclasses.is_dataclass + functools.cmp_to_key functools.reduce + hashlib.blake2b hashlib.md5 hashlib.sha1 hashlib.sha512 hashlib.sha256 + hashlib.sha3_256 hashlib.shake_256 + heapq.heapify heapq.heappop heapq.heappush heapq.merge importlib.import_module + inspect.signature itertools.count json.dumps json.loads + logging.critical logging.debug logging.error logging.exception logging.info logging.log + logging.warning math.ceil math.log math.log2 math.sqrt + os.getcwd os.getenv os.path.abspath os.path.basename os.path.dirname os.path.join + os.path.normpath os.path.realpath pathlib.Path pickle.dumps pickle.loads + portion.closedopen portion.empty pprint.pprint pydantic.Field + random.Random random.choice random.choices random.randint random.random random.randrange + random.sample random.shuffle random.uniform re.fullmatch sys.intern threading.Lock + types.MappingProxyType time.time typing.cast uuid.uuid4 + """.split() +) +IMPORTED_ARGUMENT_MUTATORS = set( + "bisect.insort bisect.insort_left bisect.insort_right heapq.heapify " + "heapq.heappop heapq.heappush heapq.heappushpop heapq.heapreplace random.shuffle".split() +) +# These APIs expose or change state shared by all simulated processes. This is +# a static check for known violations, not a sandbox for arbitrary Python code. +SHARED_RUNTIME_APIS = frozenset(""" + atexit.register atexit.unregister codecs.register codecs.unregister + copyreg.pickle copyreg.constructor copyreg.add_extension copyreg.remove_extension + csv.register_dialect csv.unregister_dialect csv.field_size_limit + decimal.getcontext decimal.setcontext decimal.localcontext + gc.enable gc.disable gc.set_threshold gc.callbacks.append + locale.setlocale locale.resetlocale + logging.getLogger logging.basicConfig logging.disable logging.setLoggerClass + logging.setLogRecordFactory logging.addLevelName logging.captureWarnings + logging.config.dictConfig logging.config.fileConfig logging.config.listen + os.chdir os.fchdir os.putenv os.unsetenv os.umask os.kill os.killpg + random.seed random.setstate random.getstate + signal.signal signal.set_wakeup_fd signal.alarm signal.setitimer + sys.settrace sys.setprofile sys.setrecursionlimit sys.setswitchinterval + sys.set_asyncgen_hooks sys.set_coroutine_origin_tracking_depth sys.addaudithook + threading.settrace threading.setprofile threading.stack_size + warnings.filterwarnings warnings.simplefilter warnings.resetwarnings +""".split()) + + +def _imported_call_allowed(name: str) -> bool: + if name in SHARED_RUNTIME_APIS: + return False + if name in ALLOWED_IMPORTED_CALLS: + return True + parts = name.split(".") + # Private interpreter objects are not part of the stdlib API. Builtins + # include reflection and execution primitives, checked separately. + return ( + len(parts) >= 2 + and parts[0] in sys.stdlib_module_names + and parts[0] != "builtins" + and all(not part.startswith("_") for part in parts) + ) + + +ALLOWED_MUTABLE_METADATA = {"__all__"} +TRUSTED_STAR_EXPORTS = { + "dataclasses": {"dataclass", "field"}, + "enum": {"Enum", "Flag", "IntEnum", "IntFlag", "StrEnum", "auto"}, + "functools": {"cache", "lru_cache"}, + "types": {"MappingProxyType"}, + "typing": { + *"Annotated Any Callable ClassVar Dict FrozenSet List Literal Mapping NewType " + "Optional ParamSpec Sequence Set Tuple Type TypeVar Union".split(), + }, +} +TRUSTED_BUILTINS = set( + "bool bytearray bytes complex dict float frozenset int list range set str tuple type".split() +) + + +def _resolved_attribute(node: ast.AST) -> tuple[ast.AST, str] | None: + if isinstance(node, ast.Attribute): + return node.value, node.attr + if ( + isinstance(node, ast.Call) + and isinstance(node.func, ast.Name) + and node.func.id == "getattr" + and len(node.args) >= 2 + and isinstance(node.args[1], ast.Constant) + and isinstance(node.args[1].value, str) + ): + return node.args[0], node.args[1].value + return None + + +def _call_name(node: ast.AST) -> str | None: + if isinstance(node, ast.Name): + return node.id + resolved = _resolved_attribute(node) + return resolved[1] if resolved is not None else None + + +def _symbol(node: ast.AST, symbols: dict[str, str]) -> str | None: + if isinstance(node, ast.Name): + return symbols.get(node.id) + resolved = _resolved_attribute(node) + if resolved is not None: + value, attribute = resolved + parent = _symbol(value, symbols) + return f"{parent}.{attribute}" if parent is not None else None + return None + + +def _root_name(node: ast.AST) -> str | None: + while (resolved := _resolved_attribute(node)) is not None: + node = resolved[0] + return node.id if isinstance(node, ast.Name) else None + + +def _assignment_parts(node: ast.AST) -> tuple[ast.AST, list[ast.AST]] | None: + if isinstance(node, ast.Assign): + return node.value, node.targets + if isinstance(node, (ast.AnnAssign, ast.NamedExpr)) and node.value is not None: + return node.value, [node.target] + return None + + +def _call_returns_shared( + node: ast.AST, + is_shared: Callable[[ast.AST], bool], + shared_functions: set[str], + returned_parameters: dict[str, set[int]], +) -> bool: + if not isinstance(node, ast.Call): + return False + name = node.func.id if isinstance(node.func, ast.Name) else None + if name in shared_functions: + return True + return any( + index < len(node.args) and is_shared(node.args[index]) + for index in returned_parameters.get(name or "", set()) + ) + + +def _is_instance_path(node: ast.AST, instance_names: set[str]) -> bool: + while True: + if isinstance(node, ast.Subscript): + node = node.value + elif (resolved := _resolved_attribute(node)) is not None: + node = resolved[0] + else: + break + return isinstance(node, ast.Name) and node.id in instance_names + + +def _static_string(node: ast.AST, names: dict[str, str]) -> str | None: + if isinstance(node, ast.Constant) and isinstance(node.value, str): + return node.value + if isinstance(node, ast.Name): + return names.get(node.id) + if isinstance(node, ast.BinOp) and isinstance(node.op, ast.Add): + left = _static_string(node.left, names) + right = _static_string(node.right, names) + return left + right if left is not None and right is not None else None + if isinstance(node, ast.JoinedStr): + parts: list[str] = [] + for value in node.values: + if isinstance(value, ast.Constant) and isinstance(value.value, str): + parts.append(value.value) + elif isinstance(value, ast.FormattedValue): + part = _static_string(value.value, names) + if part is None: + return None + parts.append(part) + else: + return None + return "".join(parts) + return None + + +def _static_string_values(tree: ast.Module) -> dict[str, str]: + assignments: list[tuple[str, ast.AST]] = [] + for node in ast.walk(tree): + if isinstance(node, ast.Assign): + for target in node.targets: + assignments.extend((name, node.value) for name in _target_names(target)) + elif isinstance(node, (ast.AnnAssign, ast.NamedExpr)) and node.value is not None: + assignments.extend((name, node.value) for name in _target_names(node.target)) + + values: dict[str, str] = {} + ambiguous: set[str] = set() + changed = True + while changed: + changed = False + for name, value_node in assignments: + value = _static_string(value_node, values) + if value is None or name in ambiguous: + continue + if name in values and values[name] != value: + values.pop(name) + ambiguous.add(name) + changed = True + elif name not in values: + values[name] = value + changed = True + return values + + +def _dynamic_import_names(tree: ast.Module) -> tuple[set[str], set[str]]: + modules = {"importlib"} + functions: set[str] = set() + for node in ast.walk(tree): + if isinstance(node, ast.Import): + for alias in node.names: + if alias.name == "importlib": + modules.add(alias.asname or "importlib") + elif isinstance(node, ast.ImportFrom) and node.module == "importlib": + for alias in node.names: + if alias.name == "import_module": + functions.add(alias.asname or alias.name) + assignments = [node for node in ast.walk(tree) if isinstance(node, ast.Assign)] + changed = True + while changed: + changed = False + for assignment in assignments: + names = set().union(*(_target_names(target) for target in assignment.targets)) + if isinstance(assignment.value, ast.Name): + if assignment.value.id in modules: + before = len(modules) + modules.update(names) + changed |= len(modules) != before + if assignment.value.id in functions: + before = len(functions) + functions.update(names) + changed |= len(functions) != before + elif ( + isinstance(assignment.value, ast.Attribute) + and assignment.value.attr == "import_module" + and isinstance(assignment.value.value, ast.Name) + and assignment.value.value.id in modules + ): + before = len(functions) + functions.update(names) + changed |= len(functions) != before + return modules, functions + + +def _is_dynamic_import_call( + node: ast.Call, importlib_modules: set[str], import_module_functions: set[str] +) -> bool: + return ( + isinstance(node.func, ast.Name) + and (node.func.id == "__import__" or node.func.id in import_module_functions) + ) or ( + (resolved := _resolved_attribute(node.func)) is not None + and resolved[1] == "import_module" + and isinstance(resolved[0], ast.Name) + and resolved[0].id in importlib_modules + ) + + +def _class_has_mutation_escape(class_node: ast.ClassDef) -> bool: + for statement in class_node.body: + if not isinstance(statement, (ast.FunctionDef, ast.AsyncFunctionDef)): + continue + positional = [*statement.args.posonlyargs, *statement.args.args] + if not positional: + continue + instance_names = {positional[0].arg} + for node in _function_nodes(statement): + if isinstance(node, (ast.Assign, ast.AnnAssign, ast.AugAssign, ast.Delete)): + targets = ( + node.targets + if isinstance(node, (ast.Assign, ast.Delete)) + else [node.target] + ) + if any(_is_instance_path(target, instance_names) for target in targets): + return True + if not isinstance(node, ast.Call): + continue + if ( + isinstance(node.func, ast.Name) + and node.func.id in {"delattr", "setattr"} + and node.args + and _is_instance_path(node.args[0], instance_names) + ): + return True + if ( + (resolved := _resolved_attribute(node.func)) is not None + and resolved[1] in UNBOUND_MUTATOR_METHODS + and node.args + and _is_instance_path(node.args[0], instance_names) + ): + return True + if ( + (resolved := _resolved_attribute(node.func)) is not None + and resolved[1] in MUTATOR_METHODS + and _is_instance_path(resolved[0], instance_names) + ): + return True + return False + + +def _target_names(node: ast.AST) -> set[str]: + if isinstance(node, ast.Name): + return {node.id} + if isinstance(node, (ast.Tuple, ast.List)): + result: set[str] = set() + for element in node.elts: + result.update(_target_names(element)) + return result + return set() + + +def _dict_is_deeply_immutable( + node: ast.Dict, safe_names: set[str], symbols: dict[str, str] +) -> bool: + return all( + key is not None + and _is_immutable_expr(key, safe_names, symbols) + and _is_immutable_expr(value, safe_names, symbols) + for key, value in zip(node.keys, node.values) + ) + + +def _sequence_is_deeply_immutable( + node: ast.List | ast.Set | ast.Tuple, + safe_names: set[str], + symbols: dict[str, str], +) -> bool: + return all(_is_immutable_expr(element, safe_names, symbols) for element in node.elts) + + +def _is_immutable_expr( + node: ast.AST | None, safe_names: set[str], symbols: dict[str, str] +) -> bool: + if node is None or isinstance(node, ast.Constant): + return True + if isinstance(node, ast.Name): + return node.id in safe_names + if isinstance(node, ast.Attribute): + root = _root_name(node) + if root is not None and root in symbols and symbols[root] != f"builtins.{root}": + return node.attr.isupper() + return _is_immutable_expr(node.value, safe_names, symbols) + if isinstance(node, ast.Subscript): + return _symbol(node.value, symbols) in TYPE_SUBSCRIPT_NAMES + if isinstance(node, ast.Tuple): + return _sequence_is_deeply_immutable(node, safe_names, symbols) + if isinstance(node, ast.UnaryOp): + return _is_immutable_expr(node.operand, safe_names, symbols) + if isinstance(node, ast.BinOp): + return _is_immutable_expr(node.left, safe_names, symbols) and _is_immutable_expr( + node.right, safe_names, symbols + ) + if isinstance(node, ast.BoolOp): + return all(_is_immutable_expr(value, safe_names, symbols) for value in node.values) + if isinstance(node, ast.Compare): + return _is_immutable_expr(node.left, safe_names, symbols) and all( + _is_immutable_expr(value, safe_names, symbols) for value in node.comparators + ) + if isinstance(node, ast.IfExp): + return _is_immutable_expr(node.body, safe_names, symbols) and _is_immutable_expr( + node.orelse, safe_names, symbols + ) + if isinstance(node, ast.JoinedStr): + return all( + not isinstance(value, ast.FormattedValue) + or _is_immutable_expr(value.value, safe_names, symbols) + for value in node.values + ) + if isinstance(node, ast.Lambda): + return True + if isinstance(node, ast.Call): + name = _symbol(node.func, symbols) + if name in TYPE_FACTORIES: + return True + local_name = _call_name(node.func) + if local_name is not None and f"frozen-dataclass:{local_name}" in safe_names: + return all( + _is_immutable_expr(arg, safe_names, symbols) for arg in node.args + ) and all( + _is_immutable_expr(keyword.value, safe_names, symbols) + for keyword in node.keywords + ) + if name in IMMUTABLE_SCALAR_CONSTRUCTORS: + return all(_is_immutable_expr(arg, safe_names, symbols) for arg in node.args) + if name in KNOWN_IMMUTABLE_CALLS or ( + name is not None + and name.split(".")[0] in {"math", "cmath"} + and len(name.split(".")) == 2 + and not name.split(".")[1].startswith("_") + ): + return True + if name in {"builtins.tuple", "builtins.frozenset"}: + if not node.args: + return True + if len(node.args) != 1 or node.keywords: + return False + value = node.args[0] + return isinstance(value, (ast.List, ast.Set, ast.Tuple)) and ( + _sequence_is_deeply_immutable(value, safe_names, symbols) + ) + if name == "types.MappingProxyType": + return ( + len(node.args) == 1 + and not node.keywords + and isinstance(node.args[0], ast.Dict) + and _dict_is_deeply_immutable(node.args[0], safe_names, symbols) + ) + return False + + +def _is_mutable_expr( + node: ast.AST | None, safe_names: set[str], symbols: dict[str, str] +) -> bool: + if node is None: + return False + if isinstance(node, MUTABLE_NODES): + return True + if isinstance(node, ast.Call) and _symbol(node.func, symbols) in MUTABLE_CONSTRUCTORS: + return True + return not _is_immutable_expr(node, safe_names, symbols) + + +def _scope_statements(body: Iterable[ast.stmt]) -> Iterable[ast.stmt]: + for statement in body: + yield statement + nested: list[list[ast.stmt]] = [] + if isinstance(statement, (ast.If, ast.For, ast.AsyncFor, ast.While)): + nested.extend([statement.body, statement.orelse]) + elif isinstance(statement, (ast.With, ast.AsyncWith)): + nested.append(statement.body) + elif isinstance(statement, (ast.Try, ast.TryStar)): + nested.extend([statement.body, statement.orelse, statement.finalbody]) + nested.extend(handler.body for handler in statement.handlers) + elif isinstance(statement, ast.Match): + nested.extend(case.body for case in statement.cases) + for statements in nested: + yield from _scope_statements(statements) + + +def _function_nodes(function: ast.FunctionDef | ast.AsyncFunctionDef) -> list[ast.AST]: + result: list[ast.AST] = [] + stack: list[ast.AST] = list(reversed(function.body)) + while stack: + node = stack.pop() + result.append(node) + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef, ast.Lambda)): + continue + stack.extend(reversed(list(ast.iter_child_nodes(node)))) + return result + + +def _shared_class_nodes( + body: Iterable[ast.stmt], prefix: str = "" +) -> Iterable[tuple[str, ast.ClassDef]]: + for statement in _scope_statements(body): + if not isinstance(statement, ast.ClassDef): + continue + qualified_name = f"{prefix}.{statement.name}" if prefix else statement.name + yield qualified_name, statement + yield from _shared_class_nodes(statement.body, qualified_name) + + +def _is_classvar_annotation(node: ast.AST, symbols: dict[str, str]) -> bool: + value = node.value if isinstance(node, ast.Subscript) else node + return _symbol(value, symbols) == "typing.ClassVar" + + +class _FileChecker: + def __init__( + self, + tree: ast.Module, + path: Path, + _target_classes: set[str], + root: Path | None = None, + ): + self.tree = tree + self.path = path + self.root = (root or path.parent).resolve() + self.violations: list[tuple[int, int, str]] = [] + self.safe_names: set[str] = { + "False", + "None", + "True", + "bool", + "bytes", + "complex", + "float", + "frozenset", + "int", + "range", + "str", + "tuple", + } + self.symbols = {name: f"builtins.{name}" for name in TRUSTED_BUILTINS} + self.import_names: set[str] = set() + self.definition_names: set[str] = set() + self.module_binding_names: set[str] = set() + self.module_mutable_names: set[str] = set() + self.classes: dict[str, ast.ClassDef] = {} + self.shared_classes = list(_shared_class_nodes(tree.body)) + self.functions: dict[str, ast.FunctionDef | ast.AsyncFunctionDef] = {} + self.lambdas: dict[str, ast.Lambda] = {} + self.shared_closure_factories: set[str] = set() + self.shared_returning_functions: set[str] = set() + self.returned_parameters: dict[str, set[int]] = {} + self.stateful_decorator_factories: set[str] = set() + self.shared_instance_attributes: dict[str, set[str]] = {} + + def add(self, node: ast.AST, message: str) -> None: + self.violations.append( + (getattr(node, "lineno", 1), getattr(node, "col_offset", 0) + 1, message) + ) + + def collect_symbols(self) -> None: + shadowed_names: set[str] = set() + for statement in _scope_statements(self.tree.body): + if isinstance(statement, ast.Import): + for alias in statement.names: + name = alias.asname or alias.name.split(".")[0] + self.import_names.add(name) + self.module_binding_names.add(name) + if _candidate_module(self.root, alias.name.split(".")) is None: + self.symbols[name] = alias.name if alias.asname else name + else: + self.symbols.pop(name, None) + elif isinstance(statement, ast.ImportFrom): + local_module = ( + statement.level > 0 + or statement.module is None + or _candidate_module(self.root, statement.module.split(".")) + is not None + ) + for alias in statement.names: + if alias.name == "*": + if statement.module is not None and not local_module: + for export in TRUSTED_STAR_EXPORTS.get( + statement.module, set() + ): + self.import_names.add(export) + self.module_binding_names.add(export) + self.symbols[export] = f"{statement.module}.{export}" + if statement.module == "typing": + self.safe_names.add(export) + else: + name = alias.asname or alias.name + self.import_names.add(name) + self.module_binding_names.add(name) + if statement.module is not None and not local_module: + self.symbols[name] = f"{statement.module}.{alias.name}" + else: + self.symbols.pop(name, None) + if ( + name.isupper() + or statement.module in {"collections.abc", "typing"} + or ( + statement.module == "math" + and alias.name in {"e", "inf", "nan", "pi", "tau"} + ) + ): + self.safe_names.add(name) + elif isinstance(statement, (ast.FunctionDef, ast.AsyncFunctionDef)): + self.definition_names.add(statement.name) + self.module_binding_names.add(statement.name) + shadowed_names.add(statement.name) + self.functions[statement.name] = statement + elif isinstance(statement, ast.ClassDef): + self.definition_names.add(statement.name) + self.module_binding_names.add(statement.name) + shadowed_names.add(statement.name) + self.classes[statement.name] = statement + elif isinstance(statement, ast.Assign): + for target in statement.targets: + names = _target_names(target) + self.module_binding_names.update(names) + shadowed_names.update(names) + if isinstance(statement.value, ast.Lambda): + for name in names: + self.lambdas[name] = statement.value + elif isinstance(statement, (ast.AnnAssign, ast.AugAssign)): + names = _target_names(statement.target) + self.module_binding_names.update(names) + shadowed_names.update(names) + for name in shadowed_names: + self.symbols.pop(name, None) + self.safe_names.difference_update(shadowed_names) + self.safe_names.update(self.definition_names) + for _, class_node in self.shared_classes: + frozen = any( + isinstance(decorator, ast.Call) + and _symbol(decorator.func, self.symbols) == "dataclasses.dataclass" + and any( + keyword.arg == "frozen" + and isinstance(keyword.value, ast.Constant) + and keyword.value.value is True + for keyword in decorator.keywords + ) + for decorator in class_node.decorator_list + ) + immutable_defaults = all( + not isinstance(statement, ast.AnnAssign) + or statement.value is None + or _is_immutable_expr(statement.value, self.safe_names, self.symbols) + for statement in class_node.body + ) + if frozen and immutable_defaults and not _class_has_mutation_escape(class_node): + self.safe_names.add(f"frozen-dataclass:{class_node.name}") + + def mark_shared_closure_factories(self) -> None: + def consider(value: ast.AST) -> None: + if not isinstance(value, ast.Call): + return + name = _call_name(value.func) + function = self.functions.get(name or "") + if function is not None and any( + isinstance(node, ast.Nonlocal) for node in ast.walk(function) + ): + self.shared_closure_factories.add(function.name) + + scopes = [self.tree.body] + scopes.extend(class_node.body for class_node in self.classes.values()) + for body in scopes: + for statement in _scope_statements(body): + if isinstance(statement, ast.Assign): + consider(statement.value) + elif isinstance(statement, ast.AnnAssign) and statement.value is not None: + consider(statement.value) + + for node in ast.walk(self.tree): + if not isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + continue + for decorator in node.decorator_list: + name = _call_name(decorator.func if isinstance(decorator, ast.Call) else decorator) + function = self.functions.get(name or "") + if function is not None and any( + isinstance(child, ast.Nonlocal) for child in ast.walk(function) + ): + self.shared_closure_factories.add(function.name) + + def collect_function_summaries(self) -> None: + """Summarize simple wrappers without executing student code.""" + + callables: dict[ + str, ast.FunctionDef | ast.AsyncFunctionDef | ast.Lambda + ] = {**self.functions, **self.lambdas} + changed = True + while changed: + changed = False + for function_name, function in callables.items(): + positional = [*function.args.posonlyargs, *function.args.args] + local_names = { + argument.arg + for argument in [*positional, *function.args.kwonlyargs] + } + if function.args.vararg is not None: + local_names.add(function.args.vararg.arg) + if function.args.kwarg is not None: + local_names.add(function.args.kwarg.arg) + + if isinstance(function, ast.Lambda): + nodes = [function.body] + return_values = [function.body] + else: + nodes = _function_nodes(function) + return_values = [ + node.value + for node in nodes + if isinstance(node, ast.Return) and node.value is not None + ] + assignments = [ + node + for node in nodes + if _assignment_parts(node) is not None + ] + for assignment in assignments: + _, targets = _assignment_parts(assignment) or (assignment, []) + for target in targets: + local_names.update(_target_names(target)) + + shared_aliases: set[str] = set() + parameter_aliases = { + argument.arg: {index} for index, argument in enumerate(positional) + } + + def returned_parameter_indexes(node: ast.AST) -> set[int]: + if isinstance(node, ast.Name): + return parameter_aliases.get(node.id, set()) + resolved = _resolved_attribute(node) + if resolved is not None: + return returned_parameter_indexes(resolved[0]) + if isinstance(node, ast.Subscript): + return returned_parameter_indexes(node.value) + if isinstance(node, ast.Call): + name = node.func.id if isinstance(node.func, ast.Name) else None + return set().union( + *( + returned_parameter_indexes(node.args[index]) + for index in self.returned_parameters.get( + name or "", set() + ) + if index < len(node.args) + ), + set(), + ) + return set() + + def returns_shared(node: ast.AST) -> bool: + if _is_immutable_expr(node, self.safe_names, self.symbols): + return False + if isinstance(node, ast.Name): + return ( + node.id in self.module_binding_names + and node.id not in local_names + ) or node.id in shared_aliases + resolved = _resolved_attribute(node) + if resolved is not None: + return returns_shared(resolved[0]) + if isinstance(node, ast.Subscript): + return returns_shared(node.value) + return _call_returns_shared( + node, + returns_shared, + self.shared_returning_functions, + self.returned_parameters, + ) or ( + isinstance(node, ast.Call) + and _symbol(node.func, self.symbols) + == "importlib.import_module" + ) + + aliases_changed = True + while aliases_changed: + aliases_changed = False + for assignment in assignments: + value, targets = _assignment_parts(assignment) or (assignment, []) + indexes = returned_parameter_indexes(value) + shared = returns_shared(value) + before = ( + len(shared_aliases), + sum(map(len, parameter_aliases.values())), + ) + for target in targets: + for name in _target_names(target): + if shared: + shared_aliases.add(name) + if indexes: + parameter_aliases.setdefault(name, set()).update(indexes) + aliases_changed |= before != ( + len(shared_aliases), + sum(map(len, parameter_aliases.values())), + ) + + indexes = set().union( + *(returned_parameter_indexes(value) for value in return_values), set() + ) + known_indexes = self.returned_parameters.setdefault(function_name, set()) + before = len(known_indexes) + known_indexes.update(indexes) + changed |= len(known_indexes) != before + if any(returns_shared(value) for value in return_values): + if function_name not in self.shared_returning_functions: + self.shared_returning_functions.add(function_name) + changed = True + for value in return_values: + if isinstance(value, ast.Call): + called = _symbol(value.func, self.symbols) + local_called = ( + value.func.id + if isinstance(value.func, ast.Name) + else None + ) + if ( + called in STATEFUL_DECORATORS + or local_called in self.stateful_decorator_factories + ) and function_name not in self.stateful_decorator_factories: + self.stateful_decorator_factories.add(function_name) + changed = True + + def decorator_is_stateful( + self, decorator: ast.AST, symbols: dict[str, str] + ) -> bool: + expression = decorator.func if isinstance(decorator, ast.Call) else decorator + return _symbol(expression, symbols) in STATEFUL_DECORATORS or ( + isinstance(expression, ast.Name) + and expression.id in self.stateful_decorator_factories + ) + + def check_module_bindings(self) -> None: + for statement in _scope_statements(self.tree.body): + value: ast.AST | None = None + targets: set[str] = set() + if isinstance(statement, ast.Assign): + value = statement.value + for target in statement.targets: + targets.update(_target_names(target)) + elif isinstance(statement, ast.AnnAssign): + value = statement.value + targets.update(_target_names(statement.target)) + elif isinstance(statement, ast.AugAssign): + value = statement.value + targets.update(_target_names(statement.target)) + else: + continue + + if value is None: + continue + if _is_immutable_expr(value, self.safe_names, self.symbols): + self.safe_names.update(targets) + continue + for name in sorted(targets - ALLOWED_MUTABLE_METADATA): + self.module_mutable_names.add(name) + self.add( + statement, + f"module-level value '{name}' is mutable or stateful and can be shared between processes", + ) + + def check_class_bindings(self) -> None: + for class_name, class_node in self.shared_classes: + class_safe_names = set(self.safe_names) + is_dataclass = any( + _symbol( + decorator.func if isinstance(decorator, ast.Call) else decorator, + self.symbols, + ) + == "dataclasses.dataclass" + for decorator in class_node.decorator_list + ) + is_pydantic_model = any( + ( + isinstance(base, ast.Name) + and _symbol(base, self.symbols) + in {"pydantic.BaseModel", "pydantic.main.BaseModel"} + ) + or ( + isinstance(base, ast.Attribute) + and _symbol(base, self.symbols) + in {"pydantic.BaseModel", "pydantic.main.BaseModel"} + ) + for base in class_node.bases + ) + is_enum = any( + ( + isinstance(base, ast.Name) + and _symbol(base, self.symbols) in ENUM_BASE_NAMES + ) + or ( + isinstance(base, ast.Attribute) + and _symbol(base, self.symbols) in ENUM_BASE_NAMES + ) + for base in class_node.bases + ) + class_symbols = dict(self.symbols) + class_import_symbols: dict[str, str] = {} + class_shadowed_names: set[str] = set() + for class_statement in _scope_statements(class_node.body): + names: set[str] = set() + if isinstance( + class_statement, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef) + ): + names.add(class_statement.name) + elif isinstance(class_statement, ast.Assign): + for target in class_statement.targets: + names.update(_target_names(target)) + elif isinstance(class_statement, (ast.AnnAssign, ast.AugAssign)): + names.update(_target_names(class_statement.target)) + elif isinstance(class_statement, ast.Import): + for alias in class_statement.names: + name = alias.asname or alias.name.split(".")[0] + if _candidate_module(self.root, alias.name.split(".")) is None: + class_import_symbols[name] = ( + alias.name if alias.asname else name + ) + elif isinstance(class_statement, ast.ImportFrom): + local_module = ( + class_statement.level > 0 + or class_statement.module is None + or _candidate_module( + self.root, class_statement.module.split(".") + ) + is not None + ) + if class_statement.module is not None and not local_module: + for alias in class_statement.names: + if alias.name == "*": + for export in TRUSTED_STAR_EXPORTS.get( + class_statement.module, set() + ): + class_import_symbols[export] = ( + f"{class_statement.module}.{export}" + ) + else: + name = alias.asname or alias.name + class_import_symbols[name] = ( + f"{class_statement.module}.{alias.name}" + ) + class_shadowed_names.update(names) + class_symbols.update( + (name, symbol) + for name, symbol in class_import_symbols.items() + if name not in class_shadowed_names + ) + for name in class_shadowed_names: + class_symbols.pop(name, None) + class_safe_names.discard(name) + for class_statement in class_node.body: + if not isinstance( + class_statement, (ast.FunctionDef, ast.AsyncFunctionDef) + ): + continue + for decorator in class_statement.decorator_list: + expression = decorator.func if isinstance(decorator, ast.Call) else decorator + name = _symbol(expression, class_symbols) + if self.decorator_is_stateful(decorator, class_symbols): + self.add( + decorator, + f"'{(name or _call_name(expression) or 'decorator').rsplit('.', 1)[-1]}' keeps a cache shared between process instances", + ) + for statement in _scope_statements(class_node.body): + def targets_class_namespace(target: ast.AST) -> bool: + if isinstance(target, ast.Subscript): + return ( + isinstance(target.value, ast.Call) + and isinstance(target.value.func, ast.Name) + and target.value.func.id in {"globals", "locals"} + ) + if isinstance(target, (ast.Tuple, ast.List)): + return any(targets_class_namespace(item) for item in target.elts) + return False + + if isinstance(statement, ast.Assign) and any( + targets_class_namespace(target) for target in statement.targets + ): + self.add(statement, "assignment mutates class-level shared state") + elif isinstance(statement, (ast.AnnAssign, ast.AugAssign)) and ( + targets_class_namespace(statement.target) + ): + self.add(statement, "assignment mutates class-level shared state") + elif isinstance(statement, ast.Delete) and any( + targets_class_namespace(target) for target in statement.targets + ): + self.add(statement, "deletion mutates class-level shared state") + elif isinstance(statement, ast.Expr) and isinstance( + statement.value, ast.Call + ): + call = statement.value + if ( + isinstance(call.func, ast.Attribute) + and call.func.attr in MUTATOR_METHODS | {"__setitem__"} + and isinstance(call.func.value, ast.Call) + and isinstance(call.func.value.func, ast.Name) + and call.func.value.func.id in {"globals", "locals"} + ): + self.add(call, "call mutates class-level shared state") + + value: ast.AST | None = None + targets: set[str] = set() + if isinstance(statement, ast.Assign): + value = statement.value + for target in statement.targets: + targets.update(_target_names(target)) + elif isinstance(statement, ast.AnnAssign): + value = statement.value + targets.update(_target_names(statement.target)) + else: + continue + if value is None: + continue + if _is_immutable_expr(value, class_safe_names, class_symbols): + class_safe_names.update(targets) + for name in targets: + class_symbols.pop(name, None) + continue + if ( + is_enum + and isinstance(value, ast.Call) + and _symbol(value.func, class_symbols) == "enum.auto" + and not value.args + and not value.keywords + ): + class_safe_names.update(targets) + continue + if ( + is_dataclass + and isinstance(statement, ast.AnnAssign) + and not _is_classvar_annotation(statement.annotation, class_symbols) + and isinstance(value, ast.Call) + and _symbol(value.func, class_symbols) == "dataclasses.field" + and not value.args + and all( + keyword.arg != "default" + or not _is_mutable_expr( + keyword.value, class_safe_names, class_symbols + ) + for keyword in value.keywords + ) + ): + continue + if ( + is_pydantic_model + and isinstance(statement, ast.AnnAssign) + and not _is_classvar_annotation(statement.annotation, class_symbols) + and isinstance(value, ast.Call) + and _symbol(value.func, class_symbols) == "pydantic.Field" + and not value.args + and all( + keyword.arg != "default" + or not _is_mutable_expr( + keyword.value, class_safe_names, class_symbols + ) + for keyword in value.keywords + ) + ): + continue + for name in sorted(targets): + self.add( + statement, + f"class attribute '{class_name}.{name}' is mutable or stateful and is shared by all process instances", + ) + + def check_module_mutations(self) -> None: + statements = list(_scope_statements(self.tree.body)) + class_aliases = set(self.classes) + shared_aliases: set[str] = set() + global_objects = self.module_binding_names + + def is_class_ref(node: ast.AST) -> bool: + return isinstance(node, ast.Name) and node.id in class_aliases + + def is_shared_ref(node: ast.AST) -> bool: + if isinstance(node, ast.Name): + return node.id in global_objects or node.id in shared_aliases + if isinstance(node, ast.Attribute): + return ( + is_class_ref(node.value) + or is_shared_ref(node.value) + or ( + isinstance(node.value, ast.Name) + and node.value.id in global_objects + ) + ) + if isinstance(node, ast.Subscript): + return is_shared_ref(node.value) + if ( + isinstance(node, ast.Call) + and isinstance(node.func, ast.Name) + and node.func.id == "getattr" + and node.args + ): + return is_class_ref(node.args[0]) or is_shared_ref(node.args[0]) + if ( + isinstance(node, ast.Call) + and isinstance(node.func, ast.Name) + and node.func.id == "eval" + and node.args + and isinstance(node.args[0], ast.Constant) + and isinstance(node.args[0].value, str) + ): + try: + expression = ast.parse(node.args[0].value, mode="eval").body + except SyntaxError: + return False + return is_shared_ref(expression) + return ( + isinstance(node, ast.Call) + and isinstance(node.func, ast.Name) + and node.func.id == "vars" + and len(node.args) == 1 + and (is_class_ref(node.args[0]) or is_shared_ref(node.args[0])) + ) or ( + isinstance(node, ast.Call) + and isinstance(node.func, ast.Name) + and node.func.id in {"globals", "locals"} + ) + + assignments = [ + statement + for statement in statements + if isinstance(statement, (ast.Assign, ast.AnnAssign, ast.NamedExpr)) + ] + changed = True + while changed: + changed = False + for assignment in assignments: + value, targets = _assignment_parts(assignment) or (assignment, []) + names = set().union(*map(_target_names, targets), set()) + before = (len(class_aliases), len(shared_aliases)) + if is_class_ref(value): + class_aliases.update(names) + if is_shared_ref(value): + shared_aliases.update(names) + if before != (len(class_aliases), len(shared_aliases)): + changed = True + + def target_is_shared(target: ast.AST) -> bool: + if isinstance(target, ast.Attribute): + return ( + is_class_ref(target.value) + or is_shared_ref(target.value) + or ( + isinstance(target.value, ast.Name) + and target.value.id in global_objects + ) + ) + if isinstance(target, ast.Subscript): + return is_shared_ref(target.value) + if isinstance(target, (ast.Tuple, ast.List)): + return any(target_is_shared(element) for element in target.elts) + return False + + for statement in statements: + if isinstance(statement, ast.Assign): + if any(target_is_shared(target) for target in statement.targets): + self.add(statement, "assignment creates or mutates module-level shared state") + elif isinstance(statement, (ast.AnnAssign, ast.AugAssign)): + if target_is_shared(statement.target): + self.add(statement, "assignment creates or mutates module-level shared state") + elif isinstance(statement, ast.Delete): + if any(target_is_shared(target) for target in statement.targets): + self.add(statement, "deletion mutates module-level shared state") + elif isinstance(statement, ast.Expr) and isinstance(statement.value, ast.Call): + call = statement.value + if isinstance(call.func, ast.Name) and call.func.id in { + "exec", + "globals", + "locals", + }: + self.add(call, f"'{call.func.id}' can access or modify module-level shared state") + elif ( + isinstance(call.func, ast.Name) + and call.func.id == "eval" + and is_shared_ref(call) + ): + self.add(call, "'eval' accesses module-level shared state") + elif ( + isinstance(call.func, ast.Name) + and call.func.id in {"setattr", "delattr"} + and call.args + and ( + is_class_ref(call.args[0]) + or is_shared_ref(call.args[0]) + or ( + isinstance(call.args[0], ast.Name) + and call.args[0].id in global_objects + ) + ) + ): + self.add(call, f"'{call.func.id}' mutates module-level shared state") + elif ( + _symbol(call.func, self.symbols) in OPERATOR_MUTATORS + and call.args + and is_shared_ref(call.args[0]) + ): + self.add(call, "operator call mutates module-level shared state") + elif ( + (resolved := _resolved_attribute(call.func)) is not None + and resolved[1] in UNBOUND_MUTATOR_METHODS + and call.args + and ( + is_class_ref(call.args[0]) + or is_shared_ref(call.args[0]) + or ( + isinstance(call.args[0], ast.Name) + and call.args[0].id in global_objects + ) + ) + ): + self.add(call, f"'{resolved[1]}' mutates module-level shared state") + elif ( + (resolved := _resolved_attribute(call.func)) is not None + and resolved[1] in MUTATOR_METHODS + and _symbol(call.func, self.symbols) != "operator.add" + and is_shared_ref(resolved[0]) + ): + if not ( + _symbol(resolved[0], self.symbols) == "sys.path" + and resolved[1] in {"append", "insert"} + ): + self.add( + call, + f"'{resolved[1]}' mutates module-level shared state", + ) + elif ( + (canonical := _symbol(call.func, self.symbols)) is not None + and (_root_name(call.func) or "") in self.import_names + and not _imported_call_allowed(canonical) + and canonical not in STATEFUL_DECORATORS + and canonical not in TYPE_FACTORIES + and canonical not in MUTABLE_CONSTRUCTORS + and canonical not in IMMUTABLE_SCALAR_CONSTRUCTORS + ): + self.add( + call, + f"call to imported API '{canonical}' is not known to be free of shared module-state mutation", + ) + + def check_defaults(self) -> None: + for node in ast.walk(self.tree): + if not isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.Lambda)): + continue + defaults = list(node.args.defaults) + defaults.extend(default for default in node.args.kw_defaults if default is not None) + for default in defaults: + if _is_mutable_expr(default, self.safe_names, self.symbols): + self.add(default, "mutable default argument is shared between function calls") + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + for decorator in node.decorator_list: + expression = decorator.func if isinstance(decorator, ast.Call) else decorator + name = _symbol(expression, self.symbols) + if self.decorator_is_stateful(decorator, self.symbols): + self.add( + decorator, + f"'{(name or _call_name(expression) or 'decorator').rsplit('.', 1)[-1]}' keeps a cache shared between process instances", + ) + + def check_dynamic_imports(self) -> None: + string_values = _static_string_values(self.tree) + importlib_modules, import_module_functions = _dynamic_import_names(self.tree) + for node in ast.walk(self.tree): + if ( + isinstance(node, ast.Call) + and node.args + and _is_dynamic_import_call( + node, importlib_modules, import_module_functions + ) + and _static_string(node.args[0], string_values) is None + ): + self.add( + node, + "dynamic import name cannot be resolved statically; shared state in the imported module cannot be checked", + ) + + def collect_shared_instance_attributes(self) -> None: + for class_name, class_node in self.classes.items(): + attributes = self.shared_instance_attributes.setdefault(class_name, set()) + functions = [ + statement + for statement in class_node.body + if isinstance(statement, (ast.FunctionDef, ast.AsyncFunctionDef)) + ] + changed = True + while changed: + changed = False + for function in functions: + positional = [*function.args.posonlyargs, *function.args.args] + if not positional: + continue + instance_names = {positional[0].arg} + nodes = _function_nodes(function) + assignments = [ + node + for node in nodes + if isinstance(node, (ast.Assign, ast.AnnAssign, ast.NamedExpr)) + ] + local_names = { + argument.arg + for argument in [*positional, *function.args.kwonlyargs] + } + local_import_names: set[str] = set() + for node in nodes: + if isinstance(node, ast.Assign): + for target in node.targets: + local_names.update(_target_names(target)) + elif isinstance(node, (ast.AnnAssign, ast.NamedExpr)): + local_names.update(_target_names(node.target)) + elif isinstance(node, (ast.Import, ast.ImportFrom)): + names = { + alias.asname or alias.name.split(".")[0] + for alias in node.names + if alias.name != "*" + } + local_names.update(names) + local_import_names.update(names) + + shared_aliases: set[str] = set() + + def is_instance(node: ast.AST) -> bool: + return isinstance(node, ast.Name) and node.id in instance_names + + def is_shared(node: ast.AST) -> bool: + if isinstance(node, ast.Name): + return ( + node.id in self.module_binding_names + and node.id not in local_names + ) or node.id in shared_aliases | local_import_names + if (resolved := _resolved_attribute(node)) is not None: + return ( + is_instance(resolved[0]) and resolved[1] in attributes + ) or is_shared(resolved[0]) + if isinstance(node, ast.Subscript): + return is_shared(node.value) + return _call_returns_shared( + node, + is_shared, + self.shared_returning_functions, + self.returned_parameters, + ) or ( + isinstance(node, ast.Call) + and _symbol(node.func, self.symbols) + == "importlib.import_module" + ) + + aliases_changed = True + while aliases_changed: + aliases_changed = False + for assignment in assignments: + value, targets = _assignment_parts(assignment) or (assignment, []) + if is_instance(value): + names = set().union(*(_target_names(t) for t in targets)) + before = len(instance_names) + instance_names.update(names) + aliases_changed |= len(instance_names) != before + if not is_shared(value): + continue + for target in targets: + names = _target_names(target) + before = len(shared_aliases) + shared_aliases.update(names) + aliases_changed |= len(shared_aliases) != before + if ( + isinstance(target, ast.Attribute) + and is_instance(target.value) + and target.attr not in attributes + ): + attributes.add(target.attr) + changed = True + + def check_function( + self, + function: ast.FunctionDef | ast.AsyncFunctionDef, + owner: str | None, + shared_closure: bool, + ) -> None: + nodes = _function_nodes(function) + all_class_names = set(self.classes) + instance_aliases: set[str] = set() + class_aliases: set[str] = set() + shared_aliases: set[str] = set() + shared_instance_attributes = self.shared_instance_attributes.get(owner or "", set()) + positional = [*function.args.posonlyargs, *function.args.args] + local_names = { + argument.arg + for argument in [*positional, *function.args.kwonlyargs] + } + if function.args.vararg is not None: + local_names.add(function.args.vararg.arg) + if function.args.kwarg is not None: + local_names.add(function.args.kwarg.arg) + local_import_names: set[str] = set() + local_import_symbols: dict[str, str] = {} + for node in nodes: + if isinstance(node, ast.Assign): + for target in node.targets: + local_names.update(_target_names(target)) + elif isinstance(node, (ast.AnnAssign, ast.AugAssign, ast.NamedExpr)): + local_names.update(_target_names(node.target)) + elif isinstance(node, ast.Import): + names = { + alias.asname or alias.name.split(".")[0] + for alias in node.names + } + local_names.update(names) + local_import_names.update(names) + for alias in node.names: + name = alias.asname or alias.name.split(".")[0] + if _candidate_module(self.root, alias.name.split(".")) is None: + local_import_symbols[name] = alias.name if alias.asname else name + elif isinstance(node, ast.ImportFrom): + names = { + alias.asname or alias.name + for alias in node.names + if alias.name != "*" + } + local_names.update(names) + local_import_names.update(names) + local_module = ( + node.level > 0 + or node.module is None + or _candidate_module(self.root, node.module.split(".")) is not None + ) + if node.module is not None and not local_module: + for alias in node.names: + if alias.name != "*": + local_import_symbols[alias.asname or alias.name] = ( + f"{node.module}.{alias.name}" + ) + function_symbols = dict(self.symbols) + for name in local_names - local_import_names: + function_symbols.pop(name, None) + function_symbols.update(local_import_symbols) + imported_callable_names = (self.import_names - local_names) | local_import_names + if owner is not None and positional: + first = positional[0].arg + if first == "cls" or any( + isinstance(decorator, ast.Name) and decorator.id == "classmethod" + for decorator in function.decorator_list + ): + class_aliases.add(first) + else: + instance_aliases.add(first) + + def is_instance_ref(node: ast.AST) -> bool: + return isinstance(node, ast.Name) and node.id in instance_aliases + + def is_class_ref(node: ast.AST) -> bool: + if isinstance(node, ast.Name): + return ( + node.id in all_class_names and node.id not in local_names + ) or node.id in class_aliases + if isinstance(node, ast.Attribute): + return node.attr == "__class__" and is_instance_ref(node.value) + return ( + isinstance(node, ast.Call) + and isinstance(node.func, ast.Name) + and node.func.id == "type" + and len(node.args) == 1 + and is_instance_ref(node.args[0]) + ) + + global_objects = self.module_binding_names + + def is_shared_ref(node: ast.AST) -> bool: + if isinstance(node, ast.Name): + return ( + node.id in global_objects and node.id not in local_names + ) or node.id in shared_aliases | local_import_names + if (resolved := _resolved_attribute(node)) is not None: + value, attribute = resolved + if is_instance_ref(value) and attribute in shared_instance_attributes: + return True + if attribute == "__dict__" and ( + is_class_ref(value) + or (isinstance(value, ast.Name) and value.id in global_objects) + ): + return True + return is_class_ref(value) or is_shared_ref(value) or ( + isinstance(value, ast.Name) and value.id in global_objects + ) + if isinstance(node, ast.Subscript): + return is_shared_ref(node.value) + if _call_returns_shared( + node, + is_shared_ref, + self.shared_returning_functions, + self.returned_parameters, + ): + return True + if ( + isinstance(node, ast.Call) + and isinstance(node.func, ast.Name) + and node.func.id == "getattr" + and node.args + ): + return ( + is_class_ref(node.args[0]) + or is_shared_ref(node.args[0]) + or ( + isinstance(node.args[0], ast.Name) + and node.args[0].id in global_objects + ) + ) + if ( + isinstance(node, ast.Call) + and isinstance(node.func, ast.Name) + and node.func.id == "eval" + and node.args + and isinstance(node.args[0], ast.Constant) + and isinstance(node.args[0].value, str) + ): + try: + expression = ast.parse(node.args[0].value, mode="eval").body + except SyntaxError: + return False + return is_shared_ref(expression) + return ( + isinstance(node, ast.Call) + and isinstance(node.func, ast.Name) + and node.func.id == "vars" + and len(node.args) == 1 + and ( + is_class_ref(node.args[0]) + or is_shared_ref(node.args[0]) + or ( + isinstance(node.args[0], ast.Name) + and node.args[0].id in global_objects + ) + ) + ) or ( + isinstance(node, ast.Call) + and isinstance(node.func, ast.Name) + and node.func.id == "globals" + ) + + assignments = [ + node + for node in nodes + if isinstance(node, (ast.Assign, ast.AnnAssign, ast.NamedExpr)) + ] + unbound_mutator_aliases: set[str] = set() + bound_mutator_aliases: set[str] = set() + imported_callable_aliases: dict[str, str] = {} + changed = True + while changed: + changed = False + for assignment in assignments: + value, targets = _assignment_parts(assignment) or (assignment, []) + names: set[str] = set() + for target in targets: + names.update(_target_names(target)) + before = ( + len(instance_aliases), + len(class_aliases), + len(shared_aliases), + len(unbound_mutator_aliases), + len(bound_mutator_aliases), + len(imported_callable_aliases), + ) + if is_instance_ref(value): + instance_aliases.update(names) + if is_class_ref(value): + class_aliases.update(names) + if is_shared_ref(value): + shared_aliases.update(names) + symbol = _symbol(value, function_symbols) + aliased_symbol = ( + imported_callable_aliases.get(value.id) + if isinstance(value, ast.Name) + else None + ) + if aliased_symbol is not None: + symbol = aliased_symbol + if ( + symbol is not None + and ( + aliased_symbol is not None + or (_root_name(value) or "") in imported_callable_names + ) + ): + for name in names: + imported_callable_aliases[name] = symbol + if symbol in OPERATOR_MUTATORS or ( + isinstance(value, ast.Name) + and value.id in unbound_mutator_aliases + ): + unbound_mutator_aliases.update(names) + if ( + (resolved := _resolved_attribute(value)) is not None + and resolved[1] in MUTATOR_METHODS + and _symbol(value, function_symbols) != "operator.add" + and is_shared_ref(resolved[0]) + ) or ( + isinstance(value, ast.Name) and value.id in bound_mutator_aliases + ): + bound_mutator_aliases.update(names) + if before != ( + len(instance_aliases), + len(class_aliases), + len(shared_aliases), + len(unbound_mutator_aliases), + len(bound_mutator_aliases), + len(imported_callable_aliases), + ): + changed = True + + def target_is_shared(target: ast.AST) -> bool: + if isinstance(target, ast.Attribute): + return ( + is_class_ref(target.value) + or is_shared_ref(target.value) + or ( + isinstance(target.value, ast.Name) + and target.value.id in global_objects + ) + ) + if isinstance(target, ast.Subscript): + return is_shared_ref(target.value) + if isinstance(target, (ast.Tuple, ast.List)): + return any(target_is_shared(element) for element in target.elts) + return False + + for node in nodes: + if isinstance(node, ast.Global): + self.add(node, "'global' state can be shared between process instances") + elif isinstance(node, ast.Nonlocal) and shared_closure: + self.add(node, "shared closure state can be accessed by multiple process instances") + elif isinstance(node, ast.Assign): + if any(target_is_shared(target) for target in node.targets): + self.add(node, "assignment mutates state outside the process instance") + elif isinstance(node, (ast.AnnAssign, ast.AugAssign)): + if target_is_shared(node.target): + self.add(node, "assignment mutates state outside the process instance") + elif isinstance(node, ast.Delete): + if any(target_is_shared(target) for target in node.targets): + self.add(node, "deletion mutates state outside the process instance") + elif isinstance(node, ast.Call): + if isinstance(node.func, ast.Name) and node.func.id in {"exec", "globals"}: + self.add(node, f"'{node.func.id}' can access or modify module-level shared state") + elif ( + isinstance(node.func, ast.Name) + and node.func.id == "eval" + and is_shared_ref(node) + ): + self.add(node, "'eval' accesses module-level shared state") + elif ( + isinstance(node.func, ast.Name) + and node.func.id in {"setattr", "delattr"} + and node.args + and ( + is_class_ref(node.args[0]) + or is_shared_ref(node.args[0]) + or ( + isinstance(node.args[0], ast.Name) + and node.args[0].id in global_objects + ) + ) + ): + self.add(node, f"'{node.func.id}' mutates shared object state") + elif ( + ( + _symbol(node.func, function_symbols) in OPERATOR_MUTATORS + or ( + isinstance(node.func, ast.Name) + and node.func.id in unbound_mutator_aliases + ) + ) + and node.args + and is_shared_ref(node.args[0]) + ): + self.add(node, "operator call mutates shared object state") + elif ( + isinstance(node.func, ast.Name) + and node.func.id in bound_mutator_aliases + ): + self.add(node, "aliased method mutates shared object state") + elif ( + ( + imported_callable_aliases.get( + node.func.id, _symbol(node.func, function_symbols) + ) + if isinstance(node.func, ast.Name) + else _symbol(node.func, function_symbols) + ) + in IMPORTED_ARGUMENT_MUTATORS + and node.args + and is_shared_ref(node.args[0]) + ): + self.add(node, "imported function mutates shared object state") + elif ( + (resolved := _resolved_attribute(node.func)) is not None + and resolved[1] in UNBOUND_MUTATOR_METHODS + and node.args + and ( + is_class_ref(node.args[0]) + or is_shared_ref(node.args[0]) + or ( + isinstance(node.args[0], ast.Name) + and node.args[0].id in global_objects + ) + ) + ): + self.add(node, f"'{resolved[1]}' mutates shared object state") + elif ( + (resolved := _resolved_attribute(node.func)) is not None + and resolved[1] in MUTATOR_METHODS + and _symbol(node.func, function_symbols) != "operator.add" + and is_shared_ref(resolved[0]) + ): + self.add(node, f"'{resolved[1]}' mutates state outside the process instance") + elif ( + ( + canonical := ( + imported_callable_aliases.get( + node.func.id, _symbol(node.func, function_symbols) + ) + if isinstance(node.func, ast.Name) + else _symbol(node.func, function_symbols) + ) + ) + is not None + and ( + (_root_name(node.func) or "") in imported_callable_names + or ( + isinstance(node.func, ast.Name) + and node.func.id in imported_callable_aliases + ) + ) + and not _imported_call_allowed(canonical) + and canonical not in STATEFUL_DECORATORS + and canonical not in TYPE_FACTORIES + and canonical not in MUTABLE_CONSTRUCTORS + and canonical not in IMMUTABLE_SCALAR_CONSTRUCTORS + ): + self.add( + node, + f"call to imported API '{canonical}' is not known to be free of shared module-state mutation", + ) + + def check_functions(self) -> None: + def visit_body( + body: Iterable[ast.stmt], + owner: str | None = None, + shared_closure: bool = False, + ) -> None: + for statement in body: + if isinstance(statement, (ast.FunctionDef, ast.AsyncFunctionDef)): + function_closure = shared_closure or ( + owner is None and statement.name in self.shared_closure_factories + ) + self.check_function(statement, owner, function_closure) + visit_body(statement.body, None, function_closure) + elif isinstance(statement, ast.ClassDef): + visit_body(statement.body, statement.name, shared_closure) + else: + for child in ast.iter_child_nodes(statement): + if isinstance(child, ast.stmt): + visit_body([child], owner, shared_closure) + + visit_body(self.tree.body) + + def run(self) -> list[tuple[int, int, str]]: + self.collect_symbols() + self.mark_shared_closure_factories() + self.check_module_bindings() + self.collect_function_summaries() + self.check_class_bindings() + self.check_module_mutations() + self.check_defaults() + self.check_dynamic_imports() + self.collect_shared_instance_attributes() + self.check_functions() + return sorted(set(self.violations)) + + +def _candidate_module(root: Path, parts: list[str]) -> Path | None: + if not parts: + return None + module = root.joinpath(*parts) + candidates = [module.with_suffix(".py"), module / "__init__.py"] + root = root.resolve() + for candidate in candidates: + try: + resolved = candidate.resolve() + resolved.relative_to(root) + except (OSError, ValueError): + continue + if resolved.is_file(): + return resolved + return None + + +def _local_imports(tree: ast.Module, current: Path, root: Path) -> set[Path]: + result: set[Path] = set() + importlib_modules, import_module_functions = _dynamic_import_names(tree) + string_values = _static_string_values(tree) + + for node in ast.walk(tree): + if isinstance(node, ast.Import): + for alias in node.names: + candidate = _candidate_module(root, alias.name.split(".")) + if candidate is not None: + result.add(candidate) + elif isinstance(node, ast.ImportFrom): + if node.level: + base = current.parent + for _ in range(node.level - 1): + base = base.parent + else: + base = root + module_parts = node.module.split(".") if node.module else [] + candidate = _candidate_module(base, module_parts) + if candidate is not None: + result.add(candidate) + for alias in node.names: + if alias.name == "*": + continue + candidate = _candidate_module(base, [*module_parts, alias.name]) + if candidate is not None: + result.add(candidate) + elif isinstance(node, ast.Call) and node.args: + module_name = _static_string(node.args[0], string_values) + if ( + _is_dynamic_import_call( + node, importlib_modules, import_module_functions + ) + and module_name is not None + and not module_name.startswith(".") + ): + candidate = _candidate_module(root, module_name.split(".")) + if candidate is not None: + result.add(candidate) + return result + + +def validate_source(source: str, filename: str, class_names: list[str]) -> list[str]: + tree = ast.parse(source, filename=filename) + path = Path(filename) + return [ + f"{filename}:{line}:{column}: {message}" + for line, column, message in _FileChecker( + tree, path, set(class_names), path.parent + ).run() + ] + + +def validate_solution(path: str, class_names: list[str]) -> list[str]: + entrypoint = Path(path).resolve() + root = entrypoint.parent + pending = [entrypoint] + visited: set[Path] = set() + violations: list[str] = [] + while pending: + current = pending.pop() + if current in visited: + continue + visited.add(current) + source = current.read_text(encoding="utf-8") + tree = ast.parse(source, filename=str(current)) + checker = _FileChecker( + tree, + current, + set(class_names) if current == entrypoint else set(), + root, + ) + violations.extend( + f"{current}:{line}:{column}: {message}" + for line, column, message in checker.run() + ) + pending.extend(sorted(_local_imports(tree, current, root) - visited)) + return sorted(set(violations)) diff --git a/homework/08-kv-replication/tests/src/common.rs b/homework/08-kv-replication/tests/src/common.rs new file mode 100644 index 0000000..194f5ab --- /dev/null +++ b/homework/08-kv-replication/tests/src/common.rs @@ -0,0 +1,139 @@ +use std::cell::OnceCell; + +use byteorder::{ByteOrder, LittleEndian}; +use rand::distributions::WeightedIndex; +use rand::prelude::*; +use rand_pcg::Pcg64; +use serde::Serialize; +use serde_json::{Map, Value}; + +use anysystem::python::{PyProcess, PyProcessFactory}; +use anysystem::{Message, Process, System}; + +use crate::count_keys::TestSystem; + +#[derive(Serialize)] +pub struct GetReqMessage<'a> { + pub key: &'a str, + pub quorum: u8, +} + +pub struct ValueResponse { + pub key: String, + pub value: Option, +} + +pub fn parse_value_response(msg: &Message, proc: &str) -> Result { + let tip = &msg.tip; + let mut data: Map = serde_json::from_str(&msg.data) + .map_err(|err| format!("Invalid {tip} from {proc}: {err}"))?; + let Some(Value::String(key)) = data.remove("key") else { + return Err(format!("Missing or invalid key in {tip} from {proc}")); + }; + let value = match data.remove("value") { + Some(Value::String(value)) => Some(value), + Some(Value::Null) => None, + None => { + return Err(format!( + "Missing field 'value' in {tip} from {proc} for key {key}" + )) + } + Some(_) => { + return Err(format!( + "Invalid 'value' in {tip} from {proc} for key {key}: expected a string or null" + )) + } + }; + Ok(ValueResponse { key, value }) +} + +#[derive(Serialize)] +pub struct PutReqMessage<'a> { + pub key: &'a str, + pub value: &'a str, + pub quorum: u8, +} + +#[derive(Serialize)] +pub struct DeleteReqMessage<'a> { + pub key: &'a str, + pub quorum: u8, +} + +#[derive(Copy, Clone)] +pub struct TestConfig<'a> { + pub proc_factory: &'a PyProcessFactory, + pub count_keys_error: &'a OnceCell, + pub proc_count: u32, + pub seed: u64, +} + +pub const MIN_NETWORK_DELAY: f64 = 0.01; + +pub fn build_system<'a>(config: &TestConfig<'a>) -> TestSystem<'a> { + build_system_with(config, |proc| Box::new(proc)) +} + +pub fn build_system_with<'a>( + config: &TestConfig<'a>, + wrap: impl Fn(PyProcess) -> Box, +) -> TestSystem<'a> { + let mut sys = System::new(config.seed); + sys.network().set_delays(MIN_NETWORK_DELAY, 0.1); + let mut proc_names = Vec::new(); + for n in 0..config.proc_count { + proc_names.push(format!("{n}")); + } + for proc_name in proc_names.iter() { + let proc = config + .proc_factory + .build((proc_name, proc_names.clone()), config.seed); + // process and node on which it runs have the same name + let node_name = proc_name.clone(); + sys.add_node(&node_name); + sys.add_process(proc_name, wrap(proc), &node_name); + } + TestSystem::new(sys, config.count_keys_error) +} + +pub fn key_replicas(key: &str, sys: &System) -> Vec { + let proc_count = sys.process_names().len(); + let mut replicas = Vec::new(); + let hash = md5::compute(key); + let hash128 = LittleEndian::read_u128(&hash.0); + let mut replica = (hash128 % proc_count as u128) as usize; + for _ in 0..3 { + replicas.push(replica.to_string()); + replica += 1; + if replica == proc_count { + replica = 0; + } + } + replicas +} + +pub fn key_non_replicas(key: &str, sys: &System) -> Vec { + let proc_count = sys.process_names().len(); + let replicas = key_replicas(key, sys); + let last_primary = replicas.last().unwrap().parse::().unwrap(); + (1..=proc_count - replicas.len()) + .map(|offset| ((last_primary + offset) % proc_count).to_string()) + .collect() +} + +const SYMBOLS: [char; 36] = [ + 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', + 't', 'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', +]; +const WEIGHTS: [usize; 36] = [ + 13, 16, 3, 8, 8, 5, 6, 23, 4, 8, 24, 12, 2, 1, 1, 10, 5, 8, 10, 1, 24, 3, 1, 8, 12, 22, 5, 20, + 18, 5, 5, 2, 1, 3, 16, 22, +]; + +pub fn random_string(length: usize, rand: &mut Pcg64) -> String { + let dist = WeightedIndex::new(WEIGHTS).unwrap(); + rand.sample_iter(&dist) + .take(length) + .map(|x| SYMBOLS[x]) + .collect() +} diff --git a/homework/08-kv-replication/tests/src/count_keys.rs b/homework/08-kv-replication/tests/src/count_keys.rs new file mode 100644 index 0000000..e7b2557 --- /dev/null +++ b/homework/08-kv-replication/tests/src/count_keys.rs @@ -0,0 +1,80 @@ +use std::cell::OnceCell; +use std::collections::HashSet; +use std::ops::{Deref, DerefMut}; + +use anysystem::logger::LogEntry; +use anysystem::System; + +// One test case can build several simulations. Check each finished trace once, +// including callbacks that ran after the client received COUNT_KEYS_RESP. +pub struct TestSystem<'a> { + system: System, + count_keys_error: &'a OnceCell, +} + +impl<'a> TestSystem<'a> { + pub fn new(system: System, count_keys_error: &'a OnceCell) -> Self { + Self { + system, + count_keys_error, + } + } +} + +impl Deref for TestSystem<'_> { + type Target = System; + + fn deref(&self) -> &System { + &self.system + } +} + +impl DerefMut for TestSystem<'_> { + fn deref_mut(&mut self) -> &mut System { + &mut self.system + } +} + +impl Drop for TestSystem<'_> { + fn drop(&mut self) { + if !std::thread::panicking() && self.count_keys_error.get().is_none() { + if let Some(sender) = count_keys_network_sender(self.system.logger().trace()) { + let _ = self.count_keys_error.set(format!( + "COUNT_KEYS on {sender} sent a message to another node" + )); + } + } + } +} + +fn count_keys_network_sender(trace: &[LogEntry]) -> Option<&str> { + let mut timers = HashSet::new(); + let mut self_messages = HashSet::new(); + let mut handling_count = false; + for entry in trace { + match entry { + LogEntry::LocalMessageReceived { msg, .. } => handling_count = msg.tip == "COUNT_KEYS", + LogEntry::TimerFired { timer_id, .. } => handling_count = timers.remove(timer_id), + LogEntry::MessageReceived { msg_id, .. } => { + handling_count = self_messages.remove(msg_id) + } + // AnySystem gives every scheduled timer a new ID, even when the name is reused. + LogEntry::TimerSet { timer_id, .. } if handling_count => { + timers.insert(timer_id); + } + LogEntry::MessageSent { + src_node, + dst_node, + msg_id, + .. + } if handling_count => { + if src_node != dst_node { + return Some(src_node); + } + self_messages.insert(msg_id); + } + _ => {} + } + } + None +} diff --git a/homework/08-kv-replication/tests/src/grading.rs b/homework/08-kv-replication/tests/src/grading.rs new file mode 100644 index 0000000..70029c5 --- /dev/null +++ b/homework/08-kv-replication/tests/src/grading.rs @@ -0,0 +1,171 @@ +use indexmap::IndexMap; + +use anysystem::test::TestResult; + +pub type Results = IndexMap<&'static str, TestResult>; + +// Each item is worth half a point. Paired tests check the same skill. +pub const CORE: &[(&str, &[&[&str]])] = &[ + ( + "Operations and placement", + &[ + &["BASIC"], + &["MC BASIC"], + &["REPLICAS CHECK"], + &["DATA PLACEMENT"], + &["CONCURRENT GET PUT", "CONCURRENT GETS"], + &["DELETE ABSENT"], + ], + ), + ( + "Conflicts and repair", + &[ + &["CONCURRENT WRITES", "MC CONCURRENT WRITES"], + &["CONCURRENT WRITES TIE"], + &["STALE REPLICA", "DIVERGED REPLICAS"], + &["STALE REPLICA DELETE", "READ REPAIR DELETE"], + &["CONCURRENT DELETES"], + ], + ), + ( + "Quorums and partitions", + &[ + &["SLOPPY QUORUM READ", "SLOPPY QUORUM WRITE"], + &["SLOPPY QUORUM TRICKY"], + &["SLOPPY QUORUM DELETE"], + &["PARTITION CLIENTS", "PARTITION MIXED"], + &["HINTED HANDOFF", "MC SLOPPY QUORUM HINTED HANDOFF"], + ], + ), +]; + +pub const TRACKS: &[(&str, &[&str])] = &[ + ( + "A", + &[ + "CONCURRENT PUT DELETE TIE", + "READ REPAIR PUT DELETE TIE", + "READ REPAIR LATE", + "READ REPAIR TIE", + "READ REPAIR MULTI KEY", + "DATA PLACEMENT MULTI KEY", + ], + ), + ( + "B", + &[ + "SLOPPY QUORUM GET DISTINCT", + "SLOPPY QUORUM PUT DISTINCT", + "SLOPPY QUORUM DELETE DISTINCT", + "SLOPPY QUORUM GET ORDER", + "SLOPPY QUORUM PUT ORDER", + "SLOPPY QUORUM DELETE ORDER", + "SLOPPY QUORUM DELETE ORDER PARTITION", + "SLOPPY QUORUM DELETE PARTITION", + ], + ), + ( + "C", + &[ + "HINTED HANDOFF PUT DELETE TIE", + "HINTED HANDOFF MULTI KEY", + "HINTED HANDOFF RETENTION", + "HINTED HANDOFF LWW", + "SLOPPY QUORUM READ REPAIR", + "SLOPPY QUORUM READ REPAIR DELETE", + "SLOPPY QUORUM READ REPAIR MULTI KEY", + ], + ), +]; + +pub fn all_passed(results: &Results, names: &[&str]) -> bool { + names + .iter() + .all(|name| results.get(name).is_some_and(Result::is_ok)) +} + +pub fn core_points(results: &Results, items: &[&[&str]]) -> f32 { + items + .iter() + .filter(|item| all_passed(results, item)) + .count() as f32 + / 2.0 +} + +pub fn score(core: f32, results: &Results) -> f32 { + if core < 8.0 { + return core; + } + let complete = TRACKS + .iter() + .filter(|(_, names)| all_passed(results, names)) + .count(); + core + complete.min(2) as f32 +} + +#[cfg(test)] +mod tests { + use super::*; + use std::collections::HashSet; + + #[test] + fn rubric_covers_each_case_once() { + let mut names = HashSet::new(); + for &(_, items) in CORE { + for item in items { + for &name in *item { + assert!(names.insert(name), "{name}"); + } + } + } + assert_eq!(names.len(), 23); + for &(_, track) in TRACKS { + for &name in track { + assert!(names.insert(name), "{name}"); + } + } + assert_eq!(names.len(), 44); + assert_eq!(names, crate::CASES.iter().map(|&(name, _)| name).collect()); + } + + #[test] + fn independent_core_credits_and_advanced_thresholds() { + let reference: Results = crate::CASES + .iter() + .map(|&(name, _)| (name, Ok(true))) + .collect(); + let total = |results: &Results| { + score( + CORE.iter() + .map(|(_, items)| core_points(results, items)) + .sum(), + results, + ) + }; + for (i, &(_, items)) in CORE.iter().enumerate() { + let mut results = reference.clone(); + for item in items { + // A failure on either half of a pair loses only this item's credit. + for &name in *item { + results.insert(name, Err(String::new())); + assert_eq!(total(&results), 7.5); + results.insert(name, Ok(true)); + } + } + for item in items { + results.insert(item[0], Err(String::new())); + } + assert_eq!(total(&results), [5.0, 5.5, 5.5][i]); + } + for mask in 0_u32..8 { + let mut results = reference.clone(); + for (i, &(_, names)) in TRACKS.iter().enumerate() { + if mask & (1 << i) == 0 { + results.shift_remove(names[0]); + } + } + assert_eq!(total(&results), 8.0 + mask.count_ones().min(2) as f32); + } + assert_eq!(total(&Results::new()), 0.0); + } +} diff --git a/homework/08-kv-replication/tests/src/main.rs b/homework/08-kv-replication/tests/src/main.rs new file mode 100644 index 0000000..32947ad --- /dev/null +++ b/homework/08-kv-replication/tests/src/main.rs @@ -0,0 +1,252 @@ +mod common; +mod count_keys; +mod grading; +mod python_errors; +mod shared_state; +mod tests; +mod tests_mc; + +use std::collections::HashSet; +use std::env; +use std::io::Write; +use std::path::Path; + +use clap::Parser; +use env_logger::Builder; +use log::LevelFilter; + +use crate::grading::{core_points, score, Results, CORE, TRACKS}; +use anysystem::test::TestSuite; + +use crate::python_errors::{run_case, Case, CaseConfig, Solution}; +use crate::tests::*; +use crate::tests_mc::*; + +/// Replicated KV Store Homework Tests +#[derive(Parser, Debug)] +#[clap(about, long_about = None)] +struct Args { + /// Path to Python file with solution + #[clap(long = "impl", short = 'i', default_value = "solution/node.py")] + solution_path: String, + + /// Test to run (optional) + #[clap(long = "test", short)] + test: Option, + + /// Run only the 23 core tests + #[clap(long, conflicts_with_all = &["all", "test"])] + core: bool, + + /// Run all 44 tests for diagnostics, ignoring track selection and the core prerequisite + #[clap(long, conflicts_with = "test")] + all: bool, + + /// Print execution trace + #[clap(long, short)] + debug: bool, + + /// Number of nodes used in tests + #[clap(long, short, default_value = "6")] + node_count: u32, + + /// Random seed used in tests + #[clap(long, short, default_value = "123")] + seed: u64, +} + +fn main() { + let args = Args::parse(); + // Initialize CPython with a 32-bit hash seed; the simulator uses the full u64. + env::set_var("PYTHONHASHSEED", (args.seed as u32).to_string()); + python_errors::install_panic_hook(); + let validation = shared_state::validate_or_report(&args.solution_path, &["StorageNode"]); + if matches!(validation, Ok(false)) { + return; + } + if args.debug { + Builder::new() + .filter(Some("anysystem"), LevelFilter::Debug) + .format(|buf, record| writeln!(buf, "{}", record.args())) + .init(); + } + let solution = validation.and_then(|_| Solution::load(&args.solution_path)); + let config = CaseConfig { + solution: &solution, + proc_count: args.node_count, + seed: args.seed, + }; + if let Some(test) = &args.test { + let mut suite = TestSuite::new(); + for &(name, case) in CASES { + suite.add(name, run_case, (&config, case)); + } + suite.run_test(test); + return; + } + + let mut results = Results::new(); + let mut core = 0.0; + for &(block, items) in CORE { + for item in items { + for name in *item { + run_named_case(name, &config, &mut results); + } + } + let points = core_points(&results, items); + println!("{block}: {points}/{}", items.len() as f32 / 2.0); + core += points; + } + println!("Core: {core}/8"); + + if args.all || (!args.core && core == 8.0) { + let selected = if args.all { + Ok(None) + } else { + selected_tracks(Path::new(&args.solution_path)) + }; + match selected { + Ok(selected) => { + for &(track, names) in TRACKS { + if selected.as_ref().is_some_and(|ids| !ids.contains(track)) { + println!("Track {track}: skipped (not selected)"); + continue; + } + for name in names { + run_named_case(name, &config, &mut results); + } + let passed = names.iter().filter(|name| results[*name].is_ok()).count(); + println!("Track {track}: {passed}/{}", names.len()); + } + } + Err(error) => eprintln!("Cannot select advanced tracks: {error}. Grading core only."), + } + } else if args.core { + println!("Advanced tracks skipped (--core)"); + } else { + println!("Advanced tracks skipped (full core required)"); + } + let passed = results.values().filter(|result| result.is_ok()).count(); + println!("\nPassed {passed} from {} tests", results.len()); + for (name, result) in &results { + if let Err(error) = result { + println!("- {name}: {error}"); + } + } + println!("\nSCORE: {}\n", score(core, &results)); +} + +fn run_named_case(name: &'static str, config: &CaseConfig, results: &mut Results) { + let &(_, case) = CASES + .iter() + .find(|(test, _)| *test == name) + .expect("registered case"); + println!("\n--- {name} ---\n"); + let result = run_case(&(config, case)); + match &result { + Ok(_) => println!("\nPASSED\n"), + Err(error) => println!("\nFAILED: {error}\n"), + } + results.insert(name, result); +} + +// A missing file selects all tracks; an empty file selects only the core. +fn selected_tracks(solution: &Path) -> Result>, String> { + let path = solution.with_file_name("tracks.txt"); + let contents = match std::fs::read_to_string(&path) { + Ok(contents) => contents, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(error) => return Err(format!("{}: {error}", path.display())), + }; + let mut selected = HashSet::new(); + for word in contents.trim_start_matches('\u{feff}').split_whitespace() { + let id = word.to_ascii_uppercase(); + if !TRACKS.iter().any(|(track, _)| *track == id) { + return Err(format!( + "unknown track {word:?} in tracks.txt (expected A, B or C)" + )); + } + selected.insert(id); + } + Ok(Some(selected)) +} + +const CASES: &[(&str, Case)] = &[ + ("BASIC", test_basic), + ("REPLICAS CHECK", test_replicas_check), + ("DATA PLACEMENT", test_data_placement), + ("DATA PLACEMENT MULTI KEY", test_data_placement_multi_key), + ("CONCURRENT GET PUT", test_concurrent_get_put), + ("CONCURRENT GETS", test_concurrent_gets), + ("MC BASIC", test_mc_basic), + ("CONCURRENT WRITES", test_concurrent_writes), + ("CONCURRENT WRITES TIE", test_concurrent_writes_tie), + ("CONCURRENT PUT DELETE TIE", test_concurrent_put_delete_tie), + ( + "READ REPAIR PUT DELETE TIE", + test_read_repair_put_delete_tie, + ), + ( + "HINTED HANDOFF PUT DELETE TIE", + test_hinted_handoff_put_delete_tie, + ), + ("MC CONCURRENT WRITES", test_mc_concurrent_writes), + ("STALE REPLICA", test_stale_replica), + ("STALE REPLICA DELETE", test_stale_replica_delete), + ("DIVERGED REPLICAS", test_diverged_replicas), + ("DELETE ABSENT", test_delete_absent), + ("CONCURRENT DELETES", test_concurrent_deletes), + ("SLOPPY QUORUM DELETE", test_sloppy_quorum_delete), + ( + "SLOPPY QUORUM DELETE PARTITION", + test_sloppy_quorum_delete_partition, + ), + ( + "SLOPPY QUORUM DELETE DISTINCT", + test_sloppy_quorum_delete_distinct, + ), + ("READ REPAIR LATE", test_read_repair_late), + ("READ REPAIR DELETE", test_read_repair_delete), + ("READ REPAIR TIE", test_read_repair_tie), + ("READ REPAIR MULTI KEY", test_read_repair_multi_key), + ("SLOPPY QUORUM READ REPAIR", test_sloppy_quorum_read_repair), + ( + "SLOPPY QUORUM READ REPAIR DELETE", + test_sloppy_quorum_read_repair_delete, + ), + ( + "SLOPPY QUORUM READ REPAIR MULTI KEY", + test_sloppy_quorum_read_repair_multi_key, + ), + ("SLOPPY QUORUM GET ORDER", test_sloppy_quorum_get_order), + ( + "SLOPPY QUORUM GET DISTINCT", + test_sloppy_quorum_get_distinct, + ), + ("SLOPPY QUORUM PUT ORDER", test_sloppy_quorum_put_order), + ( + "SLOPPY QUORUM PUT DISTINCT", + test_sloppy_quorum_put_distinct, + ), + ( + "SLOPPY QUORUM DELETE ORDER", + test_sloppy_quorum_delete_order, + ), + ( + "SLOPPY QUORUM DELETE ORDER PARTITION", + test_sloppy_quorum_delete_order_partition, + ), + ("SLOPPY QUORUM READ", test_sloppy_quorum_read), + ("SLOPPY QUORUM WRITE", test_sloppy_quorum_write), + ("SLOPPY QUORUM TRICKY", test_sloppy_quorum_tricky), + ( + "MC SLOPPY QUORUM HINTED HANDOFF", + test_mc_sloppy_quorum_hinted_handoff, + ), + ("HINTED HANDOFF", test_hinted_handoff), + ("HINTED HANDOFF MULTI KEY", test_hinted_handoff_multi_key), + ("HINTED HANDOFF RETENTION", test_hinted_handoff_retention), + ("HINTED HANDOFF LWW", test_hinted_handoff_lww), + ("PARTITION CLIENTS", test_partition_clients), + ("PARTITION MIXED", test_partition_mixed), +]; diff --git a/homework/08-kv-replication/tests/src/python_errors.rs b/homework/08-kv-replication/tests/src/python_errors.rs new file mode 100644 index 0000000..8b65126 --- /dev/null +++ b/homework/08-kv-replication/tests/src/python_errors.rs @@ -0,0 +1,111 @@ +use std::cell::OnceCell; +use std::ffi::CString; +use std::panic::{catch_unwind, resume_unwind, AssertUnwindSafe}; +use std::path::Path; + +use anysystem::python::PyProcessFactory; +use anysystem::test::TestResult; +use pyo3::prelude::*; +use pyo3::types::{PyList, PyModule}; + +use crate::common::TestConfig; + +// AnySystem 0.3 prints the Python traceback and then unwraps the same error. +// Suppress this duplicate diagnostic only at those bridge sites. +pub fn install_panic_hook() { + let default = std::panic::take_hook(); + std::panic::set_hook(Box::new(move |info| { + let site = info + .location() + .map(|l| l.file().replace('\\', "/")) + .unwrap_or_default(); + let message = info.payload_as_str().unwrap_or(""); + let python = site.ends_with("/anysystem-0.3.0/src/python/mod.rs") + && (message.contains("PyErr {") + || message.contains("Error when creating Python process")); + let callback = (site.ends_with("/anysystem-0.3.0/src/node.rs") + || site.ends_with("/anysystem-0.3.0/src/mc/node.rs")) + && message.contains("Error when calling process"); + if !python && !callback { + default(info); + } + })); +} + +pub struct Solution { + factory: PyProcessFactory, + errors: Py, +} + +impl Solution { + pub fn load(path: &str) -> Result { + let path = Path::new(path).canonicalize().map_err(|e| e.to_string())?; + let path = path.to_str().ok_or("Solution path is not UTF-8")?; + // AnySystem unwraps Python errors during import. Preserve unrelated loader panics. + let factory = catch_unwind(|| PyProcessFactory::new(path, "StorageNode")) + .map_err(|panic| format!("Cannot load StorageNode: {}", python_panic(panic)))?; + let errors = Python::attach(|py| -> PyResult<_> { + // AnySystem 0.3 registers the solution under its canonical filename without ".py". + let modules = PyModule::import(py, "sys")?.getattr("modules")?; + let class = modules + .get_item(path.replace(".py", ""))? + .getattr("StorageNode")?; + let code = CString::new(include_str!("../python_errors.py")).unwrap(); + let adapter = + PyModule::from_code(py, &code, c"python_errors.py", c"hw8_python_errors")?; + Ok(adapter + .getattr("watch")? + .call1((class,))? + .extract::>()?) + }) + .map_err(|error| error.to_string())?; + Ok(Self { factory, errors }) + } +} + +fn python_panic(panic: Box) -> String { + if let Some(message) = panic.downcast_ref::() { + if let Some(start) = message.find("PyErr {") { + return message[start..].to_string(); + } + } + resume_unwind(panic) +} + +pub type Case = fn(&TestConfig) -> TestResult; + +pub struct CaseConfig<'a> { + pub solution: &'a Result, + pub proc_count: u32, + pub seed: u64, +} + +pub fn run_case((config, test): &(&CaseConfig, Case)) -> TestResult { + let solution = config.solution.as_ref().map_err(Clone::clone)?; + let count_keys_error = OnceCell::new(); + let config = TestConfig { + count_keys_error: &count_keys_error, + proc_factory: &solution.factory, + proc_count: config.proc_count, + seed: config.seed, + }; + // Each case owns its simulator. Unwinding drops all nodes from the failed case. + match catch_unwind(AssertUnwindSafe(|| test(&config))) { + Ok(result) => count_keys_error.into_inner().map_or(result, Err), + Err(panic) => { + let error = Python::attach(|py| { + solution + .errors + .bind(py) + .call_method0("pop") + .and_then(|item| item.extract::()) + .ok() + }); + match error { + Some(error) => Err(format!("Python exception in {error}")), + // PyO3 can also report Python errors outside a callback, for example in deepcopy. + None => Err(format!("Python exception: {}", python_panic(panic))), + } + } + } +} diff --git a/homework/08-kv-replication/tests/src/shared_state.rs b/homework/08-kv-replication/tests/src/shared_state.rs new file mode 100644 index 0000000..cc97e7d --- /dev/null +++ b/homework/08-kv-replication/tests/src/shared_state.rs @@ -0,0 +1,76 @@ +use std::ffi::CString; + +use pyo3::prelude::*; +use pyo3::types::{PyList, PyModule}; + +const VALIDATOR_CODE: &str = include_str!("../shared_state_validator.py"); + +fn validator_module<'py>(py: Python<'py>) -> PyResult> { + let code = CString::new(VALIDATOR_CODE).expect("validator source contains a null byte"); + PyModule::from_code( + py, + &code, + c"shared_state_validator.py", + c"shared_state_validator", + ) +} + +#[cfg(test)] +fn validate_source( + source: &str, + filename: &str, + class_names: &[&str], +) -> Result, String> { + Python::attach(|py| { + let module = validator_module(py).map_err(|error| error.to_string())?; + let class_names = PyList::new(py, class_names).map_err(|error| error.to_string())?; + module + .getattr("validate_source") + .and_then(|function| function.call1((source, filename, class_names))) + .and_then(|result| result.extract::>()) + .map_err(|error| error.to_string()) + }) +} + +fn validate_solution(path: &str, class_names: &[&str]) -> Result, String> { + Python::attach(|py| { + let module = validator_module(py).map_err(|error| error.to_string())?; + let class_names = PyList::new(py, class_names).map_err(|error| error.to_string())?; + module + .getattr("validate_solution") + .and_then(|function| function.call1((path, class_names))) + .and_then(|result| result.extract::>()) + .map_err(|error| error.to_string()) + }) +} + +pub fn validate_or_report(path: &str, class_names: &[&str]) -> Result { + let violations = validate_solution(path, class_names)?; + if violations.is_empty() { + return Ok(true); + } + println!("ERROR: shared state between AnySystem processes is forbidden"); + for violation in violations { + println!("{violation}"); + } + println!("\nSCORE: 0\n"); + Ok(false) +} + +#[cfg(test)] +mod tests { + use super::validate_source; + + #[test] + fn embeds_validator_and_rejects_shared_state() { + let violations = validate_source( + "class ProcessImpl:\n shared = {}\n", + "solution.py", + &["ProcessImpl"], + ) + .unwrap(); + + assert_eq!(violations.len(), 1); + assert!(violations[0].contains("ProcessImpl.shared")); + } +} diff --git a/homework/08-kv-replication/tests/src/tests.rs b/homework/08-kv-replication/tests/src/tests.rs new file mode 100644 index 0000000..bb43d63 --- /dev/null +++ b/homework/08-kv-replication/tests/src/tests.rs @@ -0,0 +1,2323 @@ +use assertables::{assume, assume_eq}; +use rand::prelude::*; +use rand_pcg::Pcg64; + +use anysystem::test::TestResult; +use anysystem::{Message, System}; + +use crate::common::*; + +pub fn test_basic(config: &TestConfig) -> TestResult { + let mut sys = build_system(config); + let procs = sys.process_names(); + let mut rand = Pcg64::seed_from_u64(config.seed); + + let key = random_string(8, &mut rand).to_uppercase(); + let replicas = key_replicas(&key, &sys); + let non_replicas = key_non_replicas(&key, &sys); + println!("Key {key} replicas: {replicas:?}"); + println!("Key {key} non-replicas: {non_replicas:?}"); + + // get key from the first node + check_get(&mut sys, &procs[0], &key, 2, None, 100)?; + + // put key from the first replica + let value = random_string(8, &mut rand); + check_put(&mut sys, &replicas[0], &key, &value, 2, 100)?; + + // get key from the last replica + check_get(&mut sys, &replicas[2], &key, 2, Some(&value), 100)?; + + // get key from the first non-replica + check_get(&mut sys, &non_replicas[0], &key, 2, Some(&value), 100)?; + + // update key from the last non-replica + let value2 = random_string(8, &mut rand); + check_put(&mut sys, &non_replicas[2], &key, &value2, 2, 100)?; + + // get key from the first node + check_get(&mut sys, &procs[0], &key, 2, Some(&value2), 100)?; + + // delete key from the second non-replica + check_delete(&mut sys, &non_replicas[1], &key, 2, Some(&value2), 100)?; + + // get key from the last replica + check_get(&mut sys, &replicas[2], &key, 2, None, 100)?; + + // get key from the first non-replica + check_get(&mut sys, &non_replicas[0], &key, 2, None, 100)?; + + Ok(true) +} + +pub fn test_replicas_check(config: &TestConfig) -> TestResult { + let mut sys = build_system(config); + let mut rand = Pcg64::seed_from_u64(config.seed); + + let key = random_string(8, &mut rand).to_uppercase(); + let replicas = key_replicas(&key, &sys); + + // put key from the first replica with quorum 3 + let value = random_string(8, &mut rand); + check_put(&mut sys, &replicas[0], &key, &value, 3, 100)?; + + // disconnect each replica and check the stored value + for replica in replicas.iter() { + sys.network().disconnect_node(replica); + check_get(&mut sys, replica, &key, 1, Some(&value), 100)?; + } + Ok(true) +} + +pub fn test_data_placement(config: &TestConfig) -> TestResult { + for quorum in 1..=3 { + for coordinator in 0..config.proc_count { + let mut sys = build_system(config); + let mut rand = Pcg64::seed_from_u64(config.seed); + let key = random_string(8, &mut rand).to_uppercase(); + let value = random_string(8, &mut rand); + let replacement = format!("{value}-new"); + let client = coordinator.to_string(); + println!( + "Data placement: key={key}, replicas={:?}, coordinator={client}, quorum={quorum}", + key_replicas(&key, &sys) + ); + check_live_key_counts(&mut sys, &[(&key, None)])?; + + // Every node coordinates a write, including all three primaries + // and every non-replica. Overwrites still occupy one local key. + for value in [value.as_str(), "", replacement.as_str()] { + send_put(&mut sys, &client, &key, value, quorum); + wait_for_responses( + &mut sys, + &[ExpectedResponse::put(&client, &key, &[Some(value)])], + 100, + )?; + // Quorum one may reply before the remaining copies arrive. + finish_background_work(&mut sys, "Healthy PUT replication")?; + check_live_key_counts(&mut sys, &[(&key, Some(value))])?; + } + // COUNT_KEYS must be repeatable and must not remove stored data. + check_live_key_counts(&mut sys, &[(&key, Some(&replacement))])?; + // Isolated reads are last: reconnecting their retrying requests + // could create hints and spoil the healthy-network scenario. + check_isolated_reads(&mut sys, &[(&key, Some(&replacement))])?; + } + } + Ok(true) +} + +pub fn test_data_placement_multi_key(config: &TestConfig) -> TestResult { + // Include wraparound and two-digit IDs, with additional nodes that do not + // own any of the written keys and therefore must remain empty. + let config = TestConfig { + proc_count: config.proc_count.max(12), + ..*config + }; + let mut sys = build_system(&config); + let mut rand = Pcg64::seed_from_u64(config.seed); + let primary = config.proc_count - 1; + let mut keys = Vec::new(); + for owner in [primary, primary, 0, 2, primary] { + keys.push(unused_key_on_primary( + &sys, + &owner.to_string(), + &keys, + &mut rand, + )); + } + let client = primary.to_string(); + let reader = "2"; + let base = random_string(8, &mut rand); + let first = format!("{base}-first"); + let second = format!("{base}-second"); + let guard = format!("{base}-guard"); + let updated = format!("{base}-updated"); + let mut expected: Vec<(&str, Option<&str>)> = + keys.iter().map(|key| (key.as_str(), None)).collect(); + println!("Multi-key placement: keys={keys:?}, coordinator={client}"); + check_live_key_counts(&mut sys, &expected)?; + + // Same, overlapping and disjoint primary sets. This coordinator is a + // primary for the first two keys and a non-replica for the next two. + for (index, value) in [first.as_str(), second.as_str(), "", guard.as_str()] + .into_iter() + .enumerate() + { + send_put(&mut sys, &client, &keys[index], value, 3); + wait_for_responses( + &mut sys, + &[ExpectedResponse::put(&client, &keys[index], &[Some(value)])], + 100, + )?; + finish_background_work(&mut sys, "Multi-key PUT replication")?; + expected[index].1 = Some(value); + check_live_key_counts(&mut sys, &expected)?; + } + + // Reads on a node outside a key's primary set must not leave extra copies. + // Reading a missing key must not add a live record either. + for index in [0, 3, 4] { + send_get(&mut sys, reader, &keys[index], 3); + } + let responses: Vec<_> = [0, 3, 4] + .into_iter() + .map(|index| ExpectedResponse::get(reader, &keys[index], &[expected[index].1])) + .collect(); + wait_for_responses(&mut sys, &responses, 200)?; + finish_background_work(&mut sys, "Healthy GET placement")?; + check_live_key_counts(&mut sys, &expected)?; + + let other_client = &key_non_replicas(&keys[0], &sys)[1]; + for (index, value) in [(0, updated.as_str()), (1, "")] { + send_put(&mut sys, other_client, &keys[index], value, 2); + wait_for_responses( + &mut sys, + &[ExpectedResponse::put( + other_client, + &keys[index], + &[Some(value)], + )], + 100, + )?; + finish_background_work(&mut sys, "Multi-key overwrite")?; + expected[index].1 = Some(value); + check_live_key_counts(&mut sys, &expected)?; + } + + // Remove an empty string, repeat the deletion and delete a missing key. + // Tombstones and request metadata must not increase the live key counts. + for index in [1, 1, 4] { + send_delete(&mut sys, other_client, &keys[index], 3); + wait_for_responses( + &mut sys, + &[ExpectedResponse::delete( + other_client, + &keys[index], + &[expected[index].1], + )], + 100, + )?; + finish_background_work(&mut sys, "Multi-key DELETE placement")?; + expected[index].1 = None; + check_live_key_counts(&mut sys, &expected)?; + } + + send_put(&mut sys, other_client, &keys[1], "", 3); + wait_for_responses( + &mut sys, + &[ExpectedResponse::put(other_client, &keys[1], &[Some("")])], + 100, + )?; + finish_background_work(&mut sys, "Recreated key placement")?; + expected[1].1 = Some(""); + check_live_key_counts(&mut sys, &expected)?; + check_isolated_reads(&mut sys, &expected) +} + +pub fn test_concurrent_get_put(config: &TestConfig) -> TestResult { + let mut sys = build_system(config); + let mut rand = Pcg64::seed_from_u64(config.seed); + + let key = random_string(8, &mut rand).to_uppercase(); + let value = random_string(8, &mut rand); + let non_replicas = key_non_replicas(&key, &sys); + let client = &non_replicas[0]; + + // Both operations are outstanding on the same coordinator. Either response + // may arrive first, and GET may observe the concurrent PUT or miss it. + send_put(&mut sys, client, &key, &value, 2); + sys.send_local_message( + client, + Message::json( + "GET", + &GetReqMessage { + key: &key, + quorum: 2, + }, + ), + ); + wait_for_responses( + &mut sys, + &[ + ExpectedResponse::get(client, &key, &[None, Some(&value)]), + ExpectedResponse::put(client, &key, &[Some(&value)]), + ], + 100, + )?; + + // The completed PUT must be visible both to its coordinator and to a fresh reader. + check_concurrent_final_value(&mut sys, &[client, &non_replicas[1]], &key, &value) +} + +pub fn test_concurrent_gets(config: &TestConfig) -> TestResult { + let mut sys = build_system(config); + let mut rand = Pcg64::seed_from_u64(config.seed); + let key = random_string(8, &mut rand).to_uppercase(); + let value = random_string(8, &mut rand); + let non_replicas = key_non_replicas(&key, &sys); + let client = &non_replicas[0]; + let writer = &non_replicas[1]; + + for stored in [None, Some(value.as_str())] { + if let Some(value) = stored { + check_put(&mut sys, writer, &key, value, 3, 100)?; + } + // The coordinator receives both local requests before any replica can reply. + send_get(&mut sys, client, &key, 2); + send_get(&mut sys, client, &key, 2); + let pair = ExpectedResponse { + count: 2, + ..ExpectedResponse::get(client, &key, &[stored]) + }; + wait_for_responses(&mut sys, &[pair], 100)?; + finish_background_work(&mut sys, "Concurrent GETs")?; + } + Ok(true) +} + +pub fn test_concurrent_writes(config: &TestConfig) -> TestResult { + let mut sys = build_system(config); + let mut rand = Pcg64::seed_from_u64(config.seed); + + let key = random_string(8, &mut rand).to_uppercase(); + let non_replicas = key_non_replicas(&key, &sys); + let base = random_string(8, &mut rand); + // The later value is lexicographically smaller: time must take precedence. + let value = format!("{base}-z"); + let value2 = format!("{base}-a"); + send_put(&mut sys, &non_replicas[0], &key, &value, 2); + sys.step_for_duration(0.01); + send_put(&mut sys, &non_replicas[1], &key, &value2, 2); + + // The earlier PUT may already observe the later write. The later PUT + // cannot return a value older than its own write. + wait_for_responses( + &mut sys, + &[ + ExpectedResponse::put(&non_replicas[0], &key, &[Some(&value), Some(&value2)]), + ExpectedResponse::put(&non_replicas[1], &key, &[Some(&value2)]), + ], + 100, + )?; + check_concurrent_final_value(&mut sys, &[&non_replicas[2]], &key, &value2) +} + +pub fn test_concurrent_writes_tie(config: &TestConfig) -> TestResult { + for larger_first in [false, true] { + let mut sys = build_system(config); + let mut rand = Pcg64::seed_from_u64(config.seed); + + let key = random_string(8, &mut rand).to_uppercase(); + let non_replicas = key_non_replicas(&key, &sys); + let base = random_string(8, &mut rand); + let smaller = format!("{base}-a"); + let larger = format!("{base}-z"); + let (first, second) = if larger_first { + (&larger, &smaller) + } else { + (&smaller, &larger) + }; + + // Equal timestamps: the larger value wins regardless of which client + // submits it. A smaller PUT may finish before observing its competitor. + send_put(&mut sys, &non_replicas[0], &key, first, 2); + send_put(&mut sys, &non_replicas[1], &key, second, 2); + wait_for_responses( + &mut sys, + &[ + ExpectedResponse::put(&non_replicas[0], &key, &[Some(first), Some(&larger)]), + ExpectedResponse::put(&non_replicas[1], &key, &[Some(second), Some(&larger)]), + ], + 100, + )?; + check_concurrent_final_value(&mut sys, &[&non_replicas[2]], &key, &larger)?; + } + Ok(true) +} + +mod ordered_delivery; + +pub fn test_concurrent_put_delete_tie(config: &TestConfig) -> TestResult { + ordered_delivery::check_put_before_later_delete(config)?; + for quorum in 1..=3 { + for empty in [false, true] { + for present in [false, true] { + for same_client in [false, true] { + for delete_first in [false, true] { + let mut sys = build_system(config); + let mut rand = Pcg64::seed_from_u64(config.seed); + let key = random_string(8, &mut rand).to_uppercase(); + let others = key_non_replicas(&key, &sys); + let base = random_string(8, &mut rand); + let old = format!("{base}-z"); + let value = if empty { + String::new() + } else { + format!("{base}-a") + }; + let writer = &others[0]; + let deleter = if same_client { writer } else { &others[1] }; + println!( + "PUT/DELETE tie: quorum={quorum}, empty={empty}, present={present}, same_client={same_client}, delete_first={delete_first}" + ); + if present { + check_put(&mut sys, &others[2], &key, &old, 3, 100)?; + finish_background_work(&mut sys, "PUT/DELETE tie setup")?; + } + // No setup for an absent key: also exercise timestamp zero. + send_tied_put_delete( + &mut sys, + &key, + &value, + (writer, quorum), + (deleter, quorum), + delete_first, + ); + wait_for_responses( + &mut sys, + &[ + ExpectedResponse::put(writer, &key, &[Some(&value)]), + ExpectedResponse::delete( + deleter, + &key, + &[None, Some(&value), present.then_some(old.as_str())], + ), + ], + 200, + )?; + finish_background_work(&mut sys, "Concurrent PUT/DELETE tie")?; + // Check each local copy before a quorum GET can repair it. + check_live_key_counts(&mut sys, &[(&key, Some(&value))])?; + check_isolated_reads(&mut sys, &[(&key, Some(&value))])?; + } + } + } + } + } + // The tie-breaker must not make PUT win over a strictly later DELETE. + // Conversely, a later PUT must be able to recreate a deleted key. + for empty in [false, true] { + for delete_later in [false, true] { + let mut sys = build_system(config); + let mut rand = Pcg64::seed_from_u64(config.seed); + let key = random_string(8, &mut rand).to_uppercase(); + let others = key_non_replicas(&key, &sys); + let value = if empty { + String::new() + } else { + random_string(8, &mut rand) + }; + println!("PUT/DELETE time precedence: empty={empty}, delete_later={delete_later}"); + if delete_later { + check_put(&mut sys, &others[0], &key, &value, 3, 100)?; + } else { + check_delete(&mut sys, &others[0], &key, 3, None, 100)?; + } + finish_background_work(&mut sys, "LWW time setup")?; + sys.step_for_duration(0.1); + if delete_later { + check_delete(&mut sys, &others[1], &key, 3, Some(&value), 100)?; + } else { + check_put(&mut sys, &others[1], &key, &value, 3, 100)?; + } + finish_background_work(&mut sys, "LWW later operation")?; + let expected = (!delete_later).then_some(value.as_str()); + check_live_key_counts(&mut sys, &[(&key, expected)])?; + check_isolated_reads(&mut sys, &[(&key, expected)])?; + } + } + Ok(true) +} + +pub fn test_read_repair_put_delete_tie(config: &TestConfig) -> TestResult { + let mut rand = Pcg64::seed_from_u64(config.seed); + let key = random_string(8, &mut rand).to_uppercase(); + let base = random_string(8, &mut rand); + let previous = format!("{base}-z"); + let incoming = format!("{base}-a"); + for winner in 0..3 { + for quorum in 1..=3 { + for (old, value) in [ + (previous.as_str(), incoming.as_str()), + (&previous, ""), + ("", &incoming), + ] { + for delete_first in [false, true] { + let mut sys = build_system(config); + let replicas = key_replicas(&key, &sys); + let others = key_non_replicas(&key, &sys); + println!( + "PUT/DELETE tie repair: winner={winner}, quorum={quorum}, old={old:?}, value={value:?}, delete_first={delete_first}" + ); + check_put(&mut sys, &others[2], &key, old, 3, 100)?; + finish_background_work(&mut sys, "PUT/DELETE repair setup")?; + // One primary keeps PUT, the other two keep a tombstone + // with exactly the same timestamp. Neither writer survives. + partition_between(&mut sys, &[&replicas[winner], &others[0]]); + send_tied_put_delete( + &mut sys, + &key, + value, + (&others[0], 1), + (&others[1], 2), + delete_first, + ); + wait_for_responses( + &mut sys, + &[ + ExpectedResponse::put(&others[0], &key, &[Some(value)]), + ExpectedResponse::delete(&others[1], &key, &[Some(old)]), + ], + 200, + )?; + sys.crash_node(&others[0]); + sys.crash_node(&others[1]); + // Verify the split while links are still disabled: a valid + // anti-entropy implementation may sync immediately on healing. + for (index, replica) in replicas.iter().enumerate() { + check_count_keys(&mut sys, replica, usize::from(index == winner))?; + } + sys.network().reset(); + send_get(&mut sys, &others[2], &key, quorum); + let values = if quorum == 3 { + vec![Some(value)] + } else { + vec![None, Some(value)] + }; + wait_for_responses( + &mut sys, + &[ExpectedResponse::get(&others[2], &key, &values)], + 100, + )?; + finish_read_repair(&mut sys)?; + check_live_key_counts(&mut sys, &[(&key, Some(value))])?; + check_isolated_reads(&mut sys, &[(&key, Some(value))])?; + } + } + } + } + Ok(true) +} + +pub fn test_hinted_handoff_put_delete_tie(config: &TestConfig) -> TestResult { + let mut rand = Pcg64::seed_from_u64(config.seed); + let key = random_string(8, &mut rand).to_uppercase(); + let base = random_string(8, &mut rand); + let previous = format!("{base}-z"); + let incoming = format!("{base}-a"); + for primary_put in [false, true] { + for (old, value) in [ + (previous.as_str(), incoming.as_str()), + (&previous, ""), + ("", &incoming), + ] { + for delete_first in [false, true] { + let mut sys = build_system(config); + let replicas = key_replicas(&key, &sys); + let backups = key_non_replicas(&key, &sys); + let primary_client = &replicas[2]; + let backup_client = &backups[2]; + println!( + "PUT/DELETE tie handoff: primary_put={primary_put}, old={old:?}, value={value:?}, delete_first={delete_first}" + ); + check_put(&mut sys, primary_client, &key, old, 3, 100)?; + finish_background_work(&mut sys, "PUT/DELETE handoff setup")?; + partition_between( + &mut sys, + &replicas.iter().map(String::as_str).collect::>(), + ); + let (writer, deleter) = if primary_put { + (primary_client, backup_client) + } else { + (backup_client, primary_client) + }; + // Exercise both directions: a value replaces an equal-time + // tombstone, while an equal-time tombstone cannot erase a value. + send_tied_put_delete( + &mut sys, + &key, + value, + (writer, 3), + (deleter, 3), + delete_first, + ); + // Only the primary partition held the old value before the split. + let deleted = (!primary_put).then_some(old); + wait_for_responses( + &mut sys, + &[ + ExpectedResponse::put(writer, &key, &[Some(value)]), + ExpectedResponse::delete(deleter, &key, &[deleted]), + ], + 500, + )?; + // Let in-flight writes settle inside each partition. Keep all + // hint holders and their intended owners alive when links heal. + finish_background_work(&mut sys, "PUT/DELETE before handoff")?; + sys.network().reset(); + finish_background_work(&mut sys, "PUT/DELETE equal-time handoff")?; + check_handoff_state(&mut sys, &[(&key, Some(value))])?; + } + } + } + Ok(true) +} + +fn send_tied_put_delete( + sys: &mut System, + key: &str, + value: &str, + put: (&str, u8), + delete: (&str, u8), + delete_first: bool, +) { + // Submitting both local requests without stepping preserves equal times. + if delete_first { + send_delete(sys, delete.0, key, delete.1); + // Delay the initial PUT messages so a two-phase DELETE can write first. + let max_delay = sys.network().max_delay(); + sys.network().set_delay(4.0 * max_delay); + send_put(sys, put.0, key, value, put.1); + sys.network().set_delays(MIN_NETWORK_DELAY, max_delay); + } else { + send_put(sys, put.0, key, value, put.1); + send_delete(sys, delete.0, key, delete.1); + } +} + +fn partition_between(sys: &mut System, left: &[&str]) { + let procs = sys.process_names(); + for first in &procs { + for second in &procs { + if left.contains(&first.as_str()) != left.contains(&second.as_str()) { + sys.network().disable_link(first, second); + } + } + } +} + +pub fn test_sloppy_quorum_get_order(config: &TestConfig) -> TestResult { + // Two-digit IDs distinguish numeric ring order from lexicographic sorting. + let config = TestConfig { + proc_count: config.proc_count.max(12), + ..*config + }; + for primary in [0, config.proc_count - 6, config.proc_count - 4] { + for (missing, skipped) in [(3, 0), (3, 1), (3, 2), (1, 0)] { + let mut sys = build_system(&config); + let mut rand = Pcg64::seed_from_u64(config.seed); + let key = unused_key_on_primary(&sys, &primary.to_string(), &[], &mut rand); + let value = random_string(8, &mut rand); + let replicas = key_replicas(&key, &sys); + let backups = key_non_replicas(&key, &sys); + let first_available = &backups[skipped]; + let writer = &backups[backups.len() - 1]; + let reader = &backups[backups.len() - 2]; + println!( + "GET order: replicas={replicas:?}, backups={backups:?}, missing={missing}, skipped={skipped}" + ); + + // Seed only the first available backup, then stop the writer so + // its retries cannot supply the value to a wrongly chosen backup. + connect_only(&mut sys, &[writer, first_available]); + check_put(&mut sys, writer, &key, &value, 1, 500)?; + check_count_keys(&mut sys, first_available, 1)?; + sys.crash_node(writer); + sys.network().reset(); + for proc in replicas[..missing].iter().chain(backups[..skipped].iter()) { + sys.network().disconnect_node(proc); + } + + // Keep the value on the backup until GET, whichever primary its hint targets. + for replica in &replicas[missing..] { + sys.network().disable_link(first_available, replica); + } + // With two live primaries, only the first backup is needed. Choosing + // the second instead yields three empty replies. + send_get(&mut sys, reader, &key, 3); + wait_for_responses( + &mut sys, + &[ExpectedResponse::get(reader, &key, &[Some(&value)])], + 500, + )?; + finish_background_work(&mut sys, "GET backup order")?; + } + } + Ok(true) +} + +pub fn test_sloppy_quorum_put_order(config: &TestConfig) -> TestResult { + let config = TestConfig { + proc_count: config.proc_count.max(12), + ..*config + }; + for primary in [0, config.proc_count - 6, config.proc_count - 4] { + for (missing, skipped) in [(1, 0), (2, 0), (1, 1), (1, 2), (2, 1)] { + let mut sys = build_system(&config); + let mut rand = Pcg64::seed_from_u64(config.seed); + let key = unused_key_on_primary(&sys, &primary.to_string(), &[], &mut rand); + let value = random_string(8, &mut rand); + let replicas = key_replicas(&key, &sys); + let backups = key_non_replicas(&key, &sys); + let client = &replicas[2]; + println!( + "PUT order: replicas={replicas:?}, backups={backups:?}, missing={missing}, skipped={skipped}" + ); + for proc in replicas[..missing].iter().chain(backups[..skipped].iter()) { + sys.network().disconnect_node(proc); + } + + send_put(&mut sys, client, &key, &value, 3); + wait_for_responses( + &mut sys, + &[ExpectedResponse::put(client, &key, &[Some(&value)])], + 500, + )?; + // A later replacement can reply before an earlier one. Drain + // in-flight requests before inspecting placement without reads. + let delivery_window = 2.0 * sys.network().max_delay(); + sys.step_for_duration(delivery_window); + for (index, proc) in replicas.iter().enumerate() { + check_count_keys(&mut sys, proc, usize::from(index >= missing))?; + } + for (index, proc) in backups.iter().enumerate() { + if (skipped..skipped + missing).contains(&index) { + check_count_keys(&mut sys, proc, 1)?; + } else if index < skipped || skipped == 0 { + // When the first replacements are alive, no further + // backups are needed. Unreachable backups stay empty. + check_count_keys(&mut sys, proc, 0)?; + } + } + finish_background_work(&mut sys, "PUT backup order")?; + } + } + Ok(true) +} + +pub fn test_sloppy_quorum_delete_order(config: &TestConfig) -> TestResult { + let config = TestConfig { + proc_count: config.proc_count.max(12), + ..*config + }; + for primary in [0, config.proc_count - 6, config.proc_count - 4] { + for (missing, skipped) in [(1, 0), (2, 0), (1, 1), (1, 2), (2, 1)] { + for empty in [false, true] { + let mut sys = build_system(&config); + let mut rand = Pcg64::seed_from_u64(config.seed); + let key = unused_key_on_primary(&sys, &primary.to_string(), &[], &mut rand); + let value = if empty { + String::new() + } else { + random_string(8, &mut rand) + }; + let replicas = key_replicas(&key, &sys); + let backups = key_non_replicas(&key, &sys); + let writer = backups.last().unwrap(); + let client = &replicas[2]; + println!( + "DELETE order: replicas={replicas:?}, backups={backups:?}, missing={missing}, skipped={skipped}, empty={empty}" + ); + + // Seed old primary copies and then the backups that DELETE + // must visit. Stop the writer before observing the deletion. + check_put(&mut sys, writer, &key, &value, 3, 100)?; + finish_background_work(&mut sys, "DELETE order primary setup")?; + for proc in replicas[..missing].iter().chain(backups[..skipped].iter()) { + sys.network().disconnect_node(proc); + } + check_put(&mut sys, writer, &key, &value, 3, 500)?; + finish_background_work(&mut sys, "DELETE order backup setup")?; + for backup in &backups[skipped..skipped + missing] { + check_count_keys(&mut sys, backup, 1)?; + } + sys.crash_node(writer); + + let mut counts: Vec<(&str, usize)> = replicas + .iter() + .enumerate() + .map(|(index, proc)| (proc.as_str(), usize::from(index < missing))) + .collect(); + counts.extend( + backups[skipped..skipped + missing] + .iter() + .map(|proc| (proc.as_str(), 0)), + ); + check_delete_order(&mut sys, client, &key, &value, &counts)?; + + // A surviving DELETE coordinator must not hide missing hints + // by retrying the operation when the primary owners recover. + sys.crash_node(client); + sys.network().reset(); + finish_background_work(&mut sys, "Ordered DELETE handoff")?; + check_handoff_state(&mut sys, &[(&key, None)])?; + } + } + } + Ok(true) +} + +pub fn test_sloppy_quorum_delete_order_partition(config: &TestConfig) -> TestResult { + let config = TestConfig { + proc_count: config.proc_count.max(12), + ..*config + }; + for primary in [0, config.proc_count - 6, config.proc_count - 4] { + for skipped in 0..3 { + for empty in [false, true] { + let mut sys = build_system(&config); + let mut rand = Pcg64::seed_from_u64(config.seed); + let key = unused_key_on_primary(&sys, &primary.to_string(), &[], &mut rand); + let value = if empty { + String::new() + } else { + random_string(8, &mut rand) + }; + let replicas = key_replicas(&key, &sys); + let backups = key_non_replicas(&key, &sys); + let writer = &replicas[2]; + let client = backups.last().unwrap(); + println!( + "Partitioned DELETE order: replicas={replicas:?}, backups={backups:?}, skipped={skipped}, empty={empty}" + ); + check_put(&mut sys, writer, &key, &value, 3, 100)?; + finish_background_work(&mut sys, "Partitioned DELETE primary setup")?; + + // Seed the same key on every backup using ordinary sloppy PUTs: + // one live primary and two live backups give exactly quorum 3. + // Unselected copies make an unnecessary DELETE broadcast visible. + for index in (0..backups.len()).step_by(2) { + let pair = [&backups[index], &backups[(index + 1) % backups.len()]]; + connect_only(&mut sys, &[writer, pair[0], pair[1]]); + check_put(&mut sys, writer, &key, &value, 3, 500)?; + finish_background_work(&mut sys, "Partitioned DELETE backup setup")?; + } + sys.crash_node(writer); + sys.network().reset(); + for proc in replicas.iter().chain(backups[..skipped].iter()) { + sys.network().disconnect_node(proc); + } + for backup in &backups { + check_count_keys(&mut sys, backup, 1)?; + } + + let mut counts = Vec::new(); + for (index, proc) in backups.iter().enumerate() { + if (skipped..skipped + 3).contains(&index) { + counts.push((proc.as_str(), 0)); + } else if index < skipped || skipped == 0 { + // With no failed backups, only the first three are + // needed. Failed backups must keep their old copies. + // After skipped nodes, allow extra in-flight retries. + counts.push((proc.as_str(), 1)); + } + } + check_delete_order(&mut sys, client, &key, &value, &counts)?; + sys.crash_node(client); + sys.network().reset(); + finish_background_work(&mut sys, "Partitioned ordered DELETE handoff")?; + check_handoff_state(&mut sys, &[(&key, None)])?; + } + } + } + Ok(true) +} + +fn check_delete_order( + sys: &mut System, + client: &str, + key: &str, + value: &str, + counts: &[(&str, usize)], +) -> TestResult { + send_delete(sys, client, key, 3); + // Every intended quorum member held this value before the single client DELETE. + wait_for_responses( + sys, + &[ExpectedResponse::delete(client, key, &[Some(value)])], + 500, + )?; + let delivery_window = 2.0 * sys.network().max_delay(); + sys.step_for_duration(delivery_window); + // COUNT_KEYS exposes which live copies DELETE removed, without GET repair + // or reconnection changing placement before the assertion. + for (proc, count) in counts { + check_count_keys(sys, proc, *count).map_err(|err| format!("DELETE backup order: {err}"))?; + } + finish_background_work(sys, "DELETE backup order") +} + +pub fn test_sloppy_quorum_get_distinct(config: &TestConfig) -> TestResult { + check_distinct_quorum(config, false) +} + +pub fn test_sloppy_quorum_put_distinct(config: &TestConfig) -> TestResult { + check_distinct_quorum(config, true) +} + +// (quorum, connected primaries, connected backups, coordinator is a primary). +// Each group starts one distinct responder short, including the coordinator. +const DISTINCT_QUORUM_CASES: &[(u8, usize, usize, bool)] = &[ + (2, 1, 0, true), + (3, 2, 0, true), + (3, 1, 1, true), + (2, 0, 1, false), + (3, 1, 1, false), + (3, 0, 2, false), +]; + +fn check_distinct_quorum(config: &TestConfig, writing: bool) -> TestResult { + let operation = if writing { "PUT" } else { "GET" }; + for present in [false, true] { + if writing && !present { + continue; + } + // A coordinator counts once, as does a backup serving requests for + // several unavailable primaries. Test both primary and backup replies. + for &(quorum, primary_count, backup_count, primary_client) in DISTINCT_QUORUM_CASES { + let mut sys = build_system(config); + let mut rand = Pcg64::seed_from_u64(config.seed); + let key = random_string(8, &mut rand).to_uppercase(); + let value = random_string(8, &mut rand); + let replicas = key_replicas(&key, &sys); + let backups = key_non_replicas(&key, &sys); + let (client, recovered) = if primary_client { + (&replicas[0], &backups[backup_count]) + } else { + (&backups[0], &replicas[primary_count]) + }; + println!( + "{operation} distinct: quorum={quorum}, primaries={primary_count}, backups={backup_count}, primary_client={primary_client}, present={present}" + ); + if !writing && present { + check_put(&mut sys, client, &key, &value, 3, 100)?; + finish_background_work(&mut sys, "GET setup")?; + } + let connected: Vec<&str> = replicas[..primary_count] + .iter() + .chain(backups[..backup_count].iter()) + .map(String::as_str) + .collect(); + connect_only(&mut sys, &connected); + // Force repeated inter-node replies, rather than relying on a + // particular solution's retry policy to produce duplicates. + sys.network().set_dupl_rate(1.0); + if writing { + send_put(&mut sys, client, &key, &value, quorum); + } else { + send_get(&mut sys, client, &key, quorum); + } + sys.steps(200); + for proc in sys.process_names() { + assume!( + sys.read_local_messages(&proc).is_empty(), + format!( + "{operation} replied with fewer than {quorum} distinct available nodes (response from {proc})" + ) + )?; + } + + // Add exactly one node: the operation must now complete without + // a new client request. A returning primary also supplies the value + // for a GET that started with only backups connected. + sys.network().connect_node(recovered); + let expected = if writing { + ExpectedResponse::put(client, &key, &[Some(&value)]) + } else { + ExpectedResponse::get(client, &key, &[present.then_some(value.as_str())]) + }; + wait_for_responses(&mut sys, &[expected], 500)?; + finish_background_work(&mut sys, &format!("{operation} quorum recovery"))?; + } + } + Ok(true) +} + +pub fn test_hinted_handoff(config: &TestConfig) -> TestResult { + let mut sys = build_system(config); + let mut rand = Pcg64::seed_from_u64(config.seed); + let key = random_string(8, &mut rand).to_uppercase(); + let replicas = key_replicas(&key, &sys); + let backup = &key_non_replicas(&key, &sys)[0]; + let writer = &replicas[1]; + let value = random_string(8, &mut rand); + + sys.network().disconnect_node(&replicas[0]); + check_put(&mut sys, writer, &key, &value, 3, 200)?; + check_count_keys(&mut sys, backup, 1)?; + // A stopped writer cannot repair a missing handoff through PUT retries. + sys.crash_node(writer); + sys.network().reset(); + finish_background_work(&mut sys, "Hinted handoff")?; + check_handoff_state(&mut sys, &[(&key, Some(&value))]) +} + +pub fn test_hinted_handoff_multi_key(config: &TestConfig) -> TestResult { + let mut sys = build_system(config); + let mut rand = Pcg64::seed_from_u64(config.seed); + let key = random_string(8, &mut rand).to_uppercase(); + let replicas = key_replicas(&key, &sys); + let mut keys = vec![key]; + for primary in [&replicas[0], &replicas[0], &replicas[1]] { + keys.push(unused_key_on_primary(&sys, primary, &keys, &mut rand)); + } + let backup = &key_non_replicas(&keys[0], &sys)[0]; + let writer = &replicas[1]; + let old = format!("{}-old", random_string(8, &mut rand)); + let new = format!("{}-new", random_string(8, &mut rand)); + let guard = format!("{}-guard", random_string(8, &mut rand)); + + // The backup for keys[0..3] is a primary for keys[3]. Cleanup must retain + // that record while returning several temporary records to the same owner. + check_put(&mut sys, writer, &keys[3], &guard, 3, 100)?; + check_put(&mut sys, writer, &keys[2], &old, 3, 100)?; + sys.network().disconnect_node(&replicas[0]); + check_put(&mut sys, writer, &keys[0], &old, 3, 200)?; + check_put(&mut sys, writer, &keys[1], "", 3, 200)?; + check_put(&mut sys, writer, &keys[0], &new, 3, 200)?; + send_delete(&mut sys, writer, &keys[2], 3); + wait_for_responses( + &mut sys, + &[ExpectedResponse::delete(writer, &keys[2], &[Some(&old)])], + 200, + )?; + // Two live temporary values plus the node's own primary value. + check_count_keys(&mut sys, backup, 3)?; + sys.crash_node(writer); + sys.network().reset(); + finish_background_work(&mut sys, "Multi-key hinted handoff")?; + check_handoff_state( + &mut sys, + &[ + (&keys[0], Some(&new)), + (&keys[1], Some("")), + (&keys[2], None), + (&keys[3], Some(&guard)), + ], + ) +} + +pub fn test_hinted_handoff_retention(config: &TestConfig) -> TestResult { + let mut sys = build_system(config); + let mut rand = Pcg64::seed_from_u64(config.seed); + let key = random_string(8, &mut rand).to_uppercase(); + let replicas = key_replicas(&key, &sys); + let backup = &key_non_replicas(&key, &sys)[0]; + let writer = &replicas[2]; + let old = format!("{}-old", random_string(8, &mut rand)); + let new = format!("{}-new", random_string(8, &mut rand)); + + sys.network().disconnect_node(&replicas[0]); + check_put(&mut sys, writer, &key, &old, 3, 200)?; + // Let new PUTs reach the first owner, but delay the backup's handoff to it. + sys.network().connect_node(&replicas[0]); + sys.network().disable_link(backup, &replicas[0]); + sys.network().disconnect_node(&replicas[1]); + check_put(&mut sys, writer, &key, &new, 3, 200)?; + // Multiple pending destinations still occupy only one local key. + check_count_keys(&mut sys, backup, 1)?; + sys.crash_node(writer); + sys.network().enable_link(backup, &replicas[0]); + finish_background_work(&mut sys, "Partial hinted handoff")?; + // This one key still has to reach the second, unavailable owner. + check_count_keys(&mut sys, backup, 1)?; + sys.network().connect_node(&replicas[1]); + finish_background_work(&mut sys, "Remaining hinted handoff")?; + check_handoff_state(&mut sys, &[(&key, Some(&new))]) +} + +pub fn test_hinted_handoff_lww(config: &TestConfig) -> TestResult { + for delete in [false, true] { + let mut sys = build_system(config); + let mut rand = Pcg64::seed_from_u64(config.seed); + let key = random_string(8, &mut rand).to_uppercase(); + let replicas = key_replicas(&key, &sys); + let backup = &key_non_replicas(&key, &sys)[0]; + let writer = &replicas[1]; + let base = random_string(8, &mut rand); + let old = format!("{base}-z"); + let new = format!("{base}-a"); + + sys.network().disconnect_node(&replicas[0]); + check_put(&mut sys, writer, &key, &old, 3, 200)?; + check_count_keys(&mut sys, backup, 1)?; + // Keep the old hint while the primaries receive a newer write/delete. + sys.network().disconnect_node(backup); + sys.network().connect_node(&replicas[0]); + if delete { + send_delete(&mut sys, writer, &key, 3); + wait_for_responses( + &mut sys, + &[ExpectedResponse::delete(writer, &key, &[Some(&old)])], + 100, + )?; + } else { + check_put(&mut sys, writer, &key, &new, 3, 100)?; + } + sys.crash_node(writer); + sys.network().reset(); + finish_background_work(&mut sys, "Stale hinted handoff")?; + let expected = if delete { None } else { Some(new.as_str()) }; + check_handoff_state(&mut sys, &[(&key, expected)])?; + } + Ok(true) +} + +pub fn test_delete_absent(config: &TestConfig) -> TestResult { + for quorum in 1..=3 { + for use_replica in [false, true] { + let mut sys = build_system(config); + let mut rand = Pcg64::seed_from_u64(config.seed); + let key = random_string(8, &mut rand).to_uppercase(); + let replicas = key_replicas(&key, &sys); + let others = key_non_replicas(&key, &sys); + let client = if use_replica { + &replicas[0] + } else { + &others[0] + }; + + // Both a never-written key and an already-deleted key return null. + send_delete(&mut sys, client, &key, quorum); + wait_for_responses( + &mut sys, + &[ExpectedResponse::delete(client, &key, &[None])], + 100, + )?; + finish_background_work(&mut sys, "DELETE")?; + send_delete(&mut sys, &others[1], &key, quorum); + wait_for_responses( + &mut sys, + &[ExpectedResponse::delete(&others[1], &key, &[None])], + 100, + )?; + finish_background_work(&mut sys, "DELETE")?; + check_isolated_reads(&mut sys, &[(&key, None)])?; + + // Deleting an absent key must not prevent a later write. + // The empty string is a present value and differs from null. + sys.network().reset(); + sys.step_for_duration(0.1); + check_put(&mut sys, &others[1], &key, "", 3, 100)?; + send_delete(&mut sys, client, &key, quorum); + wait_for_responses( + &mut sys, + &[ExpectedResponse::delete(client, &key, &[Some("")])], + 100, + )?; + finish_background_work(&mut sys, "DELETE")?; + check_isolated_reads(&mut sys, &[(&key, None)])?; + } + } + Ok(true) +} + +pub fn test_concurrent_deletes(config: &TestConfig) -> TestResult { + // Cover equal and different timestamps and all supported quorum sizes. + for quorum in 1..=3 { + for delay in [0.0, 0.01] { + let mut sys = build_system(config); + let mut rand = Pcg64::seed_from_u64(config.seed); + let key = random_string(8, &mut rand).to_uppercase(); + let others = key_non_replicas(&key, &sys); + let value = random_string(8, &mut rand); + check_put(&mut sys, &others[2], &key, &value, 3, 100)?; + finish_background_work(&mut sys, "PUT setup")?; + + send_delete(&mut sys, &others[0], &key, quorum); + if delay > 0.0 { + sys.step_for_duration(delay); + } + send_delete(&mut sys, &others[1], &key, quorum); + // Each read-before-delete can observe either the original value + // or a concurrent tombstone. No fixed response order is required. + wait_for_responses( + &mut sys, + &[ + ExpectedResponse::delete(&others[0], &key, &[Some(&value), None]), + ExpectedResponse::delete(&others[1], &key, &[Some(&value), None]), + ], + 200, + )?; + finish_background_work(&mut sys, "Concurrent DELETE")?; + // Probe before any quorum GET could repair an undeleted copy. + check_isolated_reads(&mut sys, &[(&key, None)])?; + + sys.network().reset(); + send_delete(&mut sys, &others[2], &key, 3); + wait_for_responses( + &mut sys, + &[ExpectedResponse::delete(&others[2], &key, &[None])], + 100, + )?; + finish_background_work(&mut sys, "Repeated DELETE")?; + let replacement = format!("{value}-new"); + check_put(&mut sys, &others[2], &key, &replacement, 3, 100)?; + finish_background_work(&mut sys, "PUT after DELETE")?; + check_isolated_reads(&mut sys, &[(&key, Some(&replacement))])?; + } + } + Ok(true) +} + +pub fn test_sloppy_quorum_delete(config: &TestConfig) -> TestResult { + let mut sys = build_system(config); + let mut rand = Pcg64::seed_from_u64(config.seed); + let key = random_string(8, &mut rand).to_uppercase(); + let replicas = key_replicas(&key, &sys); + let others = key_non_replicas(&key, &sys); + let old = format!("{}-old", random_string(8, &mut rand)); + let new = format!("{}-new", random_string(8, &mut rand)); + let writer = &others[1]; + let deleter = &replicas[1]; + + check_put(&mut sys, writer, &key, &old, 3, 100)?; + sys.network().disconnect_node(&replicas[0]); + check_put(&mut sys, writer, &key, &new, 2, 100)?; + sys.crash_node(writer); + // Two live primaries and a backup must contribute distinct responses. + // The absent backup's response must not hide the latest stored value. + send_delete(&mut sys, deleter, &key, 3); + wait_for_responses( + &mut sys, + &[ExpectedResponse::delete(deleter, &key, &[Some(&new)])], + 200, + )?; + + // Stop the coordinator so its retries cannot substitute for hinted handoff. + // It is a primary, so the backup holding the deletion remains alive. + sys.crash_node(deleter); + sys.network().reset(); + finish_background_work(&mut sys, "DELETE handoff")?; + connect_only(&mut sys, &[]); + for replica in [&replicas[0], &replicas[2]] { + send_get(&mut sys, replica, &key, 1); + wait_for_responses( + &mut sys, + &[ExpectedResponse::get(replica, &key, &[None])], + 100, + )?; + } + Ok(true) +} + +pub fn test_sloppy_quorum_delete_partition(config: &TestConfig) -> TestResult { + let mut sys = build_system(config); + let mut rand = Pcg64::seed_from_u64(config.seed); + let key = random_string(8, &mut rand).to_uppercase(); + let replicas = key_replicas(&key, &sys); + let others = key_non_replicas(&key, &sys); + let old = format!("{}-old", random_string(8, &mut rand)); + let new = format!("{}-hinted", random_string(8, &mut rand)); + + check_put(&mut sys, &others[0], &key, &old, 3, 100)?; + finish_background_work(&mut sys, "PUT setup")?; + for replica in &replicas { + sys.network().disconnect_node(replica); + } + // With all primaries unavailable, both operations must use backup storage. + check_put(&mut sys, &others[0], &key, &new, 3, 200)?; + send_delete(&mut sys, &others[2], &key, 3); + wait_for_responses( + &mut sys, + &[ExpectedResponse::delete(&others[2], &key, &[Some(&new)])], + 200, + )?; + sys.network().reset(); + // No GET before the replicas recover: read repair must not mask lost hints. + finish_background_work(&mut sys, "Partitioned DELETE handoff")?; + check_isolated_reads(&mut sys, &[(&key, None)]) +} + +pub fn test_sloppy_quorum_delete_distinct(config: &TestConfig) -> TestResult { + // Keep exactly quorum - 1 nodes alive, including the coordinator. Cover + // primary-only, mixed and backup-only groups and both coordinator roles. + for &(quorum, primary_count, backup_count, primary_client) in DISTINCT_QUORUM_CASES { + for value_kind in ["absent", "empty", "present"] { + for recover_primary in [false, true] { + let mut sys = build_system(config); + let mut rand = Pcg64::seed_from_u64(config.seed); + let key = random_string(8, &mut rand).to_uppercase(); + let value = random_string(8, &mut rand); + let initial = match value_kind { + "absent" => None, + "empty" => Some(""), + _ => Some(value.as_str()), + }; + let replicas = key_replicas(&key, &sys); + let backups = key_non_replicas(&key, &sys); + let client = if primary_client { + &replicas[0] + } else { + &backups[0] + }; + println!( + "DELETE distinct: quorum={quorum}, primaries={primary_count}, backups={backup_count}, primary_client={primary_client}, value={value_kind}, recover_primary={recover_primary}" + ); + let connected: Vec<&str> = replicas[..primary_count] + .iter() + .chain(backups[..backup_count].iter()) + .map(String::as_str) + .collect(); + let recovered = if recover_primary { + &replicas[primary_count] + } else { + &backups[backup_count] + }; + if let Some(value) = initial { + // Seed exactly the nodes that will answer DELETE, including + // backup-only quorums. Every responder must have seen the value. + let mut setup = connected.clone(); + setup.push(recovered); + connect_only(&mut sys, &setup); + check_put(&mut sys, client, &key, value, quorum, 500)?; + finish_background_work(&mut sys, "DELETE quorum setup")?; + } + connect_only(&mut sys, &connected); + // Duplicate both requests and replies without depending on + // the solution's message names or its particular retry policy. + sys.network().set_dupl_rate(1.0); + send_delete(&mut sys, client, &key, quorum); + sys.steps(200); + for proc in sys.process_names() { + assume!( + sys.read_local_messages(&proc).is_empty(), + format!( + "DELETE replied with fewer than {quorum} distinct available nodes (response from {proc})" + ) + )?; + } + + // Add exactly one distinct responder, keeping all other nodes + // unavailable and duplication enabled. No new request is sent. + sys.network().connect_node(recovered); + wait_for_responses( + &mut sys, + &[ExpectedResponse::delete(client, &key, &[initial])], + 500, + )?; + finish_background_work(&mut sys, "DELETE quorum recovery")?; + + // A later request has its own quorum and must observe deletion. + // Strict collectors also reject repeated client responses. + send_delete(&mut sys, client, &key, quorum); + wait_for_responses( + &mut sys, + &[ExpectedResponse::delete(client, &key, &[None])], + 500, + )?; + finish_background_work(&mut sys, "Repeated DELETE quorum")?; + } + } + } + Ok(true) +} + +pub fn test_read_repair_late(config: &TestConfig) -> TestResult { + // Rotate the only up-to-date replica so both stale and fresh replies can + // arrive after the client has received its quorum-one response. + for fresh_index in 0..3 { + let mut sys = build_system(config); + let mut rand = Pcg64::seed_from_u64(config.seed); + let key = random_string(8, &mut rand).to_uppercase(); + let replicas = key_replicas(&key, &sys); + let others = key_non_replicas(&key, &sys); + let writer = &others[0]; + let reader = &others[1]; + let old = format!("{}-old", random_string(8, &mut rand)); + let new = format!("{}-new", random_string(8, &mut rand)); + + check_put(&mut sys, writer, &key, &old, 3, 100)?; + connect_only(&mut sys, &[writer, &replicas[fresh_index]]); + check_put(&mut sys, writer, &key, &new, 1, 100)?; + // No live writer or backup can finish this PUT after reconnection. + sys.crash_node(writer); + sys.network().reset(); + + send_get(&mut sys, reader, &key, 1); + wait_for_responses( + &mut sys, + &[ExpectedResponse::get( + reader, + &key, + &[Some(&old), Some(&new)], + )], + 100, + )?; + finish_read_repair(&mut sys)?; + check_isolated_reads(&mut sys, &[(&key, Some(&new))])?; + } + Ok(true) +} + +pub fn test_read_repair_delete(config: &TestConfig) -> TestResult { + let mut sys = build_system(config); + let mut rand = Pcg64::seed_from_u64(config.seed); + let key = random_string(8, &mut rand).to_uppercase(); + let replicas = key_replicas(&key, &sys); + let others = key_non_replicas(&key, &sys); + let writer = &others[0]; + let reader = &others[1]; + let value = random_string(8, &mut rand); + + check_put(&mut sys, writer, &key, &value, 3, 100)?; + connect_only(&mut sys, &[writer, &replicas[0], &replicas[1]]); + // Only this key has been written. Check the isolated replica has a live copy + // without starting a GET that could later repair it on reconnection. + check_count_keys(&mut sys, &replicas[2], 1) + .map_err(|err| format!("Missing live copy before DELETE on {}: {err}", replicas[2]))?; + check_delete(&mut sys, writer, &key, 2, Some(&value), 100)?; + sys.crash_node(writer); + sys.network().reset(); + + send_get(&mut sys, reader, &key, 3); + wait_for_responses( + &mut sys, + &[ExpectedResponse::get(reader, &key, &[None])], + 100, + )?; + finish_read_repair(&mut sys)?; + // Check the stale replica alone: a quorum read could hide resurrection. + check_isolated_reads(&mut sys, &[(&key, None)]) +} + +pub fn test_read_repair_tie(config: &TestConfig) -> TestResult { + let mut sys = build_system(config); + let mut rand = Pcg64::seed_from_u64(config.seed); + let key = random_string(8, &mut rand).to_uppercase(); + let replicas = key_replicas(&key, &sys); + let others = key_non_replicas(&key, &sys); + let base = random_string(8, &mut rand); + let smaller = format!("{base}-a"); + let larger = format!("{base}-z"); + + // Two partitions store different values with exactly the same timestamp. + // Separate coordinators are stopped before the partitions reconnect. + let left = vec![replicas[0].clone(), replicas[1].clone(), others[0].clone()]; + let right: Vec<_> = sys + .process_names() + .into_iter() + .filter(|p| !left.contains(p)) + .collect(); + for first in &left { + for second in &right { + sys.network().disable_link(first, second); + sys.network().disable_link(second, first); + } + } + send_put(&mut sys, &others[0], &key, &larger, 2); + send_put(&mut sys, &others[1], &key, &smaller, 1); + wait_for_responses( + &mut sys, + &[ + ExpectedResponse::put(&others[0], &key, &[Some(&larger)]), + ExpectedResponse::put(&others[1], &key, &[Some(&smaller)]), + ], + 100, + )?; + sys.crash_node(&others[0]); + sys.crash_node(&others[1]); + sys.network().reset(); + + send_get(&mut sys, &others[2], &key, 3); + wait_for_responses( + &mut sys, + &[ExpectedResponse::get(&others[2], &key, &[Some(&larger)])], + 100, + )?; + finish_read_repair(&mut sys)?; + check_isolated_reads(&mut sys, &[(&key, Some(&larger))]) +} + +pub fn test_read_repair_multi_key(config: &TestConfig) -> TestResult { + let mut sys = build_system(config); + let mut rand = Pcg64::seed_from_u64(config.seed); + let key = random_string(8, &mut rand).to_uppercase(); + let replicas = key_replicas(&key, &sys); + let mut keys = vec![key]; + // Two repaired keys, an unchanged key on the same replicas, an unchanged + // key on overlapping replicas, and a key that has never been written. + for primary in [&replicas[0], &replicas[0], &replicas[1], &replicas[0]] { + keys.push(unused_key_on_primary(&sys, primary, &keys, &mut rand)); + } + let others = key_non_replicas(&keys[0], &sys); + // others[0] is a primary replica of keys[3]; keep both coordinators outside + // the primary sets so stopping the writer loses no stored replica. + let writer = &others[1]; + let reader = &others[2]; + let old = format!("{}-old", random_string(8, &mut rand)); + let deleted = format!("{}-deleted", random_string(8, &mut rand)); + let guard = format!("{}-guard", random_string(8, &mut rand)); + let other_guard = format!("{}-other-guard", random_string(8, &mut rand)); + let new = format!("{}-new", random_string(8, &mut rand)); + for (key, value) in [ + (&keys[2], &guard), + (&keys[3], &other_guard), + (&keys[0], &old), + (&keys[1], &deleted), + ] { + check_put(&mut sys, writer, key, value, 3, 100)?; + } + connect_only(&mut sys, &[writer, &replicas[0], &replicas[1]]); + check_put(&mut sys, writer, &keys[0], &new, 2, 100)?; + check_delete(&mut sys, writer, &keys[1], 2, Some(&deleted), 100)?; + sys.crash_node(writer); + sys.network().reset(); + + // Concurrent reads on one coordinator must keep each key's context apart. + send_get(&mut sys, reader, &keys[0], 3); + send_get(&mut sys, reader, &keys[1], 3); + wait_for_responses( + &mut sys, + &[ + ExpectedResponse::get(reader, &keys[0], &[Some(&new)]), + ExpectedResponse::get(reader, &keys[1], &[None]), + ], + 200, + )?; + finish_read_repair(&mut sys)?; + check_isolated_reads( + &mut sys, + &[ + (&keys[0], Some(&new)), + (&keys[1], None), + (&keys[2], Some(&guard)), + (&keys[3], Some(&other_guard)), + (&keys[4], None), + ], + ) +} + +pub fn test_sloppy_quorum_read_repair(config: &TestConfig) -> TestResult { + check_backup_read_repair(config, BackupRepairScenario::Value) +} + +pub fn test_sloppy_quorum_read_repair_delete(config: &TestConfig) -> TestResult { + check_backup_read_repair(config, BackupRepairScenario::Delete) +} + +pub fn test_sloppy_quorum_read_repair_multi_key(config: &TestConfig) -> TestResult { + check_backup_read_repair(config, BackupRepairScenario::MultiKey) +} + +#[derive(Clone, Copy, Debug)] +enum BackupRepairScenario { + Value, + Delete, + MultiKey, +} + +fn check_backup_read_repair(config: &TestConfig, scenario: BackupRepairScenario) -> TestResult { + for fresh_index in 0..3 { + for stale_backups in [false, true] { + let mut sys = build_system(config); + let mut rand = Pcg64::seed_from_u64(config.seed); + let key = random_string(8, &mut rand).to_uppercase(); + let replicas = key_replicas(&key, &sys); + let others = key_non_replicas(&key, &sys); + let fresh = &replicas[fresh_index]; + let backups = &others[..2]; + let writer = others.last().unwrap(); + let multi_key = matches!(scenario, BackupRepairScenario::MultiKey); + let mut keys = vec![key]; + if multi_key { + // Three changed keys, an untouched key on the same primaries, + // and a guard owned by both backups as primary replicas. + for primary in [&replicas[0], &replicas[0], &replicas[0], &replicas[2]] { + keys.push(unused_key_on_primary(&sys, primary, &keys, &mut rand)); + } + } + let base = random_string(8, &mut rand); + let old = format!("{base}-z"); + let new = format!("{base}-a"); + let guard = format!("{base}-guard"); + let other_guard = format!("{base}-other-guard"); + let changes = match scenario { + BackupRepairScenario::Value => vec![(keys[0].as_str(), Some(new.as_str()))], + BackupRepairScenario::Delete => vec![(keys[0].as_str(), None)], + BackupRepairScenario::MultiKey => vec![ + (keys[0].as_str(), Some(new.as_str())), + (keys[1].as_str(), Some("")), + (keys[2].as_str(), None), + ], + }; + let mut expected = changes.clone(); + if multi_key { + expected.push((&keys[3], Some(&guard))); + expected.push((&keys[4], Some(&other_guard))); + } + println!( + "Backup read repair: scenario={scenario:?}, fresh={fresh}, stale_backups={stale_backups}, keys={keys:?}" + ); + + for (key, _) in &changes { + check_put(&mut sys, writer, key, &old, 3, 100)?; + } + if multi_key { + check_put(&mut sys, writer, &keys[3], &guard, 3, 100)?; + check_put(&mut sys, writer, &keys[4], &other_guard, 3, 100)?; + } + finish_background_work(&mut sys, "Backup read repair setup")?; + + if stale_backups { + // Establish old hints for both unavailable owners. The later + // read must replace these copies, including with tombstones. + connect_only(&mut sys, &[writer, fresh, &backups[0], &backups[1]]); + for (key, _) in &changes { + check_put(&mut sys, writer, key, &old, 3, 200)?; + } + finish_background_work(&mut sys, "Stale backup setup")?; + } + + // Only one primary receives the latest versions. Stop the writer + // before it can retry PUT/DELETE on any other primary or backup. + connect_only(&mut sys, &[writer, fresh]); + for (key, value) in &changes { + if let Some(value) = value { + check_put(&mut sys, writer, key, value, 1, 100)?; + } else { + check_delete(&mut sys, writer, key, 1, Some(&old), 100)?; + } + } + sys.crash_node(writer); + for backup in backups { + let expected_count = + usize::from(multi_key) + if stale_backups { changes.len() } else { 0 }; + check_count_keys(&mut sys, backup, expected_count)?; + } + + // Quorum three needs the fresh primary and both backups. Several + // keys are read concurrently through the same coordinator. + connect_only(&mut sys, &[fresh, &backups[0], &backups[1]]); + for (key, _) in &changes { + send_get(&mut sys, fresh, key, 3); + } + let responses: Vec<_> = changes + .iter() + .map(|(key, value)| ExpectedResponse::get(fresh, key, &[*value])) + .collect(); + wait_for_responses(&mut sys, &responses, 500)?; + finish_background_work(&mut sys, "Read repair on backups")?; + // Remove both the fresh data source and the GET coordinator. + // Neither retries nor late read repair may replace hinted handoff. + sys.crash_node(fresh); + for backup in backups { + let expected_count = + usize::from(multi_key) + changes.iter().filter(|(_, v)| v.is_some()).count(); + check_count_keys(&mut sys, backup, expected_count) + .map_err(|error| format!("Repaired backup {backup}: {error}"))?; + } + + // Do not probe backups with GET: that could create the very hints + // this test is meant to check. Return both stale primaries, allow + // handoff, then inspect counts and isolated primary copies. + sys.network().reset(); + finish_background_work(&mut sys, "Handoff after read repair")?; + check_handoff_state(&mut sys, &expected) + .map_err(|error| format!("Handoff after read repair: {error}"))?; + } + } + Ok(true) +} + +pub fn test_stale_replica(config: &TestConfig) -> TestResult { + let mut sys = build_system(config); + let mut rand = Pcg64::seed_from_u64(config.seed); + + let key = random_string(8, &mut rand).to_uppercase(); + let replicas = key_replicas(&key, &sys); + + // put key from the first replica with quorum 3 + let value = random_string(8, &mut rand); + check_put(&mut sys, &replicas[0], &key, &value, 3, 100)?; + + // disconnect the last replica + sys.network().disconnect_node(&replicas[2]); + + // update key from the first replica with quorum 2 + let value2 = random_string(8, &mut rand); + check_put(&mut sys, &replicas[0], &key, &value2, 2, 100)?; + + // disconnect the first replica + sys.network().disconnect_node(&replicas[0]); + // connect the last replica + sys.network().connect_node(&replicas[2]); + + // read key from the second replica with quorum 2 + // should update the last replica via read repair or anti-entropy + check_get(&mut sys, &replicas[1], &key, 2, Some(&value2), 100)?; + + // step for a while and check whether the last replica got the recent value + sys.steps(100); + sys.network().disconnect_node(&replicas[2]); + check_get(&mut sys, &replicas[2], &key, 1, Some(&value2), 100) +} + +pub fn test_stale_replica_delete(config: &TestConfig) -> TestResult { + let mut sys = build_system(config); + let mut rand = Pcg64::seed_from_u64(config.seed); + + let key = random_string(8, &mut rand).to_uppercase(); + let replicas = key_replicas(&key, &sys); + + // put key from the first replica with quorum 3 + let value = random_string(8, &mut rand); + check_put(&mut sys, &replicas[0], &key, &value, 3, 100)?; + + // disconnect the last replica + sys.network().disconnect_node(&replicas[2]); + + // update key from the first replica with quorum 2 + let value2 = random_string(8, &mut rand); + check_put(&mut sys, &replicas[0], &key, &value2, 2, 100)?; + + // disconnect the first replica + sys.network().disconnect_node(&replicas[0]); + // connect the last replica + sys.network().connect_node(&replicas[2]); + + // delete key from the last replica (should return the last-written value) + check_delete(&mut sys, &replicas[2], &key, 2, Some(&value2), 100)?; + + // connect the first replica + sys.network().connect_node(&replicas[0]); + + // get key from the first replica (should return None) + check_get(&mut sys, &replicas[0], &key, 2, None, 100) +} + +pub fn test_diverged_replicas(config: &TestConfig) -> TestResult { + let mut sys = build_system(config); + let mut rand = Pcg64::seed_from_u64(config.seed); + + let key = random_string(8, &mut rand).to_uppercase(); + let replicas = key_replicas(&key, &sys); + let non_replicas = key_non_replicas(&key, &sys); + + // put key from the first replica with quorum 3 + let value = random_string(8, &mut rand); + check_put(&mut sys, &replicas[0], &key, &value, 3, 100)?; + + // disconnect each replica and update key from it with quorum 1 + let mut new_values = Vec::new(); + for replica in replicas.iter() { + sys.network().disconnect_node(replica); + let value2 = random_string(8, &mut rand); + check_put(&mut sys, replica, &key, &value2, 1, 100)?; + new_values.push(value2); + // read some key to advance the time + // (make sure that the isolated replica is not among this key's replicas) + loop { + let some_key = random_string(8, &mut rand).to_uppercase(); + if !key_replicas(&some_key, &sys).contains(replica) { + check_get(&mut sys, &non_replicas[0], &some_key, 3, None, 100)?; + break; + } + } + sys.network().connect_node(replica); + } + + // read key from the first replica with quorum 3 + // (the last written value should win) + let expected = new_values.last().unwrap(); + check_get(&mut sys, &replicas[0], &key, 3, Some(expected), 100) +} + +pub fn test_sloppy_quorum_read(config: &TestConfig) -> TestResult { + let mut sys = build_system(config); + let mut rand = Pcg64::seed_from_u64(config.seed); + + let key = random_string(8, &mut rand).to_uppercase(); + let replicas = key_replicas(&key, &sys); + let non_replicas = key_non_replicas(&key, &sys); + + // disconnect the first replica + sys.network().disconnect_node(&replicas[0]); + + // put key from the second replica with quorum 2 + let value = random_string(8, &mut rand); + check_put(&mut sys, &replicas[1], &key, &value, 2, 100)?; + + // disconnect the second replica + sys.network().disconnect_node(&replicas[1]); + + // read key from the last non-replica with quorum 2 (should use sloppy quorum) + // since non-replicas do not store any value, the last replica's value should win + // the reading node could also do read repair on non-replicas to fix them + check_get(&mut sys, &non_replicas[2], &key, 2, Some(&value), 100) +} + +pub fn test_sloppy_quorum_write(config: &TestConfig) -> TestResult { + let mut sys = build_system(config); + let procs = sys.process_names(); + let mut rand = Pcg64::seed_from_u64(config.seed); + + let key = random_string(8, &mut rand).to_uppercase(); + let replicas = key_replicas(&key, &sys); + + // put key from the first node with quorum 3 + let value = random_string(8, &mut rand); + check_put(&mut sys, &procs[0], &key, &value, 3, 100)?; + + // temporarily disconnect the first replica + sys.network().disconnect_node(&replicas[0]); + + // update key from the second replica with quorum 3 (should use sloppy quorum) + let value2 = random_string(8, &mut rand); + check_put(&mut sys, &replicas[1], &key, &value2, 3, 100)?; + + // read key from the last replica with quorum 3 (should use sloppy quorum) + check_get(&mut sys, &replicas[2], &key, 3, Some(&value2), 100)?; + + // reconnect the first replica and let it receive the update + sys.network().connect_node(&replicas[0]); + sys.steps(100); + + // check if the first replica got update + sys.network().disconnect_node(&replicas[0]); + check_get(&mut sys, &replicas[0], &key, 1, Some(&value2), 100) +} + +pub fn test_sloppy_quorum_tricky(config: &TestConfig) -> TestResult { + let mut sys = build_system(config); + let procs = sys.process_names(); + let mut rand = Pcg64::seed_from_u64(config.seed); + + let key = random_string(8, &mut rand).to_uppercase(); + let replicas = key_replicas(&key, &sys); + let non_replicas = key_non_replicas(&key, &sys); + + // put key from the first node with quorum 3 + let value = random_string(8, &mut rand); + check_put(&mut sys, &procs[0], &key, &value, 3, 100)?; + + // temporarily disconnect the first replica + sys.network().disconnect_node(&replicas[0]); + + // update key from the second replica with quorum 3 (should use sloppy quorum) + let value2 = random_string(8, &mut rand); + check_put(&mut sys, &replicas[1], &key, &value2, 3, 100)?; + + // disconnect all members of the previous sloppy quorum + sys.network().disconnect_node(&replicas[1]); + sys.network().disconnect_node(&replicas[2]); + sys.network().disconnect_node(&non_replicas[0]); + + // reconnect the first replica + sys.network().connect_node(&replicas[0]); + + // now we have only one node storing the key value: + // - second replica: value (outdated) + // all connected non-replicas do not store the key + + // read key from the last non-replica with quorum 2 + // (will receive old value only from the first replica and probably read repair it) + check_get(&mut sys, &non_replicas[2], &key, 2, Some(&value), 100)?; + + // reconnect the second replica + sys.network().connect_node(&replicas[1]); + + // read key from the last non-replica with quorum 2 + // (should try to contact the main replicas first and receive the new value) + check_get(&mut sys, &non_replicas[2], &key, 2, Some(&value2), 100) +} + +pub fn test_partition_clients(config: &TestConfig) -> TestResult { + let mut sys = build_system(config); + let mut rand = Pcg64::seed_from_u64(config.seed); + + let key = random_string(8, &mut rand).to_uppercase(); + let replicas = key_replicas(&key, &sys); + let non_replicas = key_non_replicas(&key, &sys); + + // partition clients from all replicas + let client1 = &non_replicas[0]; + let client2 = &non_replicas[1]; + let part1: Vec<&str> = replicas.iter().map(|s| &**s).collect(); + let part2: Vec<&str> = non_replicas.iter().map(|s| &**s).collect(); + sys.network().make_partition(&part1, &part2); + + // put key from client1 with quorum 2 (should use sloppy quorum without any normal replica) + let value = random_string(8, &mut rand); + check_put(&mut sys, client1, &key, &value, 2, 100)?; + + // read key from client2 with quorum 2 (should use sloppy quorum without any normal replica) + check_get(&mut sys, client2, &key, 2, Some(&value), 100) +} + +pub fn test_partition_mixed(config: &TestConfig) -> TestResult { + let mut sys = build_system(config); + let nodes = sys.process_names(); + let mut rand = Pcg64::seed_from_u64(config.seed); + + let key = random_string(8, &mut rand).to_uppercase(); + let replicas = key_replicas(&key, &sys); + let replica1 = &replicas[0]; + let replica2 = &replicas[1]; + let replica3 = &replicas[2]; + let non_replicas = key_non_replicas(&key, &sys); + let client1 = &non_replicas[0]; + let client2 = &non_replicas[1]; + let client3 = &non_replicas[2]; + + // put key from the first node with quorum 3 + let value = random_string(8, &mut rand); + check_put(&mut sys, &nodes[0], &key, &value, 3, 100)?; + + // partition clients and replicas + let part1: Vec<&str> = vec![client1, client2, replica1]; + let part2: Vec<&str> = vec![client3, replica2, replica3]; + sys.network().make_partition(&part1, &part2); + + // partition 1 + check_get(&mut sys, client1, &key, 2, Some(&value), 100)?; + let mut value2 = format!("{value}-1"); + check_put(&mut sys, client1, &key, &value2, 2, 100)?; + check_get(&mut sys, client2, &key, 2, Some(&value2), 100)?; + value2 = format!("{value2}-2"); + check_put(&mut sys, client2, &key, &value2, 2, 100)?; + check_get(&mut sys, client2, &key, 2, Some(&value2), 100)?; + + // partition 2 + check_get(&mut sys, client3, &key, 2, Some(&value), 100)?; + let value3 = format!("{value}-3"); + check_put(&mut sys, client3, &key, &value3, 2, 100)?; + check_get(&mut sys, client3, &key, 2, Some(&value3), 100)?; + + // heal partition + sys.network().reset(); + sys.steps(100); + + // read key from all clients (should return the last-written value) + check_get(&mut sys, client1, &key, 2, Some(&value3), 100)?; + check_get(&mut sys, client2, &key, 2, Some(&value3), 100)?; + check_get(&mut sys, client3, &key, 2, Some(&value3), 100)?; + + // check all replicas (should return the last-written value) + for replica in replicas.iter() { + sys.network().disconnect_node(replica); + check_get(&mut sys, replica, &key, 1, Some(&value3), 100)?; + } + Ok(true) +} + +// UTILS --------------------------------------------------------------------------------------------------------------- + +// Group indistinguishable client replies by (process, type, key) and count them. +// Drain every local outbox so response batches and different delivery orders +// are handled without discarding messages. +struct ExpectedResponse<'a> { + proc: &'a str, + tip: &'a str, + key: &'a str, + values: Vec>, + count: usize, +} + +impl<'a> ExpectedResponse<'a> { + fn get(proc: &'a str, key: &'a str, values: &[Option<&'a str>]) -> Self { + Self { + proc, + tip: "GET_RESP", + key, + values: values.to_vec(), + count: 1, + } + } + + fn delete(proc: &'a str, key: &'a str, values: &[Option<&'a str>]) -> Self { + Self { + proc, + tip: "DELETE_RESP", + key, + values: values.to_vec(), + count: 1, + } + } + + fn put(proc: &'a str, key: &'a str, values: &[Option<&'a str>]) -> Self { + Self { + proc, + tip: "PUT_RESP", + key, + values: values.to_vec(), + count: 1, + } + } +} + +fn accept_response( + expected: &[ExpectedResponse<'_>], + remaining: &mut [usize], + proc: &str, + msg: &Message, +) -> Result<(), String> { + if !["GET_RESP", "PUT_RESP", "DELETE_RESP"].contains(&msg.tip.as_str()) { + return Err(format!("Unexpected local message {} from {proc}", msg.tip)); + } + let ValueResponse { key, value } = parse_value_response(msg, proc)?; + let value = value.as_deref(); + let index = expected + .iter() + .position(|item| item.proc == proc && item.tip == msg.tip && item.key == key) + .ok_or_else(|| format!("Unexpected {} from {proc} for key {key}", msg.tip))?; + if remaining[index] == 0 { + return Err(format!("Duplicate {} from {proc} for key {key}", msg.tip)); + } + if !expected[index].values.contains(&value) { + return Err(format!( + "Wrong {} value from {proc} for key {key}: {value:?}, expected one of {:?}", + msg.tip, expected[index].values + )); + } + remaining[index] -= 1; + Ok(()) +} + +fn wait_for_responses( + sys: &mut System, + expected: &[ExpectedResponse<'_>], + max_steps: u32, +) -> TestResult { + let procs = sys.process_names(); + let mut remaining: Vec<_> = expected.iter().map(|item| item.count).collect(); + for step in 0..=max_steps { + for proc in &procs { + for msg in sys.read_local_messages(proc) { + accept_response(expected, &mut remaining, proc, &msg)?; + } + } + if remaining.iter().all(|count| *count == 0) { + return Ok(true); + } + if step == max_steps || !sys.step() { + break; + } + } + let missing: Vec<_> = expected + .iter() + .zip(&remaining) + .filter(|(_, count)| **count > 0) + .map(|(item, count)| { + format!( + "{count} {} from {} for key {}", + item.tip, item.proc, item.key + ) + }) + .collect(); + Err(format!( + "Responses not received within {max_steps} steps: {}", + missing.join(", ") + )) +} + +fn check_concurrent_final_value( + sys: &mut System, + readers: &[&str], + key: &str, + value: &str, +) -> TestResult { + for &reader in readers { + send_get(sys, reader, key, 2); + wait_for_responses( + sys, + &[ExpectedResponse::get(reader, key, &[Some(value)])], + 100, + )?; + } + // Let background replication and timers run, then reject late duplicates. + sys.steps(100); + for proc in sys.process_names() { + assume!( + sys.read_local_messages(&proc).is_empty(), + format!("Unexpected extra local response from {proc}") + )?; + } + Ok(true) +} + +fn connect_only(sys: &mut System, connected: &[&str]) { + sys.network().reset(); + for proc in sys.process_names() { + if !connected.contains(&proc.as_str()) { + sys.network().disconnect_node(&proc); + } + } +} + +fn unused_key_on_primary(sys: &System, primary: &str, used: &[String], rand: &mut Pcg64) -> String { + loop { + let key = random_string(8, rand).to_uppercase(); + if !used.contains(&key) && key_replicas(&key, sys)[0] == primary { + return key; + } + } +} + +fn send_get(sys: &mut System, proc: &str, key: &str, quorum: u8) { + sys.send_local_message(proc, Message::json("GET", &GetReqMessage { key, quorum })); +} + +fn send_delete(sys: &mut System, proc: &str, key: &str, quorum: u8) { + sys.send_local_message( + proc, + Message::json("DELETE", &DeleteReqMessage { key, quorum }), + ); +} + +fn finish_read_repair(sys: &mut System) -> TestResult { + finish_background_work(sys, "Read repair") +} + +fn finish_background_work(sys: &mut System, phase: &str) -> TestResult { + // Allow asynchronous repair/anti-entropy to complete after the client reply. + sys.steps(200); + for proc in sys.process_names() { + assume!( + sys.read_local_messages(&proc).is_empty(), + format!("{phase} produced an extra local response from {proc}") + )?; + } + Ok(true) +} + +fn check_count_keys(sys: &mut System, proc: &str, expected: usize) -> TestResult { + let sent_before = sys.network().network_message_count(); + sys.send_local_message(proc, Message::json("COUNT_KEYS", &serde_json::json!({}))); + // The local handler runs synchronously. Later simulation steps may send unrelated traffic. + assume!( + sys.network().network_message_count() == sent_before, + format!("COUNT_KEYS on {proc} sent a message to another node") + )?; + let messages = sys + .step_until_local_message_max_steps(proc, 10) + .map_err(|error| format!("COUNT_KEYS_RESP not received from {proc}: {error}"))?; + assume_eq!(messages.len(), 1)?; + let message = &messages[0]; + assume_eq!(message.tip, "COUNT_KEYS_RESP")?; + let data: serde_json::Value = serde_json::from_str(&message.data) + .map_err(|error| format!("Invalid COUNT_KEYS_RESP from {proc}: {error}"))?; + let count = data + .get("count") + .and_then(serde_json::Value::as_u64) + .ok_or_else(|| format!("Missing or invalid count in COUNT_KEYS_RESP from {proc}"))?; + assume!( + count == expected as u64, + format!("Wrong COUNT_KEYS_RESP from {proc}: got {count}, expected {expected}") + )?; + Ok(true) +} + +fn check_live_key_counts(sys: &mut System, values: &[(&str, Option<&str>)]) -> TestResult { + for proc in sys.process_names() { + if sys.node_is_crashed(&proc) { + continue; + } + let expected = values + .iter() + .filter(|(key, value)| value.is_some() && key_replicas(key, sys).contains(&proc)) + .count(); + check_count_keys(sys, &proc, expected)?; + } + Ok(true) +} + +fn check_handoff_state(sys: &mut System, values: &[(&str, Option<&str>)]) -> TestResult { + // COUNT_KEYS and isolated probes cannot repair missing or stale copies. + connect_only(sys, &[]); + check_live_key_counts(sys, values)?; + for (key, value) in values { + for replica in key_replicas(key, sys) { + if !sys.node_is_crashed(&replica) { + send_get(sys, &replica, key, 1); + wait_for_responses(sys, &[ExpectedResponse::get(&replica, key, &[*value])], 100)?; + } + } + } + Ok(true) +} + +fn check_isolated_reads(sys: &mut System, values: &[(&str, Option<&str>)]) -> TestResult { + // Isolate every node before the first probe. Neither read repair triggered + // by a probe nor a fresh majority may hide a stale/corrupted local copy. + connect_only(sys, &[]); + for (key, value) in values { + for replica in key_replicas(key, sys) { + send_get(sys, &replica, key, 1); + wait_for_responses(sys, &[ExpectedResponse::get(&replica, key, &[*value])], 100) + .map_err(|error| format!("Local copy of {key} at {replica}: {error}"))?; + } + } + Ok(true) +} + +fn check_get( + sys: &mut System, + proc: &str, + key: &str, + quorum: u8, + expected: Option<&str>, + max_steps: u32, +) -> TestResult { + sys.send_local_message(proc, Message::json("GET", &GetReqMessage { key, quorum })); + let res = sys.step_until_local_message_max_steps(proc, max_steps); + assume!(res.is_ok(), format!("GET_RESP is not returned by {}", proc))?; + let msgs = res.unwrap(); + let msg = msgs.first().unwrap(); + assume_eq!(msg.tip, "GET_RESP")?; + let data = parse_value_response(msg, proc)?; + assume_eq!(data.key, key)?; + assume_eq!(data.value.as_deref(), expected)?; + Ok(true) +} + +fn send_put(sys: &mut System, proc: &str, key: &str, value: &str, quorum: u8) { + sys.send_local_message( + proc, + Message::json("PUT", &PutReqMessage { key, value, quorum }), + ); +} + +fn check_put_result( + sys: &mut System, + proc: &str, + key: &str, + value: &str, + max_steps: u32, +) -> TestResult { + let res = sys.step_until_local_message_max_steps(proc, max_steps); + assume!(res.is_ok(), format!("PUT_RESP is not returned by {}", proc))?; + let msgs = res.unwrap(); + let msg = msgs.first().unwrap(); + assume_eq!(msg.tip, "PUT_RESP")?; + let data = parse_value_response(msg, proc)?; + assume_eq!(data.key, key)?; + assume!( + data.value.as_deref() == Some(value), + format!( + "Wrong PUT_RESP value from {proc} for key {key}: {:?}, expected {value:?}", + data.value + ) + )?; + Ok(true) +} + +fn check_put( + sys: &mut System, + proc: &str, + key: &str, + value: &str, + quorum: u8, + max_steps: u32, +) -> TestResult { + send_put(sys, proc, key, value, quorum); + check_put_result(sys, proc, key, value, max_steps) +} + +fn check_delete( + sys: &mut System, + proc: &str, + key: &str, + quorum: u8, + expected: Option<&str>, + max_steps: u32, +) -> TestResult { + sys.send_local_message( + proc, + Message::json("DELETE", &DeleteReqMessage { key, quorum }), + ); + let res = sys.step_until_local_message_max_steps(proc, max_steps); + assume!( + res.is_ok(), + format!("DELETE_RESP is not returned by {}", proc) + )?; + let msgs = res.unwrap(); + let msg = msgs.first().unwrap(); + assume_eq!(msg.tip, "DELETE_RESP")?; + let data = parse_value_response(msg, proc)?; + assume_eq!(data.key, key)?; + assume_eq!(data.value.as_deref(), expected)?; + Ok(true) +} + +#[cfg(test)] +mod concurrent_response_tests { + use super::*; + + fn response(tip: &str, key: &str, value: Option<&str>) -> Message { + Message::json(tip, &serde_json::json!({"key": key, "value": value})) + } + + #[test] + fn accepts_either_response_order_and_read_value() { + let expected = [ + ExpectedResponse::get("client", "K", &[None, Some("v")]), + ExpectedResponse::put("client", "K", &[Some("v")]), + ]; + for value in [None, Some("v")] { + for put_first in [false, true] { + let mut remaining = [1; 2]; + let get = response("GET_RESP", "K", value); + let put = response("PUT_RESP", "K", Some("v")); + let messages = if put_first { [put, get] } else { [get, put] }; + for msg in messages { + accept_response(&expected, &mut remaining, "client", &msg).unwrap(); + } + assert_eq!(remaining, [0, 0]); + } + } + } + + #[test] + fn rejects_invalid_and_duplicate_responses() { + let expected = [ExpectedResponse::put("client", "K", &[Some("v")])]; + for (proc, msg) in [ + ("wrong-client", response("PUT_RESP", "K", Some("v"))), + ("client", response("PUT_RESP", "wrong-key", Some("v"))), + ("client", response("GET_RESP", "K", Some("v"))), + ("client", response("DELETE_RESP", "K", Some("v"))), + ("client", response("PUT_RESP", "K", Some("wrong-value"))), + ("client", response("PUT_RESP", "K", None)), + ( + "client", + Message::json("PUT_RESP", &serde_json::json!({"key": "K"})), + ), + ] { + assert!(accept_response(&expected, &mut [1], proc, &msg).is_err()); + } + let get_expected = [ExpectedResponse::get("client", "K", &[None, Some("v")])]; + let missing_value = Message::json("GET_RESP", &serde_json::json!({"key": "K"})); + assert!(accept_response(&get_expected, &mut [1], "client", &missing_value).is_err()); + let malformed = Message::new("GET_RESP", "{"); + assert!(accept_response(&get_expected, &mut [1], "client", &malformed).is_err()); + let mut remaining = [1]; + let msg = response("PUT_RESP", "K", Some("v")); + accept_response(&expected, &mut remaining, "client", &msg).unwrap(); + assert!(accept_response(&expected, &mut remaining, "client", &msg).is_err()); + } + + #[test] + fn accepts_nullable_delete_responses_without_losing_duplicates() { + let expected = [ + ExpectedResponse::delete("first", "K", &[None, Some("v")]), + ExpectedResponse::delete("second", "K", &[None, Some("v")]), + ]; + for value in [None, Some("v")] { + for first in [0, 1] { + let mut remaining = [1; 2]; + for index in [first, 1 - first] { + let msg = response("DELETE_RESP", "K", value); + accept_response(&expected, &mut remaining, expected[index].proc, &msg).unwrap(); + assert!( + accept_response(&expected, &mut remaining, expected[index].proc, &msg) + .is_err() + ); + } + assert_eq!(remaining, [0, 0]); + } + } + let missing = Message::json("DELETE_RESP", &serde_json::json!({"key": "K"})); + assert!(accept_response(&expected, &mut [1; 2], "first", &missing).is_err()); + } +} diff --git a/homework/08-kv-replication/tests/src/tests/ordered_delivery.rs b/homework/08-kv-replication/tests/src/tests/ordered_delivery.rs new file mode 100644 index 0000000..863d75e --- /dev/null +++ b/homework/08-kv-replication/tests/src/tests/ordered_delivery.rs @@ -0,0 +1,100 @@ +use anysystem::python::PyProcess; +use anysystem::{Context, Process}; + +use super::*; + +// Hold network deliveries from one coordinator at their destinations. This also +// covers retries and extra protocol rounds without inspecting student messages. +#[derive(Clone)] +struct DeliveryGate { + proc: PyProcess, + held_sender: Option, + pending: Vec, +} + +impl Process for DeliveryGate { + fn on_start(&mut self, ctx: &mut Context) -> Result<(), String> { + self.proc.on_start(ctx) + } + + fn on_message(&mut self, msg: Message, from: String, ctx: &mut Context) -> Result<(), String> { + if self.held_sender.as_ref() == Some(&from) { + self.pending.push(msg); + return Ok(()); + } + self.proc.on_message(msg, from, ctx) + } + + fn on_local_message(&mut self, msg: Message, ctx: &mut Context) -> Result<(), String> { + match msg.tip.as_str() { + "HOLD_FROM" => self.held_sender = Some(msg.data), + "RELEASE" => { + if let Some(from) = self.held_sender.take() { + for msg in self.pending.drain(..) { + self.proc.on_message(msg, from.clone(), ctx)?; + } + } + } + _ => return self.proc.on_local_message(msg, ctx), + } + Ok(()) + } + + fn on_timer(&mut self, timer: String, ctx: &mut Context) -> Result<(), String> { + self.proc.on_timer(timer, ctx) + } +} + +pub(super) fn check_put_before_later_delete(config: &TestConfig) -> TestResult { + // Two non-replica coordinators and three primaries suffice. Any two + // quorums of three intersect, even if a retry reaches a backup. + let config = TestConfig { + proc_count: 5, + ..*config + }; + for value in ["abc", ""] { + for hold_put in [false, true] { + let mut sys = build_system_with(&config, |proc| { + Box::new(DeliveryGate { + proc, + held_sender: None, + pending: Vec::new(), + }) + }); + let mut rand = Pcg64::seed_from_u64(config.seed); + let key = random_string(8, &mut rand).to_uppercase(); + let others = key_non_replicas(&key, &sys); + let (writer, deleter) = (&others[0], &others[1]); + let held = if hold_put { writer } else { deleter }; + let procs = sys.process_names(); + for proc in &procs { + sys.send_local_message(proc, Message::new("HOLD_FROM", held)); + } + + // PUT is older, but both clients submit before the network delivers anything. + send_put(&mut sys, writer, &key, value, 3); + sys.step_for_duration(MIN_NETWORK_DELAY / 2.0); + send_delete(&mut sys, deleter, &key, 3); + + let expected = [(!hold_put).then_some(value)]; + let put = ExpectedResponse::put(writer, &key, &expected); + let delete = ExpectedResponse::delete(deleter, &key, &expected); + let (first, second, delivery) = if hold_put { + (delete, put, "DELETE then PUT") + } else { + (put, delete, "PUT then DELETE") + }; + let order = format!("replicas receive {delivery}, value={value:?}"); + println!("Earlier PUT, later DELETE: {order}"); + wait_for_responses(&mut sys, &[first], 300).map_err(|err| format!("{order}: {err}"))?; + for proc in &procs { + sys.send_local_message(proc, Message::new("RELEASE", "")); + } + wait_for_responses(&mut sys, &[second], 300) + .map_err(|err| format!("{order}: {err}"))?; + finish_background_work(&mut sys, "Concurrent PUT before later DELETE")?; + check_isolated_reads(&mut sys, &[(&key, None)])?; + } + } + Ok(true) +} diff --git a/homework/08-kv-replication/tests/src/tests_mc.rs b/homework/08-kv-replication/tests/src/tests_mc.rs new file mode 100644 index 0000000..cc329ae --- /dev/null +++ b/homework/08-kv-replication/tests/src/tests_mc.rs @@ -0,0 +1,483 @@ +use std::collections::HashSet; + +use rand::prelude::*; +use rand_pcg::Pcg64; + +use anysystem::logger::LogEntry; +use anysystem::mc::{ + predicates::{collects, goals, prunes}, + strategies::bfs::Bfs, + EventOrderingMode, InvariantFn, McState, McStats, McSystem, ModelChecker, StrategyConfig, +}; +use anysystem::test::TestResult; +use anysystem::{Message, System}; + +use crate::common::*; + +pub fn test_mc_basic(config: &TestConfig) -> TestResult { + let sys = build_system(config); + let procs = sys.process_names(); + let mut rand = Pcg64::seed_from_u64(config.seed); + + let mut mc = ModelChecker::new(&sys); + + let key = random_string(8, &mut rand).to_uppercase(); + let replicas = key_replicas(&key, &sys); + let non_replicas = key_non_replicas(&key, &sys); + println!("Key {key} replicas: {replicas:?}"); + println!("Key {key} non-replicas: {non_replicas:?}"); + + // stage 1: get key from the first node + let stage1_strategy = mc_query_strategy(&procs[0], McQuery::Get(key.clone(), None)); + let stage1_msg = Message::json( + "GET", + &GetReqMessage { + key: &key, + quorum: 2, + }, + ); + let stage1_states = run_mc( + &mut mc, + stage1_strategy, + &procs[0], + stage1_msg, + McNetworkChange::None, + None, + )? + .collected_states; + println!("stage 1: {}", stage1_states.len()); + if stage1_states.is_empty() { + return Err("stage 1 - GET response is not received".to_owned()); + } + + // stage 2: put key to the first replica + let value = random_string(8, &mut rand); + let sys = build_system(config); + mc = ModelChecker::new(&sys); + + let stage2_strategy = mc_query_strategy(&replicas[0], McQuery::Put(key.clone(), value.clone())); + let stage2_msg = Message::json( + "PUT", + &PutReqMessage { + key: &key, + value: &value, + quorum: 2, + }, + ); + let stage2_states = run_mc( + &mut mc, + stage2_strategy, + &replicas[0], + stage2_msg, + McNetworkChange::None, + None, + )? + .collected_states; + println!("stage 2: {}", stage2_states.len()); + if stage2_states.is_empty() { + return Err("stage 2 - PUT response is not received".to_owned()); + } + + // stage 3: get key from the last replica + let stage3_strategy = mc_query_strategy(&replicas[2], McQuery::Get(key.clone(), Some(value))); + let stage3_msg = Message::json( + "GET", + &GetReqMessage { + key: &key, + quorum: 2, + }, + ); + let stage3_states = run_mc( + &mut mc, + stage3_strategy, + &replicas[2], + stage3_msg, + McNetworkChange::None, + Some(stage2_states), + )? + .collected_states; + println!("stage 3: {}", stage3_states.len()); + if stage3_states.is_empty() { + return Err("stage 3 - GET response is not received".to_owned()); + } + Ok(true) +} + +pub fn test_mc_concurrent_writes(config: &TestConfig) -> TestResult { + let sys = build_system(config); + let mut rand = Pcg64::seed_from_u64(config.seed); + + let key = random_string(8, &mut rand).to_uppercase(); + let replicas = key_replicas(&key, &sys); + let non_replicas = key_non_replicas(&key, &sys); + println!("Key {key} replicas: {replicas:?}"); + println!("Key {key} non-replicas: {non_replicas:?}"); + + // isolate all processes + for proc in sys.process_names() { + sys.network().disconnect_node(&proc); + } + let mut mc = ModelChecker::new(&sys); + + // put (key, value) to the first replica + // and then put (key, value2) to the second replica + let value = random_string(8, &mut rand); + let value2 = random_string(8, &mut rand); + + let strategy_config = mc_query_strategy(&replicas[0], McQuery::Put(key.clone(), value.clone())); + let msg1 = Message::json( + "PUT", + &PutReqMessage { + key: &key, + value: &value, + quorum: 1, + }, + ); + let states = run_mc( + &mut mc, + strategy_config, + &replicas[0], + msg1, + McNetworkChange::None, + None, + )? + .collected_states; + if states.is_empty() { + return Err(format!("put({key}, {value}) response is not received")); + } + println!("put({key}, {value}): {} states collected", states.len()); + + let strategy_config = + mc_query_strategy(&replicas[1], McQuery::Put(key.clone(), value2.clone())); + let msg2 = Message::json( + "PUT", + &PutReqMessage { + key: &key, + value: &value2, + quorum: 1, + }, + ); + let states = run_mc( + &mut mc, + strategy_config, + &replicas[1], + msg2, + McNetworkChange::None, + Some(states), + )? + .collected_states; + if states.is_empty() { + return Err(format!("put({key}, {value2}) response is not received")); + } + println!("put({key}, {value2}): {} states collected", states.len()); + + // now reset the network state and ask the third replica about the key's value + // we expect the value written later (value2) to be returned + let strategy_config = mc_query_strategy( + &replicas[2], + McQuery::Get(key.to_string(), Some(value2.to_string())), + ); + let msg = Message::json( + "GET", + &GetReqMessage { + key: &key, + quorum: 3, + }, + ); + let states = run_mc( + &mut mc, + strategy_config, + &replicas[2], + msg, + McNetworkChange::Reset, + Some(states), + )? + .collected_states; + if states.is_empty() { + return Err(format!("get({key}) response is not received")); + } + println!("get({key}): {} states collected", states.len()); + Ok(true) +} + +pub fn test_mc_sloppy_quorum_hinted_handoff(config: &TestConfig) -> TestResult { + let mut sys = build_system(config); + let mut rand = Pcg64::seed_from_u64(config.seed); + + let mut mc = ModelChecker::new(&sys); + + let key = random_string(8, &mut rand).to_uppercase(); + let replicas = key_replicas(&key, &sys); + let non_replicas = key_non_replicas(&key, &sys); + println!("Key {key} replicas: {replicas:?}"); + println!("Key {key} non-replicas: {non_replicas:?}"); + + // stage 1: get key from the first replica (during the network partition) + let stage1_strategy = mc_query_strategy(&replicas[0], McQuery::Get(key.clone(), None)); + let stage1_msg = Message::json( + "GET", + &GetReqMessage { + key: &key, + quorum: 2, + }, + ); + let stage1_states = run_mc( + &mut mc, + stage1_strategy, + &replicas[0], + stage1_msg, + McNetworkChange::Partition([ + vec![ + replicas[0].clone(), + non_replicas[0].clone(), + non_replicas[1].clone(), + non_replicas[2].clone(), + ], + vec![replicas[1].clone(), replicas[2].clone()], + ]), + None, + )? + .collected_states; + println!("stage 1: {}", stage1_states.len()); + if stage1_states.is_empty() { + return Err("stage 1 - GET response is not received".to_owned()); + } + + // stage 2: put key from the first replica (network partition still exists) + let value = random_string(8, &mut rand); + sys = build_system(config); + mc = ModelChecker::new(&sys); + + let stage2_strategy = mc_query_strategy(&replicas[0], McQuery::Put(key.clone(), value.clone())); + let stage2_msg = Message::json( + "PUT", + &PutReqMessage { + key: &key, + value: &value, + quorum: 2, + }, + ); + let stage2_states = run_mc( + &mut mc, + stage2_strategy, + &replicas[0], + stage2_msg, + McNetworkChange::Partition([ + vec![ + replicas[0].clone(), + non_replicas[0].clone(), + non_replicas[1].clone(), + non_replicas[2].clone(), + ], + vec![replicas[1].clone(), replicas[2].clone()], + ]), + None, + )? + .collected_states; + println!("stage 2: {}", stage2_states.len()); + if stage2_states.is_empty() { + return Err("stage 2 - PUT response is not received".to_owned()); + } + + // stage 3: recover network and let data propagate to all replicas + let stage3_states = mc_stabilize(&mut sys, stage2_states)?.collected_states; + println!("stage 3: {}", stage3_states.len()); + if stage3_states.is_empty() { + return Err( + "stage 3 - no states found during the exploration phase with recovered network" + .to_owned(), + ); + } + + // stage 4: get key from the last replica (again during the network partition) + let stage4_strategy = mc_query_strategy(&replicas[2], McQuery::Get(key.clone(), Some(value))); + let stage4_msg = Message::json( + "GET", + &GetReqMessage { + key: &key, + quorum: 2, + }, + ); + let stage4_states = run_mc( + &mut mc, + stage4_strategy, + &replicas[2], + stage4_msg, + McNetworkChange::Partition([ + vec![ + replicas[0].clone(), + non_replicas[0].clone(), + non_replicas[1].clone(), + non_replicas[2].clone(), + ], + vec![replicas[1].clone(), replicas[2].clone()], + ]), + Some(stage3_states), + )? + .collected_states; + println!("stage 4: {}", stage4_states.len()); + if stage4_states.is_empty() { + return Err("stage 4 - GET response is not received".to_owned()); + } + Ok(true) +} + +// UTILS --------------------------------------------------------------------------------------------------------------- + +enum McQuery { + Get(String, Option), + Put(String, String), +} + +fn mc_query_strategy(proc: &str, query_data: McQuery) -> StrategyConfig { + let proc_name = proc.to_string(); + + let invariant = match query_data { + McQuery::Get(key, expected) => mc_get_invariant(proc, key, expected), + McQuery::Put(key, value) => mc_put_invariant(proc, key, value), + }; + + StrategyConfig::default() + .prune(prunes::any_prune(vec![ + prunes::event_happened_n_times_current_run(LogEntry::is_mc_timer_fired, 5_usize), + prunes::event_happened_n_times_current_run(LogEntry::is_mc_message_received, 10_usize), + ])) + .goal(goals::event_happened_n_times_current_run( + LogEntry::is_mc_local_message_sent, + 1, + )) + .invariant(invariant) + .collect(collects::event_happened_n_times_current_run( + move |log_entry| match log_entry { + LogEntry::McLocalMessageSent { proc, .. } => proc == &proc_name, + _ => false, + }, + 1, + )) +} + +fn mc_get_invariant(proc: S, key: String, expected: Option) -> InvariantFn +where + S: Into, +{ + let proc_name = proc.into(); + Box::new(move |state: &McState| -> Result<(), String> { + for entry in state.current_run_trace().iter() { + if let LogEntry::McLocalMessageSent { msg, proc } = entry { + if &proc_name != proc { + return Err("local message received on wrong process".to_string()); + } + if msg.tip != "GET_RESP" { + return Err(format!("wrong type {}", msg.tip)); + } + let data = parse_value_response(msg, proc)?; + if data.key != key { + return Err(format!("wrong key {}", data.key)); + } + if data.value != expected { + return Err(format!("wrong value {:?}", data.value)); + } + } + } + Ok(()) + }) +} + +fn mc_put_invariant(proc: S, key: String, value: String) -> InvariantFn +where + S: Into, +{ + let proc_name = proc.into(); + Box::new(move |state: &McState| -> Result<(), String> { + for entry in state.current_run_trace().iter() { + if let LogEntry::McLocalMessageSent { msg, proc } = entry { + if &proc_name != proc { + return Err("local message received on wrong process".to_string()); + } + if msg.tip != "PUT_RESP" { + return Err(format!("wrong type {}", msg.tip)); + } + let data = parse_value_response(msg, proc)?; + if data.key != key { + return Err(format!("wrong key {}", data.key)); + } + if data.value.as_deref() != Some(value.as_str()) { + return Err(format!("wrong value {:?}", data.value)); + } + } + } + Ok(()) + }) +} + +enum McNetworkChange { + None, + Reset, + Partition([Vec; 2]), +} + +fn run_mc( + mc: &mut ModelChecker, + strategy_config: StrategyConfig, + proc: S, + msg: Message, + network_change: McNetworkChange, + states: Option>, +) -> Result +where + S: Into, +{ + let proc = proc.into(); + + let callback = |sys: &mut McSystem| { + match &network_change { + McNetworkChange::Partition([part1, part2]) => sys.network().partition(part1, part2), + McNetworkChange::Reset => sys.network().reset(), + McNetworkChange::None => {} + } + sys.set_event_ordering_mode(EventOrderingMode::MessagesFirst); + sys.send_local_message(proc.clone(), proc.clone(), msg.clone()); + }; + + let res = if let Some(states) = states { + mc.run_from_states_with_change::(strategy_config, states, callback) + } else { + mc.run_with_change::(strategy_config, callback) + }; + match res { + Err(e) => { + e.print_trace(); + Err(e.message()) + } + Ok(stats) => Ok(stats), + } +} + +fn mc_stabilize(sys: &mut System, states: HashSet) -> Result { + let strategy_config = StrategyConfig::default() + .prune(prunes::any_prune(vec![ + prunes::event_happened_n_times_current_run(LogEntry::is_mc_timer_fired, 6), + prunes::event_happened_n_times_current_run(LogEntry::is_mc_message_received, 24), + ])) + .goal(goals::any_goal(vec![ + goals::depth_reached(30), + goals::no_events(), + ])) + .collect(collects::any_collect(vec![ + collects::no_events(), + collects::event_happened_n_times_current_run(LogEntry::is_mc_timer_fired, 6), + collects::event_happened_n_times_current_run(LogEntry::is_mc_message_received, 24), + ])); + let mut mc = ModelChecker::new(sys); + let res = mc.run_from_states_with_change::(strategy_config, states, |sys| { + sys.network().reset(); + sys.set_event_ordering_mode(EventOrderingMode::MessagesFirst); + }); + match res { + Err(e) => { + e.print_trace(); + Err(e.message()) + } + Ok(stats) => Ok(stats), + } +}