diff --git a/homework/06-membership/readme.md b/homework/06-membership/readme.md new file mode 100644 index 0000000..65cce24 --- /dev/null +++ b/homework/06-membership/readme.md @@ -0,0 +1,138 @@ +# Group Membership + +В этом задании вам предстоит реализовать распределённый сервис для отслеживания состава группы. Он должен позволять процессам вступать в группу, покидать её и узнавать её текущий состав. + +В системе _N_ узлов, на каждом из которых запущен процесс с уникальным идентификатором _id_. Процессы вступают в _группу_ по команде `JOIN` и покидают её по команде `LEAVE`. Сам по себе запуск процесса не делает его участником группы. + +Помимо добровольного выхода, состав группы может меняться из-за отказов. В нашей системе возможны: + +- _Падение процесса или всего узла_. Процесс прекращает выполнение и теряет состояние в памяти. Позднее он может быть перезапущен с тем же _id_ и чистым состоянием. Это модель _crash-recovery_ из лекции 6. Работающий процесс будем называть _живым_. +- _Потери сообщений_ между некоторыми парами узлов, в обоих или только в одном направлении. Живые процессы могут оказаться полностью или частично изолированы от остальных. Сетевые сбои могут быть временными, после чего связь восстанавливается. + +Для обнаружения недоступных участников вам потребуется _детектор отказов_ (см. лекцию 6). Он сообщает о подозрении: отсутствие ответа может быть вызвано как падением процесса, так и проблемами сети. От детектора требуется _полнота_: падение участника должно в конечном счёте обнаруживаться всеми оставшимися участниками. Подберите баланс между скоростью обнаружения, числом ложных подозрений и расходами на обмен сообщениями. + +Списки участников на разных процессах могут временно различаться после входа, выхода, отказа или восстановления. Требуется _согласованность в конечном счёте_: когда условия работы перестают меняться, списки должны стабилизироваться. Каждый участник должен видеть в своём списке себя и всех живых участников, с которыми возможен обмен сообщениями в обе стороны, в том числе через других участников. + +Упавшие и добровольно вышедшие процессы должны автоматически исключаться из списков остальных участников. То же относится к живому процессу, который не может обмениваться сообщениями в обе стороны ни с одним другим участником. + +Если сеть распалась на подгруппы, между которыми нет двусторонней связи, происходит _разделение сети_. В этом случае списки должны совпадать внутри каждой подгруппы, включая меньшую, и содержать всех её участников, без процессов из других подгрупп. В списке участников изолированного процесса должен остаться только он сам. Односторонней связи между подгруппами недостаточно для сохранения общей группы. + +После перезапуска процесс снова вступает в группу по команде `JOIN`, и остальные участники должны вернуть его в свои списки. При восстановлении связи с временно недоступным живым участником они должны сделать это автоматически, без новой команды `JOIN`. Это нужно учитывать и при использовании идей SWIM из лекции: исключение из локального списка не должно навсегда препятствовать возвращению участника. + +Чтобы получить максимальный балл, нужно также обеспечить масштабируемость решения. При увеличении числа узлов в _X_ раз сетевой трафик и число сообщений в единицу времени должны увеличиваться не более чем в _2X_ раз. Нагрузка на процессы измеряется суммой отправленных и полученных сообщений; отношение максимальной нагрузки к минимальной не должно превышать 5. + +Подробные сценарии проверок приведены в тестах — они являются частью условия. + +## Реализация + +Для реализации и тестирования решения используется фреймворк AnySystem (см. материалы первого семинара). В папке `solution` размещена заготовка для решения [member.py](solution/member.py). Вам надо доработать реализацию процесса в классе `GroupMember` так, чтобы проходили все тесты. + +Из библиотек разрешены только AnySystem и стандартная библиотека Python. Можно добавлять собственные вспомогательные модули. Сжимать сетевые сообщения запрещено, в том числе с помощью `zlib`, `gzip`, `bz2`, `lzma` или собственного алгоритма сжатия. Снижайте сетевые расходы за счёт устройства протокола: например, передавайте только изменения состава группы вместо полного списка. + +Ваши реализации процессов AnySystem должны соблюдать правило [изоляции процессов](../readme.md#изоляция-процессов-anysystem): изменяемое состояние каждого процесса должно храниться только в `self`, а общая изменяемая память между процессами запрещена. Все взаимодействия между процессами должны происходить через сообщения, как и в реальной распределённой системе. + +При инициализации процессу передается его уникальный _id_. Процесс должен поддерживать обработку следующих локальных сообщений: + +- _JOIN_ - команда присоединиться к группе. В результате обработки этой команды процесс должен стать участником группы. В поле `seed` передается идентификатор одного из живых участников группы. Если в `seed` находится идентификатор самого процесса, то он должен создать новую пустую группу и добавить в неё себя. Ответ на это сообщение не нужен. +- _LEAVE_ - команда покинуть группу. В результате обработки этой команды процесс должен перестать быть участником группы. Ответ на это сообщение не нужен. +- _GET_MEMBERS_ - запрос локального списка участников группы. Процесс должен сразу ответить одним локальным сообщением _MEMBERS_ с хранимым списком идентификаторов (см. заготовку). Обращаться к другим процессам или ждать таймера для ответа нельзя. Эта семантика проверяется отдельным тестом. + +Для взаимодействия между процессами вы можете использовать любые собственные типы сообщений. + +**Важно!** Для измерения прошедшего времени в коде процесса используйте метод `ctx.time()`, возвращающий локальное время "внутри симуляции". Использовать обычные способы, например `time.time()`, некорректно, так как скорость течения времени в симуляции отличается от реального времени. Также не стоит опираться на синхронизацию часов в системе - у каждого узла свои локальные часы, и показания `ctx.time()` у разных процессов могут расходиться. После перезапуска узла смещение его часов может измениться. + +В коде процесса можно использовать генератор случайных чисел из `random`, но не следует его явно инициализировать с помощью `random.seed()`. Это уже делается в тестах на основе переданного random seed, а дополнительная инициализация может нарушить воспроизводимость результатов. + +## Тестирование + +### Локальное тестирование + +Тесты находятся в папке `tests`. Есть два варианта их запуска. + +Рекомендуемый вариант запуска тестов - через готовый Docker-образ. В этом случае используемое окружение будет аналогично тестирующей системе. Для запуска тестов выполните команду: + +```commandline +docker run --pull always --rm -t -v ./solution:/solution distsys.ru/course/membership:latest [ЗДЕСЬ МОЖНО УКАЗАТЬ ОПЦИИ] +``` + +Для запуска полного набора тестов с теми же параметрами, что и в тестирующей системе, выполните команду: + +```commandline +docker run --pull always --rm -t -v ./solution:/solution distsys.ru/course/membership:latest -m 1000 +``` + +Вы также можете собрать и запустить тесты непосредственно в Linux с помощью компилятора Rust. На Windows можно использовать WSL или Docker, на macOS — Docker. Сборка из исходников может быть удобна, если вы хотите лучше изучить или доработать тесты. Скомпилируйте тесты с помощью команды `cargo install --locked --path tests`. Для запуска тестов выполните команду: + +```commandline +distsys-membership [ЗДЕСЬ МОЖНО УКАЗАТЬ ОПЦИИ] +``` + +Запустить один тест можно с помощью опции `-t` (например, `... -t "PROCESS CRASH"`). По умолчанию тестер выводит результаты проверок, диагностику долгих запусков и итоговую сводку. Флаг `-d` добавляет трассу событий, а `-h` показывает все доступные опции. При отдельном запуске теста масштабируемости через `-t` предварительное прохождение функциональных тестов не требуется. + +### Что проверяют тесты + +Сначала выполняются обычные функциональные тесты, затем `CHAOS MONKEY`, `MODEL CHECKING` и тесты масштабируемости. + +На каждом этапе обычных тестов списки участников должны сойтись к ожидаемому составу за 300 секунд симуляции. В тестах изоляции проверяются и оставшаяся группа, и изолированный участник: в его списке должен остаться только он сам. + +Тесты `NETWORK PARTITION UNEQUAL` и `NETWORK PARTITION UNEQUAL RECOVER` разделяют сеть на две непустые подгруппы разного размера: 4 и 6 при десяти процессах. Состав группы должен стабилизироваться в обеих подгруппах, включая меньшую. Второй тест затем восстанавливает связь и проверяет, что все процессы снова видят общую группу. + +Тесты `TWO PROCESSES CANNOT COMMUNICATE` и `TWO PROCESSES CANNOT COMMUNICATE IN TRIANGLE` отключают связь между парой узлов, сохраняя связь через остальных участников. Во втором тесте всегда три узла. Все участники должны остаться в группе. + +В этих двух тестах, а также после восстановления в `PROCESS CRASH RECOVER`, `PROCESS OFFLINE RECOVER`, `NETWORK PARTITION RECOVER` и `NETWORK PARTITION UNEQUAL RECOVER` проверяется непрерывная стабильность состава группы. За 300 секунд симуляции должен найтись интервал длительностью 30 секунд, в котором списки участников остаются корректными. Внутри этого интервала списки проверяются после каждого события. Переходные ошибки до стабилизации допускаются. + +`CHAOS MONKEY` проверяет последовательности отказов. По умолчанию выполняются 100 прогонов, а в тестирующей системе — 1000 (`-m 1000`). Каждый прогон включает до шести шагов с отказами; при стандартных десяти процессах в проверяемой группе может остаться четыре участника. + +Для отладки `CHAOS MONKEY` используйте флаг `-v` (`--verbose`): он выводит параметры запуска, порядок входа процессов и отказы с номерами процессов и временем в симуляции. Этот вывод также включается с `-d`. Чтобы повторить запуск, укажите то же значение `-s` (`--seed`) и то же число процессов `-p`, а в `-m` — число не меньше номера нужного прогона. Например, `... -t "CHAOS MONKEY (run 850)" -m 1000 -s 123 -p 10 -v`. В тестах выбор процессов воспроизводим при одинаковых параметрах; случайность внутри вашего решения также должна быть воспроизводимой. + +`MODEL CHECKING` (MC) проверяет сборку группы из трёх процессов без отказов и ответы на `GET_MEMBERS` при разных порядках доставки сообщений. Полный состав ожидается в состояниях, где каждый процесс получил не менее трёх сетевых сообщений или больше нет событий. В тестирующей системе MC включён; для быстрой локальной проверки его можно отключить флагом `--disable-mc-tests`. + +Тесты `SCALABILITY NORMAL` и `SCALABILITY CRASH` измеряют расходы на поддержание группы. В первом тесте измерение длится 10 секунд симуляции после сборки группы, во втором — от падения одного процесса до стабилизации состава. Расходы на первоначальную сборку группы не учитываются; во втором тесте равномерность нагрузки проверяется среди оставшихся живых процессов. + +При стандартных параметрах оба теста масштабируемости запускаются для групп из 10, 20, 50 и 100 узлов. Сетевой трафик и число сообщений в единицу времени сравниваются между последовательными запусками: 10 → 20, 20 → 50 и 50 → 100. По правилу _2X_ допустимый рост расходов составляет соответственно 4, 5 и 4 раза. + +### Лимиты и диагностика + +Тесты выполняются в отдельных процессах; прогоны `CHAOS MONKEY` объединены в серию с сохранением результата каждого прогона. При зависании, исчерпании памяти или аварийном завершении исполнителя проверка продолжается, пока остаётся общий бюджет времени. Баллы за пройденные функциональные компоненты и сетевые подгруппы сохраняются. Непроверенные тесты отмечаются в сводке и не считаются пройденными. + +По умолчанию и в тестирующей системе действуют следующие ограничения реального времени. Все проверки делят общий бюджет 540 секунд: + +| Проверка | Лимит | Опция | +| --- | --- | --- | +| Обычный тест или один прогон CHAOS MONKEY | 15 секунд | `--test-timeout` | +| Все прогоны CHAOS MONKEY вместе | 300 секунд | `--chaos-timeout` | +| Обе фазы MODEL CHECKING вместе | 300 секунд | `--mc-timeout` | +| Каждый тест SCALABILITY | 60 секунд | `--scalability-timeout` | +| Полный набор тестов | 540 секунд | `--suite-timeout` | + +Адресное пространство исполнителя ограничено 4 ГиБ для обычных тестов, CHAOS MONKEY и SCALABILITY (`--memory-limit 4096`) и 14 ГиБ для MODEL CHECKING (`--mc-memory-limit 14336`). В тестирующей системе эти пределы фиксированы. Окружение должно также иметь запас 256 МиБ для управляющего процесса. + +Если памяти окружения недостаточно для заданного предела и запаса для управляющего процесса, тестер завершится с сообщением об ошибке окружения без оценки. Для локального запуска в меньшем объёме памяти можно явно уменьшить соответствующий предел, но результат может отличаться от проверки на сервере. + +При остановке тестер сообщает причину и последний этап проверки: например, размер собираемой группы или фазу model checking. Долгие проверки также периодически выводят текущий этап и расход памяти. + +Бонусы за пробелы в тестах начисляются по [общим правилам](../readme.md#бонусы-за-пробелы-в-тестах). + +### Проверка в тестирующей системе + +Отправьте решение в тестирующую систему по [инструкции](../readme.md) и дождитесь результатов. + +## Оценивание + +Это задание имеет повышенный вес 1.5, то есть максимум можно набрать **15 баллов**. + +Базовый функционал, отказы процессов и четыре сетевые подгруппы оцениваются независимо друг от друга. Внутри каждого компонента или подгруппы нужно пройти все перечисленные обычные тесты; исключения для MC описаны ниже. Масштабируемость даёт баллы только при зачёте всей функциональной части. + +- **Базовый функционал без отказов — до 4 баллов:** + - **3 балла** за прохождение всех пяти тестов: `SIMPLE`, `GET MEMBERS SEMANTICS`, `RANDOM SEED`, `PROCESS JOIN`, `PROCESS LEAVE`. + - **Ещё 1 балл**, если пройдены и все пять обычных тестов, и `MODEL CHECKING`. Отключение MC при локальном запуске или остановка уже запущенного MC по лимиту времени (в том числе общего бюджета) или памяти сохраняет первые 3 балла. + - **0 баллов за весь компонент**, если MC обнаружил ошибку, не запустился из-за исчерпания общего бюджета или его исполнитель аварийно завершился с неизвестной причиной. +- **Отказы процессов — 3 балла:** `PROCESS CRASH`, `SEED PROCESS CRASH`, `PROCESS CRASH RECOVER`. +- **Сетевые и комбинированные отказы — до 4 баллов, по 1 баллу за каждую подгруппу:** + - **Отключения и разделения сети с восстановлением:** `PROCESS OFFLINE`, `SEED PROCESS OFFLINE`, `PROCESS OFFLINE RECOVER`, `NETWORK PARTITION`, `NETWORK PARTITION RECOVER`, `NETWORK PARTITION UNEQUAL`, `NETWORK PARTITION UNEQUAL RECOVER`. + - **Односторонняя и частичная связность:** `PROCESS CANNOT RECEIVE`, `PROCESS JOIN CANNOT RECEIVE`, `PROCESS CANNOT SEND`, `TWO PROCESSES CANNOT COMMUNICATE`, `TWO PROCESSES CANNOT COMMUNICATE IN TRIANGLE`. + - **Задержки и потери сообщений:** `SLOW NETWORK`, `FLAKY NETWORK`, `FLAKY NETWORK ON START`, `FLAKY NETWORK AND CRASH`. + - **Последовательности отказов:** все 1000 прогонов `CHAOS MONKEY`. +- **Масштабируемость — 4 балла:** `SCALABILITY NORMAL`, `SCALABILITY CRASH`. Эти тесты запускаются в полном наборе только после зачёта всех функциональных компонентов: базовой группы (не менее 3 баллов), отказов процессов и всех четырёх сетевых подгрупп. Остановка уже запущенного MC по времени или памяти и его отключение при локальном запуске не блокируют масштабируемость. Если функциональные компоненты не зачтены, тесты масштабируемости пропускаются с указанием причины. Для получения 4 баллов нужно пройти оба теста; при исчерпании общего бюджета незапущенные тесты не считаются пройденными. + +К решению приложите отчёт в файле `solution/readme.md` по [общим требованиям](../readme.md#отчёт). Без отчёта автоматические тесты запускаются, но защита не проводится и решение не засчитывается. diff --git a/homework/06-membership/solution/member.py b/homework/06-membership/solution/member.py new file mode 100644 index 0000000..df142da --- /dev/null +++ b/homework/06-membership/solution/member.py @@ -0,0 +1,34 @@ +from anysystem import Context, Message, Process + + +class GroupMember(Process): + def __init__(self, proc_id: str): + self._id = proc_id + + def on_local_message(self, msg: Message, ctx: Context): + if msg.type == 'JOIN': + # Add process to the group + seed = msg['seed'] + if seed == self._id: + # create new empty group and add process to it + pass + else: + # join existing group + pass + elif msg.type == 'LEAVE': + # Remove process from the group + pass + elif msg.type == 'GET_MEMBERS': + # Get a list of group members + # - return the list of all known alive processes in MEMBERS message + ctx.send_local(Message('MEMBERS', {'members': [self._id]})) + + def on_start(self, ctx: Context): + pass + + def on_message(self, msg: Message, sender: str, ctx: Context): + # Implement communication between processes using any message types + pass + + def on_timer(self, timer_name: str, ctx: Context): + pass diff --git a/homework/06-membership/tests/.dockerignore b/homework/06-membership/tests/.dockerignore new file mode 100644 index 0000000..d291891 --- /dev/null +++ b/homework/06-membership/tests/.dockerignore @@ -0,0 +1,5 @@ +target +**/target +**/__pycache__ +**/.pytest_cache +*.pyc diff --git a/homework/06-membership/tests/Cargo.lock b/homework/06-membership/tests/Cargo.lock new file mode 100644 index 0000000..a1dbfbd --- /dev/null +++ b/homework/06-membership/tests/Cargo.lock @@ -0,0 +1,993 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "aho-corasick" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" +dependencies = [ + "memchr", +] + +[[package]] +name = "anysystem" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e01badd3bc0d042e4399f8f685c394a79a72e7f233b3d3b5eabb3b94734f4dbe" +dependencies = [ + "colored", + "downcast-rs", + "dyn-clone", + "indexmap 2.14.0", + "lazy_static", + "log", + "ordered-float", + "pyo3", + "rand", + "rand_pcg", + "regex", + "rstest", + "serde", + "serde_json", + "simcore", + "sugars", +] + +[[package]] +name = "assertables" +version = "3.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d84b91d63c67e6c0b1a74728dad0057c4a6124b74e671ff91b7fe9f8bbd671aa" + +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi", + "libc", + "winapi", +] + +[[package]] +name = "autocfg" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "cfg-if" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2fd1289c04a9ea8cb22300a459a72a385d7c73d3259e2ed7dcb2af674838cfa9" + +[[package]] +name = "clap" +version = "3.2.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ea181bf566f71cb9a5d17a59e1871af638180a18fb0035c92ae62b705207123" +dependencies = [ + "atty", + "bitflags", + "clap_derive", + "clap_lex", + "indexmap 1.9.3", + "once_cell", + "strsim", + "termcolor", + "textwrap", +] + +[[package]] +name = "clap_derive" +version = "3.2.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae6371b8bdc8b7d3959e9cf7b22d4435ef3e79e138688421ec654acf8c81b008" +dependencies = [ + "heck 0.4.1", + "proc-macro-error", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "clap_lex" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2850f2f5a82cbf437dd5af4d49848fbdfc27c157c3d010345776f952765261c5" +dependencies = [ + "os_str_bytes", +] + +[[package]] +name = "colored" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "117725a109d387c937a1533ce01b450cbde6b88abceea8473c4d7a85853cda3c" +dependencies = [ + "lazy_static", + "windows-sys 0.59.0", +] + +[[package]] +name = "distsys-membership" +version = "0.1.0" +dependencies = [ + "anysystem", + "assertables", + "clap", + "env_logger", + "indexmap 2.14.0", + "libc", + "log", + "pyo3", + "rand", + "rand_pcg", + "serde", + "serde_json", + "sugars", +] + +[[package]] +name = "downcast-rs" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75b325c5dbd37f80359721ad39aca5a29fb04c89279657cffdda8736d0c0b9d2" + +[[package]] +name = "dyn-clone" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" + +[[package]] +name = "env_logger" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a12e6657c4c97ebab115a42dcee77225f7f482cdd841cf7088c657a42e9e00e7" +dependencies = [ + "atty", + "humantime", + "log", + "regex", + "termcolor", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "erased-serde" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e004d887f51fcb9fef17317a2f3525c887d8aa3f4f50fed920816a688284a5b7" +dependencies = [ + "serde", + "typeid", +] + +[[package]] +name = "futures" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" + +[[package]] +name = "futures-executor" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" + +[[package]] +name = "futures-macro" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.106", +] + +[[package]] +name = "futures-sink" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" + +[[package]] +name = "futures-task" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" + +[[package]] +name = "futures-timer" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f288b0a4f20f9a56b5d1da57e2227c661b7b16168e2f72365f57b63326e29b24" + +[[package]] +name = "futures-util" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "pin-utils", + "slab", +] + +[[package]] +name = "getrandom" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "335ff9f135e4384c8150d6f27c6daed433577f86b4750418338c01a1a2528592" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "glob" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" + +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "hermit-abi" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +dependencies = [ + "libc", +] + +[[package]] +name = "humantime" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424" + +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown 0.12.3", +] + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown 0.17.1", +] + +[[package]] +name = "itoa" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "libc" +version = "0.2.175" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a82ae493e598baaea5209805c49bbf2ea7de956d50d7da0da1164f9c6d28543" + +[[package]] +name = "log" +version = "0.4.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34080505efa8e45a4b816c349525ebe327ceaa8559756f0356cba97ef3bf7432" + +[[package]] +name = "memchr" +version = "2.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a282da65faaf38286cf3be983213fcf1d2e2a58700e808f83f4ea9a4804bc0" + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "once_cell" +version = "1.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" + +[[package]] +name = "ordered-float" +version = "4.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7bb71e1b3fa6ca1c61f383464aaf2bb0e2f8e772a1f01d486832464de363b951" +dependencies = [ + "num-traits", + "rand", + "serde", +] + +[[package]] +name = "os_str_bytes" +version = "6.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2355d85b9a3786f481747ced0e0ff2ba35213a1f9bd406ed906554d7af805a1" + +[[package]] +name = "pin-project-lite" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "portable-atomic" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "proc-macro-crate" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edce586971a4dfaa28950c6f18ed55e0406c1ab88bbce2c6f6293a7aaba73d35" +dependencies = [ + "toml_edit", +] + +[[package]] +name = "proc-macro-error" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" +dependencies = [ + "proc-macro-error-attr", + "proc-macro2", + "quote", + "syn 1.0.109", + "version_check", +] + +[[package]] +name = "proc-macro-error-attr" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" +dependencies = [ + "proc-macro2", + "quote", + "version_check", +] + +[[package]] +name = "proc-macro2" +version = "1.0.101" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89ae43fd86e4158d6db51ad8e2b80f313af9cc74f5c0e03ccb87de09998732de" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "pyo3" +version = "0.29.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4688ddedf473e32662b9b067670129a8afb8c18e351482c70d62ba4a88171e8b" +dependencies = [ + "libc", + "once_cell", + "portable-atomic", + "pyo3-build-config", + "pyo3-ffi", + "pyo3-macros", +] + +[[package]] +name = "pyo3-build-config" +version = "0.29.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f41027e41b4bd03f6e60f9f417fe24a6341a6bb744edd62b6f709f2a52ea30e9" +dependencies = [ + "target-lexicon", +] + +[[package]] +name = "pyo3-ffi" +version = "0.29.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e591a95526fead067432c3b3a33fc74770b87b1e04e73671090d9c2055a2b327" +dependencies = [ + "libc", + "pyo3-build-config", +] + +[[package]] +name = "pyo3-macros" +version = "0.29.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73225868fc1cd84eef2c3c230ddb91273bf1de46aeb8a4248da76d32a0924a1c" +dependencies = [ + "proc-macro2", + "pyo3-macros-backend", + "quote", + "syn 2.0.106", +] + +[[package]] +name = "pyo3-macros-backend" +version = "0.29.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "571575aa3749fa6216757dd47d2a3e7ef360f329a40f0666a9fbd14889024952" +dependencies = [ + "heck 0.5.0", + "proc-macro2", + "quote", + "syn 2.0.106", +] + +[[package]] +name = "quote" +version = "1.0.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", + "serde", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom", + "serde", +] + +[[package]] +name = "rand_pcg" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59cad018caf63deb318e5a4586d99a24424a364f40f1e5778c29aca23f4fc73e" +dependencies = [ + "rand_core", +] + +[[package]] +name = "regex" +version = "1.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23d7fd106d8c02486a8d64e778353d1cffe08ce79ac2e82f540c86d0facf6912" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b9458fa0bfeeac22b5ca447c63aaf45f28439a709ccd244698632f9aa6394d6" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caf4aa5b0f434c91fe5c7f1ecb6a5ece2130b02ad2a590589dda5146df959001" + +[[package]] +name = "relative-path" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba39f3699c378cd8970968dcbff9c43159ea4cfbd88d43c00b22f2ef10a435d2" + +[[package]] +name = "rstest" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9afd55a67069d6e434a95161415f5beeada95a01c7b815508a82dcb0e1593682" +dependencies = [ + "futures", + "futures-timer", + "rstest_macros", + "rustc_version", +] + +[[package]] +name = "rstest_macros" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4165dfae59a39dd41d8dec720d3cbfbc71f69744efb480a3920f5d4e0cc6798d" +dependencies = [ + "cfg-if", + "glob", + "proc-macro-crate", + "proc-macro2", + "quote", + "regex", + "relative-path", + "rustc_version", + "syn 2.0.106", + "unicode-ident", +] + +[[package]] +name = "rustc-hash" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" + +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + +[[package]] +name = "ryu" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" + +[[package]] +name = "semver" +version = "1.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56e6fa9c48d24d85fb3de5ad847117517440f6beceb7798af16b4a87d616b8d0" + +[[package]] +name = "serde" +version = "1.0.219" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f0e2c6ed6606019b4e29e69dbaba95b11854410e5347d525002456dbbb786b6" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.219" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.106", +] + +[[package]] +name = "serde_json" +version = "1.0.143" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d401abef1d108fbd9cbaebc3e46611f4b1021f714a0597a71f41ee463f5f4a5a" +dependencies = [ + "indexmap 2.14.0", + "itoa", + "memchr", + "ryu", + "serde", +] + +[[package]] +name = "serde_type_name" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92acc4cd6ae20767c54d6cf1a075624e7f4d9e99d7ebc685398ff243144d8714" +dependencies = [ + "serde", +] + +[[package]] +name = "simcore" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a20dfec85e94e569fa5b810b2d992c68b361f31bfba471447427536488c9c292" +dependencies = [ + "colored", + "downcast-rs", + "dyn-clone", + "erased-serde", + "futures", + "log", + "rand", + "rand_pcg", + "rustc-hash", + "serde", + "serde_json", + "serde_type_name", +] + +[[package]] +name = "slab" +version = "0.4.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a2ae44ef20feb57a68b23d846850f861394c2e02dc425a50098ae8c90267589" + +[[package]] +name = "strsim" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" + +[[package]] +name = "sugars" +version = "3.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc0db74f9ee706e039d031a560bd7d110c7022f016051b3d33eeff9583e3e67a" + +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "2.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ede7c438028d4436d71104916910f5bb611972c5cfd7f89b8300a8186e6fada6" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "target-lexicon" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca" + +[[package]] +name = "termcolor" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "textwrap" +version = "0.16.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c13547615a44dc9c452a8a534638acdf07120d4b6847c8178705da06306a3057" + +[[package]] +name = "toml_datetime" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" + +[[package]] +name = "toml_edit" +version = "0.22.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" +dependencies = [ + "indexmap 2.14.0", + "toml_datetime", + "winnow", +] + +[[package]] +name = "typeid" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc7d623258602320d5c55d1bc22793b57daff0ec7efc270ea7d55ce1d5f5471c" + +[[package]] +name = "unicode-ident" +version = "1.0.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f63a545481291138910575129486daeaf8ac54aee4387fe7906919f7830c7d9d" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.61.0", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-link" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45e46c0661abb7180e7b9c281db115305d49ca1709ab8242adf09666d2173c65" + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.61.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e201184e40b2ede64bc2ea34968b28e33622acdbbf37104f0e4a33f7abe657aa" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "winnow" +version = "0.7.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21a0236b59786fed61e2a80582dd500fe61f18b5dca67a4a067d0bc9039339cf" +dependencies = [ + "memchr", +] + +[[package]] +name = "zerocopy" +version = "0.8.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0894878a5fa3edfd6da3f88c4805f4c8558e2b996227a3d864f47fe11e38282c" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88d2b8d9c68ad2b9e4340d7832716a4d21a22a1154777ad56ea55c51a9cf3831" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.106", +] diff --git a/homework/06-membership/tests/Cargo.toml b/homework/06-membership/tests/Cargo.toml new file mode 100644 index 0000000..31b7180 --- /dev/null +++ b/homework/06-membership/tests/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "distsys-membership" +version = "0.1.0" +edition = "2021" +resolver = "3" + +[dependencies] +anysystem = "=0.3.0" +indexmap = "2.12" +assertables = "3.2.2" +clap = { version = "3.2.17", features = ["cargo", "derive"] } +env_logger = "0.9.0" +log = "0.4.17" +pyo3 = { version = "=0.29.2", features = ["auto-initialize"] } +rand = "0.8.5" +rand_pcg = "0.3.1" +serde = { version = "1.0.142", features = ["derive"] } +serde_json = "1.0.83" +sugars = "3.0.1" + +[target.'cfg(target_os = "linux")'.dependencies] +libc = "0.2.175" diff --git a/homework/06-membership/tests/Dockerfile b/homework/06-membership/tests/Dockerfile new file mode 100644 index 0000000..a720287 --- /dev/null +++ b/homework/06-membership/tests/Dockerfile @@ -0,0 +1,19 @@ +# syntax=docker/dockerfile:1 +FROM rust:1.97.1-bookworm@sha256:0e2bcaef56d041a486784e54104a81aebe0da44bd03019bd70bc0401e42e4a97 AS builder + +RUN apt-get update && apt-get install -y build-essential python3-dev + +WORKDIR /tests +COPY . . +RUN --mount=type=cache,id=distsys-course-cargo-registry,target=/usr/local/cargo/registry,sharing=locked \ + --mount=type=cache,id=distsys-course-cargo-target-rust-1-97-1,target=/tests/target,sharing=locked \ + cargo install --locked --path . --root /opt/course + +FROM debian:bookworm-slim + +RUN apt-get update && apt-get install -y python3-dev && rm -rf /var/lib/apt/lists/* + +COPY --from=builder /opt/course/bin/distsys-membership /usr/local/bin/distsys-membership + +WORKDIR /solution +ENTRYPOINT ["timeout", "--verbose", "-k", "10", "600", "distsys-membership", "-i", "member.py"] diff --git a/homework/06-membership/tests/library_policy.py b/homework/06-membership/tests/library_policy.py new file mode 100644 index 0000000..e1a01c1 --- /dev/null +++ b/homework/06-membership/tests/library_policy.py @@ -0,0 +1,134 @@ +"""HW6 library rules; inspect student source without importing it. + +The general shared-state checker is passed in by the embedded Rust wrapper so +its import resolution is shared without changing the policy of other homework. +This checks common compression APIs, not arbitrary compression algorithms. +""" + +import ast +from pathlib import Path +import sys + +COMPRESSION_APIS = { + "zlib": {"compress", "compressobj", "decompress", "decompressobj"}, + "gzip": {"compress", "decompress", "open", "GzipFile"}, + "bz2": {"compress", "decompress", "open", "BZ2File", "BZ2Compressor", "BZ2Decompressor"}, + "lzma": {"compress", "decompress", "open", "LZMAFile", "LZMACompressor", "LZMADecompressor"}, +} + + +def _check_file(tree, current, root, checker): + violations = set() + pending = checker._local_imports(tree, current, root) + modules, functions = checker._dynamic_import_names(tree) + strings = checker._static_string_values(tree) + symbols = {} + + def report(node, message): + violations.add(f"{current}:{node.lineno}:{node.col_offset + 1}: {message}") + + def check_module(name, node, base=root): + parts = name.split(".") + local = checker._candidate_module(base, parts) + # Inspect package initializers as well as the explicitly imported file. + for index in range(1, len(parts) + 1): + candidate = checker._candidate_module(base, parts[:index]) + if candidate is not None: + pending.add(candidate) + namespace = base.joinpath(*parts).resolve() + local_namespace = namespace.is_relative_to(root) and namespace.is_dir() + if local is None and not local_namespace and parts[0] not in sys.stdlib_module_names and parts[0] != "anysystem": + report(node, f"library '{name}' is not allowed: use AnySystem and the Python standard library") + + def dynamic_name(node): + argument = node.args[0] if node.args else next( + (item.value for item in node.keywords if item.arg == "name"), None + ) + return checker._static_string(argument, strings) if argument is not None else None + + def symbol(node): + if isinstance(node, ast.Name): + return symbols.get(node.id) + if isinstance(node, ast.Call) and checker._is_dynamic_import_call(node, modules, functions): + return dynamic_name(node) + resolved = checker._resolved_attribute(node) + if resolved is not None: + parent = symbol(resolved[0]) + if parent is not None: + return f"{parent}.{resolved[1]}" + return None + + def check_compression(name, node): + parts = name.split(".") + if len(parts) >= 2 and parts[1] in COMPRESSION_APIS.get(parts[0], set()): + report(node, f"message compression is forbidden in HW6: '{parts[0]}.{parts[1]}'") + + for node in ast.walk(tree): + if isinstance(node, ast.Import): + for alias in node.names: + check_module(alias.name, node) + symbols[alias.asname or alias.name.split(".")[0]] = alias.name if alias.asname else alias.name.split(".")[0] + elif isinstance(node, ast.ImportFrom): + base = root + if node.level: + base = current.parent + for _ in range(node.level - 1): + base = base.parent + if not base.is_relative_to(root): + report(node, "local imports must stay inside the solution directory") + continue + if node.module: + check_module(node.module, node, base) + for alias in node.names: + name = f"{node.module}.{alias.name}" if node.module else alias.name + if checker._candidate_module(base, name.split(".")) is not None: + check_module(name, node, base) + if alias.name == "*" and node.module in COMPRESSION_APIS: + report(node, f"message compression is forbidden in HW6: star import from '{node.module}'") + else: + check_compression(name, node) + symbols[alias.asname or alias.name] = name + elif isinstance(node, ast.Call) and checker._is_dynamic_import_call(node, modules, functions): + name = dynamic_name(node) + if name is None or name.startswith("."): + report(node, "dynamic imports must use a statically known absolute module name") + else: + check_module(name, node) + + # Follow aliases, including compressor functions saved for later use. + assignments = [parts for node in ast.walk(tree) if (parts := checker._assignment_parts(node)) is not None] + for _ in range(len(assignments) + 1): + changed = False + for value, targets in assignments: + name = symbol(value) + if name is None: + continue + for target in targets: + if isinstance(target, ast.Name) and target.id not in symbols: + symbols[target.id] = name + changed = True + if not changed: + break + for node in ast.walk(tree): + name = symbol(node) + if name is not None: + check_compression(name, node) + return violations, pending + + +def validate_solution(path, checker): + entrypoint = Path(path).resolve() + root = entrypoint.parent + pending = {entrypoint} + visited = set() + violations = set() + while pending: + current = pending.pop() + if current in visited: + continue + visited.add(current) + tree = ast.parse(current.read_text(encoding="utf-8-sig"), filename=str(current)) + errors, imports = _check_file(tree, current, root, checker) + violations.update(errors) + pending.update(imports - visited) + return sorted(violations) diff --git a/homework/06-membership/tests/shared_state_validator.py b/homework/06-membership/tests/shared_state_validator.py new file mode 100644 index 0000000..bf28c31 --- /dev/null +++ b/homework/06-membership/tests/shared_state_validator.py @@ -0,0 +1,1938 @@ +"""Static checks for Python state shared between AnySystem processes. + +The checker parses source files only. It never imports or executes a student's +solution. Local imports which can be resolved next to the entrypoint are checked +recursively. Standard-library calls are allowed unless they access known shared +runtime state; other external calls need an explicitly reviewed API. +""" + +from __future__ import annotations + +import ast +import sys +from pathlib import Path +from typing import Callable, Iterable + + +MUTABLE_NODES = ( + ast.List, + ast.ListComp, + ast.Dict, + ast.DictComp, + ast.Set, + ast.SetComp, + ast.GeneratorExp, +) +MUTABLE_CONSTRUCTORS = { + "builtins.bytearray", + "builtins.dict", + "builtins.list", + "builtins.set", + "collections.ChainMap", + "collections.Counter", + "collections.OrderedDict", + "collections.defaultdict", + "collections.deque", + "weakref.WeakKeyDictionary", + "weakref.WeakValueDictionary", +} +IMMUTABLE_SCALAR_CONSTRUCTORS = { + "builtins.bool", + "builtins.bytes", + "builtins.complex", + "builtins.float", + "builtins.int", + "builtins.range", + "builtins.str", +} +KNOWN_IMMUTABLE_CALLS = { + "os.getcwd", + "os.path.abspath", + "os.path.basename", + "os.path.dirname", + "os.path.join", + "os.path.normpath", + "os.path.realpath", + "random.randint", + "random.randrange", +} +TYPE_SUBSCRIPT_NAMES = { + "builtins.dict", + "builtins.frozenset", + "builtins.list", + "builtins.set", + "builtins.tuple", + "builtins.type", + "collections.abc.Mapping", + "collections.abc.Sequence", + "typing.Annotated", + "typing.Callable", + "typing.ClassVar", + "typing.Dict", + "typing.FrozenSet", + "typing.List", + "typing.Literal", + "typing.Mapping", + "typing.Optional", + "typing.Sequence", + "typing.Set", + "typing.Tuple", + "typing.Type", + "typing.Union", +} +TYPE_FACTORIES = { + "collections.namedtuple", + "typing.NewType", + "typing.ParamSpec", + "typing.TypeVar", +} +ENUM_BASE_NAMES = { + "enum.Enum", + "enum.Flag", + "enum.IntEnum", + "enum.IntFlag", + "enum.StrEnum", +} +MUTATOR_METHODS = frozenset( + "__delitem__ __setitem__ add append clear discard extend insert move_to_end " + "pop popitem remove reverse rotate setdefault sort update".split() +) +UNBOUND_MUTATOR_METHODS = {"__delattr__", "__delitem__", "__setattr__", "__setitem__"} +OPERATOR_MUTATORS = frozenset( + f"operator.{name}" + for name in "delitem iadd iand iconcat imatmul imul ior isub ixor setitem".split() +) +STATEFUL_DECORATORS = {"functools.cache", "functools.lru_cache"} +# Reviewed external APIs and a few interpreter operations used by the course. +# Ordinary stdlib functions do not need to be enumerated here: see +# _imported_call_allowed. Shared-state checks still apply to their arguments. +ALLOWED_IMPORTED_CALLS = frozenset( + """ + anysystem.Message anysystem.Message.from_json anysystem.python.anysystem.Message + array.array ast.literal_eval + bisect.bisect bisect.bisect_left bisect.bisect_right bisect.insort builtins.print + collections.defaultdict collections.deque collections.namedtuple copy.copy copy.deepcopy + dataclasses.asdict dataclasses.field dataclasses.fields dataclasses.is_dataclass + functools.cmp_to_key functools.reduce + hashlib.blake2b hashlib.md5 hashlib.sha1 hashlib.sha512 hashlib.sha256 + hashlib.sha3_256 hashlib.shake_256 + heapq.heapify heapq.heappop heapq.heappush heapq.merge importlib.import_module + inspect.signature itertools.count json.dumps json.loads + logging.critical logging.debug logging.error logging.exception logging.info logging.log + logging.warning math.ceil math.log math.log2 math.sqrt + os.getcwd os.getenv os.path.abspath os.path.basename os.path.dirname os.path.join + os.path.normpath os.path.realpath pathlib.Path pickle.dumps pickle.loads + portion.closedopen portion.empty pprint.pprint pydantic.Field + random.Random random.choice random.choices random.randint random.random random.randrange + random.sample random.shuffle random.uniform re.fullmatch sys.intern threading.Lock + types.MappingProxyType time.time typing.cast uuid.uuid4 + """.split() +) +IMPORTED_ARGUMENT_MUTATORS = set( + "bisect.insort bisect.insort_left bisect.insort_right heapq.heapify " + "heapq.heappop heapq.heappush heapq.heappushpop heapq.heapreplace random.shuffle".split() +) +# These APIs expose or change state shared by all simulated processes. This is +# a static check for known violations, not a sandbox for arbitrary Python code. +SHARED_RUNTIME_APIS = frozenset(""" + atexit.register atexit.unregister codecs.register codecs.unregister + copyreg.pickle copyreg.constructor copyreg.add_extension copyreg.remove_extension + csv.register_dialect csv.unregister_dialect csv.field_size_limit + decimal.getcontext decimal.setcontext decimal.localcontext + gc.enable gc.disable gc.set_threshold gc.callbacks.append + locale.setlocale locale.resetlocale + logging.getLogger logging.basicConfig logging.disable logging.setLoggerClass + logging.setLogRecordFactory logging.addLevelName logging.captureWarnings + logging.config.dictConfig logging.config.fileConfig logging.config.listen + os.chdir os.fchdir os.putenv os.unsetenv os.umask os.kill os.killpg + random.seed random.setstate random.getstate + signal.signal signal.set_wakeup_fd signal.alarm signal.setitimer + sys.settrace sys.setprofile sys.setrecursionlimit sys.setswitchinterval + sys.set_asyncgen_hooks sys.set_coroutine_origin_tracking_depth sys.addaudithook + threading.settrace threading.setprofile threading.stack_size + warnings.filterwarnings warnings.simplefilter warnings.resetwarnings +""".split()) + + +def _imported_call_allowed(name: str) -> bool: + if name in SHARED_RUNTIME_APIS: + return False + if name in ALLOWED_IMPORTED_CALLS: + return True + parts = name.split(".") + # Private interpreter objects are not part of the stdlib API. Builtins + # include reflection and execution primitives, checked separately. + return ( + len(parts) >= 2 + and parts[0] in sys.stdlib_module_names + and parts[0] != "builtins" + and all(not part.startswith("_") for part in parts) + ) + + +ALLOWED_MUTABLE_METADATA = {"__all__"} +TRUSTED_STAR_EXPORTS = { + "dataclasses": {"dataclass", "field"}, + "enum": {"Enum", "Flag", "IntEnum", "IntFlag", "StrEnum", "auto"}, + "functools": {"cache", "lru_cache"}, + "types": {"MappingProxyType"}, + "typing": { + *"Annotated Any Callable ClassVar Dict FrozenSet List Literal Mapping NewType " + "Optional ParamSpec Sequence Set Tuple Type TypeVar Union".split(), + }, +} +TRUSTED_BUILTINS = set( + "bool bytearray bytes complex dict float frozenset int list range set str tuple type".split() +) + + +def _resolved_attribute(node: ast.AST) -> tuple[ast.AST, str] | None: + if isinstance(node, ast.Attribute): + return node.value, node.attr + if ( + isinstance(node, ast.Call) + and isinstance(node.func, ast.Name) + and node.func.id == "getattr" + and len(node.args) >= 2 + and isinstance(node.args[1], ast.Constant) + and isinstance(node.args[1].value, str) + ): + return node.args[0], node.args[1].value + return None + + +def _call_name(node: ast.AST) -> str | None: + if isinstance(node, ast.Name): + return node.id + resolved = _resolved_attribute(node) + return resolved[1] if resolved is not None else None + + +def _symbol(node: ast.AST, symbols: dict[str, str]) -> str | None: + if isinstance(node, ast.Name): + return symbols.get(node.id) + resolved = _resolved_attribute(node) + if resolved is not None: + value, attribute = resolved + parent = _symbol(value, symbols) + return f"{parent}.{attribute}" if parent is not None else None + return None + + +def _root_name(node: ast.AST) -> str | None: + while (resolved := _resolved_attribute(node)) is not None: + node = resolved[0] + return node.id if isinstance(node, ast.Name) else None + + +def _assignment_parts(node: ast.AST) -> tuple[ast.AST, list[ast.AST]] | None: + if isinstance(node, ast.Assign): + return node.value, node.targets + if isinstance(node, (ast.AnnAssign, ast.NamedExpr)) and node.value is not None: + return node.value, [node.target] + return None + + +def _call_returns_shared( + node: ast.AST, + is_shared: Callable[[ast.AST], bool], + shared_functions: set[str], + returned_parameters: dict[str, set[int]], +) -> bool: + if not isinstance(node, ast.Call): + return False + name = node.func.id if isinstance(node.func, ast.Name) else None + if name in shared_functions: + return True + return any( + index < len(node.args) and is_shared(node.args[index]) + for index in returned_parameters.get(name or "", set()) + ) + + +def _is_instance_path(node: ast.AST, instance_names: set[str]) -> bool: + while True: + if isinstance(node, ast.Subscript): + node = node.value + elif (resolved := _resolved_attribute(node)) is not None: + node = resolved[0] + else: + break + return isinstance(node, ast.Name) and node.id in instance_names + + +def _static_string(node: ast.AST, names: dict[str, str]) -> str | None: + if isinstance(node, ast.Constant) and isinstance(node.value, str): + return node.value + if isinstance(node, ast.Name): + return names.get(node.id) + if isinstance(node, ast.BinOp) and isinstance(node.op, ast.Add): + left = _static_string(node.left, names) + right = _static_string(node.right, names) + return left + right if left is not None and right is not None else None + if isinstance(node, ast.JoinedStr): + parts: list[str] = [] + for value in node.values: + if isinstance(value, ast.Constant) and isinstance(value.value, str): + parts.append(value.value) + elif isinstance(value, ast.FormattedValue): + part = _static_string(value.value, names) + if part is None: + return None + parts.append(part) + else: + return None + return "".join(parts) + return None + + +def _static_string_values(tree: ast.Module) -> dict[str, str]: + assignments: list[tuple[str, ast.AST]] = [] + for node in ast.walk(tree): + if isinstance(node, ast.Assign): + for target in node.targets: + assignments.extend((name, node.value) for name in _target_names(target)) + elif isinstance(node, (ast.AnnAssign, ast.NamedExpr)) and node.value is not None: + assignments.extend((name, node.value) for name in _target_names(node.target)) + + values: dict[str, str] = {} + ambiguous: set[str] = set() + changed = True + while changed: + changed = False + for name, value_node in assignments: + value = _static_string(value_node, values) + if value is None or name in ambiguous: + continue + if name in values and values[name] != value: + values.pop(name) + ambiguous.add(name) + changed = True + elif name not in values: + values[name] = value + changed = True + return values + + +def _dynamic_import_names(tree: ast.Module) -> tuple[set[str], set[str]]: + modules = {"importlib"} + functions: set[str] = set() + for node in ast.walk(tree): + if isinstance(node, ast.Import): + for alias in node.names: + if alias.name == "importlib": + modules.add(alias.asname or "importlib") + elif isinstance(node, ast.ImportFrom) and node.module == "importlib": + for alias in node.names: + if alias.name == "import_module": + functions.add(alias.asname or alias.name) + assignments = [node for node in ast.walk(tree) if isinstance(node, ast.Assign)] + changed = True + while changed: + changed = False + for assignment in assignments: + names = set().union(*(_target_names(target) for target in assignment.targets)) + if isinstance(assignment.value, ast.Name): + if assignment.value.id in modules: + before = len(modules) + modules.update(names) + changed |= len(modules) != before + if assignment.value.id in functions: + before = len(functions) + functions.update(names) + changed |= len(functions) != before + elif ( + isinstance(assignment.value, ast.Attribute) + and assignment.value.attr == "import_module" + and isinstance(assignment.value.value, ast.Name) + and assignment.value.value.id in modules + ): + before = len(functions) + functions.update(names) + changed |= len(functions) != before + return modules, functions + + +def _is_dynamic_import_call( + node: ast.Call, importlib_modules: set[str], import_module_functions: set[str] +) -> bool: + return ( + isinstance(node.func, ast.Name) + and (node.func.id == "__import__" or node.func.id in import_module_functions) + ) or ( + (resolved := _resolved_attribute(node.func)) is not None + and resolved[1] == "import_module" + and isinstance(resolved[0], ast.Name) + and resolved[0].id in importlib_modules + ) + + +def _class_has_mutation_escape(class_node: ast.ClassDef) -> bool: + for statement in class_node.body: + if not isinstance(statement, (ast.FunctionDef, ast.AsyncFunctionDef)): + continue + positional = [*statement.args.posonlyargs, *statement.args.args] + if not positional: + continue + instance_names = {positional[0].arg} + for node in _function_nodes(statement): + if isinstance(node, (ast.Assign, ast.AnnAssign, ast.AugAssign, ast.Delete)): + targets = ( + node.targets + if isinstance(node, (ast.Assign, ast.Delete)) + else [node.target] + ) + if any(_is_instance_path(target, instance_names) for target in targets): + return True + if not isinstance(node, ast.Call): + continue + if ( + isinstance(node.func, ast.Name) + and node.func.id in {"delattr", "setattr"} + and node.args + and _is_instance_path(node.args[0], instance_names) + ): + return True + if ( + (resolved := _resolved_attribute(node.func)) is not None + and resolved[1] in UNBOUND_MUTATOR_METHODS + and node.args + and _is_instance_path(node.args[0], instance_names) + ): + return True + if ( + (resolved := _resolved_attribute(node.func)) is not None + and resolved[1] in MUTATOR_METHODS + and _is_instance_path(resolved[0], instance_names) + ): + return True + return False + + +def _target_names(node: ast.AST) -> set[str]: + if isinstance(node, ast.Name): + return {node.id} + if isinstance(node, (ast.Tuple, ast.List)): + result: set[str] = set() + for element in node.elts: + result.update(_target_names(element)) + return result + return set() + + +def _dict_is_deeply_immutable( + node: ast.Dict, safe_names: set[str], symbols: dict[str, str] +) -> bool: + return all( + key is not None + and _is_immutable_expr(key, safe_names, symbols) + and _is_immutable_expr(value, safe_names, symbols) + for key, value in zip(node.keys, node.values) + ) + + +def _sequence_is_deeply_immutable( + node: ast.List | ast.Set | ast.Tuple, + safe_names: set[str], + symbols: dict[str, str], +) -> bool: + return all(_is_immutable_expr(element, safe_names, symbols) for element in node.elts) + + +def _is_immutable_expr( + node: ast.AST | None, safe_names: set[str], symbols: dict[str, str] +) -> bool: + if node is None or isinstance(node, ast.Constant): + return True + if isinstance(node, ast.Name): + return node.id in safe_names + if isinstance(node, ast.Attribute): + root = _root_name(node) + if root is not None and root in symbols and symbols[root] != f"builtins.{root}": + return node.attr.isupper() + return _is_immutable_expr(node.value, safe_names, symbols) + if isinstance(node, ast.Subscript): + return _symbol(node.value, symbols) in TYPE_SUBSCRIPT_NAMES + if isinstance(node, ast.Tuple): + return _sequence_is_deeply_immutable(node, safe_names, symbols) + if isinstance(node, ast.UnaryOp): + return _is_immutable_expr(node.operand, safe_names, symbols) + if isinstance(node, ast.BinOp): + return _is_immutable_expr(node.left, safe_names, symbols) and _is_immutable_expr( + node.right, safe_names, symbols + ) + if isinstance(node, ast.BoolOp): + return all(_is_immutable_expr(value, safe_names, symbols) for value in node.values) + if isinstance(node, ast.Compare): + return _is_immutable_expr(node.left, safe_names, symbols) and all( + _is_immutable_expr(value, safe_names, symbols) for value in node.comparators + ) + if isinstance(node, ast.IfExp): + return _is_immutable_expr(node.body, safe_names, symbols) and _is_immutable_expr( + node.orelse, safe_names, symbols + ) + if isinstance(node, ast.JoinedStr): + return all( + not isinstance(value, ast.FormattedValue) + or _is_immutable_expr(value.value, safe_names, symbols) + for value in node.values + ) + if isinstance(node, ast.Lambda): + return True + if isinstance(node, ast.Call): + name = _symbol(node.func, symbols) + if name in TYPE_FACTORIES: + return True + local_name = _call_name(node.func) + if local_name is not None and f"frozen-dataclass:{local_name}" in safe_names: + return all( + _is_immutable_expr(arg, safe_names, symbols) for arg in node.args + ) and all( + _is_immutable_expr(keyword.value, safe_names, symbols) + for keyword in node.keywords + ) + if name in IMMUTABLE_SCALAR_CONSTRUCTORS: + return all(_is_immutable_expr(arg, safe_names, symbols) for arg in node.args) + if name in KNOWN_IMMUTABLE_CALLS or ( + name is not None + and name.split(".")[0] in {"math", "cmath"} + and len(name.split(".")) == 2 + and not name.split(".")[1].startswith("_") + ): + return True + if name in {"builtins.tuple", "builtins.frozenset"}: + if not node.args: + return True + if len(node.args) != 1 or node.keywords: + return False + value = node.args[0] + return isinstance(value, (ast.List, ast.Set, ast.Tuple)) and ( + _sequence_is_deeply_immutable(value, safe_names, symbols) + ) + if name == "types.MappingProxyType": + return ( + len(node.args) == 1 + and not node.keywords + and isinstance(node.args[0], ast.Dict) + and _dict_is_deeply_immutable(node.args[0], safe_names, symbols) + ) + return False + + +def _is_mutable_expr( + node: ast.AST | None, safe_names: set[str], symbols: dict[str, str] +) -> bool: + if node is None: + return False + if isinstance(node, MUTABLE_NODES): + return True + if isinstance(node, ast.Call) and _symbol(node.func, symbols) in MUTABLE_CONSTRUCTORS: + return True + return not _is_immutable_expr(node, safe_names, symbols) + + +def _scope_statements(body: Iterable[ast.stmt]) -> Iterable[ast.stmt]: + for statement in body: + yield statement + nested: list[list[ast.stmt]] = [] + if isinstance(statement, (ast.If, ast.For, ast.AsyncFor, ast.While)): + nested.extend([statement.body, statement.orelse]) + elif isinstance(statement, (ast.With, ast.AsyncWith)): + nested.append(statement.body) + elif isinstance(statement, (ast.Try, ast.TryStar)): + nested.extend([statement.body, statement.orelse, statement.finalbody]) + nested.extend(handler.body for handler in statement.handlers) + elif isinstance(statement, ast.Match): + nested.extend(case.body for case in statement.cases) + for statements in nested: + yield from _scope_statements(statements) + + +def _function_nodes(function: ast.FunctionDef | ast.AsyncFunctionDef) -> list[ast.AST]: + result: list[ast.AST] = [] + stack: list[ast.AST] = list(reversed(function.body)) + while stack: + node = stack.pop() + result.append(node) + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef, ast.Lambda)): + continue + stack.extend(reversed(list(ast.iter_child_nodes(node)))) + return result + + +def _shared_class_nodes( + body: Iterable[ast.stmt], prefix: str = "" +) -> Iterable[tuple[str, ast.ClassDef]]: + for statement in _scope_statements(body): + if not isinstance(statement, ast.ClassDef): + continue + qualified_name = f"{prefix}.{statement.name}" if prefix else statement.name + yield qualified_name, statement + yield from _shared_class_nodes(statement.body, qualified_name) + + +def _is_classvar_annotation(node: ast.AST, symbols: dict[str, str]) -> bool: + value = node.value if isinstance(node, ast.Subscript) else node + return _symbol(value, symbols) == "typing.ClassVar" + + +class _FileChecker: + def __init__( + self, + tree: ast.Module, + path: Path, + _target_classes: set[str], + root: Path | None = None, + ): + self.tree = tree + self.path = path + self.root = (root or path.parent).resolve() + self.violations: list[tuple[int, int, str]] = [] + self.safe_names: set[str] = { + "False", + "None", + "True", + "bool", + "bytes", + "complex", + "float", + "frozenset", + "int", + "range", + "str", + "tuple", + } + self.symbols = {name: f"builtins.{name}" for name in TRUSTED_BUILTINS} + self.import_names: set[str] = set() + self.definition_names: set[str] = set() + self.module_binding_names: set[str] = set() + self.module_mutable_names: set[str] = set() + self.classes: dict[str, ast.ClassDef] = {} + self.shared_classes = list(_shared_class_nodes(tree.body)) + self.functions: dict[str, ast.FunctionDef | ast.AsyncFunctionDef] = {} + self.lambdas: dict[str, ast.Lambda] = {} + self.shared_closure_factories: set[str] = set() + self.shared_returning_functions: set[str] = set() + self.returned_parameters: dict[str, set[int]] = {} + self.stateful_decorator_factories: set[str] = set() + self.shared_instance_attributes: dict[str, set[str]] = {} + + def add(self, node: ast.AST, message: str) -> None: + self.violations.append( + (getattr(node, "lineno", 1), getattr(node, "col_offset", 0) + 1, message) + ) + + def collect_symbols(self) -> None: + shadowed_names: set[str] = set() + for statement in _scope_statements(self.tree.body): + if isinstance(statement, ast.Import): + for alias in statement.names: + name = alias.asname or alias.name.split(".")[0] + self.import_names.add(name) + self.module_binding_names.add(name) + if _candidate_module(self.root, alias.name.split(".")) is None: + self.symbols[name] = alias.name if alias.asname else name + else: + self.symbols.pop(name, None) + elif isinstance(statement, ast.ImportFrom): + local_module = ( + statement.level > 0 + or statement.module is None + or _candidate_module(self.root, statement.module.split(".")) + is not None + ) + for alias in statement.names: + if alias.name == "*": + if statement.module is not None and not local_module: + for export in TRUSTED_STAR_EXPORTS.get( + statement.module, set() + ): + self.import_names.add(export) + self.module_binding_names.add(export) + self.symbols[export] = f"{statement.module}.{export}" + if statement.module == "typing": + self.safe_names.add(export) + else: + name = alias.asname or alias.name + self.import_names.add(name) + self.module_binding_names.add(name) + if statement.module is not None and not local_module: + self.symbols[name] = f"{statement.module}.{alias.name}" + else: + self.symbols.pop(name, None) + if ( + name.isupper() + or statement.module in {"collections.abc", "typing"} + or ( + statement.module == "math" + and alias.name in {"e", "inf", "nan", "pi", "tau"} + ) + ): + self.safe_names.add(name) + elif isinstance(statement, (ast.FunctionDef, ast.AsyncFunctionDef)): + self.definition_names.add(statement.name) + self.module_binding_names.add(statement.name) + shadowed_names.add(statement.name) + self.functions[statement.name] = statement + elif isinstance(statement, ast.ClassDef): + self.definition_names.add(statement.name) + self.module_binding_names.add(statement.name) + shadowed_names.add(statement.name) + self.classes[statement.name] = statement + elif isinstance(statement, ast.Assign): + for target in statement.targets: + names = _target_names(target) + self.module_binding_names.update(names) + shadowed_names.update(names) + if isinstance(statement.value, ast.Lambda): + for name in names: + self.lambdas[name] = statement.value + elif isinstance(statement, (ast.AnnAssign, ast.AugAssign)): + names = _target_names(statement.target) + self.module_binding_names.update(names) + shadowed_names.update(names) + for name in shadowed_names: + self.symbols.pop(name, None) + self.safe_names.difference_update(shadowed_names) + self.safe_names.update(self.definition_names) + for _, class_node in self.shared_classes: + frozen = any( + isinstance(decorator, ast.Call) + and _symbol(decorator.func, self.symbols) == "dataclasses.dataclass" + and any( + keyword.arg == "frozen" + and isinstance(keyword.value, ast.Constant) + and keyword.value.value is True + for keyword in decorator.keywords + ) + for decorator in class_node.decorator_list + ) + immutable_defaults = all( + not isinstance(statement, ast.AnnAssign) + or statement.value is None + or _is_immutable_expr(statement.value, self.safe_names, self.symbols) + for statement in class_node.body + ) + if frozen and immutable_defaults and not _class_has_mutation_escape(class_node): + self.safe_names.add(f"frozen-dataclass:{class_node.name}") + + def mark_shared_closure_factories(self) -> None: + def consider(value: ast.AST) -> None: + if not isinstance(value, ast.Call): + return + name = _call_name(value.func) + function = self.functions.get(name or "") + if function is not None and any( + isinstance(node, ast.Nonlocal) for node in ast.walk(function) + ): + self.shared_closure_factories.add(function.name) + + scopes = [self.tree.body] + scopes.extend(class_node.body for class_node in self.classes.values()) + for body in scopes: + for statement in _scope_statements(body): + if isinstance(statement, ast.Assign): + consider(statement.value) + elif isinstance(statement, ast.AnnAssign) and statement.value is not None: + consider(statement.value) + + for node in ast.walk(self.tree): + if not isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + continue + for decorator in node.decorator_list: + name = _call_name(decorator.func if isinstance(decorator, ast.Call) else decorator) + function = self.functions.get(name or "") + if function is not None and any( + isinstance(child, ast.Nonlocal) for child in ast.walk(function) + ): + self.shared_closure_factories.add(function.name) + + def collect_function_summaries(self) -> None: + """Summarize simple wrappers without executing student code.""" + + callables: dict[ + str, ast.FunctionDef | ast.AsyncFunctionDef | ast.Lambda + ] = {**self.functions, **self.lambdas} + changed = True + while changed: + changed = False + for function_name, function in callables.items(): + positional = [*function.args.posonlyargs, *function.args.args] + local_names = { + argument.arg + for argument in [*positional, *function.args.kwonlyargs] + } + if function.args.vararg is not None: + local_names.add(function.args.vararg.arg) + if function.args.kwarg is not None: + local_names.add(function.args.kwarg.arg) + + if isinstance(function, ast.Lambda): + nodes = [function.body] + return_values = [function.body] + else: + nodes = _function_nodes(function) + return_values = [ + node.value + for node in nodes + if isinstance(node, ast.Return) and node.value is not None + ] + assignments = [ + node + for node in nodes + if _assignment_parts(node) is not None + ] + for assignment in assignments: + _, targets = _assignment_parts(assignment) or (assignment, []) + for target in targets: + local_names.update(_target_names(target)) + + shared_aliases: set[str] = set() + parameter_aliases = { + argument.arg: {index} for index, argument in enumerate(positional) + } + + def returned_parameter_indexes(node: ast.AST) -> set[int]: + if isinstance(node, ast.Name): + return parameter_aliases.get(node.id, set()) + resolved = _resolved_attribute(node) + if resolved is not None: + return returned_parameter_indexes(resolved[0]) + if isinstance(node, ast.Subscript): + return returned_parameter_indexes(node.value) + if isinstance(node, ast.Call): + name = node.func.id if isinstance(node.func, ast.Name) else None + return set().union( + *( + returned_parameter_indexes(node.args[index]) + for index in self.returned_parameters.get( + name or "", set() + ) + if index < len(node.args) + ), + set(), + ) + return set() + + def returns_shared(node: ast.AST) -> bool: + if _is_immutable_expr(node, self.safe_names, self.symbols): + return False + if isinstance(node, ast.Name): + return ( + node.id in self.module_binding_names + and node.id not in local_names + ) or node.id in shared_aliases + resolved = _resolved_attribute(node) + if resolved is not None: + return returns_shared(resolved[0]) + if isinstance(node, ast.Subscript): + return returns_shared(node.value) + return _call_returns_shared( + node, + returns_shared, + self.shared_returning_functions, + self.returned_parameters, + ) or ( + isinstance(node, ast.Call) + and _symbol(node.func, self.symbols) + == "importlib.import_module" + ) + + aliases_changed = True + while aliases_changed: + aliases_changed = False + for assignment in assignments: + value, targets = _assignment_parts(assignment) or (assignment, []) + indexes = returned_parameter_indexes(value) + shared = returns_shared(value) + before = ( + len(shared_aliases), + sum(map(len, parameter_aliases.values())), + ) + for target in targets: + for name in _target_names(target): + if shared: + shared_aliases.add(name) + if indexes: + parameter_aliases.setdefault(name, set()).update(indexes) + aliases_changed |= before != ( + len(shared_aliases), + sum(map(len, parameter_aliases.values())), + ) + + indexes = set().union( + *(returned_parameter_indexes(value) for value in return_values), set() + ) + known_indexes = self.returned_parameters.setdefault(function_name, set()) + before = len(known_indexes) + known_indexes.update(indexes) + changed |= len(known_indexes) != before + if any(returns_shared(value) for value in return_values): + if function_name not in self.shared_returning_functions: + self.shared_returning_functions.add(function_name) + changed = True + for value in return_values: + if isinstance(value, ast.Call): + called = _symbol(value.func, self.symbols) + local_called = ( + value.func.id + if isinstance(value.func, ast.Name) + else None + ) + if ( + called in STATEFUL_DECORATORS + or local_called in self.stateful_decorator_factories + ) and function_name not in self.stateful_decorator_factories: + self.stateful_decorator_factories.add(function_name) + changed = True + + def decorator_is_stateful( + self, decorator: ast.AST, symbols: dict[str, str] + ) -> bool: + expression = decorator.func if isinstance(decorator, ast.Call) else decorator + return _symbol(expression, symbols) in STATEFUL_DECORATORS or ( + isinstance(expression, ast.Name) + and expression.id in self.stateful_decorator_factories + ) + + def check_module_bindings(self) -> None: + for statement in _scope_statements(self.tree.body): + value: ast.AST | None = None + targets: set[str] = set() + if isinstance(statement, ast.Assign): + value = statement.value + for target in statement.targets: + targets.update(_target_names(target)) + elif isinstance(statement, ast.AnnAssign): + value = statement.value + targets.update(_target_names(statement.target)) + elif isinstance(statement, ast.AugAssign): + value = statement.value + targets.update(_target_names(statement.target)) + else: + continue + + if value is None: + continue + if _is_immutable_expr(value, self.safe_names, self.symbols): + self.safe_names.update(targets) + continue + for name in sorted(targets - ALLOWED_MUTABLE_METADATA): + self.module_mutable_names.add(name) + self.add( + statement, + f"module-level value '{name}' is mutable or stateful and can be shared between processes", + ) + + def check_class_bindings(self) -> None: + for class_name, class_node in self.shared_classes: + class_safe_names = set(self.safe_names) + is_dataclass = any( + _symbol( + decorator.func if isinstance(decorator, ast.Call) else decorator, + self.symbols, + ) + == "dataclasses.dataclass" + for decorator in class_node.decorator_list + ) + is_pydantic_model = any( + ( + isinstance(base, ast.Name) + and _symbol(base, self.symbols) + in {"pydantic.BaseModel", "pydantic.main.BaseModel"} + ) + or ( + isinstance(base, ast.Attribute) + and _symbol(base, self.symbols) + in {"pydantic.BaseModel", "pydantic.main.BaseModel"} + ) + for base in class_node.bases + ) + is_enum = any( + ( + isinstance(base, ast.Name) + and _symbol(base, self.symbols) in ENUM_BASE_NAMES + ) + or ( + isinstance(base, ast.Attribute) + and _symbol(base, self.symbols) in ENUM_BASE_NAMES + ) + for base in class_node.bases + ) + class_symbols = dict(self.symbols) + class_import_symbols: dict[str, str] = {} + class_shadowed_names: set[str] = set() + for class_statement in _scope_statements(class_node.body): + names: set[str] = set() + if isinstance( + class_statement, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef) + ): + names.add(class_statement.name) + elif isinstance(class_statement, ast.Assign): + for target in class_statement.targets: + names.update(_target_names(target)) + elif isinstance(class_statement, (ast.AnnAssign, ast.AugAssign)): + names.update(_target_names(class_statement.target)) + elif isinstance(class_statement, ast.Import): + for alias in class_statement.names: + name = alias.asname or alias.name.split(".")[0] + if _candidate_module(self.root, alias.name.split(".")) is None: + class_import_symbols[name] = ( + alias.name if alias.asname else name + ) + elif isinstance(class_statement, ast.ImportFrom): + local_module = ( + class_statement.level > 0 + or class_statement.module is None + or _candidate_module( + self.root, class_statement.module.split(".") + ) + is not None + ) + if class_statement.module is not None and not local_module: + for alias in class_statement.names: + if alias.name == "*": + for export in TRUSTED_STAR_EXPORTS.get( + class_statement.module, set() + ): + class_import_symbols[export] = ( + f"{class_statement.module}.{export}" + ) + else: + name = alias.asname or alias.name + class_import_symbols[name] = ( + f"{class_statement.module}.{alias.name}" + ) + class_shadowed_names.update(names) + class_symbols.update( + (name, symbol) + for name, symbol in class_import_symbols.items() + if name not in class_shadowed_names + ) + for name in class_shadowed_names: + class_symbols.pop(name, None) + class_safe_names.discard(name) + for class_statement in class_node.body: + if not isinstance( + class_statement, (ast.FunctionDef, ast.AsyncFunctionDef) + ): + continue + for decorator in class_statement.decorator_list: + expression = decorator.func if isinstance(decorator, ast.Call) else decorator + name = _symbol(expression, class_symbols) + if self.decorator_is_stateful(decorator, class_symbols): + self.add( + decorator, + f"'{(name or _call_name(expression) or 'decorator').rsplit('.', 1)[-1]}' keeps a cache shared between process instances", + ) + for statement in _scope_statements(class_node.body): + def targets_class_namespace(target: ast.AST) -> bool: + if isinstance(target, ast.Subscript): + return ( + isinstance(target.value, ast.Call) + and isinstance(target.value.func, ast.Name) + and target.value.func.id in {"globals", "locals"} + ) + if isinstance(target, (ast.Tuple, ast.List)): + return any(targets_class_namespace(item) for item in target.elts) + return False + + if isinstance(statement, ast.Assign) and any( + targets_class_namespace(target) for target in statement.targets + ): + self.add(statement, "assignment mutates class-level shared state") + elif isinstance(statement, (ast.AnnAssign, ast.AugAssign)) and ( + targets_class_namespace(statement.target) + ): + self.add(statement, "assignment mutates class-level shared state") + elif isinstance(statement, ast.Delete) and any( + targets_class_namespace(target) for target in statement.targets + ): + self.add(statement, "deletion mutates class-level shared state") + elif isinstance(statement, ast.Expr) and isinstance( + statement.value, ast.Call + ): + call = statement.value + if ( + isinstance(call.func, ast.Attribute) + and call.func.attr in MUTATOR_METHODS | {"__setitem__"} + and isinstance(call.func.value, ast.Call) + and isinstance(call.func.value.func, ast.Name) + and call.func.value.func.id in {"globals", "locals"} + ): + self.add(call, "call mutates class-level shared state") + + value: ast.AST | None = None + targets: set[str] = set() + if isinstance(statement, ast.Assign): + value = statement.value + for target in statement.targets: + targets.update(_target_names(target)) + elif isinstance(statement, ast.AnnAssign): + value = statement.value + targets.update(_target_names(statement.target)) + else: + continue + if value is None: + continue + if _is_immutable_expr(value, class_safe_names, class_symbols): + class_safe_names.update(targets) + for name in targets: + class_symbols.pop(name, None) + continue + if ( + is_enum + and isinstance(value, ast.Call) + and _symbol(value.func, class_symbols) == "enum.auto" + and not value.args + and not value.keywords + ): + class_safe_names.update(targets) + continue + if ( + is_dataclass + and isinstance(statement, ast.AnnAssign) + and not _is_classvar_annotation(statement.annotation, class_symbols) + and isinstance(value, ast.Call) + and _symbol(value.func, class_symbols) == "dataclasses.field" + and not value.args + and all( + keyword.arg != "default" + or not _is_mutable_expr( + keyword.value, class_safe_names, class_symbols + ) + for keyword in value.keywords + ) + ): + continue + if ( + is_pydantic_model + and isinstance(statement, ast.AnnAssign) + and not _is_classvar_annotation(statement.annotation, class_symbols) + and isinstance(value, ast.Call) + and _symbol(value.func, class_symbols) == "pydantic.Field" + and not value.args + and all( + keyword.arg != "default" + or not _is_mutable_expr( + keyword.value, class_safe_names, class_symbols + ) + for keyword in value.keywords + ) + ): + continue + for name in sorted(targets): + self.add( + statement, + f"class attribute '{class_name}.{name}' is mutable or stateful and is shared by all process instances", + ) + + def check_module_mutations(self) -> None: + statements = list(_scope_statements(self.tree.body)) + class_aliases = set(self.classes) + shared_aliases: set[str] = set() + global_objects = self.module_binding_names + + def is_class_ref(node: ast.AST) -> bool: + return isinstance(node, ast.Name) and node.id in class_aliases + + def is_shared_ref(node: ast.AST) -> bool: + if isinstance(node, ast.Name): + return node.id in global_objects or node.id in shared_aliases + if isinstance(node, ast.Attribute): + return ( + is_class_ref(node.value) + or is_shared_ref(node.value) + or ( + isinstance(node.value, ast.Name) + and node.value.id in global_objects + ) + ) + if isinstance(node, ast.Subscript): + return is_shared_ref(node.value) + if ( + isinstance(node, ast.Call) + and isinstance(node.func, ast.Name) + and node.func.id == "getattr" + and node.args + ): + return is_class_ref(node.args[0]) or is_shared_ref(node.args[0]) + if ( + isinstance(node, ast.Call) + and isinstance(node.func, ast.Name) + and node.func.id == "eval" + and node.args + and isinstance(node.args[0], ast.Constant) + and isinstance(node.args[0].value, str) + ): + try: + expression = ast.parse(node.args[0].value, mode="eval").body + except SyntaxError: + return False + return is_shared_ref(expression) + return ( + isinstance(node, ast.Call) + and isinstance(node.func, ast.Name) + and node.func.id == "vars" + and len(node.args) == 1 + and (is_class_ref(node.args[0]) or is_shared_ref(node.args[0])) + ) or ( + isinstance(node, ast.Call) + and isinstance(node.func, ast.Name) + and node.func.id in {"globals", "locals"} + ) + + assignments = [ + statement + for statement in statements + if isinstance(statement, (ast.Assign, ast.AnnAssign, ast.NamedExpr)) + ] + changed = True + while changed: + changed = False + for assignment in assignments: + value, targets = _assignment_parts(assignment) or (assignment, []) + names = set().union(*map(_target_names, targets), set()) + before = (len(class_aliases), len(shared_aliases)) + if is_class_ref(value): + class_aliases.update(names) + if is_shared_ref(value): + shared_aliases.update(names) + if before != (len(class_aliases), len(shared_aliases)): + changed = True + + def target_is_shared(target: ast.AST) -> bool: + if isinstance(target, ast.Attribute): + return ( + is_class_ref(target.value) + or is_shared_ref(target.value) + or ( + isinstance(target.value, ast.Name) + and target.value.id in global_objects + ) + ) + if isinstance(target, ast.Subscript): + return is_shared_ref(target.value) + if isinstance(target, (ast.Tuple, ast.List)): + return any(target_is_shared(element) for element in target.elts) + return False + + for statement in statements: + if isinstance(statement, ast.Assign): + if any(target_is_shared(target) for target in statement.targets): + self.add(statement, "assignment creates or mutates module-level shared state") + elif isinstance(statement, (ast.AnnAssign, ast.AugAssign)): + if target_is_shared(statement.target): + self.add(statement, "assignment creates or mutates module-level shared state") + elif isinstance(statement, ast.Delete): + if any(target_is_shared(target) for target in statement.targets): + self.add(statement, "deletion mutates module-level shared state") + elif isinstance(statement, ast.Expr) and isinstance(statement.value, ast.Call): + call = statement.value + if isinstance(call.func, ast.Name) and call.func.id in { + "exec", + "globals", + "locals", + }: + self.add(call, f"'{call.func.id}' can access or modify module-level shared state") + elif ( + isinstance(call.func, ast.Name) + and call.func.id == "eval" + and is_shared_ref(call) + ): + self.add(call, "'eval' accesses module-level shared state") + elif ( + isinstance(call.func, ast.Name) + and call.func.id in {"setattr", "delattr"} + and call.args + and ( + is_class_ref(call.args[0]) + or is_shared_ref(call.args[0]) + or ( + isinstance(call.args[0], ast.Name) + and call.args[0].id in global_objects + ) + ) + ): + self.add(call, f"'{call.func.id}' mutates module-level shared state") + elif ( + _symbol(call.func, self.symbols) in OPERATOR_MUTATORS + and call.args + and is_shared_ref(call.args[0]) + ): + self.add(call, "operator call mutates module-level shared state") + elif ( + (resolved := _resolved_attribute(call.func)) is not None + and resolved[1] in UNBOUND_MUTATOR_METHODS + and call.args + and ( + is_class_ref(call.args[0]) + or is_shared_ref(call.args[0]) + or ( + isinstance(call.args[0], ast.Name) + and call.args[0].id in global_objects + ) + ) + ): + self.add(call, f"'{resolved[1]}' mutates module-level shared state") + elif ( + (resolved := _resolved_attribute(call.func)) is not None + and resolved[1] in MUTATOR_METHODS + and _symbol(call.func, self.symbols) != "operator.add" + and is_shared_ref(resolved[0]) + ): + if not ( + _symbol(resolved[0], self.symbols) == "sys.path" + and resolved[1] in {"append", "insert"} + ): + self.add( + call, + f"'{resolved[1]}' mutates module-level shared state", + ) + elif ( + (canonical := _symbol(call.func, self.symbols)) is not None + and (_root_name(call.func) or "") in self.import_names + and not _imported_call_allowed(canonical) + and canonical not in STATEFUL_DECORATORS + and canonical not in TYPE_FACTORIES + and canonical not in MUTABLE_CONSTRUCTORS + and canonical not in IMMUTABLE_SCALAR_CONSTRUCTORS + ): + self.add( + call, + f"call to imported API '{canonical}' is not known to be free of shared module-state mutation", + ) + + def check_defaults(self) -> None: + for node in ast.walk(self.tree): + if not isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.Lambda)): + continue + defaults = list(node.args.defaults) + defaults.extend(default for default in node.args.kw_defaults if default is not None) + for default in defaults: + if _is_mutable_expr(default, self.safe_names, self.symbols): + self.add(default, "mutable default argument is shared between function calls") + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + for decorator in node.decorator_list: + expression = decorator.func if isinstance(decorator, ast.Call) else decorator + name = _symbol(expression, self.symbols) + if self.decorator_is_stateful(decorator, self.symbols): + self.add( + decorator, + f"'{(name or _call_name(expression) or 'decorator').rsplit('.', 1)[-1]}' keeps a cache shared between process instances", + ) + + def check_dynamic_imports(self) -> None: + string_values = _static_string_values(self.tree) + importlib_modules, import_module_functions = _dynamic_import_names(self.tree) + for node in ast.walk(self.tree): + if ( + isinstance(node, ast.Call) + and node.args + and _is_dynamic_import_call( + node, importlib_modules, import_module_functions + ) + and _static_string(node.args[0], string_values) is None + ): + self.add( + node, + "dynamic import name cannot be resolved statically; shared state in the imported module cannot be checked", + ) + + def collect_shared_instance_attributes(self) -> None: + for class_name, class_node in self.classes.items(): + attributes = self.shared_instance_attributes.setdefault(class_name, set()) + functions = [ + statement + for statement in class_node.body + if isinstance(statement, (ast.FunctionDef, ast.AsyncFunctionDef)) + ] + changed = True + while changed: + changed = False + for function in functions: + positional = [*function.args.posonlyargs, *function.args.args] + if not positional: + continue + instance_names = {positional[0].arg} + nodes = _function_nodes(function) + assignments = [ + node + for node in nodes + if isinstance(node, (ast.Assign, ast.AnnAssign, ast.NamedExpr)) + ] + local_names = { + argument.arg + for argument in [*positional, *function.args.kwonlyargs] + } + local_import_names: set[str] = set() + for node in nodes: + if isinstance(node, ast.Assign): + for target in node.targets: + local_names.update(_target_names(target)) + elif isinstance(node, (ast.AnnAssign, ast.NamedExpr)): + local_names.update(_target_names(node.target)) + elif isinstance(node, (ast.Import, ast.ImportFrom)): + names = { + alias.asname or alias.name.split(".")[0] + for alias in node.names + if alias.name != "*" + } + local_names.update(names) + local_import_names.update(names) + + shared_aliases: set[str] = set() + + def is_instance(node: ast.AST) -> bool: + return isinstance(node, ast.Name) and node.id in instance_names + + def is_shared(node: ast.AST) -> bool: + if isinstance(node, ast.Name): + return ( + node.id in self.module_binding_names + and node.id not in local_names + ) or node.id in shared_aliases | local_import_names + if (resolved := _resolved_attribute(node)) is not None: + return ( + is_instance(resolved[0]) and resolved[1] in attributes + ) or is_shared(resolved[0]) + if isinstance(node, ast.Subscript): + return is_shared(node.value) + return _call_returns_shared( + node, + is_shared, + self.shared_returning_functions, + self.returned_parameters, + ) or ( + isinstance(node, ast.Call) + and _symbol(node.func, self.symbols) + == "importlib.import_module" + ) + + aliases_changed = True + while aliases_changed: + aliases_changed = False + for assignment in assignments: + value, targets = _assignment_parts(assignment) or (assignment, []) + if is_instance(value): + names = set().union(*(_target_names(t) for t in targets)) + before = len(instance_names) + instance_names.update(names) + aliases_changed |= len(instance_names) != before + if not is_shared(value): + continue + for target in targets: + names = _target_names(target) + before = len(shared_aliases) + shared_aliases.update(names) + aliases_changed |= len(shared_aliases) != before + if ( + isinstance(target, ast.Attribute) + and is_instance(target.value) + and target.attr not in attributes + ): + attributes.add(target.attr) + changed = True + + def check_function( + self, + function: ast.FunctionDef | ast.AsyncFunctionDef, + owner: str | None, + shared_closure: bool, + ) -> None: + nodes = _function_nodes(function) + all_class_names = set(self.classes) + instance_aliases: set[str] = set() + class_aliases: set[str] = set() + shared_aliases: set[str] = set() + shared_instance_attributes = self.shared_instance_attributes.get(owner or "", set()) + positional = [*function.args.posonlyargs, *function.args.args] + local_names = { + argument.arg + for argument in [*positional, *function.args.kwonlyargs] + } + if function.args.vararg is not None: + local_names.add(function.args.vararg.arg) + if function.args.kwarg is not None: + local_names.add(function.args.kwarg.arg) + local_import_names: set[str] = set() + local_import_symbols: dict[str, str] = {} + for node in nodes: + if isinstance(node, ast.Assign): + for target in node.targets: + local_names.update(_target_names(target)) + elif isinstance(node, (ast.AnnAssign, ast.AugAssign, ast.NamedExpr)): + local_names.update(_target_names(node.target)) + elif isinstance(node, ast.Import): + names = { + alias.asname or alias.name.split(".")[0] + for alias in node.names + } + local_names.update(names) + local_import_names.update(names) + for alias in node.names: + name = alias.asname or alias.name.split(".")[0] + if _candidate_module(self.root, alias.name.split(".")) is None: + local_import_symbols[name] = alias.name if alias.asname else name + elif isinstance(node, ast.ImportFrom): + names = { + alias.asname or alias.name + for alias in node.names + if alias.name != "*" + } + local_names.update(names) + local_import_names.update(names) + local_module = ( + node.level > 0 + or node.module is None + or _candidate_module(self.root, node.module.split(".")) is not None + ) + if node.module is not None and not local_module: + for alias in node.names: + if alias.name != "*": + local_import_symbols[alias.asname or alias.name] = ( + f"{node.module}.{alias.name}" + ) + function_symbols = dict(self.symbols) + for name in local_names - local_import_names: + function_symbols.pop(name, None) + function_symbols.update(local_import_symbols) + imported_callable_names = (self.import_names - local_names) | local_import_names + if owner is not None and positional: + first = positional[0].arg + if first == "cls" or any( + isinstance(decorator, ast.Name) and decorator.id == "classmethod" + for decorator in function.decorator_list + ): + class_aliases.add(first) + else: + instance_aliases.add(first) + + def is_instance_ref(node: ast.AST) -> bool: + return isinstance(node, ast.Name) and node.id in instance_aliases + + def is_class_ref(node: ast.AST) -> bool: + if isinstance(node, ast.Name): + return ( + node.id in all_class_names and node.id not in local_names + ) or node.id in class_aliases + if isinstance(node, ast.Attribute): + return node.attr == "__class__" and is_instance_ref(node.value) + return ( + isinstance(node, ast.Call) + and isinstance(node.func, ast.Name) + and node.func.id == "type" + and len(node.args) == 1 + and is_instance_ref(node.args[0]) + ) + + global_objects = self.module_binding_names + + def is_shared_ref(node: ast.AST) -> bool: + if isinstance(node, ast.Name): + return ( + node.id in global_objects and node.id not in local_names + ) or node.id in shared_aliases | local_import_names + if (resolved := _resolved_attribute(node)) is not None: + value, attribute = resolved + if is_instance_ref(value) and attribute in shared_instance_attributes: + return True + if attribute == "__dict__" and ( + is_class_ref(value) + or (isinstance(value, ast.Name) and value.id in global_objects) + ): + return True + return is_class_ref(value) or is_shared_ref(value) or ( + isinstance(value, ast.Name) and value.id in global_objects + ) + if isinstance(node, ast.Subscript): + return is_shared_ref(node.value) + if _call_returns_shared( + node, + is_shared_ref, + self.shared_returning_functions, + self.returned_parameters, + ): + return True + if ( + isinstance(node, ast.Call) + and isinstance(node.func, ast.Name) + and node.func.id == "getattr" + and node.args + ): + return ( + is_class_ref(node.args[0]) + or is_shared_ref(node.args[0]) + or ( + isinstance(node.args[0], ast.Name) + and node.args[0].id in global_objects + ) + ) + if ( + isinstance(node, ast.Call) + and isinstance(node.func, ast.Name) + and node.func.id == "eval" + and node.args + and isinstance(node.args[0], ast.Constant) + and isinstance(node.args[0].value, str) + ): + try: + expression = ast.parse(node.args[0].value, mode="eval").body + except SyntaxError: + return False + return is_shared_ref(expression) + return ( + isinstance(node, ast.Call) + and isinstance(node.func, ast.Name) + and node.func.id == "vars" + and len(node.args) == 1 + and ( + is_class_ref(node.args[0]) + or is_shared_ref(node.args[0]) + or ( + isinstance(node.args[0], ast.Name) + and node.args[0].id in global_objects + ) + ) + ) or ( + isinstance(node, ast.Call) + and isinstance(node.func, ast.Name) + and node.func.id == "globals" + ) + + assignments = [ + node + for node in nodes + if isinstance(node, (ast.Assign, ast.AnnAssign, ast.NamedExpr)) + ] + unbound_mutator_aliases: set[str] = set() + bound_mutator_aliases: set[str] = set() + imported_callable_aliases: dict[str, str] = {} + changed = True + while changed: + changed = False + for assignment in assignments: + value, targets = _assignment_parts(assignment) or (assignment, []) + names: set[str] = set() + for target in targets: + names.update(_target_names(target)) + before = ( + len(instance_aliases), + len(class_aliases), + len(shared_aliases), + len(unbound_mutator_aliases), + len(bound_mutator_aliases), + len(imported_callable_aliases), + ) + if is_instance_ref(value): + instance_aliases.update(names) + if is_class_ref(value): + class_aliases.update(names) + if is_shared_ref(value): + shared_aliases.update(names) + symbol = _symbol(value, function_symbols) + aliased_symbol = ( + imported_callable_aliases.get(value.id) + if isinstance(value, ast.Name) + else None + ) + if aliased_symbol is not None: + symbol = aliased_symbol + if ( + symbol is not None + and ( + aliased_symbol is not None + or (_root_name(value) or "") in imported_callable_names + ) + ): + for name in names: + imported_callable_aliases[name] = symbol + if symbol in OPERATOR_MUTATORS or ( + isinstance(value, ast.Name) + and value.id in unbound_mutator_aliases + ): + unbound_mutator_aliases.update(names) + if ( + (resolved := _resolved_attribute(value)) is not None + and resolved[1] in MUTATOR_METHODS + and _symbol(value, function_symbols) != "operator.add" + and is_shared_ref(resolved[0]) + ) or ( + isinstance(value, ast.Name) and value.id in bound_mutator_aliases + ): + bound_mutator_aliases.update(names) + if before != ( + len(instance_aliases), + len(class_aliases), + len(shared_aliases), + len(unbound_mutator_aliases), + len(bound_mutator_aliases), + len(imported_callable_aliases), + ): + changed = True + + def target_is_shared(target: ast.AST) -> bool: + if isinstance(target, ast.Attribute): + return ( + is_class_ref(target.value) + or is_shared_ref(target.value) + or ( + isinstance(target.value, ast.Name) + and target.value.id in global_objects + ) + ) + if isinstance(target, ast.Subscript): + return is_shared_ref(target.value) + if isinstance(target, (ast.Tuple, ast.List)): + return any(target_is_shared(element) for element in target.elts) + return False + + for node in nodes: + if isinstance(node, ast.Global): + self.add(node, "'global' state can be shared between process instances") + elif isinstance(node, ast.Nonlocal) and shared_closure: + self.add(node, "shared closure state can be accessed by multiple process instances") + elif isinstance(node, ast.Assign): + if any(target_is_shared(target) for target in node.targets): + self.add(node, "assignment mutates state outside the process instance") + elif isinstance(node, (ast.AnnAssign, ast.AugAssign)): + if target_is_shared(node.target): + self.add(node, "assignment mutates state outside the process instance") + elif isinstance(node, ast.Delete): + if any(target_is_shared(target) for target in node.targets): + self.add(node, "deletion mutates state outside the process instance") + elif isinstance(node, ast.Call): + if isinstance(node.func, ast.Name) and node.func.id in {"exec", "globals"}: + self.add(node, f"'{node.func.id}' can access or modify module-level shared state") + elif ( + isinstance(node.func, ast.Name) + and node.func.id == "eval" + and is_shared_ref(node) + ): + self.add(node, "'eval' accesses module-level shared state") + elif ( + isinstance(node.func, ast.Name) + and node.func.id in {"setattr", "delattr"} + and node.args + and ( + is_class_ref(node.args[0]) + or is_shared_ref(node.args[0]) + or ( + isinstance(node.args[0], ast.Name) + and node.args[0].id in global_objects + ) + ) + ): + self.add(node, f"'{node.func.id}' mutates shared object state") + elif ( + ( + _symbol(node.func, function_symbols) in OPERATOR_MUTATORS + or ( + isinstance(node.func, ast.Name) + and node.func.id in unbound_mutator_aliases + ) + ) + and node.args + and is_shared_ref(node.args[0]) + ): + self.add(node, "operator call mutates shared object state") + elif ( + isinstance(node.func, ast.Name) + and node.func.id in bound_mutator_aliases + ): + self.add(node, "aliased method mutates shared object state") + elif ( + ( + imported_callable_aliases.get( + node.func.id, _symbol(node.func, function_symbols) + ) + if isinstance(node.func, ast.Name) + else _symbol(node.func, function_symbols) + ) + in IMPORTED_ARGUMENT_MUTATORS + and node.args + and is_shared_ref(node.args[0]) + ): + self.add(node, "imported function mutates shared object state") + elif ( + (resolved := _resolved_attribute(node.func)) is not None + and resolved[1] in UNBOUND_MUTATOR_METHODS + and node.args + and ( + is_class_ref(node.args[0]) + or is_shared_ref(node.args[0]) + or ( + isinstance(node.args[0], ast.Name) + and node.args[0].id in global_objects + ) + ) + ): + self.add(node, f"'{resolved[1]}' mutates shared object state") + elif ( + (resolved := _resolved_attribute(node.func)) is not None + and resolved[1] in MUTATOR_METHODS + and _symbol(node.func, function_symbols) != "operator.add" + and is_shared_ref(resolved[0]) + ): + self.add(node, f"'{resolved[1]}' mutates state outside the process instance") + elif ( + ( + canonical := ( + imported_callable_aliases.get( + node.func.id, _symbol(node.func, function_symbols) + ) + if isinstance(node.func, ast.Name) + else _symbol(node.func, function_symbols) + ) + ) + is not None + and ( + (_root_name(node.func) or "") in imported_callable_names + or ( + isinstance(node.func, ast.Name) + and node.func.id in imported_callable_aliases + ) + ) + and not _imported_call_allowed(canonical) + and canonical not in STATEFUL_DECORATORS + and canonical not in TYPE_FACTORIES + and canonical not in MUTABLE_CONSTRUCTORS + and canonical not in IMMUTABLE_SCALAR_CONSTRUCTORS + ): + self.add( + node, + f"call to imported API '{canonical}' is not known to be free of shared module-state mutation", + ) + + def check_functions(self) -> None: + def visit_body( + body: Iterable[ast.stmt], + owner: str | None = None, + shared_closure: bool = False, + ) -> None: + for statement in body: + if isinstance(statement, (ast.FunctionDef, ast.AsyncFunctionDef)): + function_closure = shared_closure or ( + owner is None and statement.name in self.shared_closure_factories + ) + self.check_function(statement, owner, function_closure) + visit_body(statement.body, None, function_closure) + elif isinstance(statement, ast.ClassDef): + visit_body(statement.body, statement.name, shared_closure) + else: + for child in ast.iter_child_nodes(statement): + if isinstance(child, ast.stmt): + visit_body([child], owner, shared_closure) + + visit_body(self.tree.body) + + def run(self) -> list[tuple[int, int, str]]: + self.collect_symbols() + self.mark_shared_closure_factories() + self.check_module_bindings() + self.collect_function_summaries() + self.check_class_bindings() + self.check_module_mutations() + self.check_defaults() + self.check_dynamic_imports() + self.collect_shared_instance_attributes() + self.check_functions() + return sorted(set(self.violations)) + + +def _candidate_module(root: Path, parts: list[str]) -> Path | None: + if not parts: + return None + module = root.joinpath(*parts) + candidates = [module.with_suffix(".py"), module / "__init__.py"] + root = root.resolve() + for candidate in candidates: + try: + resolved = candidate.resolve() + resolved.relative_to(root) + except (OSError, ValueError): + continue + if resolved.is_file(): + return resolved + return None + + +def _local_imports(tree: ast.Module, current: Path, root: Path) -> set[Path]: + result: set[Path] = set() + importlib_modules, import_module_functions = _dynamic_import_names(tree) + string_values = _static_string_values(tree) + + for node in ast.walk(tree): + if isinstance(node, ast.Import): + for alias in node.names: + candidate = _candidate_module(root, alias.name.split(".")) + if candidate is not None: + result.add(candidate) + elif isinstance(node, ast.ImportFrom): + if node.level: + base = current.parent + for _ in range(node.level - 1): + base = base.parent + else: + base = root + module_parts = node.module.split(".") if node.module else [] + candidate = _candidate_module(base, module_parts) + if candidate is not None: + result.add(candidate) + for alias in node.names: + if alias.name == "*": + continue + candidate = _candidate_module(base, [*module_parts, alias.name]) + if candidate is not None: + result.add(candidate) + elif isinstance(node, ast.Call) and node.args: + module_name = _static_string(node.args[0], string_values) + if ( + _is_dynamic_import_call( + node, importlib_modules, import_module_functions + ) + and module_name is not None + and not module_name.startswith(".") + ): + candidate = _candidate_module(root, module_name.split(".")) + if candidate is not None: + result.add(candidate) + return result + + +def validate_source(source: str, filename: str, class_names: list[str]) -> list[str]: + tree = ast.parse(source, filename=filename) + path = Path(filename) + return [ + f"{filename}:{line}:{column}: {message}" + for line, column, message in _FileChecker( + tree, path, set(class_names), path.parent + ).run() + ] + + +def validate_solution(path: str, class_names: list[str]) -> list[str]: + entrypoint = Path(path).resolve() + root = entrypoint.parent + pending = [entrypoint] + visited: set[Path] = set() + violations: list[str] = [] + while pending: + current = pending.pop() + if current in visited: + continue + visited.add(current) + source = current.read_text(encoding="utf-8") + tree = ast.parse(source, filename=str(current)) + checker = _FileChecker( + tree, + current, + set(class_names) if current == entrypoint else set(), + root, + ) + violations.extend( + f"{current}:{line}:{column}: {message}" + for line, column, message in checker.run() + ) + pending.extend(sorted(_local_imports(tree, current, root) - visited)) + return sorted(set(violations)) diff --git a/homework/06-membership/tests/src/common.rs b/homework/06-membership/tests/src/common.rs new file mode 100644 index 0000000..8f420eb --- /dev/null +++ b/homework/06-membership/tests/src/common.rs @@ -0,0 +1,44 @@ +use serde::{Deserialize, Serialize}; +use sugars::boxed; + +use anysystem::python::PyProcessFactory; +use anysystem::System; + +#[derive(Serialize)] +pub struct JoinMessage<'a> { + pub seed: &'a str, +} + +#[derive(Serialize)] +pub struct LeaveMessage {} + +#[derive(Serialize)] +pub struct GetMembersMessage {} + +#[derive(Deserialize)] +pub struct MembersMessage { + pub members: Vec, +} + +#[derive(Clone, Copy)] +pub struct TestConfig<'a> { + pub process_factory: &'a PyProcessFactory, + pub process_count: u32, + pub seed: u64, +} + +pub fn build_system(config: &TestConfig) -> System { + crate::runner::progress(format!("creating simulation: N={}", config.process_count)); + let mut sys = System::new(config.seed); + sys.network().set_delays(0.01, 0.1); + for n in 0..config.process_count { + // process and node on which it runs have the same name + let name = n.to_string(); + sys.add_node(&name); + let clock_skew = sys.gen_range(0.0..10.0); + sys.set_node_clock_skew(&name, clock_skew); + let process = config.process_factory.build((&name,), config.seed); + sys.add_process(&name, boxed!(process), &name); + } + sys +} diff --git a/homework/06-membership/tests/src/library_policy.rs b/homework/06-membership/tests/src/library_policy.rs new file mode 100644 index 0000000..c9aa636 --- /dev/null +++ b/homework/06-membership/tests/src/library_policy.rs @@ -0,0 +1,36 @@ +use std::ffi::CString; + +use pyo3::prelude::*; +use pyo3::types::PyModule; + +const POLICY_CODE: &str = include_str!("../library_policy.py"); + +fn validate_solution(path: &str) -> Result, String> { + Python::attach(|py| { + let checker = + crate::shared_state::validator_module(py).map_err(|error| error.to_string())?; + let code = CString::new(POLICY_CODE).expect("library policy contains a null byte"); + let policy = PyModule::from_code(py, &code, c"library_policy.py", c"library_policy") + .map_err(|error| error.to_string())?; + policy + .getattr("validate_solution") + .and_then(|function| function.call1((path, checker))) + .and_then(|result| result.extract::>()) + .map_err(|error| error.to_string()) + }) +} + +pub fn validate_or_report(path: &str) -> bool { + match validate_solution(path) { + Ok(violations) if violations.is_empty() => true, + Ok(violations) => { + println!("ERROR: HW6 library policy violation"); + for violation in violations { + println!("{violation}"); + } + println!("\nSCORE: 0\n"); + false + } + Err(error) => panic!("failed to validate solution libraries: {error}"), + } +} diff --git a/homework/06-membership/tests/src/main.rs b/homework/06-membership/tests/src/main.rs new file mode 100644 index 0000000..f1b8c05 --- /dev/null +++ b/homework/06-membership/tests/src/main.rs @@ -0,0 +1,462 @@ +mod common; +mod library_policy; +mod python_memory; +mod resources; +mod runner; +mod shared_state; +mod tests; +mod tests_mc; + +use indexmap::IndexMap; +use std::collections::HashSet; + +use clap::Parser; +use runner::Outcome; + +/// Membership Homework Tests +#[derive(Parser, Debug)] +#[clap(about, long_about = None)] +struct Args { + /// Path to Python file with solution + #[clap(long = "impl", short = 'i', default_value = "solution/member.py")] + solution_path: String, + + /// Test to run (optional) + #[clap(long = "test", short)] + test: Option, + + /// Print execution trace + #[clap(long, short)] + debug: bool, + + /// Print test parameters and chaos monkey failures without the execution trace + #[clap(long, short = 'v')] + verbose: bool, + + /// Random seed used in tests + #[clap(long, short, default_value = "123")] + seed: u64, + + /// Number of processes + #[clap(long, short, default_value = "10")] + process_count: u32, + + /// Number of chaos monkey runs + #[clap(long, short, default_value = "100")] + monkeys: u32, + + /// Do not run model checking tests + #[clap(long)] + disable_mc_tests: bool, + + /// Wall-clock limit in seconds per ordinary test and per chaos run + #[clap(long, default_value = "15", value_parser = clap::value_parser!(u64).range(1..=300))] + test_timeout: u64, + + /// Shared wall-clock budget in seconds for all chaos runs + #[clap(long, default_value = "300", value_parser = clap::value_parser!(u64).range(1..=540))] + chaos_timeout: u64, + + /// Wall-clock limit in seconds per scalability test + #[clap(long, default_value = "60", value_parser = clap::value_parser!(u64).range(1..=300))] + scalability_timeout: u64, + + /// Wall-clock limit in seconds for both model-checking phases + #[clap(long, default_value = "300", value_parser = clap::value_parser!(u64).range(1..=300))] + mc_timeout: u64, + + /// Total wall-clock budget in seconds (the Docker watchdog is 600s) + #[clap(long, default_value = "540", value_parser = clap::value_parser!(u64).range(1..=540))] + suite_timeout: u64, + + /// Fixed address-space limit in MiB for ordinary, chaos and scalability workers + #[clap(long, default_value = "4096", value_parser = clap::value_parser!(u64).range(128..=65536))] + memory_limit: u64, + + /// Fixed address-space limit in MiB for the model-checking worker + #[clap(long, default_value = "14336", value_parser = clap::value_parser!(u64).range(128..=65536))] + mc_memory_limit: u64, + + #[clap(long, hide = true, requires = "test-result")] + test_worker: Option, + + #[clap(long, hide = true)] + test_result: Option, + + #[clap(long, hide = true)] + test_progress: Option, + + #[clap(long, hide = true, default_value = "1")] + chaos_start: u32, +} + +fn main() { + let args = Args::parse(); + if args.test_worker.is_some() { + runner::worker(&args); + return; + } + if !library_policy::validate_or_report(&args.solution_path) { + return; + } + if !shared_state::validate_or_report(&args.solution_path, &["GroupMember"]) { + return; + } + let results = match runner::run(&args) { + Ok(results) => results, + Err(error) => { + eprintln!("ERROR: test environment: {error}"); + std::process::exit(2); + } + }; + if args.test.is_none() { + let score = score(results, args.monkeys, args.disable_mc_tests); + println!("\nSCORE: {score}\n"); + } +} + +macro_rules! string_set { + ($($x:expr),+ $(,)?) => ( + HashSet::from([$($x.to_string()),+]) + ); +} + +fn score(results: IndexMap, monkeys: u32, disable_mc_tests: bool) -> f32 { + let mut basic_group = string_set! { + "SIMPLE", + "GET MEMBERS SEMANTICS", + "RANDOM SEED", + "PROCESS JOIN", + "PROCESS LEAVE", + "MODEL CHECKING", + }; + let process_crash_group = string_set! { + "PROCESS CRASH", + "SEED PROCESS CRASH", + "PROCESS CRASH RECOVER", + }; + let network_groups = [ + ( + "Network partitions tests", + string_set! { + "PROCESS OFFLINE", + "SEED PROCESS OFFLINE", + "PROCESS OFFLINE RECOVER", + "NETWORK PARTITION", + "NETWORK PARTITION RECOVER", + "NETWORK PARTITION UNEQUAL", + "NETWORK PARTITION UNEQUAL RECOVER", + }, + ), + ( + "Network connectivity tests", + string_set! { + "PROCESS CANNOT RECEIVE", + "PROCESS JOIN CANNOT RECEIVE", + "PROCESS CANNOT SEND", + "TWO PROCESSES CANNOT COMMUNICATE", + "TWO PROCESSES CANNOT COMMUNICATE IN TRIANGLE", + }, + ), + ( + "Network delays and losses tests", + string_set! { + "SLOW NETWORK", + "FLAKY NETWORK", + "FLAKY NETWORK ON START", + "FLAKY NETWORK AND CRASH", + }, + ), + ( + "Chaos monkey tests", + (1..=monkeys) + .map(|run| format!("CHAOS MONKEY (run {run})")) + .collect(), + ), + ]; + let scalability_group = string_set! {"SCALABILITY NORMAL", "SCALABILITY CRASH"}; + // Explicit local opt-out changes the selected basic tests. An MC skipped + // because the suite ran out of time remains required during grading. + if disable_mc_tests { + basic_group.remove("MODEL CHECKING"); + } + let mut all_groups = &(&basic_group | &process_crash_group) | &scalability_group; + let mut test_count = basic_group.len() + process_crash_group.len() + scalability_group.len(); + for (_, group) in &network_groups { + all_groups.extend(group.iter().cloned()); + test_count += group.len(); + } + assert_eq!(all_groups.len(), test_count, "overlapping scoring groups"); + assert_eq!(all_groups, results.keys().cloned().collect()); + + let scalability_skipped = scalability_group + .iter() + .all(|name| matches!(results.get(name), Some(Outcome::SkippedPrerequisites(_)))); + let mut passed = HashSet::new(); + let mut model_checking_ok = false; + for (test, result) in results { + if test == "MODEL CHECKING" && matches!(result, Outcome::Passed) { + model_checking_ok = true; + } + if result.satisfies_functional_requirement(&test) { + passed.insert(test); + } + } + + let basic_group_ok = basic_group.is_subset(&passed); + let mut score = 0.; + print!("Basic tests: "); + if basic_group_ok { + score += 3.; + println!("passed"); + } else { + let mut failed: Vec<&String> = basic_group.difference(&passed).collect(); + failed.sort(); + println!("not passed {failed:?}"); + } + print!("Process crashes tests: "); + if process_crash_group.is_subset(&passed) { + score += 3.; + println!("passed"); + } else { + let mut failed: Vec<&String> = process_crash_group.difference(&passed).collect(); + failed.sort(); + println!("not passed {failed:?}"); + } + for (name, group) in &network_groups { + print!("{name}: "); + if group.is_subset(&passed) { + score += 1.; + println!("passed"); + } else { + let mut failed: Vec<&String> = group.difference(&passed).collect(); + failed.sort(); + println!("not passed {failed:?}"); + } + } + // The basic-group eligibility already includes the resource-limit and + // explicit local opt-out exceptions for MC. + let functional_tests_ok = basic_group_ok + && process_crash_group.is_subset(&passed) + && network_groups + .iter() + .all(|(_, group)| group.is_subset(&passed)); + print!("Scalability tests: "); + if scalability_skipped { + println!("skipped (functional tests not passed)"); + } else if scalability_group.is_subset(&passed) { + if functional_tests_ok { + score += 4.; + println!("passed"); + } else { + println!("passed (no points: functional tests not passed)"); + } + } else { + let mut failed: Vec<&String> = scalability_group.difference(&passed).collect(); + failed.sort(); + println!("not passed {failed:?}"); + } + print!("Model checking test: "); + if model_checking_ok && basic_group_ok { + score += 1.; + println!("passed"); + } else if model_checking_ok { + println!("passed (no bonus: basic tests not passed)"); + } else { + println!("not passed"); + } + score +} + +#[cfg(test)] +mod score_tests { + use super::*; + + #[test] + fn functional_groups_keep_their_points_after_resource_failures() { + assert_eq!(score(runner::passed_results(2), 2, false), 15.); + for (name, expected) in [ + ("SCALABILITY NORMAL", 11.), + ("PROCESS CRASH", 8.), + ("CHAOS MONKEY (run 2)", 10.), + ] { + for failure in [ + Outcome::TimeLimit("test budget".into()), + Outcome::MemoryLimit("256 MiB".into()), + Outcome::WorkerError("SIGKILL".into()), + Outcome::Skipped("not started".into()), + ] { + let mut results = runner::passed_results(2); + results.insert(name.into(), failure); + assert_eq!(score(results, 2, false), expected, "{name}"); + } + } + } + + #[test] + fn each_network_test_only_blocks_its_subgroup_and_scalability() { + for name in [ + "PROCESS OFFLINE", + "SEED PROCESS OFFLINE", + "PROCESS OFFLINE RECOVER", + "NETWORK PARTITION", + "NETWORK PARTITION RECOVER", + "NETWORK PARTITION UNEQUAL", + "NETWORK PARTITION UNEQUAL RECOVER", + "PROCESS CANNOT RECEIVE", + "PROCESS JOIN CANNOT RECEIVE", + "PROCESS CANNOT SEND", + "TWO PROCESSES CANNOT COMMUNICATE", + "TWO PROCESSES CANNOT COMMUNICATE IN TRIANGLE", + "SLOW NETWORK", + "FLAKY NETWORK", + "FLAKY NETWORK ON START", + "FLAKY NETWORK AND CRASH", + "CHAOS MONKEY (run 1)", + "CHAOS MONKEY (run 2)", + ] { + for failure in [ + Outcome::Failed("wrong members".into()), + Outcome::TimeLimit("test budget".into()), + Outcome::MemoryLimit("4096 MiB".into()), + Outcome::WorkerError("SIGKILL".into()), + Outcome::Skipped("not started".into()), + ] { + let mut results = runner::passed_results(2); + results.insert(name.into(), failure.clone()); + assert_eq!(score(results, 2, false), 10., "{name}: {failure:?}"); + } + } + } + + #[test] + fn multiple_failures_in_one_network_subgroup_have_the_same_penalty() { + for names in [ + vec!["PROCESS OFFLINE", "NETWORK PARTITION RECOVER"], + vec!["PROCESS CANNOT RECEIVE", "TWO PROCESSES CANNOT COMMUNICATE"], + vec!["SLOW NETWORK", "FLAKY NETWORK AND CRASH"], + vec!["CHAOS MONKEY (run 1)", "CHAOS MONKEY (run 2)"], + ] { + let mut results = runner::passed_results(2); + for name in &names { + results.insert((*name).into(), Outcome::Failed("wrong members".into())); + } + assert_eq!(score(results, 2, false), 10., "{names:?}"); + } + } + + #[test] + fn unsuccessful_ordinary_basic_tests_block_mc_bonus_and_scalability() { + for name in [ + "SIMPLE", + "GET MEMBERS SEMANTICS", + "RANDOM SEED", + "PROCESS JOIN", + "PROCESS LEAVE", + ] { + for failure in [ + Outcome::Failed("wrong members".into()), + Outcome::TimeLimit("test budget".into()), + Outcome::MemoryLimit("4096 MiB".into()), + Outcome::WorkerError("SIGKILL".into()), + Outcome::Skipped("not started".into()), + ] { + let mut results = runner::passed_results(2); + results.insert(name.into(), failure.clone()); + assert_eq!(score(results, 2, false), 7., "{name}: {failure:?}"); + } + } + } + + #[test] + fn mc_outcomes_match_the_scoring_table_for_all_other_group_combinations() { + // outcome (None = explicit opt-out), basic eligibility, bonus if all basic tests pass. + let cases = [ + (Some(Outcome::Passed), true, 1.), + (None, true, 0.), + (Some(Outcome::Failed("counterexample".into())), false, 0.), + ( + Some(Outcome::WorkerError("Python exception".into())), + false, + 0., + ), + (Some(Outcome::WorkerError("SIGKILL".into())), false, 0.), + (Some(Outcome::TimeLimit("MC budget".into())), true, 0.), + ( + Some(Outcome::TimeLimit("suite budget after MC started".into())), + true, + 0., + ), + (Some(Outcome::MemoryLimit("14336 MiB".into())), true, 0.), + ( + Some(Outcome::Skipped("suite budget before MC started".into())), + false, + 0., + ), + ]; + let basic = [ + "SIMPLE", + "GET MEMBERS SEMANTICS", + "RANDOM SEED", + "PROCESS JOIN", + "PROCESS LEAVE", + ]; + let other = [ + ("PROCESS CRASH", 3.), + ("PROCESS OFFLINE", 1.), + ("PROCESS CANNOT RECEIVE", 1.), + ("SLOW NETWORK", 1.), + ("CHAOS MONKEY (run 2)", 1.), + ("SCALABILITY NORMAL", 4.), + ]; + for (mc, basic_eligible, bonus) in cases { + // All 32 combinations of the five ordinary basic tests, and all + // 64 combinations of crash, four network subgroups, and scalability. + for basic_failures in 0..32 { + for other_failures in 0..64 { + let mut results = runner::passed_results(2); + if let Some(outcome) = &mc { + results.insert("MODEL CHECKING".into(), outcome.clone()); + } else { + results.shift_remove("MODEL CHECKING"); + } + for (index, name) in basic.iter().enumerate() { + if basic_failures & (1 << index) != 0 { + results.insert((*name).into(), Outcome::Failed("wrong members".into())); + } + } + let mut expected = 0.; + if basic_eligible && basic_failures == 0 { + expected += 3. + bonus; + } + for (index, (name, points)) in other.iter().enumerate() { + if other_failures & (1 << index) != 0 { + results.insert((*name).into(), Outcome::Failed("wrong members".into())); + } else if *name != "SCALABILITY NORMAL" { + expected += points; + } + } + // Scalability needs both its own tests and every functional group. + if basic_eligible && basic_failures == 0 && other_failures == 0 { + expected += 4.; + } + let blockers = runner::scalability_blockers(&results, 2, mc.is_none()); + let eligible = + basic_eligible && basic_failures == 0 && other_failures & 31 == 0; + assert_eq!(blockers.is_empty(), eligible, + "MC={mc:?}, basic failures={basic_failures:05b}, other failures={other_failures:06b}"); + if !eligible { + for name in ["SCALABILITY NORMAL", "SCALABILITY CRASH"] { + results.insert( + name.into(), + Outcome::SkippedPrerequisites(blockers.clone()), + ); + } + } + assert_eq!(score(results, 2, mc.is_none()), expected, + "MC={mc:?}, basic failures={basic_failures:05b}, other failures={other_failures:06b}"); + } + } + } + } +} diff --git a/homework/06-membership/tests/src/python_memory.rs b/homework/06-membership/tests/src/python_memory.rs new file mode 100644 index 0000000..12d0a3d --- /dev/null +++ b/homework/06-membership/tests/src/python_memory.rs @@ -0,0 +1,83 @@ +//! Observe actual CPython allocation failures before AnySystem replaces a +//! Python exception with a generic panic. Student stdout is not a result channel. +use std::ffi::c_void; +use std::mem::MaybeUninit; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Once; + +use pyo3::ffi::{self, PyMemAllocatorDomain, PyMemAllocatorEx}; +use pyo3::Python; + +static INSTALLED: Once = Once::new(); +static FAILED: AtomicBool = AtomicBool::new(false); + +fn observe(pointer: *mut c_void, nonzero: bool) -> *mut c_void { + if nonzero && pointer.is_null() { + FAILED.store(true, Ordering::Relaxed); + } + pointer +} + +extern "C" fn malloc(ctx: *mut c_void, size: usize) -> *mut c_void { + // SAFETY: ctx points to an immutable original allocator, leaked for the + // interpreter's entire lifetime. Calls retain CPython's allocator contract. + let allocator = unsafe { &*ctx.cast::() }; + observe((allocator.malloc.unwrap())(allocator.ctx, size), size != 0) +} +extern "C" fn calloc(ctx: *mut c_void, count: usize, size: usize) -> *mut c_void { + // SAFETY: same lifetime and callback contract as malloc above. + let allocator = unsafe { &*ctx.cast::() }; + observe( + (allocator.calloc.unwrap())(allocator.ctx, count, size), + count != 0 && size != 0, + ) +} +extern "C" fn realloc(ctx: *mut c_void, pointer: *mut c_void, size: usize) -> *mut c_void { + // SAFETY: same lifetime and callback contract as malloc above. + let allocator = unsafe { &*ctx.cast::() }; + observe( + (allocator.realloc.unwrap())(allocator.ctx, pointer, size), + size != 0, + ) +} +extern "C" fn free(ctx: *mut c_void, pointer: *mut c_void) { + // SAFETY: same lifetime and callback contract as malloc above. + let allocator = unsafe { &*ctx.cast::() }; + (allocator.free.unwrap())(allocator.ctx, pointer); +} + +pub fn install() { + Python::attach(|_| { + INSTALLED.call_once(|| { + for domain in [ + PyMemAllocatorDomain::PYMEM_DOMAIN_RAW, + PyMemAllocatorDomain::PYMEM_DOMAIN_MEM, + PyMemAllocatorDomain::PYMEM_DOMAIN_OBJ, + ] { + // SAFETY: installation holds the GIL before any student code runs. + // CPython requires post-initialization hooks to wrap the previous + // allocator. Its four callbacks are supplied by the interpreter. + unsafe { + let mut original = MaybeUninit::uninit(); + ffi::PyMem_GetAllocator(domain, original.as_mut_ptr()); + let original = Box::into_raw(Box::new(original.assume_init())); + let mut wrapper = PyMemAllocatorEx { + ctx: original.cast(), + malloc: Some(malloc), + calloc: Some(calloc), + realloc: Some(realloc), + free: Some(free), + }; + ffi::PyMem_SetAllocator(domain, &mut wrapper); + } + } + }) + }); +} + +pub fn reset() { + FAILED.store(false, Ordering::Relaxed); +} +pub fn failed() -> bool { + FAILED.load(Ordering::Relaxed) +} diff --git a/homework/06-membership/tests/src/resources.rs b/homework/06-membership/tests/src/resources.rs new file mode 100644 index 0000000..4705e2d --- /dev/null +++ b/homework/06-membership/tests/src/resources.rs @@ -0,0 +1,165 @@ +//! Hard address-space limits are applied before exec, including Python imports. +//! Unlike an RSS polling loop, RLIMIT_AS also stops a single oversized allocation. +use std::fs::File; +use std::process::{Child, Command, ExitStatus}; + +#[cfg(target_os = "linux")] +const MIB: u64 = 1024 * 1024; + +#[cfg(target_os = "linux")] +mod linux { + use std::fs; + use std::os::fd::OwnedFd; + use std::os::unix::net::UnixStream; + use std::os::unix::process::CommandExt; + use std::process::Stdio; + + use super::*; + + fn number(path: &str) -> Option { + fs::read_to_string(path).ok()?.trim().parse().ok() + } + + fn memory_capacity() -> Option { + let meminfo = fs::read_to_string("/proc/meminfo").ok()?; + let host = meminfo.lines().find_map(|line| { + line.strip_prefix("MemTotal:")? + .split_whitespace() + .next()? + .parse::() + .ok() + })? * 1024; + // Both namespaced cgroup v2 and the conventional v1 Docker mount. + let container = [ + "/sys/fs/cgroup/memory.max", + "/sys/fs/cgroup/memory/memory.limit_in_bytes", + ] + .into_iter() + .filter_map(number) + .min(); + Some(container.map_or(host, |container| host.min(container))) + } + + pub fn limit(requested_mib: u64) -> Result { + // Check static capacity, never current free memory or cgroup usage. + // Insufficient capacity is an environment error, not a lower grading limit. + let capacity = memory_capacity().ok_or("cannot determine memory capacity")? / MIB; + if requested_mib + 256 > capacity { + return Err(format!("fixed worker limit is {requested_mib} MiB, but environment capacity is {capacity} MiB; provide at least {} MiB including the supervisor reserve, or explicitly select a smaller local limit", requested_mib + 256)); + } + let mut inherited = libc::rlimit { + rlim_cur: 0, + rlim_max: 0, + }; + // SAFETY: getrlimit writes into an initialized rlimit structure. + if unsafe { libc::getrlimit(libc::RLIMIT_AS, &mut inherited) } != 0 { + return Err(std::io::Error::last_os_error().to_string()); + } + if inherited.rlim_max / MIB < requested_mib { + return Err(format!("fixed worker limit is {requested_mib} MiB, but inherited address-space hard limit is {} MiB", inherited.rlim_max / MIB)); + } + Ok(requested_mib) + } + + pub fn capture(command: &mut Command) -> std::io::Result { + let (reader, writer) = UnixStream::pair()?; + command.stdout(Stdio::from(OwnedFd::from(writer.try_clone()?))); + command.stderr(Stdio::from(OwnedFd::from(writer))); + Ok(File::from(OwnedFd::from(reader))) + } + + pub fn configure(command: &mut Command, mib: u64) { + let supervisor = std::process::id() as libc::pid_t; + // SAFETY: only async-signal-safe libc calls run between fork and exec. + unsafe { + command.pre_exec(move || { + // Interrupting the supervisor must not leave a hanging worker. + if libc::prctl(libc::PR_SET_PDEATHSIG, libc::SIGKILL) != 0 { + return Err(std::io::Error::last_os_error()); + } + if libc::getppid() != supervisor { + return Err(std::io::Error::from_raw_os_error(libc::ECHILD)); + } + if libc::setsid() < 0 { + return Err(std::io::Error::last_os_error()); + } + let memory = libc::rlimit { + rlim_cur: mib * MIB, + rlim_max: mib * MIB, + }; + let core = libc::rlimit { + rlim_cur: 0, + rlim_max: 0, + }; + if libc::setrlimit(libc::RLIMIT_AS, &memory) != 0 + || libc::setrlimit(libc::RLIMIT_CORE, &core) != 0 + { + return Err(std::io::Error::last_os_error()); + } + Ok(()) + }); + } + } + + pub fn terminate(child: &mut Child) { + // The child starts a new session: clean up its descendants as well. + // SAFETY: the negative pid targets this worker's process group. + unsafe { + libc::kill(-(child.id() as i32), libc::SIGKILL); + } + let _ = child.kill(); + let _ = child.wait(); + } + + pub fn memory_usage(child: &Child) -> Option { + fs::read_to_string(format!("/proc/{}/status", child.id())) + .ok()? + .lines() + .find_map(|line| { + line.strip_prefix("VmSize:")? + .split_whitespace() + .next()? + .parse() + .ok() + }) + .map(|kib: u64| kib / 1024) + } +} + +#[cfg(target_os = "linux")] +pub use linux::{capture, configure, limit, memory_usage, terminate}; + +#[cfg(not(target_os = "linux"))] +pub fn limit(_: u64) -> Result { + Err("resource-limited workers require Linux; use the homework Docker image".into()) +} +#[cfg(not(target_os = "linux"))] +pub fn capture(_: &mut Command) -> std::io::Result { + Err(std::io::Error::other("use the homework Docker image")) +} +#[cfg(not(target_os = "linux"))] +pub fn configure(_: &mut Command, _: u64) {} +#[cfg(not(target_os = "linux"))] +pub fn terminate(child: &mut Child) { + let _ = child.kill(); + let _ = child.wait(); +} +#[cfg(not(target_os = "linux"))] +pub fn memory_usage(_: &Child) -> Option { + None +} + +pub fn allocation_crash(status: Option, output: &str) -> bool { + #[cfg(target_os = "linux")] + { + use std::os::unix::process::ExitStatusExt; + status.is_some_and(|status| status.signal() == Some(libc::SIGABRT)) + && output.contains("memory allocation of ") + && output.contains(" bytes failed") + } + #[cfg(not(target_os = "linux"))] + { + let _ = (status, output); + false + } +} diff --git a/homework/06-membership/tests/src/runner.rs b/homework/06-membership/tests/src/runner.rs new file mode 100644 index 0000000..990e8a1 --- /dev/null +++ b/homework/06-membership/tests/src/runner.rs @@ -0,0 +1,878 @@ +//! Based on HW4's killable-process runner. Only workers execute solution code. +//! Chaos runs share a worker, with an append-only result journal after every run. +use std::cell::OnceCell; +use std::fs::{self, File, OpenOptions}; +use std::io::{BufRead, BufReader, Read, Write}; +use std::path::PathBuf; +use std::process::Command; +use std::sync::OnceLock; +use std::thread; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; + +use anysystem::python::PyProcessFactory; +use anysystem::test::TestResult; +use env_logger::Builder; +use indexmap::IndexMap; +use log::{info, LevelFilter}; +use pyo3::prelude::*; +use rand::{RngCore, SeedableRng}; +use rand_pcg::Pcg64; +use serde::{Deserialize, Serialize}; + +use crate::common::TestConfig; +use crate::tests::*; +use crate::tests_mc::test_mc_group; +use crate::Args; +use crate::{python_memory, resources}; + +type TestFn = fn(&TestConfig) -> TestResult; +const ORDINARY: &[(&str, TestFn)] = &[ + ("SIMPLE", test_simple), + ("GET MEMBERS SEMANTICS", test_get_members_semantics), + ("RANDOM SEED", test_random_seed), + ("PROCESS JOIN", test_process_join), + ("PROCESS LEAVE", test_process_leave), + ("PROCESS CRASH", test_process_crash), + ("SEED PROCESS CRASH", test_seed_process_crash), + ("PROCESS CRASH RECOVER", test_process_crash_recover), + ("PROCESS OFFLINE", test_process_offline), + ("SEED PROCESS OFFLINE", test_seed_process_offline), + ("PROCESS OFFLINE RECOVER", test_process_offline_recover), + ("PROCESS CANNOT RECEIVE", test_process_cannot_receive), + ( + "PROCESS JOIN CANNOT RECEIVE", + test_process_join_cannot_receive, + ), + ("PROCESS CANNOT SEND", test_process_cannot_send), + ("NETWORK PARTITION", test_network_partition), + ("NETWORK PARTITION RECOVER", test_network_partition_recover), + ("NETWORK PARTITION UNEQUAL", test_network_partition_unequal), + ( + "NETWORK PARTITION UNEQUAL RECOVER", + test_network_partition_unequal_recover, + ), + ( + "TWO PROCESSES CANNOT COMMUNICATE", + test_two_processes_cannot_communicate, + ), + ( + "TWO PROCESSES CANNOT COMMUNICATE IN TRIANGLE", + test_two_processes_cannot_communicate_in_triangle, + ), + ("SLOW NETWORK", test_slow_network), + ("FLAKY NETWORK", test_flaky_network), + ("FLAKY NETWORK ON START", test_flaky_network_on_start), + ("FLAKY NETWORK AND CRASH", test_flaky_network_and_crash), +]; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum Outcome { + Passed, + Failed(String), + TimeLimit(String), + MemoryLimit(String), + WorkerError(String), + Skipped(String), + SkippedPrerequisites(Vec), +} + +impl Outcome { + pub fn resource_limited(&self) -> bool { + matches!(self, Self::TimeLimit(_) | Self::MemoryLimit(_)) + } + + // A resource failure before the Started record is not an attempted test. + // In particular it must not grant the MC exemption for the basic group. + fn require_started(self, started: bool) -> Self { + if !started && self.resource_limited() { + Self::Skipped(format!("test did not start; {}", self.description())) + } else { + self + } + } + + pub fn satisfies_functional_requirement(&self, name: &str) -> bool { + matches!(self, Self::Passed) || (name == "MODEL CHECKING" && self.resource_limited()) + } + + fn description(&self) -> String { + match self { + Self::Passed => "passed".into(), + Self::Failed(message) => message.clone(), + Self::TimeLimit(message) => format!("time limit exceeded: {message}"), + Self::MemoryLimit(message) => format!("memory limit exceeded: {message}"), + Self::WorkerError(message) => format!("test worker error: {message}"), + Self::Skipped(message) => format!("not checked: {message}"), + Self::SkippedPrerequisites(tests) => { + let mut names = tests.iter().take(8).cloned().collect::>().join(", "); + if tests.len() > 8 { + names.push_str(&format!(" (and {} more)", tests.len() - 8)); + } + format!("required functional tests not passed: {names}") + } + } + } +} + +#[derive(Serialize, Deserialize)] +enum Event { + Started(String), + Finished(String, Outcome), +} + +struct Test { + name: String, + run: TestFn, + seed: u64, + chaos_run: Option, +} + +fn tests(args: &Args) -> Vec { + let mut tests: Vec<_> = ORDINARY + .iter() + .map(|&(name, run)| Test { + name: name.into(), + run, + seed: args.seed, + chaos_run: None, + }) + .collect(); + let mut rand = Pcg64::seed_from_u64(args.seed); + for run in 1..=args.monkeys { + tests.push(Test { + name: format!("CHAOS MONKEY (run {run})"), + run: test_chaos_monkey, + seed: rand.next_u64(), + chaos_run: Some(run), + }); + } + for (name, run) in [ + ("MODEL CHECKING", test_mc_group as TestFn), + ("SCALABILITY NORMAL", test_scalability_normal as TestFn), + ("SCALABILITY CRASH", test_scalability_crash as TestFn), + ] { + if name != "MODEL CHECKING" || !args.disable_mc_tests { + tests.push(Test { + name: name.into(), + run, + seed: args.seed, + chaos_run: None, + }); + } + } + tests +} + +/// Missing, failed and unstarted functional tests block scalability. Only an +/// explicitly disabled MC or a resource limit after it started is exempt. +pub fn scalability_blockers( + results: &IndexMap, + monkeys: u32, + disable_mc_tests: bool, +) -> Vec { + ORDINARY + .iter() + .map(|(name, _)| (*name).to_string()) + .chain((1..=monkeys).map(|run| format!("CHAOS MONKEY (run {run})"))) + .chain((!disable_mc_tests).then(|| "MODEL CHECKING".to_string())) + .filter(|name| { + !results + .get(name) + .is_some_and(|outcome| outcome.satisfies_functional_requirement(name)) + }) + .collect() +} + +static PROGRESS: OnceLock = OnceLock::new(); + +/// Bounded, replaceable diagnostics; never mixed with the structured result stream. +pub fn progress(message: impl AsRef) { + if let Some(path) = PROGRESS.get() { + let temporary = path.with_extension("next"); + if fs::write(&temporary, message.as_ref()).is_ok() { + let _ = fs::rename(&temporary, path); + } + } +} + +fn event(file: &mut File, event: Event) { + serde_json::to_writer(&mut *file, &event).unwrap(); + writeln!(file).unwrap(); + file.flush().unwrap(); +} + +fn report(name: &str, outcome: &Outcome) { + match outcome { + Outcome::Passed => println!("\nPASSED\n"), + Outcome::Skipped(_) | Outcome::SkippedPrerequisites(_) => { + println!("\nSKIPPED: {}\n", outcome.description()) + } + _ => println!("\nFAILED: {}\n", outcome.description()), + } + if name == "MODEL CHECKING" && outcome.resource_limited() { + println!("INCOMPLETE: model checking reached its resource limit; no MC bonus, basic tests remain eligible\n"); + } + std::io::stdout().flush().unwrap(); +} + +pub fn worker(args: &Args) { + if let Some(path) = &args.test_progress { + let _ = PROGRESS.set(PathBuf::from(path)); + } + if args.debug || args.verbose { + let mut logger = Builder::new(); + logger.filter(Some("distsys_membership"), LevelFilter::Info); + if args.debug { + logger.filter(Some("anysystem"), LevelFilter::Debug); + } + logger + .format(|buf, record| writeln!(buf, "{}", record.args())) + .init(); + } + info!( + "Test parameters: seed={}, processes={}", + args.seed, args.process_count + ); + let mut journal = OpenOptions::new() + .append(true) + .open(args.test_result.as_ref().unwrap()) + .unwrap(); + let factory = OnceCell::new(); + for test in tests(args).into_iter().filter(|test| { + if args.test_worker.as_deref() == Some("CHAOS MONKEY") { + test.chaos_run.is_some_and(|run| run >= args.chaos_start) + } else { + args.test_worker.as_deref() == Some(test.name.as_str()) + } + }) { + progress("loading Python implementation"); + event(&mut journal, Event::Started(test.name.clone())); + println!("\n--- {} ---\n", test.name); + std::io::stdout().flush().unwrap(); + python_memory::reset(); + let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + python_memory::install(); + let factory = factory.get_or_init(|| { + // Match the preflight's local-import root when --impl is outside cwd. + let solution = fs::canonicalize(&args.solution_path).unwrap(); + let directory = solution.parent().unwrap().to_string_lossy(); + Python::attach(|py| { + py.import("sys") + .and_then(|sys| sys.getattr("path")) + .and_then(|path| path.call_method1("insert", (0, directory.as_ref()))) + .unwrap(); + }); + PyProcessFactory::new(&args.solution_path, "GroupMember") + }); + progress("initializing simulation"); + let config = TestConfig { + process_factory: factory, + process_count: if test.name == "MODEL CHECKING" { + 3 + } else { + args.process_count + }, + seed: test.seed, + }; + (test.run)(&config) + })); + let mut outcome = match result { + Ok(Ok(_)) => Outcome::Passed, + Ok(Err(message)) + if message == "time limit of 300s exceeded" && test.name == "MODEL CHECKING" => + { + Outcome::TimeLimit(message) + } + Ok(Err(message)) => Outcome::Failed(message), + Err(payload) => { + let message = payload + .downcast_ref::() + .map(String::as_str) + .or_else(|| payload.downcast_ref::<&str>().copied()) + .unwrap_or("test panicked") + .to_owned(); + if python_memory::failed() { + Outcome::MemoryLimit(message) + } else { + Outcome::WorkerError(message) + } + } + }; + if let Outcome::MemoryLimit(message) = &outcome { + let phase = PROGRESS + .get() + .and_then(|path| fs::read_to_string(path).ok()) + .unwrap_or_default(); + outcome = Outcome::MemoryLimit(format!( + "worker address-space limit {} MiB; {phase}; {message}", + memory_limit(args, &test) + )); + } + // Bound IPC diagnostics independently of the worker's memory budget. + match &mut outcome { + Outcome::Failed(message) + | Outcome::TimeLimit(message) + | Outcome::MemoryLimit(message) + | Outcome::WorkerError(message) + | Outcome::Skipped(message) + if message.len() > 16384 => + { + let mut end = 16384; + while !message.is_char_boundary(end) { + end -= 1; + } + message.truncate(end); + message.push_str(" [diagnostic truncated]"); + } + _ => {} + } + event( + &mut journal, + Event::Finished(test.name.clone(), outcome.clone()), + ); + // Report panics after draining their diagnostics from the output pipe. + if !matches!(outcome, Outcome::WorkerError(_)) { + report(&test.name, &outcome); + } + // A panic can leave the embedded interpreter or simulator in an unknown state. + if matches!(outcome, Outcome::WorkerError(_) | Outcome::MemoryLimit(_)) { + break; + } + } +} + +struct Files(PathBuf); +impl Files { + fn new() -> std::io::Result { + let nonce = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let path = + std::env::temp_dir().join(format!("membership-test-{}-{nonce}", std::process::id())); + fs::create_dir(&path)?; + let files = Self(path); + File::create(files.journal())?; + Ok(files) + } + fn journal(&self) -> PathBuf { + self.0.join("results.jsonl") + } + fn progress(&self) -> PathBuf { + self.0.join("progress.txt") + } +} +impl Drop for Files { + fn drop(&mut self) { + for name in ["results.jsonl", "progress.txt", "progress.next"] { + let _ = fs::remove_file(self.0.join(name)); + } + let _ = fs::remove_dir(&self.0); + } +} + +// Retain a bounded diagnostic tail for native allocator failures. Python +// allocation failures are tracked separately; output alone is not an outcome. +fn output_reader(mut pipe: impl Read + Send + 'static) -> thread::JoinHandle { + thread::spawn(move || { + let mut tail = Vec::new(); + let mut chunk = [0; 4096]; + while let Ok(count) = pipe.read(&mut chunk) { + if count == 0 { + break; + } + let _ = std::io::stdout().lock().write_all(&chunk[..count]); + tail.extend_from_slice(&chunk[..count]); + if tail.len() > 65536 { + tail.drain(..tail.len() - 65536); + } + } + String::from_utf8_lossy(&tail).into_owned() + }) +} + +struct Journal { + reader: BufReader, + pending: String, + next: usize, + started: Instant, + announced: bool, +} +impl Journal { + fn read( + &mut self, + batch: &[Test], + results: &mut IndexMap, + ) -> Result<(), String> { + loop { + let remaining = 65537_usize.saturating_sub(self.pending.len()); + let count = (&mut self.reader) + .take(remaining as u64) + .read_line(&mut self.pending) + .map_err(|error| error.to_string())?; + if self.pending.len() > 65536 { + return Err("worker result exceeds 64 KiB".into()); + } + if !self.pending.ends_with('\n') { + return Ok(()); + } + let event: Event = serde_json::from_str(&self.pending) + .map_err(|error| format!("invalid worker result: {error}"))?; + self.pending.clear(); + let name = match &event { + Event::Started(name) | Event::Finished(name, _) => name, + }; + if batch.get(self.next).map(|test| &test.name) != Some(name) { + return Err(format!("unexpected worker result for {name}")); + } + match event { + Event::Started(_) => { + self.started = Instant::now(); + self.announced = true; + } + Event::Finished(name, outcome) => { + results.insert(name, outcome); + self.next += 1; + self.started = Instant::now(); + self.announced = false; + } + } + if count == 0 { + return Ok(()); + } + } + } +} + +fn memory_limit(args: &Args, test: &Test) -> u64 { + if test.name == "MODEL CHECKING" { + args.mc_memory_limit + } else { + args.memory_limit + } +} + +fn seconds(args: &Args, test: &Test) -> u64 { + if test.name == "MODEL CHECKING" { + args.mc_timeout + } else if test.name.starts_with("SCALABILITY") { + args.scalability_timeout + } else { + args.test_timeout + } +} + +fn run_worker( + args: &Args, + batch: &[Test], + results: &mut IndexMap, + suite: Instant, + phase: Instant, +) -> usize { + let setup = || -> Result<_, String> { + let files = Files::new().map_err(|error| error.to_string())?; + let limit = resources::limit(memory_limit(args, &batch[0]))?; + let mut command = Command::new(std::env::current_exe().map_err(|error| error.to_string())?); + command + .arg("--test-worker") + .arg(if batch.len() > 1 { + "CHAOS MONKEY" + } else { + &batch[0].name + }) + .args([ + "-i", + &args.solution_path, + "-s", + &args.seed.to_string(), + "-p", + &args.process_count.to_string(), + "-m", + &args.monkeys.to_string(), + "--chaos-start", + &batch[0].chaos_run.unwrap_or(1).to_string(), + "--memory-limit", + &args.memory_limit.to_string(), + "--mc-memory-limit", + &args.mc_memory_limit.to_string(), + "--test-result", + ]) + .arg(files.journal()) + .arg("--test-progress") + .arg(files.progress()) + .env("PYTHONHASHSEED", (args.seed as u32).to_string()) + .env("PYTHONUNBUFFERED", "1"); + if args.debug { + command.arg("-d"); + } + if args.verbose { + command.arg("-v"); + } + if args.disable_mc_tests { + command.arg("--disable-mc-tests"); + } + resources::configure(&mut command, limit); + let reader = + BufReader::new(File::open(files.journal()).map_err(|error| error.to_string())?); + let output = resources::capture(&mut command).map_err(|error| error.to_string())?; + let child = command.spawn().map_err(|error| error.to_string())?; + Ok((files, limit, reader, output, child)) + }; + let (files, limit, reader, output, mut child) = match setup() { + Ok(setup) => setup, + Err(error) => { + let outcome = Outcome::WorkerError(error); + println!("\n--- {} ---\n", batch[0].name); + report(&batch[0].name, &outcome); + results.insert(batch[0].name.clone(), outcome); + return 1; + } + }; + let stderr = output_reader(output); + let mut journal = Journal { + reader, + pending: String::new(), + next: 0, + started: Instant::now(), + announced: false, + }; + let mut last_progress = Instant::now(); + let mut detail = "starting worker".to_owned(); + let mut stopped = None; + let status = loop { + if let Err(error) = journal.read(batch, results) { + stopped = Some(Outcome::WorkerError(error)); + break None; + } + match child.try_wait() { + Ok(Some(status)) => break Some(status), + Err(error) => { + stopped = Some(Outcome::WorkerError(error.to_string())); + break None; + } + Ok(None) => {} + } + if let Ok(message) = fs::read_to_string(files.progress()) { + detail = message; + } + let now = Instant::now(); + let current = &batch[journal.next.min(batch.len() - 1)]; + let limit_seconds = seconds(args, current); + if now >= suite + || now >= phase + || now.duration_since(journal.started).as_secs() >= limit_seconds + { + let budget = if now >= suite { + format!("suite budget of {}s", args.suite_timeout) + } else if now >= phase { + format!("chaos budget of {}s", args.chaos_timeout) + } else { + format!("test budget of {limit_seconds}s") + }; + stopped = Some(Outcome::TimeLimit(format!("{budget}; {detail}"))); + break None; + } + if last_progress.elapsed() >= Duration::from_secs(5) { + let memory = resources::memory_usage(&child).map_or(String::new(), |mib| { + format!("; address space {mib}/{limit} MiB") + }); + eprintln!( + "RUNNING: {}; {}; elapsed {:.0}s{}", + current.name, + detail, + journal.started.elapsed().as_secs_f64(), + memory + ); + last_progress = now; + } + thread::sleep(Duration::from_millis(10)); + }; + resources::terminate(&mut child); + let stderr = stderr.join().unwrap_or_default(); + // The worker may have committed another result between the last poll and exit. + if let Err(error) = journal.read(batch, results) { + stopped = Some(Outcome::WorkerError(error)); + } + if journal.next > 0 { + let finished = &batch[journal.next - 1]; + if let Some(Outcome::WorkerError(message)) = results.get(&finished.name) { + let phase = fs::read_to_string(files.progress()).unwrap_or_default(); + let outcome = Outcome::WorkerError(format!("{message}; {phase}")); + report(&finished.name, &outcome); + results.insert(finished.name.clone(), outcome); + } + } + if journal.next == batch.len() { + return journal.next; + } + // A handled panic/allocation failure ends the batch cleanly. Restart the + // remaining runs without inventing a failure for a test that never started. + if stopped.is_none() && journal.next > 0 && !journal.announced { + return journal.next; + } + if let Ok(message) = fs::read_to_string(files.progress()) { + detail = message; + } + let test = &batch[journal.next]; + let outcome = stopped.unwrap_or_else(|| { + if resources::allocation_crash(status, &stderr) { + Outcome::MemoryLimit(format!("worker address-space limit {limit} MiB; {detail}")) + } else { + Outcome::WorkerError(format!( + "{}; {detail}", + status.map_or("worker stopped".into(), |status| format!( + "exited with {status} without a result" + )) + )) + } + }); + // Read the final journal first: Started may have arrived after the timeout + // poll. Only an actually started test is eligible for a resource exemption. + let outcome = outcome.require_started(journal.announced); + if !journal.announced { + println!("\n--- {} ---\n", test.name); + } + report(&test.name, &outcome); + println!( + "REPRODUCE: -t {:?} -s {} -p {} -m {} (effective run seed: {})\n", + test.name, args.seed, args.process_count, args.monkeys, test.seed + ); + results.insert(test.name.clone(), outcome); + journal.next + 1 +} + +pub fn run(args: &Args) -> Result, String> { + let selected: Vec<_> = tests(args) + .into_iter() + .filter(|test| args.test.as_ref().is_none_or(|name| name == &test.name)) + .collect(); + if selected.is_empty() { + println!("FAILED: unknown test {:?}", args.test); + return Ok(IndexMap::new()); + } + let maximum_memory = selected + .iter() + .map(|test| memory_limit(args, test)) + .max() + .unwrap(); + resources::limit(maximum_memory)?; + let suite = Instant::now() + Duration::from_secs(args.suite_timeout); + let mut results: IndexMap<_, _> = selected + .iter() + .map(|test| { + ( + test.name.clone(), + Outcome::Skipped(format!( + "suite budget of {}s exhausted before this test started", + args.suite_timeout + )), + ) + }) + .collect(); + let mut next = 0; + while next < selected.len() && Instant::now() < suite { + // A single explicitly selected test remains independently runnable. + if args.test.is_none() && selected[next].name.starts_with("SCALABILITY") { + let blockers = scalability_blockers(&results, args.monkeys, args.disable_mc_tests); + if !blockers.is_empty() { + for test in &selected[next..] { + let outcome = Outcome::SkippedPrerequisites(blockers.clone()); + println!("\n--- {} ---\n", test.name); + report(&test.name, &outcome); + results.insert(test.name.clone(), outcome); + } + break; + } + } + let mut end = next + 1; + let chaos = selected[next].chaos_run.is_some(); + if chaos { + while end < selected.len() && selected[end].chaos_run.is_some() { + end += 1; + } + } + let phase = if chaos { + Instant::now() + Duration::from_secs(args.chaos_timeout) + } else { + suite + }; + while next < end && Instant::now() < suite && Instant::now() < phase { + next += run_worker(args, &selected[next..end], &mut results, suite, phase); + } + if next < end { + if Instant::now() < suite { + for test in &selected[next..end] { + results.insert( + test.name.clone(), + Outcome::Skipped(format!( + "chaos budget of {}s exhausted before this run started", + args.chaos_timeout + )), + ); + } + } + next = end; + } + } + if args.test.is_none() { + let passed = results + .values() + .filter(|result| matches!(result, Outcome::Passed)) + .count(); + println!("-------------------------------------------------------------------------------\n\nPassed {passed} from {} tests\n", selected.len()); + for (name, result) in &results { + if !matches!(result, Outcome::Passed) { + println!("- {name}: {}", result.description()); + } + } + println!(); + } + Ok(results) +} + +#[cfg(test)] +pub fn passed_results(monkeys: u32) -> IndexMap { + let args = ::parse_from(["membership", "-m", &monkeys.to_string()]); + tests(&args) + .into_iter() + .map(|test| (test.name, Outcome::Passed)) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn resource_failure_requires_a_started_record_even_at_the_deadline() { + // Model the final journal drain on either side of the startup boundary. + // With no Started record, the result must remain ineligible for MC credit. + for resource in [ + Outcome::TimeLimit("suite budget exhausted during startup".into()), + Outcome::MemoryLimit("allocation failed during startup".into()), + ] { + let files = Files::new().unwrap(); + let args = ::parse_from(["membership", "-m", "0"]); + let registry = tests(&args); + let mc_index = registry + .iter() + .position(|test| test.name == "MODEL CHECKING") + .unwrap(); + let batch = ®istry[mc_index..mc_index + 1]; + let mut journal = Journal { + reader: BufReader::new(File::open(files.journal()).unwrap()), + pending: String::new(), + next: 0, + started: Instant::now(), + announced: false, + }; + let mut results = IndexMap::new(); + journal.read(batch, &mut results).unwrap(); + let outcome = resource.clone().require_started(journal.announced); + assert!(matches!(outcome, Outcome::Skipped(_))); + assert!(!outcome.resource_limited()); + + // Started committed just before termination is picked up by the + // final drain, and makes a genuine interrupted run resource-limited. + let mut writer = OpenOptions::new() + .append(true) + .open(files.journal()) + .unwrap(); + event(&mut writer, Event::Started(batch[0].name.clone())); + journal.read(batch, &mut results).unwrap(); + assert!(resource + .require_started(journal.announced) + .resource_limited()); + } + } + + #[test] + fn default_memory_limits_are_separate_for_mc_and_simulation() { + let args = ::parse_from(["membership"]); + for test in tests(&args) { + assert_eq!( + memory_limit(&args, &test), + if test.name == "MODEL CHECKING" { + 14336 + } else { + 4096 + } + ); + } + } + + #[test] + fn registry_preserves_all_tests_and_chaos_seeds() { + let args = ::parse_from(["membership", "-m", "1000"]); + let registry = tests(&args); + assert_eq!(registry.len(), 1027); + let ordinary = ORDINARY.len(); + assert_eq!(registry[0].name, "SIMPLE"); + assert_eq!(registry[ordinary].name, "CHAOS MONKEY (run 1)"); + assert_eq!(registry[ordinary + 999].name, "CHAOS MONKEY (run 1000)"); + assert_eq!(registry[ordinary + 1000].name, "MODEL CHECKING"); + assert_eq!(registry[ordinary + 1001].name, "SCALABILITY NORMAL"); + assert_eq!(registry[ordinary + 1002].name, "SCALABILITY CRASH"); + let mut rand = Pcg64::seed_from_u64(123); + for test in ®istry[ordinary..ordinary + 1000] { + assert_eq!(test.seed, rand.next_u64()); + } + } + + #[test] + fn oversized_ipc_records_do_not_exhaust_supervisor_memory() { + let files = Files::new().unwrap(); + fs::write(files.journal(), vec![b'x'; 100_000]).unwrap(); + let args = ::parse_from(["membership"]); + let mut journal = Journal { + reader: BufReader::new(File::open(files.journal()).unwrap()), + pending: String::new(), + next: 0, + started: Instant::now(), + announced: false, + }; + let error = journal + .read(&tests(&args), &mut IndexMap::new()) + .unwrap_err(); + assert_eq!(error, "worker result exceeds 64 KiB"); + assert!(journal.pending.len() <= 65537); + } + + #[test] + fn journal_keeps_completed_results_and_waits_for_partial_records() { + let files = Files::new().unwrap(); + let args = ::parse_from(["membership", "-m", "2"]); + let registry = tests(&args); + let batch = ®istry[ORDINARY.len()..ORDINARY.len() + 2]; + let mut writer = OpenOptions::new() + .append(true) + .open(files.journal()) + .unwrap(); + event(&mut writer, Event::Started(batch[0].name.clone())); + event( + &mut writer, + Event::Finished(batch[0].name.clone(), Outcome::Passed), + ); + let mut journal = Journal { + reader: BufReader::new(File::open(files.journal()).unwrap()), + pending: String::new(), + next: 0, + started: Instant::now(), + announced: false, + }; + let mut results = IndexMap::new(); + journal.read(batch, &mut results).unwrap(); + assert_eq!(journal.next, 1); + assert!(matches!(results[&batch[0].name], Outcome::Passed)); + let record = serde_json::to_string(&Event::Started(batch[1].name.clone())).unwrap(); + writer.write_all(record.as_bytes()).unwrap(); + writer.flush().unwrap(); + journal.read(batch, &mut results).unwrap(); + assert!(!journal.announced); + writeln!(writer).unwrap(); + writer.flush().unwrap(); + journal.read(batch, &mut results).unwrap(); + assert!(journal.announced); + assert_eq!(results.len(), 1); + } +} diff --git a/homework/06-membership/tests/src/shared_state.rs b/homework/06-membership/tests/src/shared_state.rs new file mode 100644 index 0000000..35b5d1c --- /dev/null +++ b/homework/06-membership/tests/src/shared_state.rs @@ -0,0 +1,78 @@ +use std::ffi::CString; + +use pyo3::prelude::*; +use pyo3::types::{PyList, PyModule}; + +const VALIDATOR_CODE: &str = include_str!("../shared_state_validator.py"); + +pub(crate) fn validator_module<'py>(py: Python<'py>) -> PyResult> { + let code = CString::new(VALIDATOR_CODE).expect("validator source contains a null byte"); + PyModule::from_code( + py, + &code, + c"shared_state_validator.py", + c"shared_state_validator", + ) +} + +#[cfg(test)] +fn validate_source( + source: &str, + filename: &str, + class_names: &[&str], +) -> Result, String> { + Python::attach(|py| { + let module = validator_module(py).map_err(|error| error.to_string())?; + let class_names = PyList::new(py, class_names).map_err(|error| error.to_string())?; + module + .getattr("validate_source") + .and_then(|function| function.call1((source, filename, class_names))) + .and_then(|result| result.extract::>()) + .map_err(|error| error.to_string()) + }) +} + +fn validate_solution(path: &str, class_names: &[&str]) -> Result, String> { + Python::attach(|py| { + let module = validator_module(py).map_err(|error| error.to_string())?; + let class_names = PyList::new(py, class_names).map_err(|error| error.to_string())?; + module + .getattr("validate_solution") + .and_then(|function| function.call1((path, class_names))) + .and_then(|result| result.extract::>()) + .map_err(|error| error.to_string()) + }) +} + +pub fn validate_or_report(path: &str, class_names: &[&str]) -> bool { + match validate_solution(path, class_names) { + Ok(violations) if violations.is_empty() => true, + Ok(violations) => { + println!("ERROR: shared state between AnySystem processes is forbidden"); + for violation in violations { + println!("{violation}"); + } + println!("\nSCORE: 0\n"); + false + } + Err(error) => panic!("failed to validate solution for shared state: {error}"), + } +} + +#[cfg(test)] +mod tests { + use super::validate_source; + + #[test] + fn embeds_validator_and_rejects_shared_state() { + let violations = validate_source( + "class ProcessImpl:\n shared = {}\n", + "solution.py", + &["ProcessImpl"], + ) + .unwrap(); + + assert_eq!(violations.len(), 1); + assert!(violations[0].contains("ProcessImpl.shared")); + } +} diff --git a/homework/06-membership/tests/src/tests.rs b/homework/06-membership/tests/src/tests.rs new file mode 100644 index 0000000..b211081 --- /dev/null +++ b/homework/06-membership/tests/src/tests.rs @@ -0,0 +1,813 @@ +use std::collections::{BTreeSet, HashMap}; + +use assertables::{assume, assume_eq}; +use log::info; +use rand::prelude::*; +use rand_pcg::Pcg64; +use sugars::boxed; + +use anysystem::test::TestResult; +use anysystem::{Message, System}; + +use crate::common::*; + +pub fn test_simple(config: &TestConfig) -> TestResult { + let mut sys = build_system(config); + let group = sys.process_names(); + let seed = &group[0]; + initialize_group(&mut sys, &group, seed) +} + +pub fn test_get_members_semantics(config: &TestConfig) -> TestResult { + let mut sys = build_system(config); + let group = sys.process_names(); + let seed = &group[0]; + initialize_group(&mut sys, &group, seed)?; + for proc in sys.process_names() { + get_members(&mut sys, &proc)?; + } + Ok(true) +} + +pub fn test_random_seed(config: &TestConfig) -> TestResult { + let mut rand = Pcg64::seed_from_u64(config.seed); + let mut sys = build_system(config); + let mut group = Vec::new(); + for proc in sys.process_names() { + let seed = match group.len() { + 0 => &proc, + _ => group.choose(&mut rand).unwrap(), + }; + sys.send_local_message(&proc, Message::json("JOIN", &JoinMessage { seed })); + group.push(proc); + } + step_until_stabilized(&mut sys, group.into_iter().collect()) +} + +pub fn test_process_join(config: &TestConfig) -> TestResult { + let mut rand = Pcg64::seed_from_u64(config.seed); + let mut sys = build_system(config); + let mut group = sys.process_names(); + group.shuffle(&mut rand); + let new_proc = group.remove(rand.gen_range(0..group.len())); + let seed = &group[0]; + initialize_group(&mut sys, &group, seed)?; + + // process joins the system + sys.send_local_message(&new_proc, Message::json("JOIN", &JoinMessage { seed })); + group.push(new_proc); + step_until_stabilized(&mut sys, group.into_iter().collect()) +} + +pub fn test_process_leave(config: &TestConfig) -> TestResult { + let mut rand = Pcg64::seed_from_u64(config.seed); + let mut sys = build_system(config); + let mut group = sys.process_names(); + group.shuffle(&mut rand); + let seed = &group[0]; + initialize_group(&mut sys, &group, seed)?; + + // process leaves the system + let left_proc = group.remove(rand.gen_range(0..group.len())); + sys.send_local_message(&left_proc, Message::json("LEAVE", &LeaveMessage {})); + step_until_stabilized(&mut sys, group.into_iter().collect()) +} + +pub fn test_process_crash(config: &TestConfig) -> TestResult { + let mut rand = Pcg64::seed_from_u64(config.seed); + let mut sys = build_system(config); + let mut group = sys.process_names(); + group.shuffle(&mut rand); + let seed = &group[0]; + initialize_group(&mut sys, &group, seed)?; + + // process crashes + let crashed = group.remove(rand.gen_range(0..group.len())); + crash_process(&crashed, &mut sys); + step_until_stabilized(&mut sys, group.into_iter().collect()) +} + +pub fn test_seed_process_crash(config: &TestConfig) -> TestResult { + let mut rand = Pcg64::seed_from_u64(config.seed); + let mut sys = build_system(config); + let mut group = sys.process_names(); + group.shuffle(&mut rand); + let seed = &group[0].clone(); + initialize_group(&mut sys, &group, seed)?; + + // seed process crashes + group.remove(0); + crash_process(seed, &mut sys); + step_until_stabilized(&mut sys, group.into_iter().collect()) +} + +pub fn test_process_crash_recover(config: &TestConfig) -> TestResult { + let mut rand = Pcg64::seed_from_u64(config.seed); + let mut sys = build_system(config); + let mut group = sys.process_names(); + group.shuffle(&mut rand); + let seed = &group[0].clone(); + initialize_group(&mut sys, &group, seed)?; + + // process crashes + let crashed = group.remove(rand.gen_range(0..group.len())); + crash_process(&crashed, &mut sys); + step_until_stabilized(&mut sys, group.clone().into_iter().collect())?; + + // process recovers + recover_process(&crashed, &mut sys, config); + sys.send_local_message(&crashed, Message::json("JOIN", &JoinMessage { seed })); + + group.push(crashed); + step_until_membership_stable(&mut sys, group.into_iter().collect(), "process recovery") +} + +pub fn test_process_offline(config: &TestConfig) -> TestResult { + let mut rand = Pcg64::seed_from_u64(config.seed); + let mut sys = build_system(config); + let mut group = sys.process_names(); + group.shuffle(&mut rand); + let seed = &group[0]; + initialize_group(&mut sys, &group, seed)?; + + // process goes offline + let offline_proc = group.remove(rand.gen_range(0..group.len())); + sys.network().disconnect_node(&offline_proc); + step_until_stabilized(&mut sys, group.into_iter().collect())?; + step_until_stabilized_at(&mut sys, BTreeSet::from([offline_proc]), "isolated process") +} + +pub fn test_seed_process_offline(config: &TestConfig) -> TestResult { + let mut rand = Pcg64::seed_from_u64(config.seed); + let mut sys = build_system(config); + let mut group = sys.process_names(); + group.shuffle(&mut rand); + let seed = &group[0].clone(); + initialize_group(&mut sys, &group, seed)?; + + // seed process goes offline + group.remove(0); + sys.network().disconnect_node(seed); + step_until_stabilized(&mut sys, group.into_iter().collect())?; + step_until_stabilized_at(&mut sys, BTreeSet::from([seed.clone()]), "isolated process") +} + +pub fn test_process_offline_recover(config: &TestConfig) -> TestResult { + let mut rand = Pcg64::seed_from_u64(config.seed); + let mut sys = build_system(config); + let mut group = sys.process_names(); + group.shuffle(&mut rand); + let seed = &group[0]; + initialize_group(&mut sys, &group, seed)?; + + // process goes offline + let offline_proc = group.remove(rand.gen_range(0..group.len())); + sys.network().disconnect_node(&offline_proc); + step_until_stabilized(&mut sys, group.clone().into_iter().collect())?; + step_until_stabilized_at( + &mut sys, + BTreeSet::from([offline_proc.clone()]), + "isolated process", + )?; + + // process goes back online + sys.network().connect_node(&offline_proc); + group.push(offline_proc); + step_until_membership_stable( + &mut sys, + group.into_iter().collect(), + "network reconnection", + ) +} + +pub fn test_network_partition(config: &TestConfig) -> TestResult { + run_network_partition(config, false, false) +} + +pub fn test_network_partition_recover(config: &TestConfig) -> TestResult { + run_network_partition(config, false, true) +} + +pub fn test_network_partition_unequal(config: &TestConfig) -> TestResult { + run_network_partition(config, true, false) +} + +pub fn test_network_partition_unequal_recover(config: &TestConfig) -> TestResult { + run_network_partition(config, true, true) +} + +fn run_network_partition(config: &TestConfig, unequal: bool, recover: bool) -> TestResult { + if unequal { + assume!( + config.process_count >= 3, + "Unequal network partition requires at least 3 processes" + )?; + } + let mut rand = Pcg64::seed_from_u64(config.seed); + let mut sys = build_system(config); + let mut group = sys.process_names(); + group.shuffle(&mut rand); + let seed = &group[0]; + initialize_group(&mut sys, &group, seed)?; + + let (group1, group2): (Vec<_>, Vec<_>) = if unequal { + // Always a nonempty strict minority, including odd process counts. + let (minority, majority) = group.split_at((group.len() - 1) / 2); + ( + minority.iter().map(String::as_str).collect(), + majority.iter().map(String::as_str).collect(), + ) + } else { + // Preserve the existing random partition and its seeds. + group + .iter() + .map(String::as_str) + .partition(|_| rand.gen_range(0.0..1.0) > 0.6) + }; + info!( + "Network partition: [{}] ({} processes) | [{}] ({} processes)", + group1.join(", "), + group1.len(), + group2.join(", "), + group2.len() + ); + sys.network().make_partition(&group1, &group2); + step_until_stabilized_at( + &mut sys, + group1.into_iter().map(String::from).collect(), + "network partition: first subgroup", + )?; + step_until_stabilized_at( + &mut sys, + group2.into_iter().map(String::from).collect(), + "network partition: second subgroup", + )?; + + if recover { + sys.network().reset(); + step_until_membership_stable(&mut sys, group.into_iter().collect(), "network recovery")?; + } + Ok(true) +} + +pub fn test_process_cannot_receive(config: &TestConfig) -> TestResult { + let mut rand = Pcg64::seed_from_u64(config.seed); + let mut sys = build_system(config); + let mut group = sys.process_names(); + group.shuffle(&mut rand); + let seed = &group[0]; + initialize_group(&mut sys, &group, seed)?; + + // process goes partially offline (cannot receive incoming messages) + let blocked_proc = group.remove(rand.gen_range(0..group.len())); + sys.network().drop_incoming(&blocked_proc); + step_until_stabilized(&mut sys, group.into_iter().collect())?; + step_until_stabilized_at(&mut sys, BTreeSet::from([blocked_proc]), "isolated process") +} + +pub fn test_process_join_cannot_receive(config: &TestConfig) -> TestResult { + let mut rand = Pcg64::seed_from_u64(config.seed); + let mut sys = build_system(config); + let mut group = sys.process_names(); + group.shuffle(&mut rand); + let new_proc = group.remove(rand.gen_range(0..group.len())); + let seed = &group[0]; + initialize_group(&mut sys, &group, seed)?; + + // process tries to join the system but cannot receive messages + // it must not be kept in the group + sys.network().drop_incoming(&new_proc); + sys.send_local_message(&new_proc, Message::json("JOIN", &JoinMessage { seed })); + step_until_stabilized(&mut sys, group.into_iter().collect()) +} + +pub fn test_process_cannot_send(config: &TestConfig) -> TestResult { + let mut rand = Pcg64::seed_from_u64(config.seed); + let mut sys = build_system(config); + let mut group = sys.process_names(); + group.shuffle(&mut rand); + let seed = &group[0]; + initialize_group(&mut sys, &group, seed)?; + + // process goes partially offline (cannot send outgoing messages) + let blocked_proc = group.remove(rand.gen_range(0..group.len())); + sys.network().drop_outgoing(&blocked_proc); + step_until_stabilized(&mut sys, group.into_iter().collect())?; + step_until_stabilized_at(&mut sys, BTreeSet::from([blocked_proc]), "isolated process") +} + +pub fn test_two_processes_cannot_communicate(config: &TestConfig) -> TestResult { + run_pairwise_link_failure(config, false) +} + +pub fn test_two_processes_cannot_communicate_in_triangle(config: &TestConfig) -> TestResult { + let config = TestConfig { + process_count: 3, + ..*config + }; + run_pairwise_link_failure(&config, true) +} + +fn run_pairwise_link_failure(config: &TestConfig, triangle: bool) -> TestResult { + assume!( + config.process_count >= 3, + "Pairwise link failure with a connected group requires at least 3 processes" + )?; + let mut rand = Pcg64::seed_from_u64(config.seed); + let mut sys = build_system(config); + let mut group = sys.process_names(); + if triangle { + group.sort(); + } else { + group.shuffle(&mut rand); + } + let seed = &group[0]; + initialize_group(&mut sys, &group, seed)?; + + // In a triangle, the seed remains the bridge between the other two nodes. + let proc1 = if triangle { &group[1] } else { seed }; + let proc2 = if triangle { + &group[2] + } else { + group.get(rand.gen_range(1..group.len())).unwrap() + }; + info!( + "Network links disabled: {proc1} <-> {proc2}; group remains connected (N={})", + group.len() + ); + sys.network().disable_link(proc1, proc2); + sys.network().disable_link(proc2, proc1); + // Preserve the original test's execution before checking convergence. + sys.steps(1000); + step_until_membership_stable( + &mut sys, + group.into_iter().collect(), + "pairwise link failure", + ) +} + +/// Once membership is correct, observe every event without advancing time in GET_MEMBERS. +/// A transient mismatch restarts the stability window, allowing eventual convergence. +fn step_until_membership_stable( + sys: &mut System, + group: BTreeSet, + stage: &str, +) -> TestResult { + const STABLE_WINDOW: f64 = 30.0; + const CONVERGENCE_LIMIT: f64 = 300.0; + let deadline = sys.time() + CONVERGENCE_LIMIT; + let mut stable_since = None; + let mut checked_events = 0_u64; + let mut search_exhausted = false; + let mut last_mismatch = String::from("no correct membership snapshot"); + loop { + let now = sys.time(); + // The next event may lie beyond the observed window. Its effects must + // not invalidate the unchanged state inside the completed window. + if stable_since + .is_some_and(|since| now > since + STABLE_WINDOW && since + STABLE_WINDOW <= deadline) + { + break; + } + if now > deadline { + return Err(format!( + "Group members lists are not stable for {STABLE_WINDOW}s within {CONVERGENCE_LIMIT}s; {last_mismatch}" + )); + } + if checked_events.is_multiple_of(100) { + crate::runner::progress(format!( + "{stage}: membership stability: N={}, simulation time={now:.1}, stable for {:.1}/{STABLE_WINDOW}s, checked events={checked_events}", + group.len(), stable_since.map_or(0.0, |since| now - since) + )); + } + let mut correct = true; + for proc in &group { + let data = get_members(sys, proc)?; + let members: BTreeSet<_> = data.members.into_iter().collect(); + if members != group { + correct = false; + last_mismatch = format!( + "process {proc} at simulation time {now:.3}: expected {group:?}, got {members:?}" + ); + } + } + if correct { + let since = *stable_since.get_or_insert(now); + if now - since >= STABLE_WINDOW { + break; + } + } else { + stable_since = None; + } + if !correct && !search_exhausted && now < deadline { + // Search for the next candidate snapshot at the usual 5s cadence. + // Only a correct candidate starts event-by-event observation. This + // avoids filling AnySystem's retained trace with failed queries. + search_exhausted = !sys.step_for_duration(5.0_f64.min(deadline - now)); + continue; + } + if !correct { + return Err(format!( + "Group members lists are not stable for {STABLE_WINDOW}s within {CONVERGENCE_LIMIT}s; {last_mismatch}" + )); + } + if !sys.step() { + // With no pending events, a correct membership cannot change. + break; + } + checked_events += 1; + } + info!( + "Membership remained stable for {STABLE_WINDOW}s ({stage}; checked after {checked_events} simulation events)" + ); + Ok(true) +} + +pub fn test_slow_network(config: &TestConfig) -> TestResult { + let mut rand = Pcg64::seed_from_u64(config.seed); + let mut sys = build_system(config); + let mut group = sys.process_names(); + group.shuffle(&mut rand); + let seed = &group[0]; + initialize_group(&mut sys, &group, seed)?; + + // slow down network for a while + sys.network().set_delays(0.1, 1.0); + sys.steps(200); + sys.network().set_delays(0.01, 0.1); + step_until_stabilized(&mut sys, group.into_iter().collect()) +} + +pub fn test_flaky_network(config: &TestConfig) -> TestResult { + let mut rand = Pcg64::seed_from_u64(config.seed); + let mut sys = build_system(config); + let mut group = sys.process_names(); + group.shuffle(&mut rand); + let seed = &group[0]; + initialize_group(&mut sys, &group, seed)?; + + // make network unreliable for a while + sys.network().set_drop_rate(0.5); + sys.steps(1000); + sys.network().set_drop_rate(0.0); + step_until_stabilized(&mut sys, group.into_iter().collect()) +} + +pub fn test_flaky_network_on_start(config: &TestConfig) -> TestResult { + let mut rand = Pcg64::seed_from_u64(config.seed); + let mut sys = build_system(config); + let mut group = sys.process_names(); + group.shuffle(&mut rand); + let seed = &group[0]; + + // make network unreliable from the start + sys.network().set_drop_rate(0.2); + for proc in &group { + sys.send_local_message(proc, Message::json("JOIN", &JoinMessage { seed })); + } + sys.steps(1000); + sys.network().set_drop_rate(0.0); + step_until_stabilized(&mut sys, group.into_iter().collect()) +} + +pub fn test_flaky_network_and_crash(config: &TestConfig) -> TestResult { + let mut rand = Pcg64::seed_from_u64(config.seed); + let mut sys = build_system(config); + let mut group = sys.process_names(); + group.shuffle(&mut rand); + let seed = &group[0]; + initialize_group(&mut sys, &group, seed)?; + + // make network unreliable for a while + crash process + sys.network().set_drop_rate(0.5); + let crashed = group.remove(rand.gen_range(0..group.len())); + crash_process(&crashed, &mut sys); + sys.steps(1000); + sys.network().set_drop_rate(0.0); + step_until_stabilized(&mut sys, group.into_iter().collect()) +} + +pub fn test_chaos_monkey(config: &TestConfig) -> TestResult { + let mut rand = Pcg64::seed_from_u64(config.seed); + let mut sys = build_system(config); + let mut group = sys.process_names(); + group.shuffle(&mut rand); + let seed = &group[0]; + initialize_group(&mut sys, &group, seed)?; + + info!( + "CHAOS MONKEY run seed: {}; join order: [{}]; seed process: {seed}", + config.seed, + group.join(", ") + ); + // Keep at least two participants; pairwise link failures must not isolate + // a participant still expected in the control group. + let failure_count = 6.min(group.len().saturating_sub(2)); + for step in 1..=failure_count { + let p = rand.gen_range(0.0..1.0); + // do some nasty things + match p { + p if p < 0.25 => { + // crash process + let victim = group.remove(rand.gen_range(0..group.len())); + info!( + "- Step {step} at {:.2}: process {victim} crashes", + sys.time() + ); + crash_process(&victim, &mut sys); + } + p if p < 0.5 => { + // disconnect process + let victim = group.remove(rand.gen_range(0..group.len())); + info!( + "- Step {step} at {:.2}: process {victim} disconnects", + sys.time() + ); + sys.network().disconnect_node(&victim); + } + p if p < 0.75 => { + // partially disconnect process (cannot receive) + let victim = group.remove(rand.gen_range(0..group.len())); + info!( + "- Step {step} at {:.2}: process {victim} cannot receive messages", + sys.time() + ); + sys.network().drop_incoming(&victim); + } + _ => { + // two processes cannot communicate with each other + let proc1 = group.get(rand.gen_range(0..group.len())).unwrap(); + let mut proc2 = group.get(rand.gen_range(0..group.len())).unwrap(); + while proc1 == proc2 { + proc2 = group.get(rand.gen_range(0..group.len())).unwrap(); + } + info!( + "- Step {step} at {:.2}: processes {proc1} and {proc2} cannot communicate", + sys.time() + ); + sys.network().disable_link(proc1, proc2); + sys.network().disable_link(proc2, proc1); + } + } + info!( + "Control group after step {step}: [{}] ({} processes)", + group.join(", "), + group.len() + ); + step_until_stabilized(&mut sys, group.clone().into_iter().collect())?; + } + Ok(true) +} + +pub fn test_scalability_normal(config: &TestConfig) -> TestResult { + let sys_sizes = [ + config.process_count, + config.process_count * 2, + config.process_count * 5, + config.process_count * 10, + ]; + let mut measurements = Vec::new(); + for size in sys_sizes { + let mut run_config = *config; + run_config.process_count = size; + let mut rand = Pcg64::seed_from_u64(config.seed); + let mut sys = build_system(&run_config); + let mut group = sys.process_names(); + group.shuffle(&mut rand); + let seed = &group[0]; + initialize_group(&mut sys, &group, seed)?; + + let init_time = sys.time(); + let init_net_traffic = sys.network().traffic(); + let init_msg_count = sys.network().network_message_count(); + let mut init_loads = HashMap::new(); + for proc in sys.process_names() { + init_loads.insert( + proc.clone(), + sys.sent_message_count(&proc) + sys.received_message_count(&proc), + ); + } + + crate::runner::progress(format!( + "steady-state measurement: N={size}, simulation time={:.1}", + sys.time() + )); + sys.step_for_duration(10.0); + + let mut loads = Vec::new(); + for proc in sys.process_names() { + let load = sys.sent_message_count(&proc) + sys.received_message_count(&proc); + loads.push(load - init_loads.get(&proc).unwrap()); + } + let min_load = *loads.iter().min().unwrap(); + let max_load = *loads.iter().max().unwrap(); + let duration = sys.time() - init_time; + let traffic = sys.network().traffic(); + let message_count = sys.network().network_message_count(); + measurements.push(( + duration, + (traffic - init_net_traffic) as f64 / duration, + (message_count - init_msg_count) as f64 / duration, + max_load as f64 / duration, + max_load as f64 / min_load as f64, + )); + } + let mut scaling_ok = true; + let mut load_ratio_ok = true; + for i in 0..sys_sizes.len() { + let (time, traffic, message_count, max_load, load_ratio) = measurements[i]; + println!( + "- N = {}: time - {:.2}, traffic/s - {:.2}, messages/s - {:.2}, max load - {:.2}, max/min load - {:.2}", + sys_sizes[i], time, traffic, message_count, max_load, load_ratio + ); + if load_ratio > 5.0 { + load_ratio_ok = false; + } + if i > 0 { + let size_ratio = sys_sizes[i] as f64 / sys_sizes[i - 1] as f64; + let traffic_ratio = traffic / measurements[i - 1].1; + let messages_ratio = message_count / measurements[i - 1].2; + if traffic_ratio > 2.0 * size_ratio || messages_ratio > 2.0 * size_ratio { + scaling_ok = false; + } + } + } + assume!(scaling_ok, "Bad network load scaling")?; + assume!(load_ratio_ok, "Bad max/min process load")?; + Ok(true) +} + +pub fn test_scalability_crash(config: &TestConfig) -> TestResult { + let sys_sizes = [ + config.process_count, + config.process_count * 2, + config.process_count * 5, + config.process_count * 10, + ]; + let mut measurements = Vec::new(); + for size in sys_sizes { + let mut run_config = *config; + run_config.process_count = size; + let mut rand = Pcg64::seed_from_u64(config.seed); + let mut sys = build_system(&run_config); + let mut group = sys.process_names(); + group.shuffle(&mut rand); + let seed = &group[0]; + initialize_group(&mut sys, &group, seed)?; + + let init_time = sys.time(); + let init_net_traffic = sys.network().traffic(); + let init_msg_count = sys.network().network_message_count(); + let mut init_loads = HashMap::new(); + for proc in sys.process_names() { + init_loads.insert( + proc.clone(), + sys.sent_message_count(&proc) + sys.received_message_count(&proc), + ); + } + + let crashed = group.remove(rand.gen_range(0..group.len())); + crash_process(&crashed, &mut sys); + step_until_stabilized_at( + &mut sys, + group.clone().into_iter().collect(), + "stabilization after crash", + )?; + + let mut loads = Vec::new(); + for proc in sys.process_names() { + if proc != crashed { + let load = sys.sent_message_count(&proc) + sys.received_message_count(&proc); + loads.push(load - init_loads.get(&proc).unwrap()); + } + } + let min_load = *loads.iter().min().unwrap(); + let max_load = *loads.iter().max().unwrap(); + let duration = sys.time() - init_time; + let traffic = sys.network().traffic(); + let message_count = sys.network().network_message_count(); + measurements.push(( + duration, + (traffic - init_net_traffic) as f64 / duration, + (message_count - init_msg_count) as f64 / duration, + max_load as f64 / duration, + max_load as f64 / min_load as f64, + )); + } + let mut scaling_ok = true; + let mut load_ratio_ok = true; + for i in 0..sys_sizes.len() { + let (time, traffic, message_count, max_load, load_ratio) = measurements[i]; + println!( + "- N = {}: time - {:.2}, traffic/s - {:.2}, messages/s - {:.2}, max load - {:.2}, max/min load - {:.2}", + sys_sizes[i], time, traffic, message_count, max_load, load_ratio + ); + if load_ratio > 5.0 { + load_ratio_ok = false; + } + if i > 0 { + let size_ratio = sys_sizes[i] as f64 / sys_sizes[i - 1] as f64; + let traffic_ratio = traffic / measurements[i - 1].1; + let messages_ratio = message_count / measurements[i - 1].2; + if traffic_ratio > 2.0 * size_ratio || messages_ratio > 2.0 * size_ratio { + scaling_ok = false; + } + } + } + assume!(scaling_ok, "Bad network load scaling")?; + assume!(load_ratio_ok, "Bad max/min process load")?; + Ok(true) +} + +fn initialize_group(sys: &mut System, group: &Vec, seed: &str) -> TestResult { + for proc in group { + sys.send_local_message(proc, Message::json("JOIN", &JoinMessage { seed })); + } + step_until_stabilized_at( + sys, + group.clone().into_iter().collect(), + "initial group assembly", + ) +} + +fn crash_process(name: &str, sys: &mut System) { + // we just crash the node on which the process is running + sys.crash_node(name); +} + +fn recover_process(name: &str, sys: &mut System, config: &TestConfig) { + sys.recover_node(name); + let clock_skew = sys.gen_range(0.0..10.0); + // A reboot may change the local clock. Apply it before on_start is called. + sys.set_node_clock_skew(name, clock_skew); + info!( + "Process {name} recovers at simulation time {:.6} with clock skew {clock_skew:.6}", + sys.time() + ); + let process = config.process_factory.build((name,), config.seed); + sys.add_process(name, boxed!(process), name); +} + +fn get_members(sys: &mut System, proc: &str) -> Result { + let sent_before = sys.sent_message_count(proc); + sys.send_local_message(proc, Message::json("GET_MEMBERS", &GetMembersMessage {})); + assume_eq!( + sys.sent_message_count(proc), + sent_before, + format!("GET_MEMBERS must not send network messages: process {proc}") + )?; + let messages = sys + .step_until_local_message_max_steps(proc, 0) + .map_err(|_| { + format!("GET_MEMBERS must return immediately without simulation steps: process {proc}") + })?; + assume_eq!(messages.len(), 1, "expected exactly one MEMBERS message")?; + assume_eq!(messages[0].tip, "MEMBERS", "Wrong message type")?; + serde_json::from_str(&messages[0].data) + .map_err(|error| format!("Invalid MEMBERS message from {proc}: {error}")) +} + +fn step_until_stabilized(sys: &mut System, group: BTreeSet) -> TestResult { + step_until_stabilized_at(sys, group, "membership stabilization") +} + +fn step_until_stabilized_at(sys: &mut System, group: BTreeSet, stage: &str) -> TestResult { + let max_time = sys.time() + 300.; // timeout is 5 minutes + let mut stabilized = BTreeSet::new(); + let mut memberlists = HashMap::new(); + + while stabilized.len() < group.len() && sys.time() < max_time { + crate::runner::progress(format!("{stage}: N={}, correct members lists={}/{}, simulation time={:.1}, network messages={}", + group.len(), stabilized.len(), group.len(), sys.time(), sys.network().network_message_count())); + let cont = sys.step_for_duration(5.); + stabilized.clear(); + // Query processes in a stable order without advancing simulation time. + for proc in group.iter() { + let data = get_members(sys, proc)?; + let members: BTreeSet = data.members.clone().into_iter().collect(); + if members.eq(&group) { + stabilized.insert(proc.clone()); + } + memberlists.insert(proc.clone(), data.members); + } + if !cont { + break; + } + } + + if stabilized != group && group.len() <= 10 { + println!("Members lists:"); + for proc in sys.process_names() { + if group.contains(&proc) { + let members = memberlists.get_mut(&proc).unwrap(); + members.sort(); + println!("- [{}] {}", proc, members.join(", ")); + } + } + let mut expected = group.clone().into_iter().collect::>(); + expected.sort(); + println!("Expected group: {}", expected.join(", ")); + } + assume_eq!(stabilized, group, "Group members lists are not stabilized")?; + Ok(true) +} diff --git a/homework/06-membership/tests/src/tests_mc.rs b/homework/06-membership/tests/src/tests_mc.rs new file mode 100644 index 0000000..c6f2c6d --- /dev/null +++ b/homework/06-membership/tests/src/tests_mc.rs @@ -0,0 +1,159 @@ +use std::collections::HashSet; +use std::time::Duration; + +use rand::prelude::*; +use rand_pcg::Pcg64; +use sugars::boxed; + +use anysystem::logger::{LogEntry, LogEntry::McMessageReceived}; +use anysystem::mc::{ + predicates::{collects, goals, invariants, prunes}, + strategies::Bfs, + CollectFn, EventOrderingMode, InvariantFn, McState, ModelChecker, StrategyConfig, +}; +use anysystem::test::TestResult; +use anysystem::{Message, System}; + +use crate::common::*; + +pub fn test_mc_group(config: &TestConfig) -> TestResult { + let mut rand = Pcg64::seed_from_u64(config.seed); + let mut sys = build_system(config); + let group = sys.process_names(); + let seed = group.choose(&mut rand).unwrap(); + + crate::runner::progress("MC: exploring group assembly, N=3"); + let collected_states = mc_explore_after_joins(&mut sys, seed.to_string())?; + if collected_states.is_empty() { + return Err("no states collected during explore stage".to_string()); + } + crate::runner::progress(format!( + "MC: checking membership in {} collected states, N=3", + collected_states.len() + )); + mc_check_members(&mut sys, collected_states) +} + +fn mc_explore_after_joins(sys: &mut System, seed_proc: String) -> Result, String> { + let procs = sys.process_names(); + let mut mc = ModelChecker::new(sys); + + let strategy_config = StrategyConfig::default() + // Explore only states with up to 2 timer firings per process + // (we expect each process to communicate with others at most 3 times: + // 1 time on a local message and 2 times on a timer). + .prune(prunes::events_limit_per_proc( + |entry: &LogEntry, proc_name: &String| match entry { + LogEntry::McTimerFired { proc, .. } => proc_name == proc, + _ => false, + }, + procs.clone(), + 2, + )) + // Stop when no events left or reached depth 20 (steps of simulation). + .goal(goals::any_goal(vec![ + goals::no_events(), + goals::depth_reached(20), + ])) + // Inner exploration safeguard; the supervisor also bounds both MC phases. + .invariant(invariants::time_limit(Duration::from_secs(300))) + // Collect states in which the group should be stabilized, namely: + // either no events left, or every process received at least 3 messages. + // + // Considering a system with 3 processes and a stable network, we can show this is enough: + // * seed process (A) should get information from both other processes (B & C) + // * because C joins later than B, it will know the full group as well + // * B is able to get information about the full group either from A or C, depending on implementation + // + // Technically, there can be execution A <-> B (x3), A <-> C (x3) where B's information becomes outdated, + // but we consider solution wrong if it allows such execution. + .collect(collects::any_collect(vec![ + collects::no_events(), + collects::all_collects( + procs + .clone() + .into_iter() + .map(|proc_name| { + collects::event_happened_n_times_current_run( + move |log_entry| match log_entry { + McMessageReceived { dst: proc, .. } => proc == &proc_name, + _ => false, + }, + 3, + ) + }) + .collect::>(), + ), + ])); + + let res = mc.run_with_change::(strategy_config, |sys| { + // Use event ordering mode which prioritizes messages over timers + // (this emulates perfect network where timeouts do not occur). + sys.set_event_ordering_mode(EventOrderingMode::MessagesFirst); + for proc in &procs { + sys.send_local_message( + proc.clone(), + proc.clone(), + Message::json("JOIN", &JoinMessage { seed: &seed_proc }), + ); + } + }); + match res { + Err(e) => { + e.print_trace(); + Err(e.message()) + } + Ok(stats) => { + // println!("collected {} states", stats.collected_states.len()); + Ok(stats.collected_states) + } + } +} + +fn mc_check_members(sys: &mut System, collected: HashSet) -> TestResult { + let procs = sys.process_names(); + let mut mc = ModelChecker::new(sys); + let strategy_config = StrategyConfig::default() + .invariant(mc_invariant_check_stabilized(procs.clone())) + .goal(goals::always_ok()); + + let res = mc.run_from_states_with_change::(strategy_config, collected, |sys| { + for node in procs.iter() { + sys.send_local_message( + node, + node, + Message::json("GET_MEMBERS", &GetMembersMessage {}), + ); + } + }); + if let Err(e) = res { + e.print_trace(); + Err(e.message()) + } else { + Ok(true) + } +} + +fn mc_invariant_check_stabilized(group: Vec) -> InvariantFn { + boxed!(move |state| { + let group = group.clone().into_iter().collect::>(); + for node in state.node_states.keys() { + if let Some(msg) = state.node_states[node].proc_states[node] + .local_outbox + .first() + { + if msg.tip != "MEMBERS" { + return Err("wrong message type".to_owned()); + } + let data: MembersMessage = serde_json::from_str(&msg.data).unwrap(); + let members = HashSet::from_iter(data.members.into_iter()); + if !members.eq(&group) { + return Err(format!( + "expected a stabilized group {group:?} but got {members:?}" + )); + } + } + } + Ok(()) + }) +}