diff --git a/homework/07-kv-sharding/readme.md b/homework/07-kv-sharding/readme.md new file mode 100644 index 0000000..623eb43 --- /dev/null +++ b/homework/07-kv-sharding/readme.md @@ -0,0 +1,95 @@ +# Распределенное key-value хранилище с шардингом + +В этом задании вам предстоит реализовать распределенную систему, в которой можно хранить записи вида _(ключ, значение)_. Ключами и значениями могут быть произвольные строки. В системе может храниться только одна запись с данным ключом. Для каких-то ключей записи могут отсутствовать. Распределение используемых ключей может быть любым, необязательно равномерным. Система должна поддерживать стандартные операции для работы с данными в подобных key-value хранилищах: записать значение для ключа, прочитать значение по ключу и удалить ключ. + +Система состоит из нескольких идентичных узлов, на каждом из которых запущен процесс _StorageNode_. Для поддержки горизонтального масштабирования каждый узел отвечает за хранение только некоторой части данных. А именно, для любого возможного значения ключа должен быть определен (один и только один) узел, "отвечающий" за хранение соответствующей записи. Таким образом, каждый узел в системе отвечает за некоторое подмножество ключей и хранение связанных с этими ключами записей. Хранить данные на диске не требуется, достаточно держать все записи в памяти. + +Нагрузка на узлы системы должна распределяться равномерно. А именно, каждый узел должен хранить примерно одинаковую долю записей - в идеале _R/N_ записей, где _R_ - суммарное число хранимых записей, а _N_ - число узлов. В тестах допустимым считается отклонение не более чем на 10% от идеального значения. + +В ходе работы системы в неё могут добавляться новые узлы, а существующие - удаляться. Добавление и удаление узлов происходит по команде администратора. При этом каждому узлу в системе отправляется соответствующее уведомление. Как обычно, узлы также могут внезапно отказывать. Чтобы не создавать зависимость от предыдущего ДЗ, будем считать, что отказы узлов уже обнаруживаются и обрабатываются каким-то образом. Соответственно, от вас не требуется реализовывать функционал group membership. В случае отказа узла и невозможности быстро восстановить его работу, этот узел может быть удален из системы через описанный выше механизм. Хранимые на отказавшем узле записи становятся временно недоступны или теряются насовсем. Эту проблему мы будем решать позже в рамках следующего задания. + +При изменении состава узлов должна происходить _перебалансировка_ - перераспределение ключей (зон ответственности) и записей (хранимых данных) между узлами для восстановления ранее описанных свойств. Во-первых, для любого возможного значения ключа должен существовать узел, отвечающий за него. Во-вторых, каждый узел по-прежнему должен хранить примерно одинаковую долю записей. Таким образом, при добавлении нового узла, он должен принять на себя некоторую часть ключей и хранимых записей. А при отключении узла, ключи, за которые он отвечал, и хранимые им записи должны распределяться между оставшимися узлами. + +Перебалансировка сопряжена с расходами на перемещение хранимых данных между узлами. Требуется минимизировать эти расходы - в идеале должно перемещаться только _R/N_ записей. В тестах допустимым считается отклонение не более чем на 10% от идеального значения. + +Клиенты системы не знают о распределении данных между узлами и могут отправлять свои запросы к любому узлу. Соответственно узел должен уметь обслуживать запросы на чтение и запись к _любым_ ключам. Если узел не отвечает за данный ключ, то он должен переслать запрос правильному узлу, дождаться ответа и вернуть его локальному клиенту. Таймауты и ошибки в ответах поддерживать необязательно. + +Каждый узел должен локально определять владельца любого ключа, без необходимости делать дополнительные пересылки. Такая маршрутизация называется _zero-hop_. Для полного балла поиск владельца должен занимать _O(log N)_ или меньше, где _N_ - число узлов. Объем используемой для поиска памяти не должен зависеть от числа хранимых записей, поэтому просто хранить набор пар «ключ → узел» для всех ключей нельзя. + +Если вам плохо понятны некоторые требования, изучите соответствующие тесты - это часть условия задания. + +## Реализация + +Для реализации и тестирования решения используется фреймворк AnySystem (см. материалы первого семинара). В папке `solution` размещена заготовка для решения [node.py](solution/node.py). Вам надо доработать реализацию класса `StorageNode` так, чтобы проходили все тесты. + +При инициализации узлу передается его уникальный id, а также список id всех узлов в системе на данный момент. В начале теста узлы инициализируются одновременно, то есть каждый из них получает полный список узлов на момент старта системы. + +Узел должен поддерживать обработку следующих локальных сообщений (форматы запросов и ответов описаны в заготовке): +- _GET(key)_ - вернуть значение записи с ключом `key` или `null`, если ключ отсутствует, +- _PUT(key, value)_ - сохранить запись с ключом `key` и значением `value`, +- _DELETE(key)_ - удалить запись с ключом `key` и вернуть прежнее значение или `null`, если ключ отсутствует, +- _NODE_ADDED_ - уведомление о добавлении нового узла в систему, +- _NODE_REMOVED_ - уведомление об удалении узла из системы, +- _COUNT_RECORDS_ - возвращает число записей, хранимых на данном узле (используется для тестирования), +- _DUMP_KEYS_ - возвращает ключи записей, хранимых на данном узле (используется для тестирования). + +Для взаимодействия между узлами вы можете использовать любые собственные типы сообщений. + +Ваши реализации процессов AnySystem должны соблюдать правило [изоляции процессов](../readme.md#изоляция-процессов-anysystem): изменяемое состояние каждого процесса должно храниться только в `self`, а общая изменяемая память между процессами запрещена. Все взаимодействия между процессами должны происходить через сообщения, как и в реальной распределённой системе. + +### Рекомендации и замечания + +1. Как вы могли догадаться, распределять по шардам необходимо только запросы _GET_, _PUT_ и _DELETE_. Кроме того, необходимо реализовать перебалансировку ключей при получении _NODE_ADDED_ или _NODE_REMOVED_. +2. Вашей реализации необязательно поддерживать узлы в консистентном состоянии в любой момент времени в процессе перебалансировки - достаточно, чтобы они пришли в такое состояние через какое-то ограниченное число сообщений. +3. Во время перебалансировки при добавлении узла система должна обрабатывать _GET_, _PUT_ и _DELETE_, поступающие на любой узел, включая новый. В тестах NODE ADDED DURING... проверяются такие запросы одного типа. Ответ можно отложить до завершения переноса. Перенос данных не должен затирать новые значения или восстанавливать удалённые записи. +4. Учтите, что в тестах окончание перебалансировки определяется по косвенным признакам, таким как отсутствие изменений числа хранимых записей на узлах (см. функцию `step_until_stabilized`), поскольку мы не стали требовать от вас возвращать явные ответы на _NODE_ADDED_ и _NODE_REMOVED_. +5. Не требуется реализовывать подтверждения и ретраи передаваемых между узлами запросов, в тестах сеть надежная. + +## Тестирование + +### Локальное тестирование + +Тесты находятся в папке `tests`. Есть два варианта их запуска. + +Рекомендуемый вариант запуска тестов - через готовый Docker-образ. В этом случае используемое окружение будет аналогично тестирующей системе. Для запуска тестов выполните команду: + +```commandline +docker run --pull always --rm -t -v ./solution:/solution distsys.ru/course/kv-sharding:latest [ЗДЕСЬ МОЖНО УКАЗАТЬ ОПЦИИ] +``` + +Для запуска полного набора тестов с теми же параметрами, что и в тестирующей системе, выполните команду: + +```commandline +docker run --pull always --rm -t -v ./solution:/solution distsys.ru/course/kv-sharding:latest +``` + +Вы также можете запустить тесты, скомпилировав их локально с помощью компилятора Rust. Такой вариант может быть удобен, если вы хотите лучше изучить или доработать тесты. Скомпилируйте тесты с помощью команды `cargo install --locked --path tests`. Для запуска тестов выполните команду: + +```commandline +distsys-kv-sharding [ЗДЕСЬ МОЖНО УКАЗАТЬ ОПЦИИ] +``` + +Запустить только один из тестов можно с помощью опции `-t`. По умолчанию вывод тестов не содержит трассы (последовательности событий во время выполнения каждого из тестов), а только финальную сводку. Включить вывод трасс можно с помощью флага `-d`. Все доступные опции можно посмотреть с помощью флага `-h`. + +В тестах MC NORMAL и MC NODE REMOVED вы можете столкнуться с ошибкой "state depth exceeds maximum allowed depth...". Эти тесты основаны на model checking (см. первый семинар) и, чтобы ограничить время тестирования, глубина проверяемых состояний системы в них ограничена. Соответственно, если за заданное число шагов обработка запроса не завершится, то будет выдана эта ошибка. Ограничения на глубину заданы так, чтобы в них легко укладывалась эффективное решение. В случае такой ошибки рекомендуется доработать решение, чтобы избавиться от избыточных действий. + +Бонусы за пробелы в тестах начисляются по [общим правилам](../readme.md#бонусы-за-пробелы-в-тестах). + +### Проверка в тестирующей системе + +Отправьте ваше решение в тестирующую систему следуя [инструкции](../readme.md) и дождитесь результатов. + +## Оценивание + +Автоматические тесты приносят до 9 баллов. Если хотя бы один тест базового блока не пройден, итоговая оценка за задание - 0. После прохождения базового блока каждый из трёх остальных блоков приносит 2 балла, если пройдены все его тесты. + +| Блок | Тесты | Баллы | +|---|---|---:| +| Базовые операции, маршрутизация и память | `SINGLE NODE`, `INSERTS`, `DELETES`, `ZERO HOP`, `MEMORY GROWTH`, `MC NORMAL` | 3 | +| Добавление и удаление узлов, перенос данных | `NODE ADDED`, `NODE REMOVED`, `NODE REMOVED AFTER CRASH`, `MC NODE REMOVED`, `MIGRATION`, `SCALE UP DOWN` | 2 | +| Запросы во время добавления узла | `NODE ADDED DURING INSERTS`, `NODE ADDED DURING DELETIONS`, `NODE ADDED DURING READS` | 2 | +| Равномерность распределения и число перемещённых записей | `DISTRIBUTION`, `DISTRIBUTION NODE ADDED`, `DISTRIBUTION NODE REMOVED` | 2 | + +Ещё 1 балл можно получить на защите, если поиск владельца ключа занимает _O(log N)_ или меньше. Обоснуйте эту оценку в отчёте. Укажите также время обновления структуры маршрутизации и её расход памяти. Если используете виртуальные узлы, обоснуйте их число _V_ на физический узел и учтите _V_ в оценках сложности. + +К решению приложите отчёт в файле `solution/readme.md` по [общим требованиям](../readme.md#отчёт). Без отчёта автоматические тесты запускаются, но защита не проводится и решение не засчитывается. diff --git a/homework/07-kv-sharding/solution/node.py b/homework/07-kv-sharding/solution/node.py new file mode 100644 index 0000000..06d7096 --- /dev/null +++ b/homework/07-kv-sharding/solution/node.py @@ -0,0 +1,102 @@ +from anysystem import Context, Message, Process +from typing import List + + +class StorageNode(Process): + def __init__(self, node_id: str, nodes: List[str]): + self._id = node_id + self._nodes = set(nodes) + self._data = {} + + def on_local_message(self, msg: Message, ctx: Context): + # Get value for the key. + # Request: + # GET {"key": "some key"} + # Response: + # GET_RESP {"key": "some key", "value": "value for this key"} + # GET_RESP {"key": "some key", "value": null} - if record for this key is not found + if msg.type == 'GET': + key = msg['key'] + value = self._data.get(key) + resp = Message('GET_RESP', { + 'key': key, + 'value': value + }) + ctx.send_local(resp) + + # Store (key, value) record. + # Request: + # PUT {"key": "some key", "value: "some value"} + # Response: + # PUT_RESP {"key": "some key", "value: "some value"} + elif msg.type == 'PUT': + key = msg['key'] + value = msg['value'] + self._data[key] = value + resp = Message('PUT_RESP', { + 'key': key, + 'value': value + }) + ctx.send_local(resp) + + # Delete value for the key. + # Request: + # DELETE {"key": "some key"} + # Response: + # DELETE_RESP {"key": "some key", "value": "some value"} + elif msg.type == 'DELETE': + key = msg['key'] + value = self._data.pop(key, None) + resp = Message('DELETE_RESP', { + 'key': key, + 'value': value + }) + ctx.send_local(resp) + + # Notification that a new node is added to the system. + # Request: + # NODE_ADDED {"id": "node id"} + # Response: + # N/A + elif msg.type == 'NODE_ADDED': + self._nodes.add(msg['id']) + + # Notification that a node is removed from the system. + # Request: + # NODE_REMOVED {"id": "node id"} + # Response: + # N/A + elif msg.type == 'NODE_REMOVED': + self._nodes.remove(msg['id']) + + # Get number of records stored on the node. + # Request: + # COUNT_RECORDS {} + # Response: + # COUNT_RECORDS_RESP {"count": 100} + elif msg.type == 'COUNT_RECORDS': + resp = Message('COUNT_RECORDS_RESP', { + 'count': len(self._data) + }) + ctx.send_local(resp) + + # Get keys of records stored on the node. + # Request: + # DUMP_KEYS {} + # Response: + # DUMP_KEYS_RESP {"keys": ["key1", "key2", ...]} + elif msg.type == 'DUMP_KEYS': + resp = Message('DUMP_KEYS_RESP', { + 'keys': list(self._data.keys()) + }) + ctx.send_local(resp) + + def on_start(self, ctx: Context): + pass + + def on_message(self, msg: Message, sender: str, ctx: Context): + # Implement node-to-node communication using any message types + pass + + def on_timer(self, timer_name: str, ctx: Context): + pass diff --git a/homework/07-kv-sharding/tests/.dockerignore b/homework/07-kv-sharding/tests/.dockerignore new file mode 100644 index 0000000..d291891 --- /dev/null +++ b/homework/07-kv-sharding/tests/.dockerignore @@ -0,0 +1,5 @@ +target +**/target +**/__pycache__ +**/.pytest_cache +*.pyc diff --git a/homework/07-kv-sharding/tests/Cargo.lock b/homework/07-kv-sharding/tests/Cargo.lock new file mode 100644 index 0000000..07588e5 --- /dev/null +++ b/homework/07-kv-sharding/tests/Cargo.lock @@ -0,0 +1,1014 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "aho-corasick" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" +dependencies = [ + "memchr", +] + +[[package]] +name = "anysystem" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e01badd3bc0d042e4399f8f685c394a79a72e7f233b3d3b5eabb3b94734f4dbe" +dependencies = [ + "colored", + "downcast-rs", + "dyn-clone", + "indexmap 2.14.0", + "lazy_static", + "log", + "ordered-float", + "pyo3", + "rand", + "rand_pcg", + "regex", + "rstest", + "serde", + "serde_json", + "simcore", + "sugars", +] + +[[package]] +name = "approx" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0e60b75072ecd4168020818c0107f2857bb6c4e64252d8d3983f6263b40a5c3" +dependencies = [ + "num-traits", +] + +[[package]] +name = "assertables" +version = "3.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d84b91d63c67e6c0b1a74728dad0057c4a6124b74e671ff91b7fe9f8bbd671aa" + +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi", + "libc", + "winapi", +] + +[[package]] +name = "autocfg" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "cfg-if" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2fd1289c04a9ea8cb22300a459a72a385d7c73d3259e2ed7dcb2af674838cfa9" + +[[package]] +name = "clap" +version = "3.2.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ea181bf566f71cb9a5d17a59e1871af638180a18fb0035c92ae62b705207123" +dependencies = [ + "atty", + "bitflags", + "clap_derive", + "clap_lex", + "indexmap 1.9.3", + "once_cell", + "strsim", + "termcolor", + "textwrap", +] + +[[package]] +name = "clap_derive" +version = "3.2.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae6371b8bdc8b7d3959e9cf7b22d4435ef3e79e138688421ec654acf8c81b008" +dependencies = [ + "heck 0.4.1", + "proc-macro-error", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "clap_lex" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2850f2f5a82cbf437dd5af4d49848fbdfc27c157c3d010345776f952765261c5" +dependencies = [ + "os_str_bytes", +] + +[[package]] +name = "colored" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "117725a109d387c937a1533ce01b450cbde6b88abceea8473c4d7a85853cda3c" +dependencies = [ + "lazy_static", + "windows-sys 0.59.0", +] + +[[package]] +name = "decorum" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "281759d3c8a14f5c3f0c49363be56810fcd7f910422f97f2db850c2920fde5cf" +dependencies = [ + "approx", + "num-traits", + "serde", + "serde_derive", +] + +[[package]] +name = "distsys-kv-sharding" +version = "0.1.0" +dependencies = [ + "anysystem", + "assertables", + "clap", + "decorum", + "env_logger", + "indexmap 2.14.0", + "log", + "pyo3", + "rand", + "rand_pcg", + "serde", + "serde_json", + "sugars", +] + +[[package]] +name = "downcast-rs" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75b325c5dbd37f80359721ad39aca5a29fb04c89279657cffdda8736d0c0b9d2" + +[[package]] +name = "dyn-clone" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" + +[[package]] +name = "env_logger" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a12e6657c4c97ebab115a42dcee77225f7f482cdd841cf7088c657a42e9e00e7" +dependencies = [ + "atty", + "humantime", + "log", + "regex", + "termcolor", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "erased-serde" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e004d887f51fcb9fef17317a2f3525c887d8aa3f4f50fed920816a688284a5b7" +dependencies = [ + "serde", + "typeid", +] + +[[package]] +name = "futures" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" + +[[package]] +name = "futures-executor" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" + +[[package]] +name = "futures-macro" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.106", +] + +[[package]] +name = "futures-sink" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" + +[[package]] +name = "futures-task" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" + +[[package]] +name = "futures-timer" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f288b0a4f20f9a56b5d1da57e2227c661b7b16168e2f72365f57b63326e29b24" + +[[package]] +name = "futures-util" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "pin-utils", + "slab", +] + +[[package]] +name = "getrandom" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "335ff9f135e4384c8150d6f27c6daed433577f86b4750418338c01a1a2528592" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "glob" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" + +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "hermit-abi" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +dependencies = [ + "libc", +] + +[[package]] +name = "humantime" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424" + +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown 0.12.3", +] + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown 0.17.1", +] + +[[package]] +name = "itoa" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "libc" +version = "0.2.175" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a82ae493e598baaea5209805c49bbf2ea7de956d50d7da0da1164f9c6d28543" + +[[package]] +name = "log" +version = "0.4.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34080505efa8e45a4b816c349525ebe327ceaa8559756f0356cba97ef3bf7432" + +[[package]] +name = "memchr" +version = "2.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a282da65faaf38286cf3be983213fcf1d2e2a58700e808f83f4ea9a4804bc0" + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "once_cell" +version = "1.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" + +[[package]] +name = "ordered-float" +version = "4.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7bb71e1b3fa6ca1c61f383464aaf2bb0e2f8e772a1f01d486832464de363b951" +dependencies = [ + "num-traits", + "rand", + "serde", +] + +[[package]] +name = "os_str_bytes" +version = "6.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2355d85b9a3786f481747ced0e0ff2ba35213a1f9bd406ed906554d7af805a1" + +[[package]] +name = "pin-project-lite" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "portable-atomic" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "proc-macro-crate" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edce586971a4dfaa28950c6f18ed55e0406c1ab88bbce2c6f6293a7aaba73d35" +dependencies = [ + "toml_edit", +] + +[[package]] +name = "proc-macro-error" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" +dependencies = [ + "proc-macro-error-attr", + "proc-macro2", + "quote", + "syn 1.0.109", + "version_check", +] + +[[package]] +name = "proc-macro-error-attr" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" +dependencies = [ + "proc-macro2", + "quote", + "version_check", +] + +[[package]] +name = "proc-macro2" +version = "1.0.101" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89ae43fd86e4158d6db51ad8e2b80f313af9cc74f5c0e03ccb87de09998732de" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "pyo3" +version = "0.29.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4688ddedf473e32662b9b067670129a8afb8c18e351482c70d62ba4a88171e8b" +dependencies = [ + "libc", + "once_cell", + "portable-atomic", + "pyo3-build-config", + "pyo3-ffi", + "pyo3-macros", +] + +[[package]] +name = "pyo3-build-config" +version = "0.29.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f41027e41b4bd03f6e60f9f417fe24a6341a6bb744edd62b6f709f2a52ea30e9" +dependencies = [ + "target-lexicon", +] + +[[package]] +name = "pyo3-ffi" +version = "0.29.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e591a95526fead067432c3b3a33fc74770b87b1e04e73671090d9c2055a2b327" +dependencies = [ + "libc", + "pyo3-build-config", +] + +[[package]] +name = "pyo3-macros" +version = "0.29.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73225868fc1cd84eef2c3c230ddb91273bf1de46aeb8a4248da76d32a0924a1c" +dependencies = [ + "proc-macro2", + "pyo3-macros-backend", + "quote", + "syn 2.0.106", +] + +[[package]] +name = "pyo3-macros-backend" +version = "0.29.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "571575aa3749fa6216757dd47d2a3e7ef360f329a40f0666a9fbd14889024952" +dependencies = [ + "heck 0.5.0", + "proc-macro2", + "quote", + "syn 2.0.106", +] + +[[package]] +name = "quote" +version = "1.0.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", + "serde", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom", + "serde", +] + +[[package]] +name = "rand_pcg" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59cad018caf63deb318e5a4586d99a24424a364f40f1e5778c29aca23f4fc73e" +dependencies = [ + "rand_core", +] + +[[package]] +name = "regex" +version = "1.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23d7fd106d8c02486a8d64e778353d1cffe08ce79ac2e82f540c86d0facf6912" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b9458fa0bfeeac22b5ca447c63aaf45f28439a709ccd244698632f9aa6394d6" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caf4aa5b0f434c91fe5c7f1ecb6a5ece2130b02ad2a590589dda5146df959001" + +[[package]] +name = "relative-path" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba39f3699c378cd8970968dcbff9c43159ea4cfbd88d43c00b22f2ef10a435d2" + +[[package]] +name = "rstest" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9afd55a67069d6e434a95161415f5beeada95a01c7b815508a82dcb0e1593682" +dependencies = [ + "futures", + "futures-timer", + "rstest_macros", + "rustc_version", +] + +[[package]] +name = "rstest_macros" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4165dfae59a39dd41d8dec720d3cbfbc71f69744efb480a3920f5d4e0cc6798d" +dependencies = [ + "cfg-if", + "glob", + "proc-macro-crate", + "proc-macro2", + "quote", + "regex", + "relative-path", + "rustc_version", + "syn 2.0.106", + "unicode-ident", +] + +[[package]] +name = "rustc-hash" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" + +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + +[[package]] +name = "ryu" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" + +[[package]] +name = "semver" +version = "1.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56e6fa9c48d24d85fb3de5ad847117517440f6beceb7798af16b4a87d616b8d0" + +[[package]] +name = "serde" +version = "1.0.219" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f0e2c6ed6606019b4e29e69dbaba95b11854410e5347d525002456dbbb786b6" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.219" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.106", +] + +[[package]] +name = "serde_json" +version = "1.0.143" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d401abef1d108fbd9cbaebc3e46611f4b1021f714a0597a71f41ee463f5f4a5a" +dependencies = [ + "indexmap 2.14.0", + "itoa", + "memchr", + "ryu", + "serde", +] + +[[package]] +name = "serde_type_name" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92acc4cd6ae20767c54d6cf1a075624e7f4d9e99d7ebc685398ff243144d8714" +dependencies = [ + "serde", +] + +[[package]] +name = "simcore" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a20dfec85e94e569fa5b810b2d992c68b361f31bfba471447427536488c9c292" +dependencies = [ + "colored", + "downcast-rs", + "dyn-clone", + "erased-serde", + "futures", + "log", + "rand", + "rand_pcg", + "rustc-hash", + "serde", + "serde_json", + "serde_type_name", +] + +[[package]] +name = "slab" +version = "0.4.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a2ae44ef20feb57a68b23d846850f861394c2e02dc425a50098ae8c90267589" + +[[package]] +name = "strsim" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" + +[[package]] +name = "sugars" +version = "3.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc0db74f9ee706e039d031a560bd7d110c7022f016051b3d33eeff9583e3e67a" + +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "2.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ede7c438028d4436d71104916910f5bb611972c5cfd7f89b8300a8186e6fada6" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "target-lexicon" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca" + +[[package]] +name = "termcolor" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "textwrap" +version = "0.16.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c13547615a44dc9c452a8a534638acdf07120d4b6847c8178705da06306a3057" + +[[package]] +name = "toml_datetime" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" + +[[package]] +name = "toml_edit" +version = "0.22.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" +dependencies = [ + "indexmap 2.14.0", + "toml_datetime", + "winnow", +] + +[[package]] +name = "typeid" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc7d623258602320d5c55d1bc22793b57daff0ec7efc270ea7d55ce1d5f5471c" + +[[package]] +name = "unicode-ident" +version = "1.0.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f63a545481291138910575129486daeaf8ac54aee4387fe7906919f7830c7d9d" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.61.0", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-link" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45e46c0661abb7180e7b9c281db115305d49ca1709ab8242adf09666d2173c65" + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.61.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e201184e40b2ede64bc2ea34968b28e33622acdbbf37104f0e4a33f7abe657aa" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "winnow" +version = "0.7.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21a0236b59786fed61e2a80582dd500fe61f18b5dca67a4a067d0bc9039339cf" +dependencies = [ + "memchr", +] + +[[package]] +name = "zerocopy" +version = "0.8.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0894878a5fa3edfd6da3f88c4805f4c8558e2b996227a3d864f47fe11e38282c" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88d2b8d9c68ad2b9e4340d7832716a4d21a22a1154777ad56ea55c51a9cf3831" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.106", +] diff --git a/homework/07-kv-sharding/tests/Cargo.toml b/homework/07-kv-sharding/tests/Cargo.toml new file mode 100644 index 0000000..7d71e10 --- /dev/null +++ b/homework/07-kv-sharding/tests/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "distsys-kv-sharding" +version = "0.1.0" +edition = "2021" +resolver = "3" + +[dependencies] +anysystem = "=0.3.0" +indexmap = "2.12" +assertables = "3.2.2" +clap = { version = "3.2.17", features = ["cargo", "derive"] } +decorum = "0.3.1" +env_logger = "0.9.0" +log = "0.4.17" +pyo3 = { version = "=0.29.2", features = ["auto-initialize"] } +rand = "0.8.5" +rand_pcg = "0.3.1" +serde = { version = "1.0.142", features = ["derive"] } +serde_json = "1.0.83" +sugars = "3.0.1" diff --git a/homework/07-kv-sharding/tests/Dockerfile b/homework/07-kv-sharding/tests/Dockerfile new file mode 100644 index 0000000..8857a80 --- /dev/null +++ b/homework/07-kv-sharding/tests/Dockerfile @@ -0,0 +1,19 @@ +# syntax=docker/dockerfile:1 +FROM rust:1.97.1-bookworm@sha256:0e2bcaef56d041a486784e54104a81aebe0da44bd03019bd70bc0401e42e4a97 AS builder + +RUN apt-get update && apt-get install -y build-essential python3-dev + +WORKDIR /tests +COPY . . +RUN --mount=type=cache,id=distsys-course-cargo-registry,target=/usr/local/cargo/registry,sharing=locked \ + --mount=type=cache,id=distsys-course-cargo-target-rust-1-97-1,target=/tests/target,sharing=locked \ + cargo install --locked --path . --root /opt/course + +FROM debian:bookworm-slim + +RUN apt-get update && apt-get install -y python3-dev && rm -rf /var/lib/apt/lists/* + +COPY --from=builder /opt/course/bin/distsys-kv-sharding /usr/local/bin/distsys-kv-sharding + +WORKDIR /solution +ENTRYPOINT ["timeout", "--verbose", "-k", "10", "600", "distsys-kv-sharding", "-i", "node.py"] diff --git a/homework/07-kv-sharding/tests/shared_state_validator.py b/homework/07-kv-sharding/tests/shared_state_validator.py new file mode 100644 index 0000000..bf28c31 --- /dev/null +++ b/homework/07-kv-sharding/tests/shared_state_validator.py @@ -0,0 +1,1938 @@ +"""Static checks for Python state shared between AnySystem processes. + +The checker parses source files only. It never imports or executes a student's +solution. Local imports which can be resolved next to the entrypoint are checked +recursively. Standard-library calls are allowed unless they access known shared +runtime state; other external calls need an explicitly reviewed API. +""" + +from __future__ import annotations + +import ast +import sys +from pathlib import Path +from typing import Callable, Iterable + + +MUTABLE_NODES = ( + ast.List, + ast.ListComp, + ast.Dict, + ast.DictComp, + ast.Set, + ast.SetComp, + ast.GeneratorExp, +) +MUTABLE_CONSTRUCTORS = { + "builtins.bytearray", + "builtins.dict", + "builtins.list", + "builtins.set", + "collections.ChainMap", + "collections.Counter", + "collections.OrderedDict", + "collections.defaultdict", + "collections.deque", + "weakref.WeakKeyDictionary", + "weakref.WeakValueDictionary", +} +IMMUTABLE_SCALAR_CONSTRUCTORS = { + "builtins.bool", + "builtins.bytes", + "builtins.complex", + "builtins.float", + "builtins.int", + "builtins.range", + "builtins.str", +} +KNOWN_IMMUTABLE_CALLS = { + "os.getcwd", + "os.path.abspath", + "os.path.basename", + "os.path.dirname", + "os.path.join", + "os.path.normpath", + "os.path.realpath", + "random.randint", + "random.randrange", +} +TYPE_SUBSCRIPT_NAMES = { + "builtins.dict", + "builtins.frozenset", + "builtins.list", + "builtins.set", + "builtins.tuple", + "builtins.type", + "collections.abc.Mapping", + "collections.abc.Sequence", + "typing.Annotated", + "typing.Callable", + "typing.ClassVar", + "typing.Dict", + "typing.FrozenSet", + "typing.List", + "typing.Literal", + "typing.Mapping", + "typing.Optional", + "typing.Sequence", + "typing.Set", + "typing.Tuple", + "typing.Type", + "typing.Union", +} +TYPE_FACTORIES = { + "collections.namedtuple", + "typing.NewType", + "typing.ParamSpec", + "typing.TypeVar", +} +ENUM_BASE_NAMES = { + "enum.Enum", + "enum.Flag", + "enum.IntEnum", + "enum.IntFlag", + "enum.StrEnum", +} +MUTATOR_METHODS = frozenset( + "__delitem__ __setitem__ add append clear discard extend insert move_to_end " + "pop popitem remove reverse rotate setdefault sort update".split() +) +UNBOUND_MUTATOR_METHODS = {"__delattr__", "__delitem__", "__setattr__", "__setitem__"} +OPERATOR_MUTATORS = frozenset( + f"operator.{name}" + for name in "delitem iadd iand iconcat imatmul imul ior isub ixor setitem".split() +) +STATEFUL_DECORATORS = {"functools.cache", "functools.lru_cache"} +# Reviewed external APIs and a few interpreter operations used by the course. +# Ordinary stdlib functions do not need to be enumerated here: see +# _imported_call_allowed. Shared-state checks still apply to their arguments. +ALLOWED_IMPORTED_CALLS = frozenset( + """ + anysystem.Message anysystem.Message.from_json anysystem.python.anysystem.Message + array.array ast.literal_eval + bisect.bisect bisect.bisect_left bisect.bisect_right bisect.insort builtins.print + collections.defaultdict collections.deque collections.namedtuple copy.copy copy.deepcopy + dataclasses.asdict dataclasses.field dataclasses.fields dataclasses.is_dataclass + functools.cmp_to_key functools.reduce + hashlib.blake2b hashlib.md5 hashlib.sha1 hashlib.sha512 hashlib.sha256 + hashlib.sha3_256 hashlib.shake_256 + heapq.heapify heapq.heappop heapq.heappush heapq.merge importlib.import_module + inspect.signature itertools.count json.dumps json.loads + logging.critical logging.debug logging.error logging.exception logging.info logging.log + logging.warning math.ceil math.log math.log2 math.sqrt + os.getcwd os.getenv os.path.abspath os.path.basename os.path.dirname os.path.join + os.path.normpath os.path.realpath pathlib.Path pickle.dumps pickle.loads + portion.closedopen portion.empty pprint.pprint pydantic.Field + random.Random random.choice random.choices random.randint random.random random.randrange + random.sample random.shuffle random.uniform re.fullmatch sys.intern threading.Lock + types.MappingProxyType time.time typing.cast uuid.uuid4 + """.split() +) +IMPORTED_ARGUMENT_MUTATORS = set( + "bisect.insort bisect.insort_left bisect.insort_right heapq.heapify " + "heapq.heappop heapq.heappush heapq.heappushpop heapq.heapreplace random.shuffle".split() +) +# These APIs expose or change state shared by all simulated processes. This is +# a static check for known violations, not a sandbox for arbitrary Python code. +SHARED_RUNTIME_APIS = frozenset(""" + atexit.register atexit.unregister codecs.register codecs.unregister + copyreg.pickle copyreg.constructor copyreg.add_extension copyreg.remove_extension + csv.register_dialect csv.unregister_dialect csv.field_size_limit + decimal.getcontext decimal.setcontext decimal.localcontext + gc.enable gc.disable gc.set_threshold gc.callbacks.append + locale.setlocale locale.resetlocale + logging.getLogger logging.basicConfig logging.disable logging.setLoggerClass + logging.setLogRecordFactory logging.addLevelName logging.captureWarnings + logging.config.dictConfig logging.config.fileConfig logging.config.listen + os.chdir os.fchdir os.putenv os.unsetenv os.umask os.kill os.killpg + random.seed random.setstate random.getstate + signal.signal signal.set_wakeup_fd signal.alarm signal.setitimer + sys.settrace sys.setprofile sys.setrecursionlimit sys.setswitchinterval + sys.set_asyncgen_hooks sys.set_coroutine_origin_tracking_depth sys.addaudithook + threading.settrace threading.setprofile threading.stack_size + warnings.filterwarnings warnings.simplefilter warnings.resetwarnings +""".split()) + + +def _imported_call_allowed(name: str) -> bool: + if name in SHARED_RUNTIME_APIS: + return False + if name in ALLOWED_IMPORTED_CALLS: + return True + parts = name.split(".") + # Private interpreter objects are not part of the stdlib API. Builtins + # include reflection and execution primitives, checked separately. + return ( + len(parts) >= 2 + and parts[0] in sys.stdlib_module_names + and parts[0] != "builtins" + and all(not part.startswith("_") for part in parts) + ) + + +ALLOWED_MUTABLE_METADATA = {"__all__"} +TRUSTED_STAR_EXPORTS = { + "dataclasses": {"dataclass", "field"}, + "enum": {"Enum", "Flag", "IntEnum", "IntFlag", "StrEnum", "auto"}, + "functools": {"cache", "lru_cache"}, + "types": {"MappingProxyType"}, + "typing": { + *"Annotated Any Callable ClassVar Dict FrozenSet List Literal Mapping NewType " + "Optional ParamSpec Sequence Set Tuple Type TypeVar Union".split(), + }, +} +TRUSTED_BUILTINS = set( + "bool bytearray bytes complex dict float frozenset int list range set str tuple type".split() +) + + +def _resolved_attribute(node: ast.AST) -> tuple[ast.AST, str] | None: + if isinstance(node, ast.Attribute): + return node.value, node.attr + if ( + isinstance(node, ast.Call) + and isinstance(node.func, ast.Name) + and node.func.id == "getattr" + and len(node.args) >= 2 + and isinstance(node.args[1], ast.Constant) + and isinstance(node.args[1].value, str) + ): + return node.args[0], node.args[1].value + return None + + +def _call_name(node: ast.AST) -> str | None: + if isinstance(node, ast.Name): + return node.id + resolved = _resolved_attribute(node) + return resolved[1] if resolved is not None else None + + +def _symbol(node: ast.AST, symbols: dict[str, str]) -> str | None: + if isinstance(node, ast.Name): + return symbols.get(node.id) + resolved = _resolved_attribute(node) + if resolved is not None: + value, attribute = resolved + parent = _symbol(value, symbols) + return f"{parent}.{attribute}" if parent is not None else None + return None + + +def _root_name(node: ast.AST) -> str | None: + while (resolved := _resolved_attribute(node)) is not None: + node = resolved[0] + return node.id if isinstance(node, ast.Name) else None + + +def _assignment_parts(node: ast.AST) -> tuple[ast.AST, list[ast.AST]] | None: + if isinstance(node, ast.Assign): + return node.value, node.targets + if isinstance(node, (ast.AnnAssign, ast.NamedExpr)) and node.value is not None: + return node.value, [node.target] + return None + + +def _call_returns_shared( + node: ast.AST, + is_shared: Callable[[ast.AST], bool], + shared_functions: set[str], + returned_parameters: dict[str, set[int]], +) -> bool: + if not isinstance(node, ast.Call): + return False + name = node.func.id if isinstance(node.func, ast.Name) else None + if name in shared_functions: + return True + return any( + index < len(node.args) and is_shared(node.args[index]) + for index in returned_parameters.get(name or "", set()) + ) + + +def _is_instance_path(node: ast.AST, instance_names: set[str]) -> bool: + while True: + if isinstance(node, ast.Subscript): + node = node.value + elif (resolved := _resolved_attribute(node)) is not None: + node = resolved[0] + else: + break + return isinstance(node, ast.Name) and node.id in instance_names + + +def _static_string(node: ast.AST, names: dict[str, str]) -> str | None: + if isinstance(node, ast.Constant) and isinstance(node.value, str): + return node.value + if isinstance(node, ast.Name): + return names.get(node.id) + if isinstance(node, ast.BinOp) and isinstance(node.op, ast.Add): + left = _static_string(node.left, names) + right = _static_string(node.right, names) + return left + right if left is not None and right is not None else None + if isinstance(node, ast.JoinedStr): + parts: list[str] = [] + for value in node.values: + if isinstance(value, ast.Constant) and isinstance(value.value, str): + parts.append(value.value) + elif isinstance(value, ast.FormattedValue): + part = _static_string(value.value, names) + if part is None: + return None + parts.append(part) + else: + return None + return "".join(parts) + return None + + +def _static_string_values(tree: ast.Module) -> dict[str, str]: + assignments: list[tuple[str, ast.AST]] = [] + for node in ast.walk(tree): + if isinstance(node, ast.Assign): + for target in node.targets: + assignments.extend((name, node.value) for name in _target_names(target)) + elif isinstance(node, (ast.AnnAssign, ast.NamedExpr)) and node.value is not None: + assignments.extend((name, node.value) for name in _target_names(node.target)) + + values: dict[str, str] = {} + ambiguous: set[str] = set() + changed = True + while changed: + changed = False + for name, value_node in assignments: + value = _static_string(value_node, values) + if value is None or name in ambiguous: + continue + if name in values and values[name] != value: + values.pop(name) + ambiguous.add(name) + changed = True + elif name not in values: + values[name] = value + changed = True + return values + + +def _dynamic_import_names(tree: ast.Module) -> tuple[set[str], set[str]]: + modules = {"importlib"} + functions: set[str] = set() + for node in ast.walk(tree): + if isinstance(node, ast.Import): + for alias in node.names: + if alias.name == "importlib": + modules.add(alias.asname or "importlib") + elif isinstance(node, ast.ImportFrom) and node.module == "importlib": + for alias in node.names: + if alias.name == "import_module": + functions.add(alias.asname or alias.name) + assignments = [node for node in ast.walk(tree) if isinstance(node, ast.Assign)] + changed = True + while changed: + changed = False + for assignment in assignments: + names = set().union(*(_target_names(target) for target in assignment.targets)) + if isinstance(assignment.value, ast.Name): + if assignment.value.id in modules: + before = len(modules) + modules.update(names) + changed |= len(modules) != before + if assignment.value.id in functions: + before = len(functions) + functions.update(names) + changed |= len(functions) != before + elif ( + isinstance(assignment.value, ast.Attribute) + and assignment.value.attr == "import_module" + and isinstance(assignment.value.value, ast.Name) + and assignment.value.value.id in modules + ): + before = len(functions) + functions.update(names) + changed |= len(functions) != before + return modules, functions + + +def _is_dynamic_import_call( + node: ast.Call, importlib_modules: set[str], import_module_functions: set[str] +) -> bool: + return ( + isinstance(node.func, ast.Name) + and (node.func.id == "__import__" or node.func.id in import_module_functions) + ) or ( + (resolved := _resolved_attribute(node.func)) is not None + and resolved[1] == "import_module" + and isinstance(resolved[0], ast.Name) + and resolved[0].id in importlib_modules + ) + + +def _class_has_mutation_escape(class_node: ast.ClassDef) -> bool: + for statement in class_node.body: + if not isinstance(statement, (ast.FunctionDef, ast.AsyncFunctionDef)): + continue + positional = [*statement.args.posonlyargs, *statement.args.args] + if not positional: + continue + instance_names = {positional[0].arg} + for node in _function_nodes(statement): + if isinstance(node, (ast.Assign, ast.AnnAssign, ast.AugAssign, ast.Delete)): + targets = ( + node.targets + if isinstance(node, (ast.Assign, ast.Delete)) + else [node.target] + ) + if any(_is_instance_path(target, instance_names) for target in targets): + return True + if not isinstance(node, ast.Call): + continue + if ( + isinstance(node.func, ast.Name) + and node.func.id in {"delattr", "setattr"} + and node.args + and _is_instance_path(node.args[0], instance_names) + ): + return True + if ( + (resolved := _resolved_attribute(node.func)) is not None + and resolved[1] in UNBOUND_MUTATOR_METHODS + and node.args + and _is_instance_path(node.args[0], instance_names) + ): + return True + if ( + (resolved := _resolved_attribute(node.func)) is not None + and resolved[1] in MUTATOR_METHODS + and _is_instance_path(resolved[0], instance_names) + ): + return True + return False + + +def _target_names(node: ast.AST) -> set[str]: + if isinstance(node, ast.Name): + return {node.id} + if isinstance(node, (ast.Tuple, ast.List)): + result: set[str] = set() + for element in node.elts: + result.update(_target_names(element)) + return result + return set() + + +def _dict_is_deeply_immutable( + node: ast.Dict, safe_names: set[str], symbols: dict[str, str] +) -> bool: + return all( + key is not None + and _is_immutable_expr(key, safe_names, symbols) + and _is_immutable_expr(value, safe_names, symbols) + for key, value in zip(node.keys, node.values) + ) + + +def _sequence_is_deeply_immutable( + node: ast.List | ast.Set | ast.Tuple, + safe_names: set[str], + symbols: dict[str, str], +) -> bool: + return all(_is_immutable_expr(element, safe_names, symbols) for element in node.elts) + + +def _is_immutable_expr( + node: ast.AST | None, safe_names: set[str], symbols: dict[str, str] +) -> bool: + if node is None or isinstance(node, ast.Constant): + return True + if isinstance(node, ast.Name): + return node.id in safe_names + if isinstance(node, ast.Attribute): + root = _root_name(node) + if root is not None and root in symbols and symbols[root] != f"builtins.{root}": + return node.attr.isupper() + return _is_immutable_expr(node.value, safe_names, symbols) + if isinstance(node, ast.Subscript): + return _symbol(node.value, symbols) in TYPE_SUBSCRIPT_NAMES + if isinstance(node, ast.Tuple): + return _sequence_is_deeply_immutable(node, safe_names, symbols) + if isinstance(node, ast.UnaryOp): + return _is_immutable_expr(node.operand, safe_names, symbols) + if isinstance(node, ast.BinOp): + return _is_immutable_expr(node.left, safe_names, symbols) and _is_immutable_expr( + node.right, safe_names, symbols + ) + if isinstance(node, ast.BoolOp): + return all(_is_immutable_expr(value, safe_names, symbols) for value in node.values) + if isinstance(node, ast.Compare): + return _is_immutable_expr(node.left, safe_names, symbols) and all( + _is_immutable_expr(value, safe_names, symbols) for value in node.comparators + ) + if isinstance(node, ast.IfExp): + return _is_immutable_expr(node.body, safe_names, symbols) and _is_immutable_expr( + node.orelse, safe_names, symbols + ) + if isinstance(node, ast.JoinedStr): + return all( + not isinstance(value, ast.FormattedValue) + or _is_immutable_expr(value.value, safe_names, symbols) + for value in node.values + ) + if isinstance(node, ast.Lambda): + return True + if isinstance(node, ast.Call): + name = _symbol(node.func, symbols) + if name in TYPE_FACTORIES: + return True + local_name = _call_name(node.func) + if local_name is not None and f"frozen-dataclass:{local_name}" in safe_names: + return all( + _is_immutable_expr(arg, safe_names, symbols) for arg in node.args + ) and all( + _is_immutable_expr(keyword.value, safe_names, symbols) + for keyword in node.keywords + ) + if name in IMMUTABLE_SCALAR_CONSTRUCTORS: + return all(_is_immutable_expr(arg, safe_names, symbols) for arg in node.args) + if name in KNOWN_IMMUTABLE_CALLS or ( + name is not None + and name.split(".")[0] in {"math", "cmath"} + and len(name.split(".")) == 2 + and not name.split(".")[1].startswith("_") + ): + return True + if name in {"builtins.tuple", "builtins.frozenset"}: + if not node.args: + return True + if len(node.args) != 1 or node.keywords: + return False + value = node.args[0] + return isinstance(value, (ast.List, ast.Set, ast.Tuple)) and ( + _sequence_is_deeply_immutable(value, safe_names, symbols) + ) + if name == "types.MappingProxyType": + return ( + len(node.args) == 1 + and not node.keywords + and isinstance(node.args[0], ast.Dict) + and _dict_is_deeply_immutable(node.args[0], safe_names, symbols) + ) + return False + + +def _is_mutable_expr( + node: ast.AST | None, safe_names: set[str], symbols: dict[str, str] +) -> bool: + if node is None: + return False + if isinstance(node, MUTABLE_NODES): + return True + if isinstance(node, ast.Call) and _symbol(node.func, symbols) in MUTABLE_CONSTRUCTORS: + return True + return not _is_immutable_expr(node, safe_names, symbols) + + +def _scope_statements(body: Iterable[ast.stmt]) -> Iterable[ast.stmt]: + for statement in body: + yield statement + nested: list[list[ast.stmt]] = [] + if isinstance(statement, (ast.If, ast.For, ast.AsyncFor, ast.While)): + nested.extend([statement.body, statement.orelse]) + elif isinstance(statement, (ast.With, ast.AsyncWith)): + nested.append(statement.body) + elif isinstance(statement, (ast.Try, ast.TryStar)): + nested.extend([statement.body, statement.orelse, statement.finalbody]) + nested.extend(handler.body for handler in statement.handlers) + elif isinstance(statement, ast.Match): + nested.extend(case.body for case in statement.cases) + for statements in nested: + yield from _scope_statements(statements) + + +def _function_nodes(function: ast.FunctionDef | ast.AsyncFunctionDef) -> list[ast.AST]: + result: list[ast.AST] = [] + stack: list[ast.AST] = list(reversed(function.body)) + while stack: + node = stack.pop() + result.append(node) + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef, ast.Lambda)): + continue + stack.extend(reversed(list(ast.iter_child_nodes(node)))) + return result + + +def _shared_class_nodes( + body: Iterable[ast.stmt], prefix: str = "" +) -> Iterable[tuple[str, ast.ClassDef]]: + for statement in _scope_statements(body): + if not isinstance(statement, ast.ClassDef): + continue + qualified_name = f"{prefix}.{statement.name}" if prefix else statement.name + yield qualified_name, statement + yield from _shared_class_nodes(statement.body, qualified_name) + + +def _is_classvar_annotation(node: ast.AST, symbols: dict[str, str]) -> bool: + value = node.value if isinstance(node, ast.Subscript) else node + return _symbol(value, symbols) == "typing.ClassVar" + + +class _FileChecker: + def __init__( + self, + tree: ast.Module, + path: Path, + _target_classes: set[str], + root: Path | None = None, + ): + self.tree = tree + self.path = path + self.root = (root or path.parent).resolve() + self.violations: list[tuple[int, int, str]] = [] + self.safe_names: set[str] = { + "False", + "None", + "True", + "bool", + "bytes", + "complex", + "float", + "frozenset", + "int", + "range", + "str", + "tuple", + } + self.symbols = {name: f"builtins.{name}" for name in TRUSTED_BUILTINS} + self.import_names: set[str] = set() + self.definition_names: set[str] = set() + self.module_binding_names: set[str] = set() + self.module_mutable_names: set[str] = set() + self.classes: dict[str, ast.ClassDef] = {} + self.shared_classes = list(_shared_class_nodes(tree.body)) + self.functions: dict[str, ast.FunctionDef | ast.AsyncFunctionDef] = {} + self.lambdas: dict[str, ast.Lambda] = {} + self.shared_closure_factories: set[str] = set() + self.shared_returning_functions: set[str] = set() + self.returned_parameters: dict[str, set[int]] = {} + self.stateful_decorator_factories: set[str] = set() + self.shared_instance_attributes: dict[str, set[str]] = {} + + def add(self, node: ast.AST, message: str) -> None: + self.violations.append( + (getattr(node, "lineno", 1), getattr(node, "col_offset", 0) + 1, message) + ) + + def collect_symbols(self) -> None: + shadowed_names: set[str] = set() + for statement in _scope_statements(self.tree.body): + if isinstance(statement, ast.Import): + for alias in statement.names: + name = alias.asname or alias.name.split(".")[0] + self.import_names.add(name) + self.module_binding_names.add(name) + if _candidate_module(self.root, alias.name.split(".")) is None: + self.symbols[name] = alias.name if alias.asname else name + else: + self.symbols.pop(name, None) + elif isinstance(statement, ast.ImportFrom): + local_module = ( + statement.level > 0 + or statement.module is None + or _candidate_module(self.root, statement.module.split(".")) + is not None + ) + for alias in statement.names: + if alias.name == "*": + if statement.module is not None and not local_module: + for export in TRUSTED_STAR_EXPORTS.get( + statement.module, set() + ): + self.import_names.add(export) + self.module_binding_names.add(export) + self.symbols[export] = f"{statement.module}.{export}" + if statement.module == "typing": + self.safe_names.add(export) + else: + name = alias.asname or alias.name + self.import_names.add(name) + self.module_binding_names.add(name) + if statement.module is not None and not local_module: + self.symbols[name] = f"{statement.module}.{alias.name}" + else: + self.symbols.pop(name, None) + if ( + name.isupper() + or statement.module in {"collections.abc", "typing"} + or ( + statement.module == "math" + and alias.name in {"e", "inf", "nan", "pi", "tau"} + ) + ): + self.safe_names.add(name) + elif isinstance(statement, (ast.FunctionDef, ast.AsyncFunctionDef)): + self.definition_names.add(statement.name) + self.module_binding_names.add(statement.name) + shadowed_names.add(statement.name) + self.functions[statement.name] = statement + elif isinstance(statement, ast.ClassDef): + self.definition_names.add(statement.name) + self.module_binding_names.add(statement.name) + shadowed_names.add(statement.name) + self.classes[statement.name] = statement + elif isinstance(statement, ast.Assign): + for target in statement.targets: + names = _target_names(target) + self.module_binding_names.update(names) + shadowed_names.update(names) + if isinstance(statement.value, ast.Lambda): + for name in names: + self.lambdas[name] = statement.value + elif isinstance(statement, (ast.AnnAssign, ast.AugAssign)): + names = _target_names(statement.target) + self.module_binding_names.update(names) + shadowed_names.update(names) + for name in shadowed_names: + self.symbols.pop(name, None) + self.safe_names.difference_update(shadowed_names) + self.safe_names.update(self.definition_names) + for _, class_node in self.shared_classes: + frozen = any( + isinstance(decorator, ast.Call) + and _symbol(decorator.func, self.symbols) == "dataclasses.dataclass" + and any( + keyword.arg == "frozen" + and isinstance(keyword.value, ast.Constant) + and keyword.value.value is True + for keyword in decorator.keywords + ) + for decorator in class_node.decorator_list + ) + immutable_defaults = all( + not isinstance(statement, ast.AnnAssign) + or statement.value is None + or _is_immutable_expr(statement.value, self.safe_names, self.symbols) + for statement in class_node.body + ) + if frozen and immutable_defaults and not _class_has_mutation_escape(class_node): + self.safe_names.add(f"frozen-dataclass:{class_node.name}") + + def mark_shared_closure_factories(self) -> None: + def consider(value: ast.AST) -> None: + if not isinstance(value, ast.Call): + return + name = _call_name(value.func) + function = self.functions.get(name or "") + if function is not None and any( + isinstance(node, ast.Nonlocal) for node in ast.walk(function) + ): + self.shared_closure_factories.add(function.name) + + scopes = [self.tree.body] + scopes.extend(class_node.body for class_node in self.classes.values()) + for body in scopes: + for statement in _scope_statements(body): + if isinstance(statement, ast.Assign): + consider(statement.value) + elif isinstance(statement, ast.AnnAssign) and statement.value is not None: + consider(statement.value) + + for node in ast.walk(self.tree): + if not isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + continue + for decorator in node.decorator_list: + name = _call_name(decorator.func if isinstance(decorator, ast.Call) else decorator) + function = self.functions.get(name or "") + if function is not None and any( + isinstance(child, ast.Nonlocal) for child in ast.walk(function) + ): + self.shared_closure_factories.add(function.name) + + def collect_function_summaries(self) -> None: + """Summarize simple wrappers without executing student code.""" + + callables: dict[ + str, ast.FunctionDef | ast.AsyncFunctionDef | ast.Lambda + ] = {**self.functions, **self.lambdas} + changed = True + while changed: + changed = False + for function_name, function in callables.items(): + positional = [*function.args.posonlyargs, *function.args.args] + local_names = { + argument.arg + for argument in [*positional, *function.args.kwonlyargs] + } + if function.args.vararg is not None: + local_names.add(function.args.vararg.arg) + if function.args.kwarg is not None: + local_names.add(function.args.kwarg.arg) + + if isinstance(function, ast.Lambda): + nodes = [function.body] + return_values = [function.body] + else: + nodes = _function_nodes(function) + return_values = [ + node.value + for node in nodes + if isinstance(node, ast.Return) and node.value is not None + ] + assignments = [ + node + for node in nodes + if _assignment_parts(node) is not None + ] + for assignment in assignments: + _, targets = _assignment_parts(assignment) or (assignment, []) + for target in targets: + local_names.update(_target_names(target)) + + shared_aliases: set[str] = set() + parameter_aliases = { + argument.arg: {index} for index, argument in enumerate(positional) + } + + def returned_parameter_indexes(node: ast.AST) -> set[int]: + if isinstance(node, ast.Name): + return parameter_aliases.get(node.id, set()) + resolved = _resolved_attribute(node) + if resolved is not None: + return returned_parameter_indexes(resolved[0]) + if isinstance(node, ast.Subscript): + return returned_parameter_indexes(node.value) + if isinstance(node, ast.Call): + name = node.func.id if isinstance(node.func, ast.Name) else None + return set().union( + *( + returned_parameter_indexes(node.args[index]) + for index in self.returned_parameters.get( + name or "", set() + ) + if index < len(node.args) + ), + set(), + ) + return set() + + def returns_shared(node: ast.AST) -> bool: + if _is_immutable_expr(node, self.safe_names, self.symbols): + return False + if isinstance(node, ast.Name): + return ( + node.id in self.module_binding_names + and node.id not in local_names + ) or node.id in shared_aliases + resolved = _resolved_attribute(node) + if resolved is not None: + return returns_shared(resolved[0]) + if isinstance(node, ast.Subscript): + return returns_shared(node.value) + return _call_returns_shared( + node, + returns_shared, + self.shared_returning_functions, + self.returned_parameters, + ) or ( + isinstance(node, ast.Call) + and _symbol(node.func, self.symbols) + == "importlib.import_module" + ) + + aliases_changed = True + while aliases_changed: + aliases_changed = False + for assignment in assignments: + value, targets = _assignment_parts(assignment) or (assignment, []) + indexes = returned_parameter_indexes(value) + shared = returns_shared(value) + before = ( + len(shared_aliases), + sum(map(len, parameter_aliases.values())), + ) + for target in targets: + for name in _target_names(target): + if shared: + shared_aliases.add(name) + if indexes: + parameter_aliases.setdefault(name, set()).update(indexes) + aliases_changed |= before != ( + len(shared_aliases), + sum(map(len, parameter_aliases.values())), + ) + + indexes = set().union( + *(returned_parameter_indexes(value) for value in return_values), set() + ) + known_indexes = self.returned_parameters.setdefault(function_name, set()) + before = len(known_indexes) + known_indexes.update(indexes) + changed |= len(known_indexes) != before + if any(returns_shared(value) for value in return_values): + if function_name not in self.shared_returning_functions: + self.shared_returning_functions.add(function_name) + changed = True + for value in return_values: + if isinstance(value, ast.Call): + called = _symbol(value.func, self.symbols) + local_called = ( + value.func.id + if isinstance(value.func, ast.Name) + else None + ) + if ( + called in STATEFUL_DECORATORS + or local_called in self.stateful_decorator_factories + ) and function_name not in self.stateful_decorator_factories: + self.stateful_decorator_factories.add(function_name) + changed = True + + def decorator_is_stateful( + self, decorator: ast.AST, symbols: dict[str, str] + ) -> bool: + expression = decorator.func if isinstance(decorator, ast.Call) else decorator + return _symbol(expression, symbols) in STATEFUL_DECORATORS or ( + isinstance(expression, ast.Name) + and expression.id in self.stateful_decorator_factories + ) + + def check_module_bindings(self) -> None: + for statement in _scope_statements(self.tree.body): + value: ast.AST | None = None + targets: set[str] = set() + if isinstance(statement, ast.Assign): + value = statement.value + for target in statement.targets: + targets.update(_target_names(target)) + elif isinstance(statement, ast.AnnAssign): + value = statement.value + targets.update(_target_names(statement.target)) + elif isinstance(statement, ast.AugAssign): + value = statement.value + targets.update(_target_names(statement.target)) + else: + continue + + if value is None: + continue + if _is_immutable_expr(value, self.safe_names, self.symbols): + self.safe_names.update(targets) + continue + for name in sorted(targets - ALLOWED_MUTABLE_METADATA): + self.module_mutable_names.add(name) + self.add( + statement, + f"module-level value '{name}' is mutable or stateful and can be shared between processes", + ) + + def check_class_bindings(self) -> None: + for class_name, class_node in self.shared_classes: + class_safe_names = set(self.safe_names) + is_dataclass = any( + _symbol( + decorator.func if isinstance(decorator, ast.Call) else decorator, + self.symbols, + ) + == "dataclasses.dataclass" + for decorator in class_node.decorator_list + ) + is_pydantic_model = any( + ( + isinstance(base, ast.Name) + and _symbol(base, self.symbols) + in {"pydantic.BaseModel", "pydantic.main.BaseModel"} + ) + or ( + isinstance(base, ast.Attribute) + and _symbol(base, self.symbols) + in {"pydantic.BaseModel", "pydantic.main.BaseModel"} + ) + for base in class_node.bases + ) + is_enum = any( + ( + isinstance(base, ast.Name) + and _symbol(base, self.symbols) in ENUM_BASE_NAMES + ) + or ( + isinstance(base, ast.Attribute) + and _symbol(base, self.symbols) in ENUM_BASE_NAMES + ) + for base in class_node.bases + ) + class_symbols = dict(self.symbols) + class_import_symbols: dict[str, str] = {} + class_shadowed_names: set[str] = set() + for class_statement in _scope_statements(class_node.body): + names: set[str] = set() + if isinstance( + class_statement, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef) + ): + names.add(class_statement.name) + elif isinstance(class_statement, ast.Assign): + for target in class_statement.targets: + names.update(_target_names(target)) + elif isinstance(class_statement, (ast.AnnAssign, ast.AugAssign)): + names.update(_target_names(class_statement.target)) + elif isinstance(class_statement, ast.Import): + for alias in class_statement.names: + name = alias.asname or alias.name.split(".")[0] + if _candidate_module(self.root, alias.name.split(".")) is None: + class_import_symbols[name] = ( + alias.name if alias.asname else name + ) + elif isinstance(class_statement, ast.ImportFrom): + local_module = ( + class_statement.level > 0 + or class_statement.module is None + or _candidate_module( + self.root, class_statement.module.split(".") + ) + is not None + ) + if class_statement.module is not None and not local_module: + for alias in class_statement.names: + if alias.name == "*": + for export in TRUSTED_STAR_EXPORTS.get( + class_statement.module, set() + ): + class_import_symbols[export] = ( + f"{class_statement.module}.{export}" + ) + else: + name = alias.asname or alias.name + class_import_symbols[name] = ( + f"{class_statement.module}.{alias.name}" + ) + class_shadowed_names.update(names) + class_symbols.update( + (name, symbol) + for name, symbol in class_import_symbols.items() + if name not in class_shadowed_names + ) + for name in class_shadowed_names: + class_symbols.pop(name, None) + class_safe_names.discard(name) + for class_statement in class_node.body: + if not isinstance( + class_statement, (ast.FunctionDef, ast.AsyncFunctionDef) + ): + continue + for decorator in class_statement.decorator_list: + expression = decorator.func if isinstance(decorator, ast.Call) else decorator + name = _symbol(expression, class_symbols) + if self.decorator_is_stateful(decorator, class_symbols): + self.add( + decorator, + f"'{(name or _call_name(expression) or 'decorator').rsplit('.', 1)[-1]}' keeps a cache shared between process instances", + ) + for statement in _scope_statements(class_node.body): + def targets_class_namespace(target: ast.AST) -> bool: + if isinstance(target, ast.Subscript): + return ( + isinstance(target.value, ast.Call) + and isinstance(target.value.func, ast.Name) + and target.value.func.id in {"globals", "locals"} + ) + if isinstance(target, (ast.Tuple, ast.List)): + return any(targets_class_namespace(item) for item in target.elts) + return False + + if isinstance(statement, ast.Assign) and any( + targets_class_namespace(target) for target in statement.targets + ): + self.add(statement, "assignment mutates class-level shared state") + elif isinstance(statement, (ast.AnnAssign, ast.AugAssign)) and ( + targets_class_namespace(statement.target) + ): + self.add(statement, "assignment mutates class-level shared state") + elif isinstance(statement, ast.Delete) and any( + targets_class_namespace(target) for target in statement.targets + ): + self.add(statement, "deletion mutates class-level shared state") + elif isinstance(statement, ast.Expr) and isinstance( + statement.value, ast.Call + ): + call = statement.value + if ( + isinstance(call.func, ast.Attribute) + and call.func.attr in MUTATOR_METHODS | {"__setitem__"} + and isinstance(call.func.value, ast.Call) + and isinstance(call.func.value.func, ast.Name) + and call.func.value.func.id in {"globals", "locals"} + ): + self.add(call, "call mutates class-level shared state") + + value: ast.AST | None = None + targets: set[str] = set() + if isinstance(statement, ast.Assign): + value = statement.value + for target in statement.targets: + targets.update(_target_names(target)) + elif isinstance(statement, ast.AnnAssign): + value = statement.value + targets.update(_target_names(statement.target)) + else: + continue + if value is None: + continue + if _is_immutable_expr(value, class_safe_names, class_symbols): + class_safe_names.update(targets) + for name in targets: + class_symbols.pop(name, None) + continue + if ( + is_enum + and isinstance(value, ast.Call) + and _symbol(value.func, class_symbols) == "enum.auto" + and not value.args + and not value.keywords + ): + class_safe_names.update(targets) + continue + if ( + is_dataclass + and isinstance(statement, ast.AnnAssign) + and not _is_classvar_annotation(statement.annotation, class_symbols) + and isinstance(value, ast.Call) + and _symbol(value.func, class_symbols) == "dataclasses.field" + and not value.args + and all( + keyword.arg != "default" + or not _is_mutable_expr( + keyword.value, class_safe_names, class_symbols + ) + for keyword in value.keywords + ) + ): + continue + if ( + is_pydantic_model + and isinstance(statement, ast.AnnAssign) + and not _is_classvar_annotation(statement.annotation, class_symbols) + and isinstance(value, ast.Call) + and _symbol(value.func, class_symbols) == "pydantic.Field" + and not value.args + and all( + keyword.arg != "default" + or not _is_mutable_expr( + keyword.value, class_safe_names, class_symbols + ) + for keyword in value.keywords + ) + ): + continue + for name in sorted(targets): + self.add( + statement, + f"class attribute '{class_name}.{name}' is mutable or stateful and is shared by all process instances", + ) + + def check_module_mutations(self) -> None: + statements = list(_scope_statements(self.tree.body)) + class_aliases = set(self.classes) + shared_aliases: set[str] = set() + global_objects = self.module_binding_names + + def is_class_ref(node: ast.AST) -> bool: + return isinstance(node, ast.Name) and node.id in class_aliases + + def is_shared_ref(node: ast.AST) -> bool: + if isinstance(node, ast.Name): + return node.id in global_objects or node.id in shared_aliases + if isinstance(node, ast.Attribute): + return ( + is_class_ref(node.value) + or is_shared_ref(node.value) + or ( + isinstance(node.value, ast.Name) + and node.value.id in global_objects + ) + ) + if isinstance(node, ast.Subscript): + return is_shared_ref(node.value) + if ( + isinstance(node, ast.Call) + and isinstance(node.func, ast.Name) + and node.func.id == "getattr" + and node.args + ): + return is_class_ref(node.args[0]) or is_shared_ref(node.args[0]) + if ( + isinstance(node, ast.Call) + and isinstance(node.func, ast.Name) + and node.func.id == "eval" + and node.args + and isinstance(node.args[0], ast.Constant) + and isinstance(node.args[0].value, str) + ): + try: + expression = ast.parse(node.args[0].value, mode="eval").body + except SyntaxError: + return False + return is_shared_ref(expression) + return ( + isinstance(node, ast.Call) + and isinstance(node.func, ast.Name) + and node.func.id == "vars" + and len(node.args) == 1 + and (is_class_ref(node.args[0]) or is_shared_ref(node.args[0])) + ) or ( + isinstance(node, ast.Call) + and isinstance(node.func, ast.Name) + and node.func.id in {"globals", "locals"} + ) + + assignments = [ + statement + for statement in statements + if isinstance(statement, (ast.Assign, ast.AnnAssign, ast.NamedExpr)) + ] + changed = True + while changed: + changed = False + for assignment in assignments: + value, targets = _assignment_parts(assignment) or (assignment, []) + names = set().union(*map(_target_names, targets), set()) + before = (len(class_aliases), len(shared_aliases)) + if is_class_ref(value): + class_aliases.update(names) + if is_shared_ref(value): + shared_aliases.update(names) + if before != (len(class_aliases), len(shared_aliases)): + changed = True + + def target_is_shared(target: ast.AST) -> bool: + if isinstance(target, ast.Attribute): + return ( + is_class_ref(target.value) + or is_shared_ref(target.value) + or ( + isinstance(target.value, ast.Name) + and target.value.id in global_objects + ) + ) + if isinstance(target, ast.Subscript): + return is_shared_ref(target.value) + if isinstance(target, (ast.Tuple, ast.List)): + return any(target_is_shared(element) for element in target.elts) + return False + + for statement in statements: + if isinstance(statement, ast.Assign): + if any(target_is_shared(target) for target in statement.targets): + self.add(statement, "assignment creates or mutates module-level shared state") + elif isinstance(statement, (ast.AnnAssign, ast.AugAssign)): + if target_is_shared(statement.target): + self.add(statement, "assignment creates or mutates module-level shared state") + elif isinstance(statement, ast.Delete): + if any(target_is_shared(target) for target in statement.targets): + self.add(statement, "deletion mutates module-level shared state") + elif isinstance(statement, ast.Expr) and isinstance(statement.value, ast.Call): + call = statement.value + if isinstance(call.func, ast.Name) and call.func.id in { + "exec", + "globals", + "locals", + }: + self.add(call, f"'{call.func.id}' can access or modify module-level shared state") + elif ( + isinstance(call.func, ast.Name) + and call.func.id == "eval" + and is_shared_ref(call) + ): + self.add(call, "'eval' accesses module-level shared state") + elif ( + isinstance(call.func, ast.Name) + and call.func.id in {"setattr", "delattr"} + and call.args + and ( + is_class_ref(call.args[0]) + or is_shared_ref(call.args[0]) + or ( + isinstance(call.args[0], ast.Name) + and call.args[0].id in global_objects + ) + ) + ): + self.add(call, f"'{call.func.id}' mutates module-level shared state") + elif ( + _symbol(call.func, self.symbols) in OPERATOR_MUTATORS + and call.args + and is_shared_ref(call.args[0]) + ): + self.add(call, "operator call mutates module-level shared state") + elif ( + (resolved := _resolved_attribute(call.func)) is not None + and resolved[1] in UNBOUND_MUTATOR_METHODS + and call.args + and ( + is_class_ref(call.args[0]) + or is_shared_ref(call.args[0]) + or ( + isinstance(call.args[0], ast.Name) + and call.args[0].id in global_objects + ) + ) + ): + self.add(call, f"'{resolved[1]}' mutates module-level shared state") + elif ( + (resolved := _resolved_attribute(call.func)) is not None + and resolved[1] in MUTATOR_METHODS + and _symbol(call.func, self.symbols) != "operator.add" + and is_shared_ref(resolved[0]) + ): + if not ( + _symbol(resolved[0], self.symbols) == "sys.path" + and resolved[1] in {"append", "insert"} + ): + self.add( + call, + f"'{resolved[1]}' mutates module-level shared state", + ) + elif ( + (canonical := _symbol(call.func, self.symbols)) is not None + and (_root_name(call.func) or "") in self.import_names + and not _imported_call_allowed(canonical) + and canonical not in STATEFUL_DECORATORS + and canonical not in TYPE_FACTORIES + and canonical not in MUTABLE_CONSTRUCTORS + and canonical not in IMMUTABLE_SCALAR_CONSTRUCTORS + ): + self.add( + call, + f"call to imported API '{canonical}' is not known to be free of shared module-state mutation", + ) + + def check_defaults(self) -> None: + for node in ast.walk(self.tree): + if not isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.Lambda)): + continue + defaults = list(node.args.defaults) + defaults.extend(default for default in node.args.kw_defaults if default is not None) + for default in defaults: + if _is_mutable_expr(default, self.safe_names, self.symbols): + self.add(default, "mutable default argument is shared between function calls") + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + for decorator in node.decorator_list: + expression = decorator.func if isinstance(decorator, ast.Call) else decorator + name = _symbol(expression, self.symbols) + if self.decorator_is_stateful(decorator, self.symbols): + self.add( + decorator, + f"'{(name or _call_name(expression) or 'decorator').rsplit('.', 1)[-1]}' keeps a cache shared between process instances", + ) + + def check_dynamic_imports(self) -> None: + string_values = _static_string_values(self.tree) + importlib_modules, import_module_functions = _dynamic_import_names(self.tree) + for node in ast.walk(self.tree): + if ( + isinstance(node, ast.Call) + and node.args + and _is_dynamic_import_call( + node, importlib_modules, import_module_functions + ) + and _static_string(node.args[0], string_values) is None + ): + self.add( + node, + "dynamic import name cannot be resolved statically; shared state in the imported module cannot be checked", + ) + + def collect_shared_instance_attributes(self) -> None: + for class_name, class_node in self.classes.items(): + attributes = self.shared_instance_attributes.setdefault(class_name, set()) + functions = [ + statement + for statement in class_node.body + if isinstance(statement, (ast.FunctionDef, ast.AsyncFunctionDef)) + ] + changed = True + while changed: + changed = False + for function in functions: + positional = [*function.args.posonlyargs, *function.args.args] + if not positional: + continue + instance_names = {positional[0].arg} + nodes = _function_nodes(function) + assignments = [ + node + for node in nodes + if isinstance(node, (ast.Assign, ast.AnnAssign, ast.NamedExpr)) + ] + local_names = { + argument.arg + for argument in [*positional, *function.args.kwonlyargs] + } + local_import_names: set[str] = set() + for node in nodes: + if isinstance(node, ast.Assign): + for target in node.targets: + local_names.update(_target_names(target)) + elif isinstance(node, (ast.AnnAssign, ast.NamedExpr)): + local_names.update(_target_names(node.target)) + elif isinstance(node, (ast.Import, ast.ImportFrom)): + names = { + alias.asname or alias.name.split(".")[0] + for alias in node.names + if alias.name != "*" + } + local_names.update(names) + local_import_names.update(names) + + shared_aliases: set[str] = set() + + def is_instance(node: ast.AST) -> bool: + return isinstance(node, ast.Name) and node.id in instance_names + + def is_shared(node: ast.AST) -> bool: + if isinstance(node, ast.Name): + return ( + node.id in self.module_binding_names + and node.id not in local_names + ) or node.id in shared_aliases | local_import_names + if (resolved := _resolved_attribute(node)) is not None: + return ( + is_instance(resolved[0]) and resolved[1] in attributes + ) or is_shared(resolved[0]) + if isinstance(node, ast.Subscript): + return is_shared(node.value) + return _call_returns_shared( + node, + is_shared, + self.shared_returning_functions, + self.returned_parameters, + ) or ( + isinstance(node, ast.Call) + and _symbol(node.func, self.symbols) + == "importlib.import_module" + ) + + aliases_changed = True + while aliases_changed: + aliases_changed = False + for assignment in assignments: + value, targets = _assignment_parts(assignment) or (assignment, []) + if is_instance(value): + names = set().union(*(_target_names(t) for t in targets)) + before = len(instance_names) + instance_names.update(names) + aliases_changed |= len(instance_names) != before + if not is_shared(value): + continue + for target in targets: + names = _target_names(target) + before = len(shared_aliases) + shared_aliases.update(names) + aliases_changed |= len(shared_aliases) != before + if ( + isinstance(target, ast.Attribute) + and is_instance(target.value) + and target.attr not in attributes + ): + attributes.add(target.attr) + changed = True + + def check_function( + self, + function: ast.FunctionDef | ast.AsyncFunctionDef, + owner: str | None, + shared_closure: bool, + ) -> None: + nodes = _function_nodes(function) + all_class_names = set(self.classes) + instance_aliases: set[str] = set() + class_aliases: set[str] = set() + shared_aliases: set[str] = set() + shared_instance_attributes = self.shared_instance_attributes.get(owner or "", set()) + positional = [*function.args.posonlyargs, *function.args.args] + local_names = { + argument.arg + for argument in [*positional, *function.args.kwonlyargs] + } + if function.args.vararg is not None: + local_names.add(function.args.vararg.arg) + if function.args.kwarg is not None: + local_names.add(function.args.kwarg.arg) + local_import_names: set[str] = set() + local_import_symbols: dict[str, str] = {} + for node in nodes: + if isinstance(node, ast.Assign): + for target in node.targets: + local_names.update(_target_names(target)) + elif isinstance(node, (ast.AnnAssign, ast.AugAssign, ast.NamedExpr)): + local_names.update(_target_names(node.target)) + elif isinstance(node, ast.Import): + names = { + alias.asname or alias.name.split(".")[0] + for alias in node.names + } + local_names.update(names) + local_import_names.update(names) + for alias in node.names: + name = alias.asname or alias.name.split(".")[0] + if _candidate_module(self.root, alias.name.split(".")) is None: + local_import_symbols[name] = alias.name if alias.asname else name + elif isinstance(node, ast.ImportFrom): + names = { + alias.asname or alias.name + for alias in node.names + if alias.name != "*" + } + local_names.update(names) + local_import_names.update(names) + local_module = ( + node.level > 0 + or node.module is None + or _candidate_module(self.root, node.module.split(".")) is not None + ) + if node.module is not None and not local_module: + for alias in node.names: + if alias.name != "*": + local_import_symbols[alias.asname or alias.name] = ( + f"{node.module}.{alias.name}" + ) + function_symbols = dict(self.symbols) + for name in local_names - local_import_names: + function_symbols.pop(name, None) + function_symbols.update(local_import_symbols) + imported_callable_names = (self.import_names - local_names) | local_import_names + if owner is not None and positional: + first = positional[0].arg + if first == "cls" or any( + isinstance(decorator, ast.Name) and decorator.id == "classmethod" + for decorator in function.decorator_list + ): + class_aliases.add(first) + else: + instance_aliases.add(first) + + def is_instance_ref(node: ast.AST) -> bool: + return isinstance(node, ast.Name) and node.id in instance_aliases + + def is_class_ref(node: ast.AST) -> bool: + if isinstance(node, ast.Name): + return ( + node.id in all_class_names and node.id not in local_names + ) or node.id in class_aliases + if isinstance(node, ast.Attribute): + return node.attr == "__class__" and is_instance_ref(node.value) + return ( + isinstance(node, ast.Call) + and isinstance(node.func, ast.Name) + and node.func.id == "type" + and len(node.args) == 1 + and is_instance_ref(node.args[0]) + ) + + global_objects = self.module_binding_names + + def is_shared_ref(node: ast.AST) -> bool: + if isinstance(node, ast.Name): + return ( + node.id in global_objects and node.id not in local_names + ) or node.id in shared_aliases | local_import_names + if (resolved := _resolved_attribute(node)) is not None: + value, attribute = resolved + if is_instance_ref(value) and attribute in shared_instance_attributes: + return True + if attribute == "__dict__" and ( + is_class_ref(value) + or (isinstance(value, ast.Name) and value.id in global_objects) + ): + return True + return is_class_ref(value) or is_shared_ref(value) or ( + isinstance(value, ast.Name) and value.id in global_objects + ) + if isinstance(node, ast.Subscript): + return is_shared_ref(node.value) + if _call_returns_shared( + node, + is_shared_ref, + self.shared_returning_functions, + self.returned_parameters, + ): + return True + if ( + isinstance(node, ast.Call) + and isinstance(node.func, ast.Name) + and node.func.id == "getattr" + and node.args + ): + return ( + is_class_ref(node.args[0]) + or is_shared_ref(node.args[0]) + or ( + isinstance(node.args[0], ast.Name) + and node.args[0].id in global_objects + ) + ) + if ( + isinstance(node, ast.Call) + and isinstance(node.func, ast.Name) + and node.func.id == "eval" + and node.args + and isinstance(node.args[0], ast.Constant) + and isinstance(node.args[0].value, str) + ): + try: + expression = ast.parse(node.args[0].value, mode="eval").body + except SyntaxError: + return False + return is_shared_ref(expression) + return ( + isinstance(node, ast.Call) + and isinstance(node.func, ast.Name) + and node.func.id == "vars" + and len(node.args) == 1 + and ( + is_class_ref(node.args[0]) + or is_shared_ref(node.args[0]) + or ( + isinstance(node.args[0], ast.Name) + and node.args[0].id in global_objects + ) + ) + ) or ( + isinstance(node, ast.Call) + and isinstance(node.func, ast.Name) + and node.func.id == "globals" + ) + + assignments = [ + node + for node in nodes + if isinstance(node, (ast.Assign, ast.AnnAssign, ast.NamedExpr)) + ] + unbound_mutator_aliases: set[str] = set() + bound_mutator_aliases: set[str] = set() + imported_callable_aliases: dict[str, str] = {} + changed = True + while changed: + changed = False + for assignment in assignments: + value, targets = _assignment_parts(assignment) or (assignment, []) + names: set[str] = set() + for target in targets: + names.update(_target_names(target)) + before = ( + len(instance_aliases), + len(class_aliases), + len(shared_aliases), + len(unbound_mutator_aliases), + len(bound_mutator_aliases), + len(imported_callable_aliases), + ) + if is_instance_ref(value): + instance_aliases.update(names) + if is_class_ref(value): + class_aliases.update(names) + if is_shared_ref(value): + shared_aliases.update(names) + symbol = _symbol(value, function_symbols) + aliased_symbol = ( + imported_callable_aliases.get(value.id) + if isinstance(value, ast.Name) + else None + ) + if aliased_symbol is not None: + symbol = aliased_symbol + if ( + symbol is not None + and ( + aliased_symbol is not None + or (_root_name(value) or "") in imported_callable_names + ) + ): + for name in names: + imported_callable_aliases[name] = symbol + if symbol in OPERATOR_MUTATORS or ( + isinstance(value, ast.Name) + and value.id in unbound_mutator_aliases + ): + unbound_mutator_aliases.update(names) + if ( + (resolved := _resolved_attribute(value)) is not None + and resolved[1] in MUTATOR_METHODS + and _symbol(value, function_symbols) != "operator.add" + and is_shared_ref(resolved[0]) + ) or ( + isinstance(value, ast.Name) and value.id in bound_mutator_aliases + ): + bound_mutator_aliases.update(names) + if before != ( + len(instance_aliases), + len(class_aliases), + len(shared_aliases), + len(unbound_mutator_aliases), + len(bound_mutator_aliases), + len(imported_callable_aliases), + ): + changed = True + + def target_is_shared(target: ast.AST) -> bool: + if isinstance(target, ast.Attribute): + return ( + is_class_ref(target.value) + or is_shared_ref(target.value) + or ( + isinstance(target.value, ast.Name) + and target.value.id in global_objects + ) + ) + if isinstance(target, ast.Subscript): + return is_shared_ref(target.value) + if isinstance(target, (ast.Tuple, ast.List)): + return any(target_is_shared(element) for element in target.elts) + return False + + for node in nodes: + if isinstance(node, ast.Global): + self.add(node, "'global' state can be shared between process instances") + elif isinstance(node, ast.Nonlocal) and shared_closure: + self.add(node, "shared closure state can be accessed by multiple process instances") + elif isinstance(node, ast.Assign): + if any(target_is_shared(target) for target in node.targets): + self.add(node, "assignment mutates state outside the process instance") + elif isinstance(node, (ast.AnnAssign, ast.AugAssign)): + if target_is_shared(node.target): + self.add(node, "assignment mutates state outside the process instance") + elif isinstance(node, ast.Delete): + if any(target_is_shared(target) for target in node.targets): + self.add(node, "deletion mutates state outside the process instance") + elif isinstance(node, ast.Call): + if isinstance(node.func, ast.Name) and node.func.id in {"exec", "globals"}: + self.add(node, f"'{node.func.id}' can access or modify module-level shared state") + elif ( + isinstance(node.func, ast.Name) + and node.func.id == "eval" + and is_shared_ref(node) + ): + self.add(node, "'eval' accesses module-level shared state") + elif ( + isinstance(node.func, ast.Name) + and node.func.id in {"setattr", "delattr"} + and node.args + and ( + is_class_ref(node.args[0]) + or is_shared_ref(node.args[0]) + or ( + isinstance(node.args[0], ast.Name) + and node.args[0].id in global_objects + ) + ) + ): + self.add(node, f"'{node.func.id}' mutates shared object state") + elif ( + ( + _symbol(node.func, function_symbols) in OPERATOR_MUTATORS + or ( + isinstance(node.func, ast.Name) + and node.func.id in unbound_mutator_aliases + ) + ) + and node.args + and is_shared_ref(node.args[0]) + ): + self.add(node, "operator call mutates shared object state") + elif ( + isinstance(node.func, ast.Name) + and node.func.id in bound_mutator_aliases + ): + self.add(node, "aliased method mutates shared object state") + elif ( + ( + imported_callable_aliases.get( + node.func.id, _symbol(node.func, function_symbols) + ) + if isinstance(node.func, ast.Name) + else _symbol(node.func, function_symbols) + ) + in IMPORTED_ARGUMENT_MUTATORS + and node.args + and is_shared_ref(node.args[0]) + ): + self.add(node, "imported function mutates shared object state") + elif ( + (resolved := _resolved_attribute(node.func)) is not None + and resolved[1] in UNBOUND_MUTATOR_METHODS + and node.args + and ( + is_class_ref(node.args[0]) + or is_shared_ref(node.args[0]) + or ( + isinstance(node.args[0], ast.Name) + and node.args[0].id in global_objects + ) + ) + ): + self.add(node, f"'{resolved[1]}' mutates shared object state") + elif ( + (resolved := _resolved_attribute(node.func)) is not None + and resolved[1] in MUTATOR_METHODS + and _symbol(node.func, function_symbols) != "operator.add" + and is_shared_ref(resolved[0]) + ): + self.add(node, f"'{resolved[1]}' mutates state outside the process instance") + elif ( + ( + canonical := ( + imported_callable_aliases.get( + node.func.id, _symbol(node.func, function_symbols) + ) + if isinstance(node.func, ast.Name) + else _symbol(node.func, function_symbols) + ) + ) + is not None + and ( + (_root_name(node.func) or "") in imported_callable_names + or ( + isinstance(node.func, ast.Name) + and node.func.id in imported_callable_aliases + ) + ) + and not _imported_call_allowed(canonical) + and canonical not in STATEFUL_DECORATORS + and canonical not in TYPE_FACTORIES + and canonical not in MUTABLE_CONSTRUCTORS + and canonical not in IMMUTABLE_SCALAR_CONSTRUCTORS + ): + self.add( + node, + f"call to imported API '{canonical}' is not known to be free of shared module-state mutation", + ) + + def check_functions(self) -> None: + def visit_body( + body: Iterable[ast.stmt], + owner: str | None = None, + shared_closure: bool = False, + ) -> None: + for statement in body: + if isinstance(statement, (ast.FunctionDef, ast.AsyncFunctionDef)): + function_closure = shared_closure or ( + owner is None and statement.name in self.shared_closure_factories + ) + self.check_function(statement, owner, function_closure) + visit_body(statement.body, None, function_closure) + elif isinstance(statement, ast.ClassDef): + visit_body(statement.body, statement.name, shared_closure) + else: + for child in ast.iter_child_nodes(statement): + if isinstance(child, ast.stmt): + visit_body([child], owner, shared_closure) + + visit_body(self.tree.body) + + def run(self) -> list[tuple[int, int, str]]: + self.collect_symbols() + self.mark_shared_closure_factories() + self.check_module_bindings() + self.collect_function_summaries() + self.check_class_bindings() + self.check_module_mutations() + self.check_defaults() + self.check_dynamic_imports() + self.collect_shared_instance_attributes() + self.check_functions() + return sorted(set(self.violations)) + + +def _candidate_module(root: Path, parts: list[str]) -> Path | None: + if not parts: + return None + module = root.joinpath(*parts) + candidates = [module.with_suffix(".py"), module / "__init__.py"] + root = root.resolve() + for candidate in candidates: + try: + resolved = candidate.resolve() + resolved.relative_to(root) + except (OSError, ValueError): + continue + if resolved.is_file(): + return resolved + return None + + +def _local_imports(tree: ast.Module, current: Path, root: Path) -> set[Path]: + result: set[Path] = set() + importlib_modules, import_module_functions = _dynamic_import_names(tree) + string_values = _static_string_values(tree) + + for node in ast.walk(tree): + if isinstance(node, ast.Import): + for alias in node.names: + candidate = _candidate_module(root, alias.name.split(".")) + if candidate is not None: + result.add(candidate) + elif isinstance(node, ast.ImportFrom): + if node.level: + base = current.parent + for _ in range(node.level - 1): + base = base.parent + else: + base = root + module_parts = node.module.split(".") if node.module else [] + candidate = _candidate_module(base, module_parts) + if candidate is not None: + result.add(candidate) + for alias in node.names: + if alias.name == "*": + continue + candidate = _candidate_module(base, [*module_parts, alias.name]) + if candidate is not None: + result.add(candidate) + elif isinstance(node, ast.Call) and node.args: + module_name = _static_string(node.args[0], string_values) + if ( + _is_dynamic_import_call( + node, importlib_modules, import_module_functions + ) + and module_name is not None + and not module_name.startswith(".") + ): + candidate = _candidate_module(root, module_name.split(".")) + if candidate is not None: + result.add(candidate) + return result + + +def validate_source(source: str, filename: str, class_names: list[str]) -> list[str]: + tree = ast.parse(source, filename=filename) + path = Path(filename) + return [ + f"{filename}:{line}:{column}: {message}" + for line, column, message in _FileChecker( + tree, path, set(class_names), path.parent + ).run() + ] + + +def validate_solution(path: str, class_names: list[str]) -> list[str]: + entrypoint = Path(path).resolve() + root = entrypoint.parent + pending = [entrypoint] + visited: set[Path] = set() + violations: list[str] = [] + while pending: + current = pending.pop() + if current in visited: + continue + visited.add(current) + source = current.read_text(encoding="utf-8") + tree = ast.parse(source, filename=str(current)) + checker = _FileChecker( + tree, + current, + set(class_names) if current == entrypoint else set(), + root, + ) + violations.extend( + f"{current}:{line}:{column}: {message}" + for line, column, message in checker.run() + ) + pending.extend(sorted(_local_imports(tree, current, root) - visited)) + return sorted(set(violations)) diff --git a/homework/07-kv-sharding/tests/src/common.rs b/homework/07-kv-sharding/tests/src/common.rs new file mode 100644 index 0000000..18dd631 --- /dev/null +++ b/homework/07-kv-sharding/tests/src/common.rs @@ -0,0 +1,125 @@ +use std::collections::HashSet; + +use rand::distributions::WeightedIndex; +use rand::prelude::*; +use rand_pcg::{Lcg128Xsl64, Pcg64}; +use serde::{Deserialize, Serialize}; +use sugars::boxed; + +use anysystem::python::PyProcessFactory; +use anysystem::System; + +#[derive(Serialize)] +pub struct GetReqMessage<'a> { + pub key: &'a str, +} + +#[derive(Deserialize)] +pub struct GetRespMessage<'a> { + pub key: &'a str, + pub value: Option<&'a str>, +} + +#[derive(Serialize)] +pub struct PutReqMessage<'a> { + pub key: &'a str, + pub value: &'a str, +} + +#[derive(Deserialize)] +pub struct PutRespMessage<'a> { + pub key: &'a str, + pub value: &'a str, +} + +#[derive(Serialize)] +pub struct DeleteReqMessage<'a> { + pub key: &'a str, +} + +#[derive(Deserialize)] +pub struct DeleteRespMessage<'a> { + pub key: &'a str, + pub value: Option<&'a str>, +} + +#[derive(Serialize)] +pub struct EmptyMessage {} + +#[derive(Serialize)] +pub struct NodeMessage<'a> { + pub id: &'a str, +} + +#[derive(Deserialize)] +pub struct DumpKeysRespMessage { + pub keys: HashSet, +} + +#[derive(Deserialize)] +pub struct CountRecordsRespMessage { + pub count: u64, +} + +#[derive(Copy, Clone)] +pub struct TestConfig<'a> { + pub process_factory: &'a PyProcessFactory, + pub proc_count: u32, + pub seed: u64, +} + +pub fn build_system(config: &TestConfig, measure_max_size: bool) -> System { + let mut sys = System::new(config.seed); + sys.network().set_delays(0.01, 0.1); + let mut proc_names = Vec::new(); + for n in 0..config.proc_count { + proc_names.push(format!("{n}")); + } + for n in 0..config.proc_count { + let proc_name = proc_names[n as usize].clone(); + let mut proc = config + .process_factory + .build((proc_name.clone(), proc_names.clone()), config.seed); + if measure_max_size { + proc.set_max_size_freq(1000000); + } + // process and node on which it runs have the same name + let node_name = proc_name.clone(); + sys.add_node(&node_name); + sys.add_process(&proc_name, boxed!(proc), &node_name); + } + sys +} + +pub fn add_node(name: &str, sys: &mut System, config: &TestConfig) { + let proc_name = name.to_string(); + let mut proc_names = sys.process_names(); + proc_names.push(proc_name.clone()); + let proc = config + .process_factory + .build((proc_name.clone(), proc_names), config.seed); + let node_name = proc_name.clone(); + sys.add_node(&node_name); + sys.add_process(&proc_name, boxed!(proc), &node_name); +} + +pub fn random_proc(sys: &System, rand: &mut Lcg128Xsl64) -> String { + sys.process_names().choose(rand).unwrap().clone() +} + +const SYMBOLS: [char; 36] = [ + 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', + 't', 'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', +]; +const WEIGHTS: [usize; 36] = [ + 13, 16, 3, 8, 8, 5, 6, 23, 4, 8, 24, 12, 2, 1, 1, 10, 5, 8, 10, 1, 24, 3, 1, 8, 12, 22, 5, 20, + 18, 5, 5, 2, 1, 3, 16, 22, +]; + +pub fn random_string(length: usize, rand: &mut Pcg64) -> String { + let dist = WeightedIndex::new(WEIGHTS).unwrap(); + rand.sample_iter(&dist) + .take(length) + .map(|x| SYMBOLS[x]) + .collect() +} diff --git a/homework/07-kv-sharding/tests/src/main.rs b/homework/07-kv-sharding/tests/src/main.rs new file mode 100644 index 0000000..cd3a43d --- /dev/null +++ b/homework/07-kv-sharding/tests/src/main.rs @@ -0,0 +1,254 @@ +mod common; +mod python_seed; +mod shared_state; +mod tests; +mod tests_mc; + +use indexmap::IndexMap; +use std::io::Write; + +use clap::Parser; +use env_logger::Builder; +use log::LevelFilter; + +use anysystem::python::PyProcessFactory; +use anysystem::test::{TestResult, TestSuite}; + +use crate::common::TestConfig; +use crate::tests::*; +use crate::tests_mc::*; + +/// Sharded KV Store Homework Tests +#[derive(Parser, Debug)] +#[clap(about, long_about = None)] +struct Args { + /// Path to Python file with solution + #[clap(long = "impl", short = 'i', default_value = "solution/node.py")] + solution_path: String, + + /// Test to run (optional) + #[clap(long = "test", short)] + test: Option, + + /// Print execution trace + #[clap(long, short)] + debug: bool, + + /// Number of nodes used in tests + #[clap(long, short, default_value = "10")] + node_count: u32, + + /// Random seed used in tests + #[clap(long, short, default_value = "123")] + seed: u64, +} + +fn main() { + let args = Args::parse(); + python_seed::configure(args.seed); + if !shared_state::validate_or_report(&args.solution_path, &["StorageNode"]) { + return; + } + if args.debug { + Builder::new() + .filter(Some("anysystem"), LevelFilter::Debug) + .format(|buf, record| writeln!(buf, "{}", record.args())) + .init(); + } + let process_factory = PyProcessFactory::new(&args.solution_path, "StorageNode"); + let config = TestConfig { + process_factory: &process_factory, + proc_count: args.node_count, + seed: args.seed, + }; + let mut single_config = config; + single_config.proc_count = 1; + let mut mc_config = config; + mc_config.proc_count = 3; + let mut tests = TestSuite::new(); + + tests.add("SINGLE NODE", test_single_node, single_config); + tests.add("INSERTS", test_inserts, config); + tests.add("DELETES", test_deletes, config); + tests.add("MEMORY GROWTH", test_memory_growth, config); + tests.add("ZERO HOP", test_zero_hop, config); + tests.add("MC NORMAL", test_mc_normal, mc_config); + tests.add("NODE ADDED", test_node_added, config); + tests.add( + "NODE ADDED DURING INSERTS", + test_node_added_during_inserts, + config, + ); + tests.add( + "NODE ADDED DURING DELETIONS", + test_node_added_during_deletions, + config, + ); + tests.add( + "NODE ADDED DURING READS", + test_node_added_during_reads, + config, + ); + tests.add("NODE REMOVED", test_node_removed, config); + tests.add( + "NODE REMOVED AFTER CRASH", + test_node_removed_after_crash, + config, + ); + tests.add("MC NODE REMOVED", test_mc_node_removed, mc_config); + tests.add("MIGRATION", test_migration, config); + tests.add("SCALE UP DOWN", test_scale_up_down, config); + tests.add("DISTRIBUTION", test_distribution, config); + tests.add( + "DISTRIBUTION NODE ADDED", + test_distribution_node_added, + config, + ); + tests.add( + "DISTRIBUTION NODE REMOVED", + test_distribution_node_removed, + config, + ); + + if let Some(test) = &args.test { + tests.run_test(test); + } else { + let (_, results) = tests.run(); + let score = score(results); + println!("\nSCORE: {score}\n"); + } +} + +const BASIC_TESTS: &[&str] = &[ + "SINGLE NODE", + "INSERTS", + "DELETES", + "MEMORY GROWTH", + "ZERO HOP", + "MC NORMAL", +]; + +const ADDITIONAL_GROUPS: &[(&str, &[&str])] = &[ + ( + "Node changes", + &[ + "NODE ADDED", + "NODE REMOVED", + "NODE REMOVED AFTER CRASH", + "MC NODE REMOVED", + "MIGRATION", + "SCALE UP DOWN", + ], + ), + ( + "Requests during node addition", + &[ + "NODE ADDED DURING INSERTS", + "NODE ADDED DURING DELETIONS", + "NODE ADDED DURING READS", + ], + ), + ( + "Distribution", + &[ + "DISTRIBUTION", + "DISTRIBUTION NODE ADDED", + "DISTRIBUTION NODE REMOVED", + ], + ), +]; + +fn group_passed(name: &str, tests: &[&str], results: &mut IndexMap) -> bool { + let mut failed = Vec::new(); + for &test in tests { + let result = results + .swap_remove(test) + .unwrap_or_else(|| panic!("missing result for {test}")); + if result.is_err() { + failed.push(test); + } + } + if failed.is_empty() { + println!("{name} tests: passed"); + true + } else { + println!("{name} tests: not passed {failed:?}"); + false + } +} + +fn score(mut results: IndexMap) -> u32 { + let basic_passed = group_passed("Basic", BASIC_TESTS, &mut results); + let mut points = if basic_passed { 3 } else { 0 }; + for &(name, tests) in ADDITIONAL_GROUPS { + if group_passed(name, tests, &mut results) && basic_passed { + points += 2; + } + } + assert!(results.is_empty(), "unscored tests: {:?}", results.keys()); + points +} + +#[cfg(test)] +mod scoring_tests { + use super::*; + + fn score_with_failures(failed: &[&str]) -> u32 { + let mut results: IndexMap = BASIC_TESTS + .iter() + .chain(ADDITIONAL_GROUPS.iter().flat_map(|(_, tests)| tests.iter())) + .map(|&name| (name.to_string(), Ok(true))) + .collect(); + for &name in failed { + *results.get_mut(name).unwrap() = Err("failed".to_string()); + } + score(results) + } + + #[test] + fn every_basic_test_is_required() { + for name in [ + "SINGLE NODE", + "INSERTS", + "DELETES", + "MC NORMAL", + "ZERO HOP", + "MEMORY GROWTH", + ] { + assert_eq!(score_with_failures(&[name]), 0, "{name}"); + } + } + + #[test] + fn additional_groups_earn_two_points_each() { + let change = "NODE ADDED"; + let read = "NODE ADDED DURING READS"; + let balance = "DISTRIBUTION NODE ADDED"; + let cases: &[(&[&str], u32)] = &[ + (&[], 9), + (&[change], 7), + (&[read], 7), + (&[balance], 7), + (&[change, read], 5), + (&[change, balance], 5), + (&[read, balance], 5), + (&[change, read, balance], 3), + ]; + for &(failures, expected) in cases { + assert_eq!(score_with_failures(failures), expected, "{failures:?}"); + } + } + + #[test] + fn concurrent_requests_are_scored_as_one_group() { + let requests = [ + "NODE ADDED DURING INSERTS", + "NODE ADDED DURING DELETIONS", + "NODE ADDED DURING READS", + ]; + for name in requests { + assert_eq!(score_with_failures(&[name]), 7, "{name}"); + } + assert_eq!(score_with_failures(&requests), 7); + } +} diff --git a/homework/07-kv-sharding/tests/src/python_seed.rs b/homework/07-kv-sharding/tests/src/python_seed.rs new file mode 100644 index 0000000..4e07f97 --- /dev/null +++ b/homework/07-kv-sharding/tests/src/python_seed.rs @@ -0,0 +1,33 @@ +//! Set Python's hash seed before the interpreter (including the validator) starts. +use std::env; +use std::process::Command; + +pub fn configure(seed: u64) { + // Python accepts only 32-bit hash seeds; the simulator keeps the full u64 seed. + let hash_seed = (seed as u32).to_string(); + if env::var("PYTHONHASHSEED").as_deref() == Ok(hash_seed.as_str()) { + return; + } + + // Restart with an inherited environment instead of changing it at runtime: + // on Windows, Rust and CPython can use different copies of the environment. + let mut command = Command::new(env::current_exe().expect("cannot locate test executable")); + command + .args(env::args_os().skip(1)) + .env("PYTHONHASHSEED", hash_seed); + + // Preserve the PID so timeout/signals still reach the test process directly. + #[cfg(unix)] + { + use std::os::unix::process::CommandExt; + let error = command.exec(); + panic!("cannot restart tests with PYTHONHASHSEED: {error}"); + } + #[cfg(not(unix))] + { + let status = command + .status() + .expect("cannot restart tests with PYTHONHASHSEED"); + std::process::exit(status.code().unwrap_or(1)); + } +} diff --git a/homework/07-kv-sharding/tests/src/shared_state.rs b/homework/07-kv-sharding/tests/src/shared_state.rs new file mode 100644 index 0000000..03156d4 --- /dev/null +++ b/homework/07-kv-sharding/tests/src/shared_state.rs @@ -0,0 +1,78 @@ +use std::ffi::CString; + +use pyo3::prelude::*; +use pyo3::types::{PyList, PyModule}; + +const VALIDATOR_CODE: &str = include_str!("../shared_state_validator.py"); + +fn validator_module<'py>(py: Python<'py>) -> PyResult> { + let code = CString::new(VALIDATOR_CODE).expect("validator source contains a null byte"); + PyModule::from_code( + py, + &code, + c"shared_state_validator.py", + c"shared_state_validator", + ) +} + +#[cfg(test)] +fn validate_source( + source: &str, + filename: &str, + class_names: &[&str], +) -> Result, String> { + Python::attach(|py| { + let module = validator_module(py).map_err(|error| error.to_string())?; + let class_names = PyList::new(py, class_names).map_err(|error| error.to_string())?; + module + .getattr("validate_source") + .and_then(|function| function.call1((source, filename, class_names))) + .and_then(|result| result.extract::>()) + .map_err(|error| error.to_string()) + }) +} + +fn validate_solution(path: &str, class_names: &[&str]) -> Result, String> { + Python::attach(|py| { + let module = validator_module(py).map_err(|error| error.to_string())?; + let class_names = PyList::new(py, class_names).map_err(|error| error.to_string())?; + module + .getattr("validate_solution") + .and_then(|function| function.call1((path, class_names))) + .and_then(|result| result.extract::>()) + .map_err(|error| error.to_string()) + }) +} + +pub fn validate_or_report(path: &str, class_names: &[&str]) -> bool { + match validate_solution(path, class_names) { + Ok(violations) if violations.is_empty() => true, + Ok(violations) => { + println!("ERROR: shared state between AnySystem processes is forbidden"); + for violation in violations { + println!("{violation}"); + } + println!("\nSCORE: 0\n"); + false + } + Err(error) => panic!("failed to validate solution for shared state: {error}"), + } +} + +#[cfg(test)] +mod tests { + use super::validate_source; + + #[test] + fn embeds_validator_and_rejects_shared_state() { + let violations = validate_source( + "class ProcessImpl:\n shared = {}\n", + "solution.py", + &["ProcessImpl"], + ) + .unwrap(); + + assert_eq!(violations.len(), 1); + assert!(violations[0].contains("ProcessImpl.shared")); + } +} diff --git a/homework/07-kv-sharding/tests/src/tests.rs b/homework/07-kv-sharding/tests/src/tests.rs new file mode 100644 index 0000000..23150f0 --- /dev/null +++ b/homework/07-kv-sharding/tests/src/tests.rs @@ -0,0 +1,1199 @@ +use std::collections::{HashMap, HashSet}; + +use assertables::{assume, assume_eq}; +use decorum::R64; +use indexmap::IndexMap; +use rand::SeedableRng; +use rand_pcg::Pcg64; + +use anysystem::logger::LogEntry; +use anysystem::test::TestResult; +use anysystem::{Message, System}; + +use crate::common::*; + +pub fn test_single_node(config: &TestConfig) -> TestResult { + let mut sys = build_system(config, false); + let mut rand = Pcg64::seed_from_u64(config.seed); + + let proc = "0"; + let key = random_string(8, &mut rand).to_uppercase(); + let value = random_string(8, &mut rand); + let max_steps = 10; + + check_get(&mut sys, proc, &key, None, max_steps)?; + check_put(&mut sys, proc, &key, &value, max_steps)?; + check_get(&mut sys, proc, &key, Some(&value), max_steps)?; + let updated = format!("{value}-updated"); + check_put(&mut sys, proc, &key, &updated, max_steps)?; + check_get(&mut sys, proc, &key, Some(&updated), max_steps)?; + check_delete(&mut sys, proc, &key, Some(&updated), max_steps)?; + check_get(&mut sys, proc, &key, None, max_steps)?; + check_delete(&mut sys, proc, &key, None, max_steps) +} + +pub fn test_inserts(config: &TestConfig) -> TestResult { + let mut sys = build_system(config, false); + let mut rand = Pcg64::seed_from_u64(config.seed); + + // insert random key-value pairs from each node + let mut kv = IndexMap::new(); + for proc in sys.process_names() { + let k = random_string(8, &mut rand).to_uppercase(); + let v = random_string(8, &mut rand); + check_put(&mut sys, &proc, &k, &v, 100)?; + kv.insert(k, v); + } + + // check that all key-values can be read from each node + let procs = sys.process_names(); + check(&mut sys, &procs, &kv, true, false)?; + // Update through a different entry node and read the new values everywhere. + for (i, (key, value)) in kv.iter_mut().enumerate() { + value.push_str("-updated"); + check_put(&mut sys, &procs[(i + 1) % procs.len()], key, value, 100)?; + } + check(&mut sys, &procs, &kv, true, false) +} + +pub fn test_deletes(config: &TestConfig) -> TestResult { + let mut sys = build_system(config, false); + let mut rand = Pcg64::seed_from_u64(config.seed); + let mut kv = IndexMap::new(); + + // insert random key-value pairs from each node + for proc in sys.process_names() { + let k = random_string(8, &mut rand).to_uppercase(); + let v = random_string(8, &mut rand); + check_put(&mut sys, &proc, &k, &v, 100)?; + kv.insert(k, v); + } + + // delete each key from one node and check that key is not present from another + for (k, v) in kv.iter() { + let read_proc = random_proc(&sys, &mut rand); + let mut delete_proc = random_proc(&sys, &mut rand); + while delete_proc == read_proc { + delete_proc = random_proc(&sys, &mut rand); + } + check_get(&mut sys, &read_proc, k, Some(v), 100)?; + check_delete(&mut sys, &delete_proc, k, Some(v), 100)?; + check_get(&mut sys, &read_proc, k, None, 100)?; + } + + kv.clear(); + let procs = sys.process_names(); + check(&mut sys, &procs, &kv, false, false) +} + +const MEMORY_NEW_KEYS_PER_NODE: usize = 1000; + +pub fn test_memory_growth(config: &TestConfig) -> TestResult { + let small = memory_growth(config, 5)?; + let large = memory_growth(config, 20)?; + check_memory_growth(small, large) +} + +fn check_memory_growth(bytes_at_5: u64, bytes_at_20: u64) -> TestResult { + // Compare equal numbers of new keys before rounding either measurement. + let extra = i128::from(bytes_at_20) - 4 * i128::from(bytes_at_5); + let new_keys = (20 * MEMORY_NEW_KEYS_PER_NODE) as i128; + let difference = extra as f64 / new_keys as f64; + println!("Additional bytes per new key (20 vs 5 nodes): {difference:.5}"); + assume!( + extra <= 64 * new_keys, + format!("Memory growth exceeds 64 extra bytes per new key ({difference:.5} observed)") + ) +} + +fn memory_growth(config: &TestConfig, nodes: u32) -> Result { + let mut sys = build_system( + &TestConfig { + proc_count: nodes, + ..*config + }, + true, + ); + let mut rand = Pcg64::seed_from_u64(config.seed); + let mut keys = HashSet::new(); + let mut measure_at = |records: usize| -> Result { + while keys.len() < records { + let key = random_string(8, &mut rand).to_uppercase(); + if !keys.insert(key.clone()) { + continue; + } + let value = random_string(8, &mut rand); + let ingress = random_proc(&sys, &mut rand); + check_put(&mut sys, &ingress, &key, &value, 100)?; + } + // Allow pending work to finish without waiting forever for recurring timers. + sys.steps(1000); + let mut stored = 0; + let mut bytes = 0; + for proc in sys.process_names() { + stored += count_records(&mut sys, &proc)?; + bytes += sys.max_size(&proc); + } + assume_eq!(stored, records as u64)?; + Ok(bytes) + }; + + let before = 500 * nodes as usize; + let after = before + MEMORY_NEW_KEYS_PER_NODE * nodes as usize; + let initial = measure_at(before)?; + let final_size = measure_at(after)?; + let bytes = final_size - initial; + let growth = bytes as f64 / (after - before) as f64; + println!("{nodes} nodes, {before} to {after} records: {growth:.3} bytes per new key"); + Ok(bytes) +} + +pub fn test_zero_hop(config: &TestConfig) -> TestResult { + let mut sys = build_system(config, false); + let procs = sys.process_names(); + let mut rand = Pcg64::seed_from_u64(config.seed); + let mut kv = IndexMap::new(); + for _ in 0..16 { + let key = random_string(8, &mut rand).to_uppercase(); + let value = random_string(8, &mut rand); + let ingress = random_proc(&sys, &mut rand); + check_put(&mut sys, &ingress, &key, &value, 100)?; + kv.insert(key, value); + } + sys.steps(1000); + let mut owners = HashMap::new(); + for proc in &procs { + for key in dump_keys(&mut sys, proc)? { + assume!( + owners.insert(key, proc).is_none(), + "Key stored on multiple nodes" + )?; + } + } + for (key, value) in kv { + let owner = owners + .get(&key) + .ok_or_else(|| format!("No owner for {key}"))?; + for ingress in &procs { + let start = sys.logger().trace().len(); + check_get(&mut sys, ingress, &key, Some(&value), 100)?; + check_read_route(&sys.logger().trace()[start..], ingress, owner) + .map_err(|error| format!("GET({key}) via {ingress}, owner {owner}: {error}"))?; + } + } + Ok(true) +} + +fn check_read_route(trace: &[LogEntry], ingress: &str, owner: &str) -> TestResult { + // Follow the GET's messages and timers so unrelated background traffic + // interleaved with the read does not count as a routing hop. + let mut messages = HashSet::new(); + let mut timers = HashSet::new(); + let mut handling_read = true; + let mut reached_owner = ingress == owner; + let mut received_reply = ingress == owner; + for entry in trace { + match entry { + LogEntry::MessageSent { + msg_id, + src_proc, + dst_proc, + .. + } if handling_read => { + assume!( + dst_proc == ingress || dst_proc == owner, + format!("Indirect route through {dst_proc} (sent by {src_proc})") + )?; + messages.insert(msg_id); + } + LogEntry::MessageReceived { + msg_id, + src_proc, + dst_proc, + .. + } => { + handling_read = messages.contains(msg_id); + // A process may also dispatch queued requests from an existing timer. + reached_owner |= src_proc == ingress && dst_proc == owner; + received_reply |= reached_owner && src_proc == owner && dst_proc == ingress; + } + LogEntry::TimerSet { timer_id, .. } if handling_read => { + timers.insert(timer_id); + } + LogEntry::TimerFired { timer_id, .. } => handling_read = timers.contains(timer_id), + LogEntry::LocalMessageSent { proc, .. } if proc == ingress => break, + _ => {} + } + } + assume!( + received_reply, + "No direct exchange with the owner before GET_RESP".to_owned() + ) +} + +pub fn test_node_added(config: &TestConfig) -> TestResult { + let mut sys = build_system(config, false); + let mut rand = Pcg64::seed_from_u64(config.seed); + + // insert random key-value pairs + let keys_count = 100; + let mut kv = IndexMap::new(); + for _ in 0..keys_count { + let k = random_string(8, &mut rand).to_uppercase(); + let v = random_string(8, &mut rand); + let proc = random_proc(&sys, &mut rand); + check_put(&mut sys, &proc, &k, &v, 100)?; + kv.insert(k, v); + } + + // add new node to the system + let added = sys.process_names().len().to_string(); + add_node(&added, &mut sys, config); + send_node_added(&mut sys, &added); + + // run the system until key the distribution is stabilized + let procs = sys.process_names(); + step_until_stabilized(&mut sys, &procs, kv.len() as u64, 100, 1000)?; + + check(&mut sys, &procs, &kv, true, false) +} + +pub fn test_node_added_during_inserts(config: &TestConfig) -> TestResult { + for (scenario, delays, update_existing) in [ + ("random delays", None, false), + ("delayed PUT delivery", Some((0.1, 0.01)), false), + ("updates with delayed migration", Some((0.01, 0.1)), true), + ("updates with delayed PUT delivery", Some((0.1, 0.01)), true), + ] { + println!("Scenario: {scenario} (seed {})", config.seed); + node_added_during_inserts_scenario(config, delays, update_existing) + .map_err(|error| format!("{scenario}: {error}"))?; + } + for (scenario, delays) in [ + ("new PUTs with random delays", None), + ("new PUTs with delayed migration", Some((0.1, 0.01))), + ("new PUTs with delayed PUT delivery", Some((0.01, 0.1))), + ] { + for via_new_node in [true, false] { + let entry = if via_new_node { + "new node" + } else { + "old nodes" + }; + println!("Scenario: {scenario} (seed {}, via {entry})", config.seed); + node_added_during_new_puts_scenario(config, delays, via_new_node) + .map_err(|error| format!("{scenario}: {error}"))?; + } + } + Ok(true) +} + +fn node_added_during_inserts_scenario( + config: &TestConfig, + delays: Option<(f64, f64)>, + update_existing: bool, +) -> TestResult { + let mut sys = build_system(config, false); + let mut rand = Pcg64::seed_from_u64(config.seed); + + // Prepare unique keys and, for update scenarios, finish the initial writes. + let keys_count = 100; + let mut kv = IndexMap::new(); + let mut requests = Vec::new(); + while kv.len() < keys_count { + let k = random_string(8, &mut rand).to_uppercase(); + if kv.contains_key(&k) { + continue; + } + let v = random_string(8, &mut rand); + let proc = random_proc(&sys, &mut rand); + if update_existing { + check_put(&mut sys, &proc, &k, &format!("old-{v}"), 100)?; + } + requests.push((proc, k.clone(), v.clone())); + kv.insert(k, v); + } + if let Some((put_delay, _)) = delays { + sys.network().set_delays(put_delay, put_delay); + } + let mut expected: IndexMap> = IndexMap::new(); + for (proc, key, value) in requests { + sys.send_local_message( + &proc, + Message::json( + "PUT", + &PutReqMessage { + key: &key, + value: &value, + }, + ), + ); + expected.entry(proc).or_default().insert(key, value); + } + + // Already-sent PUTs keep their delivery times. Test both orders relative + // to migration, within the usual 0.01..0.1 delay range. + if let Some((_, migration_delay)) = delays { + sys.network().set_delays(migration_delay, migration_delay); + } + + // add new node to the system (while puts are processed) + let added = sys.process_names().len().to_string(); + add_node(&added, &mut sys, config); + send_node_added(&mut sys, &added); + if delays.is_some() { + // Forwarded PUTs can reach the new owner before a delayed snapshot. + sys.network().set_delays(0.01, 0.01); + } + + // wait for completion of puts + for (proc, responses) in expected { + wait_put_responses(&mut sys, &proc, responses, 1000)?; + } + + // run the system until key the distribution is stabilized + let procs = sys.process_names(); + step_until_stabilized(&mut sys, &procs, kv.len() as u64, 100, 1000)?; + + check(&mut sys, &procs, &kv, true, false) +} + +fn node_added_during_new_puts_scenario( + config: &TestConfig, + delays: Option<(f64, f64)>, + via_new_node: bool, +) -> TestResult { + let mut sys = build_system(config, false); + let mut rand = Pcg64::seed_from_u64(config.seed); + let mut kv = IndexMap::new(); + while kv.len() < 100 { + let key = random_string(8, &mut rand).to_uppercase(); + if kv.contains_key(&key) { + continue; + } + let value = random_string(8, &mut rand); + let proc = random_proc(&sys, &mut rand); + check_put(&mut sys, &proc, &key, &value, 100)?; + kv.insert(key, value); + } + let old_procs = sys.process_names(); + step_until_stabilized(&mut sys, &old_procs, kv.len() as u64, 100, 1000)?; + + // Update half the existing records and leave the rest unchanged. + // Each key has only one concurrent PUT, so the final value is unambiguous. + let mut requests = Vec::new(); + for (key, value) in kv.iter_mut().step_by(2) { + value.push_str("-updated"); + requests.push((key.clone(), value.clone())); + } + // Also insert new keys while the existing records are being moved. + while requests.len() < 100 { + let key = random_string(8, &mut rand).to_uppercase(); + if kv.contains_key(&key) { + continue; + } + let value = random_string(8, &mut rand); + requests.push((key.clone(), value.clone())); + kv.insert(key, value); + } + + if let Some((migration_delay, _)) = delays { + sys.network().set_delays(migration_delay, migration_delay); + } + let added = old_procs.len().to_string(); + add_node(&added, &mut sys, config); + send_node_added(&mut sys, &added); + // Do not advance the simulation: new PUTs race with pending transfers. + if let Some((_, put_delay)) = delays { + sys.network().set_delays(put_delay, put_delay); + } + let mut expected: IndexMap> = IndexMap::new(); + for (i, (key, value)) in requests.into_iter().enumerate() { + // Run the same workload through both entry paths in separate systems. + let proc = if via_new_node { + added.clone() + } else { + old_procs[i % old_procs.len()].clone() + }; + sys.send_local_message( + &proc, + Message::json( + "PUT", + &PutReqMessage { + key: &key, + value: &value, + }, + ), + ); + expected.entry(proc).or_default().insert(key, value); + } + for (proc, responses) in expected { + wait_put_responses(&mut sys, &proc, responses, 1000)?; + } + + let procs = sys.process_names(); + step_until_stabilized(&mut sys, &procs, kv.len() as u64, 100, 1000)?; + check(&mut sys, &procs, &kv, true, false) +} + +pub fn test_node_added_during_deletions(config: &TestConfig) -> TestResult { + for (scenario, delays) in [ + ("random delays", None), + ("delayed migration", Some((0.1, 0.01))), + ("delayed DELETE delivery", Some((0.01, 0.1))), + ] { + println!("Scenario: {scenario} (seed {})", config.seed); + node_added_during_deletions_scenario(config, delays) + .map_err(|error| format!("{scenario}: {error}"))?; + } + Ok(true) +} + +fn node_added_during_deletions_scenario( + config: &TestConfig, + delays: Option<(f64, f64)>, +) -> TestResult { + let mut sys = build_system(config, false); + let mut rand = Pcg64::seed_from_u64(config.seed); + let mut kv = IndexMap::new(); + for _ in 0..100 { + let key = random_string(8, &mut rand).to_uppercase(); + let value = random_string(8, &mut rand); + let proc = random_proc(&sys, &mut rand); + check_put(&mut sys, &proc, &key, &value, 100)?; + kv.insert(key, value); + } + let procs = sys.process_names(); + step_until_stabilized(&mut sys, &procs, kv.len() as u64, 100, 1000)?; + + // Keep some records to detect solutions that discard the entire transfer. + let deleted: Vec = kv.keys().step_by(2).cloned().collect(); + let mut requests: IndexMap> = deleted + .iter() + .map(|key| (key.clone(), kv.shift_remove(key))) + .collect(); + for i in 0..5 { + requests.insert(format!("missing-{i}"), None); + } + + if let Some((migration_delay, _)) = delays { + sys.network().set_delays(migration_delay, migration_delay); + } + let added = sys.process_names().len().to_string(); + add_node(&added, &mut sys, config); + send_node_added(&mut sys, &added); + // Do not advance the simulation: deletions race with the pending transfers. + if let Some((_, delete_delay)) = delays { + sys.network().set_delays(delete_delay, delete_delay); + } + let mut expected: IndexMap>> = IndexMap::new(); + for (i, (key, value)) in requests.iter().enumerate() { + let proc = if i % 2 == 0 { + added.clone() + } else { + random_proc(&sys, &mut rand) + }; + sys.send_local_message(&proc, Message::json("DELETE", &DeleteReqMessage { key })); + expected + .entry(proc) + .or_default() + .insert(key.clone(), value.clone()); + } + for (proc, responses) in expected { + wait_delete_responses(&mut sys, &proc, responses, 1000)?; + } + + let procs = sys.process_names(); + step_until_stabilized(&mut sys, &procs, kv.len() as u64, 100, 1000)?; + check(&mut sys, &procs, &kv, true, false)?; + for proc in &procs { + for key in requests.keys() { + check_get(&mut sys, proc, key, None, 100)?; + } + } + // Deleted keys must remain usable after migration (no permanent tombstones). + for key in deleted { + let proc = random_proc(&sys, &mut rand); + check_put(&mut sys, &proc, &key, "reinserted", 100)?; + kv.insert(key, "reinserted".to_string()); + } + check(&mut sys, &procs, &kv, true, false) +} + +pub fn test_node_added_during_reads(config: &TestConfig) -> TestResult { + for (scenario, delays) in [ + ("random delays", None), + ("delayed migration", Some((0.1, 0.01))), + ("delayed GET delivery", Some((0.01, 0.1))), + ] { + for via_new_node in [true, false] { + let entry = if via_new_node { + "new node" + } else { + "old nodes" + }; + println!("Scenario: {scenario} (seed {}, via {entry})", config.seed); + node_added_during_reads_scenario(config, delays, via_new_node) + .map_err(|error| format!("{scenario} via {entry}: {error}"))?; + } + } + Ok(true) +} + +fn node_added_during_reads_scenario( + config: &TestConfig, + delays: Option<(f64, f64)>, + via_new_node: bool, +) -> TestResult { + let mut sys = build_system(config, false); + let mut rand = Pcg64::seed_from_u64(config.seed); + let mut kv = IndexMap::new(); + while kv.len() < 100 { + let key = random_string(8, &mut rand).to_uppercase(); + if kv.contains_key(&key) { + continue; + } + let value = random_string(8, &mut rand); + let proc = random_proc(&sys, &mut rand); + check_put(&mut sys, &proc, &key, &value, 100)?; + kv.insert(key, value); + } + let old_procs = sys.process_names(); + step_until_stabilized(&mut sys, &old_procs, kv.len() as u64, 100, 1000)?; + + let mut requests: IndexMap> = kv + .iter() + .map(|(key, value)| (key.clone(), Some(value.clone()))) + .collect(); + for i in 0..5 { + requests.insert(format!("missing-{i}"), None); + } + + if let Some((migration_delay, _)) = delays { + sys.network().set_delays(migration_delay, migration_delay); + } + let added = old_procs.len().to_string(); + add_node(&added, &mut sys, config); + send_node_added(&mut sys, &added); + // Send every GET before advancing the simulation; no concurrent writes. + if let Some((_, get_delay)) = delays { + sys.network().set_delays(get_delay, get_delay); + } + let mut expected: IndexMap>> = IndexMap::new(); + for (i, (key, value)) in requests.into_iter().enumerate() { + let proc = if via_new_node { + added.clone() + } else { + old_procs[i % old_procs.len()].clone() + }; + sys.send_local_message(&proc, Message::json("GET", &GetReqMessage { key: &key })); + expected.entry(proc).or_default().insert(key, value); + } + for (proc, responses) in expected { + wait_get_responses(&mut sys, &proc, responses, 1000)?; + } + + // Reading must leave both values and the set of stored keys unchanged. + let procs = sys.process_names(); + step_until_stabilized(&mut sys, &procs, kv.len() as u64, 100, 1000)?; + check(&mut sys, &procs, &kv, true, false) +} + +pub fn test_node_removed(config: &TestConfig) -> TestResult { + let mut sys = build_system(config, false); + let mut rand = Pcg64::seed_from_u64(config.seed); + + // insert random key-value pairs + let keys_count = 100; + let mut kv = IndexMap::new(); + for _ in 0..keys_count { + let k = random_string(8, &mut rand).to_uppercase(); + let v = random_string(8, &mut rand); + let proc = random_proc(&sys, &mut rand); + check_put(&mut sys, &proc, &k, &v, 100)?; + kv.insert(k, v); + } + + // remove a node from the system + let removed = random_proc(&sys, &mut rand); + let count = count_records(&mut sys, &removed)?; + assume!(count > 0, "Node stores no records, bad distribution")?; + send_node_removed(&mut sys, &removed); + + // run the system until key the distribution is stabilized + let procs: Vec = sys + .process_names() + .into_iter() + .filter(|x| *x != removed) + .collect(); + step_until_stabilized(&mut sys, &procs, kv.len() as u64, 100, 1000)?; + + check(&mut sys, &procs, &kv, true, false) +} + +pub fn test_node_removed_after_crash(config: &TestConfig) -> TestResult { + let mut sys = build_system(config, false); + let mut rand = Pcg64::seed_from_u64(config.seed); + + // insert random key-value pairs + let keys_count = 100; + let mut kv = IndexMap::new(); + for _ in 0..keys_count { + let k = random_string(8, &mut rand).to_uppercase(); + let v = random_string(8, &mut rand); + let proc = random_proc(&sys, &mut rand); + check_put(&mut sys, &proc, &k, &v, 100)?; + kv.insert(k, v); + } + + // crash a node and remove it from the system (stored keys are lost) + let crashed = random_proc(&sys, &mut rand); + let crashed_keys = dump_keys(&mut sys, &crashed)?; + assume!( + !crashed_keys.is_empty(), + "Node stores no records, bad distribution" + )?; + for k in crashed_keys { + kv.shift_remove(&k); + } + sys.crash_node(&sys.proc_node_name(&crashed)); + send_node_removed(&mut sys, &crashed); + + // run the system until key the distribution is stabilized + let procs: Vec = sys + .process_names() + .into_iter() + .filter(|x| *x != crashed) + .collect(); + step_until_stabilized(&mut sys, &procs, kv.len() as u64, 100, 1000)?; + + check(&mut sys, &procs, &kv, true, false) +} + +pub fn test_migration(config: &TestConfig) -> TestResult { + let mut sys = build_system(config, false); + let mut rand = Pcg64::seed_from_u64(config.seed); + let mut procs = sys.process_names(); + + // insert random key-value pairs + let keys_count = 10000; + let mut kv = IndexMap::new(); + for _ in 0..keys_count { + let k = random_string(8, &mut rand).to_uppercase(); + let v = random_string(8, &mut rand); + let proc = random_proc(&sys, &mut rand); + check_put(&mut sys, &proc, &k, &v, 100)?; + kv.insert(k, v); + } + + // add new N nodes to the system + for i in 0..config.proc_count { + let added = format!("{}", config.proc_count + i); + add_node(&added, &mut sys, config); + send_node_added(&mut sys, &added); + procs.push(added); + step_until_stabilized(&mut sys, &procs, kv.len() as u64, 100, 1000)?; + } + + check(&mut sys, &procs, &kv, false, false)?; + + // remove old N nodes + for _ in 0..config.proc_count { + let removed = &procs[0_usize]; + send_node_removed(&mut sys, removed); + procs.remove(0); + step_until_stabilized(&mut sys, &procs, kv.len() as u64, 100, 1000)?; + } + + check(&mut sys, &procs, &kv, false, false) +} + +pub fn test_scale_up_down(config: &TestConfig) -> TestResult { + let mut sys = build_system(config, false); + let mut rand = Pcg64::seed_from_u64(config.seed); + let mut procs = sys.process_names(); + + // insert random key-value pairs + let keys_count = 1000; + let mut kv = IndexMap::new(); + for _ in 0..keys_count { + let k = random_string(8, &mut rand).to_uppercase(); + let v = random_string(8, &mut rand); + let proc = random_proc(&sys, &mut rand); + check_put(&mut sys, &proc, &k, &v, 100)?; + kv.insert(k, v); + } + + // add new N nodes to the system + for i in 0..config.proc_count { + let added = format!("{}", config.proc_count + i); + add_node(&added, &mut sys, config); + send_node_added(&mut sys, &added); + procs.push(added); + step_until_stabilized(&mut sys, &procs, kv.len() as u64, 100, 1000)?; + } + + check(&mut sys, &procs, &kv, false, false)?; + + // remove new N nodes + for _ in 0..config.proc_count { + let removed = &procs[config.proc_count as usize]; + send_node_removed(&mut sys, removed); + procs.remove(config.proc_count as usize); + step_until_stabilized(&mut sys, &procs, kv.len() as u64, 100, 1000)?; + } + + check(&mut sys, &procs, &kv, false, false) +} + +const DISTRIBUTION_KEYS: usize = 50_000; + +fn insert_distribution_keys( + sys: &mut System, + seed: u64, +) -> Result, String> { + let mut rand = Pcg64::seed_from_u64(seed); + let mut kv = IndexMap::new(); + while kv.len() < DISTRIBUTION_KEYS { + let k = random_string(8, &mut rand).to_uppercase(); + let v = random_string(8, &mut rand); + let proc = random_proc(sys, &mut rand); + check_put(sys, &proc, &k, &v, 100)?; + kv.insert(k, v); + } + Ok(kv) +} + +pub fn test_distribution(config: &TestConfig) -> TestResult { + let mut sys = build_system(config, false); + let kv = insert_distribution_keys(&mut sys, config.seed)?; + let procs = sys.process_names(); + check(&mut sys, &procs, &kv, false, true) +} + +pub fn test_distribution_node_added(config: &TestConfig) -> TestResult { + let mut sys = build_system(config, false); + let kv = insert_distribution_keys(&mut sys, config.seed)?; + let dist_before = key_distribution(&mut sys)?; + + // add new node to the system + let added = sys.process_names().len().to_string(); + add_node(&added, &mut sys, config); + send_node_added(&mut sys, &added); + + // run the system until key the distribution is stabilized + let procs = sys.process_names(); + step_until_stabilized(&mut sys, &procs, kv.len() as u64, 100, 1000)?; + let dist_after = key_distribution(&mut sys)?; + + let target_moved_keys = (kv.len() as f64 / procs.len() as f64).round() as u64; + check_moved_keys(&mut sys, &dist_before, &dist_after, target_moved_keys)?; + + check(&mut sys, &procs, &kv, false, true) +} + +pub fn test_distribution_node_removed(config: &TestConfig) -> TestResult { + let mut sys = build_system(config, false); + // Keep the topology choice independent of the number of generated keys. + let removed = random_proc(&sys, &mut Pcg64::seed_from_u64(config.seed)); + let kv = insert_distribution_keys(&mut sys, config.seed)?; + let dist_before = key_distribution(&mut sys)?; + + // remove a node from the system + println!("Removed node: {removed}"); + let count = count_records(&mut sys, &removed)?; + assume!(count > 0, "Node stores no records, bad distribution")?; + send_node_removed(&mut sys, &removed); + + // run the system until key the distribution is stabilized + let procs: Vec = sys + .process_names() + .into_iter() + .filter(|x| *x != removed) + .collect(); + step_until_stabilized(&mut sys, &procs, kv.len() as u64, 100, 1000)?; + let dist_after = key_distribution(&mut sys)?; + + let target_moved_keys = (kv.len() as f64 / (procs.len() + 1) as f64).round() as u64; + check_moved_keys(&mut sys, &dist_before, &dist_after, target_moved_keys)?; + + check(&mut sys, &procs, &kv, false, true) +} + +// UTILS --------------------------------------------------------------------------------------------------------------- + +fn check_get( + sys: &mut System, + proc: &str, + key: &str, + expected: Option<&str>, + max_steps: u32, +) -> TestResult { + sys.send_local_message(proc, Message::json("GET", &GetReqMessage { key })); + let res = sys.step_until_local_message_max_steps(proc, max_steps); + assume!(res.is_ok(), format!("GET_RESP is not returned by {}", proc))?; + let msgs = res.unwrap(); + let msg = msgs.first().unwrap(); + assume_eq!(msg.tip, "GET_RESP")?; + let data: GetRespMessage = serde_json::from_str(&msg.data).unwrap(); + assume_eq!(data.key, key)?; + assume!( + data.value == expected, + format!( + "GET({key}) via {proc}: expected {expected:?}, got {:?}", + data.value + ) + )?; + Ok(true) +} + +fn wait_get_responses( + sys: &mut System, + proc: &str, + mut expected: IndexMap>, + max_steps: u32, +) -> TestResult { + while !expected.is_empty() { + let res = sys.step_until_local_message_max_steps(proc, max_steps); + assume!(res.is_ok(), format!("GET_RESP is not returned by {proc}"))?; + for msg in res.unwrap() { + assume_eq!(msg.tip, "GET_RESP")?; + let data: GetRespMessage = serde_json::from_str(&msg.data) + .map_err(|error| format!("Invalid GET_RESP via {proc}: {error}"))?; + let value = expected.shift_remove(data.key).ok_or_else(|| { + format!("Unexpected or duplicate GET_RESP({}) via {proc}", data.key) + })?; + assume!( + data.value == value.as_deref(), + format!( + "GET({}) via {proc}: expected {value:?}, got {:?}", + data.key, data.value + ) + )?; + } + } + Ok(true) +} + +fn check_put(sys: &mut System, proc: &str, key: &str, value: &str, max_steps: u32) -> TestResult { + sys.send_local_message(proc, Message::json("PUT", &PutReqMessage { key, value })); + let res = sys.step_until_local_message_max_steps(proc, max_steps); + assume!(res.is_ok(), format!("PUT_RESP is not returned by {}", proc))?; + let msgs = res.unwrap(); + let msg = msgs.first().unwrap(); + assume_eq!(msg.tip, "PUT_RESP")?; + let data: PutRespMessage = serde_json::from_str(&msg.data).unwrap(); + assume_eq!(data.key, key)?; + assume_eq!(data.value, value)?; + Ok(true) +} + +fn wait_put_responses( + sys: &mut System, + proc: &str, + mut expected: IndexMap, + max_steps: u32, +) -> TestResult { + while !expected.is_empty() { + let res = sys.step_until_local_message_max_steps(proc, max_steps); + assume!(res.is_ok(), format!("PUT_RESP is not returned by {}", proc))?; + let msgs = res.unwrap(); + for msg in msgs { + assume_eq!(msg.tip, "PUT_RESP")?; + let data: PutRespMessage = serde_json::from_str(&msg.data) + .map_err(|error| format!("Invalid PUT_RESP via {proc}: {error}"))?; + let value = expected.shift_remove(data.key).ok_or_else(|| { + format!("Unexpected or duplicate PUT_RESP({}) via {proc}", data.key) + })?; + assume!( + data.value == value, + format!( + "PUT({}) via {proc}: expected {value:?}, got {:?}", + data.key, data.value + ) + )?; + } + } + Ok(true) +} + +fn check_delete( + sys: &mut System, + proc: &str, + key: &str, + expected: Option<&str>, + max_steps: u32, +) -> TestResult { + sys.send_local_message(proc, Message::json("DELETE", &DeleteReqMessage { key })); + let res = sys.step_until_local_message_max_steps(proc, max_steps); + assume!( + res.is_ok(), + format!("DELETE_RESP is not returned by {}", proc) + )?; + let msgs = res.unwrap(); + let msg = msgs.first().unwrap(); + assume_eq!(msg.tip, "DELETE_RESP")?; + let data: DeleteRespMessage = serde_json::from_str(&msg.data).unwrap(); + assume_eq!(data.key, key)?; + assume_eq!(data.value, expected)?; + Ok(true) +} + +fn wait_delete_responses( + sys: &mut System, + proc: &str, + mut expected: IndexMap>, + max_steps: u32, +) -> TestResult { + while !expected.is_empty() { + let res = sys.step_until_local_message_max_steps(proc, max_steps); + assume!( + res.is_ok(), + format!("DELETE_RESP is not returned by {proc}") + )?; + for msg in res.unwrap() { + assume_eq!(msg.tip, "DELETE_RESP")?; + let data: DeleteRespMessage = serde_json::from_str(&msg.data) + .map_err(|error| format!("Invalid DELETE_RESP via {proc}: {error}"))?; + let value = expected.shift_remove(data.key).ok_or_else(|| { + format!( + "Unexpected or duplicate DELETE_RESP({}) via {proc}", + data.key + ) + })?; + assume!( + data.value == value.as_deref(), + format!( + "DELETE({}) via {proc}: expected {value:?}, got {:?}", + data.key, data.value + ) + )?; + } + } + Ok(true) +} + +fn send_node_added(sys: &mut System, added: &str) { + for proc in sys.process_names() { + if !sys.proc_node_is_crashed(&proc) { + sys.send_local_message( + &proc, + Message::json("NODE_ADDED", &NodeMessage { id: added }), + ); + } + } +} + +fn send_node_removed(sys: &mut System, removed: &str) { + for proc in sys.process_names() { + if !sys.proc_node_is_crashed(&proc) { + sys.send_local_message( + &proc, + Message::json("NODE_REMOVED", &NodeMessage { id: removed }), + ); + } + } +} + +fn count_records(sys: &mut System, proc: &str) -> Result { + sys.send_local_message(proc, Message::json("COUNT_RECORDS", &EmptyMessage {})); + let res = sys.step_until_local_message_max_steps(proc, 100); + assume!( + res.is_ok(), + format!("COUNT_RECORDS_RESP is not returned by {}", proc) + )?; + let msgs = res.unwrap(); + let msg = msgs.first().unwrap(); + assume_eq!(msg.tip, "COUNT_RECORDS_RESP")?; + let data: CountRecordsRespMessage = serde_json::from_str(&msg.data).unwrap(); + Ok(data.count) +} + +fn dump_keys(sys: &mut System, proc: &str) -> Result, String> { + sys.send_local_message(proc, Message::json("DUMP_KEYS", &EmptyMessage {})); + let res = sys.step_until_local_message_max_steps(proc, 100); + assume!( + res.is_ok(), + format!("DUMP_KEYS_RESP is not returned by {}", proc) + )?; + let msgs = res.unwrap(); + let msg = msgs.first().unwrap(); + assume_eq!(msg.tip, "DUMP_KEYS_RESP")?; + let data: DumpKeysRespMessage = serde_json::from_str(&msg.data).unwrap(); + Ok(data.keys) +} + +fn key_distribution(sys: &mut System) -> Result>, String> { + let mut dist = HashMap::new(); + for proc in sys.process_names() { + dist.insert(proc.clone(), dump_keys(sys, &proc)?); + } + Ok(dist) +} + +fn step_until_stabilized( + sys: &mut System, + procs: &[String], + expected_keys: u64, + steps_per_iter: u64, + max_steps: u64, +) -> TestResult { + let mut stabilized = false; + let mut steps: u64 = 0; + let mut counts = HashMap::new(); + let mut total_count: u64 = 0; + for proc in procs.iter() { + let count = count_records(sys, proc)?; + counts.insert(proc, count); + total_count += count; + } + + while !stabilized && steps <= max_steps { + sys.steps(steps_per_iter); + steps += steps_per_iter; + total_count = 0; + let mut count_changed = false; + for proc in procs.iter() { + let count = count_records(sys, proc)?; + if *counts.get(proc).unwrap() != count { + count_changed = true; + } + counts.insert(proc, count); + total_count += count; + } + if total_count == expected_keys && !count_changed { + stabilized = true; + } + } + + assume!( + stabilized, + format!( + "Keys distribution is not stabilized (keys observed = {}, expected = {})", + total_count, expected_keys + ) + ) +} + +fn check( + sys: &mut System, + procs: &[String], + expected: &IndexMap, + check_values: bool, + check_distribution: bool, +) -> TestResult { + let mut stored_keys = HashSet::new(); + let mut proc_key_counts = Vec::new(); + for proc in procs.iter() { + let proc_count = count_records(sys, proc)?; + let proc_keys = dump_keys(sys, proc)?; + assume_eq!(proc_keys.len() as u64, proc_count)?; + stored_keys.extend(proc_keys); + proc_key_counts.push(proc_count); + } + + // all keys are stored + assume!( + expected.len() == stored_keys.len() && expected.keys().all(|k| stored_keys.contains(k)), + "Stored keys do not mach expected" + )?; + + // each key is stored on a single node + assume!( + proc_key_counts.iter().sum::() == stored_keys.len() as u64, + "Keys are not stored on a single node" + )?; + + // check values + if check_values { + println!("\nChecking values:"); + for proc in procs.iter() { + for (k, v) in expected.iter() { + check_get(sys, proc, k, Some(v), 100)?; + } + } + println!("OK") + } + + // check keys distribution + if check_distribution { + let target_count = (expected.len() as f64 / proc_key_counts.len() as f64).round(); + let max_count = *proc_key_counts.iter().max().unwrap(); + let min_count = *proc_key_counts.iter().min().unwrap(); + let deviations: Vec = proc_key_counts + .iter() + .map(|x| (target_count - *x as f64).abs() / target_count) + .collect(); + let avg_deviation = deviations.iter().sum::() / proc_key_counts.len() as f64; + let max_deviation = deviations + .iter() + .map(|x| R64::from_inner(*x)) + .max() + .unwrap(); + println!("\nStored keys per node:"); + println!(" - total: {}", stored_keys.len()); + println!(" - target: {target_count}"); + println!(" - min: {min_count}"); + println!(" - max: {max_count}"); + println!(" - average deviation from target: {avg_deviation:.3}"); + println!(" - max deviation from target: {max_deviation:.3}"); + assume!( + max_deviation <= 0.1, + "Max deviation from target is above 10%" + )?; + } + + Ok(true) +} + +fn check_moved_keys( + sys: &mut System, + before: &HashMap>, + after: &HashMap>, + target: u64, +) -> TestResult { + let mut total_count_before = 0; + let mut total_count_after = 0; + let mut not_moved_count = 0; + let empty = HashSet::new(); + for proc in sys.process_names() { + let b = before.get(&proc).unwrap_or(&empty); + total_count_before += b.len() as u64; + let a = after.get(&proc).unwrap_or(&empty); + total_count_after += a.len() as u64; + let not_moved: HashSet = a.intersection(b).cloned().collect(); + not_moved_count += not_moved.len() as u64; + } + assume_eq!( + total_count_after, + total_count_before, + format!( + "Number of stored keys has changed (before: {}, after: {})", + total_count_before, total_count_after + ), + )?; + let moved_count = total_count_before - not_moved_count; + let deviation = (moved_count as f64 - target as f64) / target as f64; + println!("\nMoved keys:"); + println!(" - target: {target}"); + println!(" - observed: {moved_count}"); + println!(" - deviation: {deviation:.3}"); + assume!( + deviation <= 0.1, + format!("Deviation from target is above 10%") + ) +} + +#[cfg(test)] +mod memory_tests { + use super::check_memory_growth; + + #[test] + fn accepts_the_limit_and_rejects_one_more_byte_in_the_cluster() { + let small = 765_301; + let at_limit = 4 * small + 64 * 20_000; + assert!(check_memory_growth(small, at_limit).is_ok()); + assert!(check_memory_growth(small, at_limit + 1).is_err()); + assert!(check_memory_growth(small, small).is_ok()); + } +} diff --git a/homework/07-kv-sharding/tests/src/tests_mc.rs b/homework/07-kv-sharding/tests/src/tests_mc.rs new file mode 100644 index 0000000..4b51525 --- /dev/null +++ b/homework/07-kv-sharding/tests/src/tests_mc.rs @@ -0,0 +1,324 @@ +use std::collections::HashSet; + +use indexmap::IndexMap; +use rand::prelude::SliceRandom; +use rand::SeedableRng; +use rand_pcg::Pcg64; +use sugars::boxed; + +use anysystem::logger::LogEntry; +use anysystem::mc::{ + predicates::{collects, goals, invariants}, + strategies::Bfs, + InvariantFn, McState, ModelChecker, StrategyConfig, +}; +use anysystem::test::TestResult; +use anysystem::Message; + +use crate::common::*; + +pub fn test_mc_normal(config: &TestConfig) -> TestResult { + let sys = build_system(config, false); + let mut mc = ModelChecker::new(&sys); + let mut rand = Pcg64::seed_from_u64(config.seed); + + let key = random_string(8, &mut rand).to_uppercase(); + let value = random_string(8, &mut rand); + let max_steps = 10; + + let mut proc = random_proc(&sys, &mut rand); + let achieved_states = check_mc_get(&mut mc, &proc, &key, None, max_steps, None)?; + proc = random_proc(&sys, &mut rand); + let achieved_states = check_mc_put( + &mut mc, + &proc, + &key, + &value, + max_steps, + Some(achieved_states), + )?; + proc = random_proc(&sys, &mut rand); + let achieved_states = check_mc_get( + &mut mc, + &proc, + &key, + Some(&value), + max_steps, + Some(achieved_states), + )?; + proc = random_proc(&sys, &mut rand); + let achieved_states = check_mc_delete( + &mut mc, + &proc, + &key, + Some(&value), + max_steps, + Some(achieved_states), + )?; + proc = random_proc(&sys, &mut rand); + let achieved_states = + check_mc_get(&mut mc, &proc, &key, None, max_steps, Some(achieved_states))?; + proc = random_proc(&sys, &mut rand); + check_mc_delete(&mut mc, &proc, &key, None, max_steps, Some(achieved_states))?; + Ok(true) +} + +pub fn test_mc_node_removed(config: &TestConfig) -> TestResult { + let sys = build_system(config, false); + let mut mc = ModelChecker::new(&sys); + let mut rand = Pcg64::seed_from_u64(config.seed); + + // insert random key-value pairs + let keys_count = 40; + let mut achieved_states = None; + let mut kv = IndexMap::new(); + for _ in 0..keys_count { + let k = random_string(8, &mut rand).to_uppercase(); + let v = random_string(8, &mut rand); + let proc = random_proc(&sys, &mut rand); + let res = check_mc_put(&mut mc, &proc, &k, &v, 10, achieved_states)?; + achieved_states = Some(res); + kv.insert(k, v); + } + + // remove a node from the system + let removed_proc = random_proc(&sys, &mut rand); + achieved_states = Some(check_mc_node_removed( + &mut mc, + sys.process_names(), + &removed_proc, + 15, + achieved_states.unwrap_or_else(|| panic!("no states found after {keys_count} GET queries")), + )?); + let alive_proc = sys + .process_names() + .into_iter() + .filter(|proc| *proc != removed_proc) + .collect::>(); + + // check that all data is still in the storage + for (k, v) in kv { + let proc = alive_proc.choose(&mut rand).unwrap().clone(); + achieved_states = Some(check_mc_get( + &mut mc, + &proc, + &k, + Some(&v), + 10, + achieved_states, + )?); + } + Ok(true) +} + +// UTILS --------------------------------------------------------------------------------------------------------------- + +fn check_mc_get( + mc: &mut ModelChecker, + proc: S, + key: S, + expected: Option, + max_steps: u64, + start_states: Option>, +) -> Result, String> +where + S: Into + Clone, +{ + let proc = proc.into(); + let key = key.into(); + let expected = expected.map(|s| s.into()); + let msg = Message::new("GET", &format!(r#"{{"key": "{key}"}}"#)); + let proc_name = proc.clone(); + let invariant = boxed!(move |state: &McState| { + for entry in state.current_run_trace().iter() { + if let LogEntry::McLocalMessageSent { + msg: message, + proc: proc_tmp, + } = entry + { + if *proc_tmp != proc { + return Err("local message received on wrong process".to_string()); + } + if message.tip != "GET_RESP" { + return Err(format!("wrong type {}", message.tip)); + } + let data: GetRespMessage = + serde_json::from_str(&message.data).map_err(|err| err.to_string())?; + if data.key != key { + return Err(format!("wrong key {}", data.key)); + } + if data.value.map(|s| s.to_string()) != expected { + return Err(format!("wrong value {:?}", data.value)); + } + } + } + Ok(()) + }); + mc_check_query(mc, proc_name, invariant, msg, start_states, max_steps) +} + +fn check_mc_put( + mc: &mut ModelChecker, + proc: S, + key: S, + value: S, + max_steps: u64, + start_states: Option>, +) -> Result, String> +where + S: Into, +{ + let proc = proc.into(); + let key = key.into(); + let value = value.into(); + let proc_name = proc.clone(); + let msg = Message::new("PUT", &format!(r#"{{"key": "{key}", "value": "{value}"}}"#)); + let invariant = Box::new(move |state: &McState| { + for entry in state.current_run_trace().iter() { + if let LogEntry::McLocalMessageSent { + msg: message, + proc: proc_tmp, + } = entry + { + if *proc_tmp != proc { + return Err("local message received on wrong process".to_string()); + } + if message.tip != "PUT_RESP" { + return Err(format!("wrong type {}", message.tip)); + } + let data: PutRespMessage = + serde_json::from_str(&message.data).map_err(|err| err.to_string())?; + if data.key != key { + return Err(format!("wrong key {}", data.key)); + } + if data.value != value { + return Err(format!("wrong value {:?}", data.value)); + } + } + } + Ok(()) + }); + mc_check_query(mc, proc_name, invariant, msg, start_states, max_steps) +} + +fn check_mc_delete( + mc: &mut ModelChecker, + proc: S, + key: S, + expected: Option, + max_steps: u64, + start_states: Option>, +) -> Result, String> +where + S: Into + Clone, +{ + let proc = proc.into(); + let key = key.into(); + let expected = expected.map(|s| s.into()); + let msg = Message::new("DELETE", &format!(r#"{{"key": "{key}"}}"#)); + let proc_name = proc.clone(); + let invariant = Box::new(move |state: &McState| { + for entry in state.current_run_trace().iter() { + if let LogEntry::McLocalMessageSent { + msg: message, + proc: proc_tmp, + } = entry + { + if *proc_tmp != proc { + return Err("local message received on wrong process".to_string()); + } + if message.tip != "DELETE_RESP" { + return Err(format!("wrong type {}", message.tip)); + } + let data: DeleteRespMessage = + serde_json::from_str(&message.data).map_err(|err| err.to_string())?; + if data.key != key { + return Err(format!("wrong key {}", data.key)); + } + if data.value.map(|s| s.to_string()) != expected { + return Err(format!("wrong value {:?}", data.value)); + } + } + } + Ok(()) + }); + mc_check_query(mc, proc_name, invariant, msg, start_states, max_steps) +} + +fn check_mc_node_removed( + mc: &mut ModelChecker, + process_names: Vec, + removed_proc: &str, + max_steps: u64, + start_states: HashSet, +) -> Result, String> { + let strategy_config = StrategyConfig::default() + .invariant(invariants::state_depth_current_run(max_steps)) + .goal(goals::no_events()) + .collect(collects::no_events()); + + let res = mc.run_from_states_with_change::(strategy_config, start_states, move |sys| { + let msg = Message::new("NODE_REMOVED", &format!(r#"{{"id": "{removed_proc}"}}"#)); + for proc in process_names.clone() { + sys.send_local_message(proc.clone(), proc, msg.clone()); + } + }); + match res { + Ok(stats) => Ok(stats + .collected_states + .into_iter() + .map(|mut state| { + state.network.disconnect_node(removed_proc); + state + }) + .collect::>()), + Err(err) => { + err.print_trace(); + Err(err.message()) + } + } +} + +fn mc_check_query( + mc: &mut ModelChecker, + proc: S, + invariant: InvariantFn, + msg: Message, + start_states: Option>, + max_depth: u64, +) -> Result, String> +where + S: Into, +{ + let proc = proc.into(); + let strategy_config = StrategyConfig::default() + .goal(goals::all_goals(vec![ + goals::event_happened_n_times_current_run(LogEntry::is_mc_local_message_sent, 1), + goals::no_events(), + ])) + .invariant(invariants::all_invariants(vec![ + invariant, + invariants::state_depth_current_run(max_depth), + ])) + .collect(collects::event_happened_n_times_current_run( + LogEntry::is_mc_local_message_sent, + 1, + )); + + let res = if let Some(start_states) = start_states { + mc.run_from_states_with_change::(strategy_config, start_states, |sys| { + sys.send_local_message(proc.clone(), proc.clone(), msg.clone()); + }) + } else { + mc.run_with_change::(strategy_config, |sys| { + sys.send_local_message(proc.clone(), proc.clone(), msg); + }) + }; + match res { + Ok(stats) => Ok(stats.collected_states), + Err(err) => { + err.print_trace(); + Err(err.message()) + } + } +}