Add week 8 materials
This commit is contained in:
osukhoroslov committed
1 parent
a987b82834
commit
0bf71ed374
38 files changed
+2908
No files matched your search
@@ -0,0 +1,30 @@
|
||||
services:
|
||||
kv1:
|
||||
build:
|
||||
context: ./storage
|
||||
ports:
|
||||
- 5001:5000
|
||||
command: -n kv1
|
||||
|
||||
kv2:
|
||||
build:
|
||||
context: ./storage
|
||||
ports:
|
||||
- 5002:5000
|
||||
command: -n kv2
|
||||
|
||||
kv3:
|
||||
build:
|
||||
context: ./storage
|
||||
ports:
|
||||
- 5003:5000
|
||||
command: -n kv3
|
||||
|
||||
proxy:
|
||||
build:
|
||||
context: ./proxy
|
||||
ports:
|
||||
- 5005:5000
|
||||
command: -c compose_config.yml
|
||||
volumes:
|
||||
- ./proxy/compose_config.yml:/app/compose_config.yml:ro
|
||||
@@ -0,0 +1,9 @@
|
||||
FROM python:3.9-slim
|
||||
|
||||
WORKDIR /app
|
||||
COPY requirements.txt .
|
||||
RUN pip install -r requirements.txt
|
||||
|
||||
COPY . .
|
||||
|
||||
ENTRYPOINT [ "python", "main.py"]
|
||||
Whitespace-only changes.
@@ -0,0 +1,8 @@
|
||||
mode: sharding
|
||||
nodes:
|
||||
- name: node1
|
||||
url: http://kv1:5000
|
||||
- name: node2
|
||||
url: http://kv2:5000
|
||||
- name: node3
|
||||
url: http://kv3:5000
|
||||
@@ -0,0 +1,76 @@
|
||||
import argparse
|
||||
import requests
|
||||
from strategies import (
|
||||
Proxy, ShardingProxy,
|
||||
ReplicatingProxy, ReplicatingQuorumProxy,
|
||||
RAID3Proxy
|
||||
)
|
||||
import yaml
|
||||
from flask import Flask, jsonify, request
|
||||
|
||||
app = Flask(__name__)
|
||||
|
||||
proxy: Proxy = None
|
||||
|
||||
|
||||
@app.route('/put', methods=['POST'])
|
||||
def put_handler():
|
||||
key = request.args.get('key')
|
||||
value = request.get_json().get('value')
|
||||
|
||||
if key is None or value is None:
|
||||
return jsonify({'error': 'Please provide key and value.'}), 400
|
||||
|
||||
return proxy.put(key, value)
|
||||
|
||||
@app.route('/get', methods=['GET'])
|
||||
def get_handler():
|
||||
key = request.args.get('key')
|
||||
|
||||
if key is None:
|
||||
return jsonify({'error': 'Please provide key.'}), 400
|
||||
|
||||
return proxy.get(key)
|
||||
|
||||
|
||||
@app.route('/state', methods=['GET'])
|
||||
def state_handler():
|
||||
results = ""
|
||||
for _, node in proxy.nodes.items():
|
||||
url = f'{node["url"]}/state'
|
||||
response = requests.get(url)
|
||||
results += f'<h1 style="font-family: monospace;">Response from {url}</h1>{response.content.decode("utf-8")}<hr>'
|
||||
|
||||
return f"<html><body>{results}</body></html>", 200
|
||||
|
||||
|
||||
def load_proxy_from_config(config_path):
|
||||
with open(config_path, 'r') as file:
|
||||
config = yaml.safe_load(file)
|
||||
|
||||
mode = config.get('mode', '')
|
||||
nodes = {}
|
||||
for node in config['nodes']:
|
||||
name = node['name']
|
||||
url = node['url']
|
||||
nodes[name] = {'url': url}
|
||||
if mode == 'sharding':
|
||||
return ShardingProxy(nodes)
|
||||
elif mode == 'replication':
|
||||
return ReplicatingProxy(nodes)
|
||||
elif mode == 'replication-quorum':
|
||||
return ReplicatingQuorumProxy(nodes)
|
||||
elif mode == 'replication-raid3':
|
||||
return RAID3Proxy(nodes)
|
||||
raise ValueError('Invalid mode, allowed values: sharding, replication, replication-quorum')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = argparse.ArgumentParser(description='Flask app acting as a proxy for key-value nodes')
|
||||
parser.add_argument('-c', '--config', type=str, required=True, help='Path to the YAML config file')
|
||||
parser.add_argument('-p', '--port', type=int, help='Port of the server')
|
||||
args = parser.parse_args()
|
||||
|
||||
proxy = load_proxy_from_config(args.config)
|
||||
|
||||
app.run('0.0.0.0', args.port)
|
||||
@@ -0,0 +1,3 @@
|
||||
Flask==2.3.3
|
||||
requests==2.31.0
|
||||
pyyaml==6.0.1
|
||||
@@ -0,0 +1,244 @@
|
||||
from dataclasses import dataclass
|
||||
import json
|
||||
from collections import defaultdict
|
||||
import math
|
||||
from typing import Union
|
||||
from flask import jsonify
|
||||
import requests
|
||||
from abc import ABC, abstractmethod
|
||||
import base64
|
||||
|
||||
from requests.exceptions import ConnectionError
|
||||
|
||||
import hashlib
|
||||
|
||||
|
||||
class Proxy(ABC):
|
||||
def __init__(self, nodes):
|
||||
self.nodes = nodes
|
||||
|
||||
@abstractmethod
|
||||
def put(self, key, value):
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def get(self, key):
|
||||
pass
|
||||
|
||||
|
||||
class ShardingProxy(Proxy):
|
||||
def __init__(self, nodes):
|
||||
super().__init__(nodes)
|
||||
|
||||
def put(self, key, value):
|
||||
node = self.get_node(key)
|
||||
if node is None:
|
||||
return {'error': 'No nodes available.'}, 500
|
||||
|
||||
url = f'{node["url"]}/put?key={key}'
|
||||
response = requests.post(url, json={'value': value})
|
||||
|
||||
return response.content, response.status_code
|
||||
|
||||
def get(self, key):
|
||||
node = self.get_node(key)
|
||||
if node is None:
|
||||
return {'error': 'No nodes available.'}, 500
|
||||
|
||||
url = f'{node["url"]}/get?key={key}'
|
||||
response = requests.get(url)
|
||||
|
||||
return response.content, response.status_code
|
||||
|
||||
def get_node(self, key):
|
||||
if len(self.nodes) == 0:
|
||||
return None
|
||||
|
||||
hash_val = hashlib.sha256(key.encode()).hexdigest()
|
||||
node_index = int(hash_val, 16) % len(self.nodes)
|
||||
return self.nodes[list(self.nodes.keys())[node_index]]
|
||||
|
||||
|
||||
class ReplicatingProxy(Proxy):
|
||||
def __init__(self, nodes):
|
||||
super().__init__(nodes)
|
||||
|
||||
def put(self, key, value):
|
||||
responses = []
|
||||
for _, node in self.nodes.items():
|
||||
url = f'{node["url"]}/put?key={key}'
|
||||
response = requests.post(url, json={'value': value})
|
||||
if response.status_code == 200:
|
||||
responses.append(response.content)
|
||||
|
||||
if len(responses) == 0:
|
||||
return {'error': 'No nodes available.'}, 500
|
||||
|
||||
return responses[0], 200
|
||||
|
||||
def get(self, key):
|
||||
non_ok_responses = dict()
|
||||
for node_name, node in self.nodes.items():
|
||||
url = f'{node["url"]}/get?key={key}'
|
||||
response = requests.get(url)
|
||||
if response.status_code == 200:
|
||||
return response.content, 200
|
||||
else:
|
||||
non_ok_responses[node_name] = response.json()
|
||||
|
||||
return jsonify({'error': 'None of the nodes responded with 200.', 'responses': non_ok_responses}), 500
|
||||
|
||||
|
||||
class ReplicatingQuorumProxy(Proxy):
|
||||
def __init__(self, nodes):
|
||||
super().__init__(nodes)
|
||||
|
||||
def put(self, key, value):
|
||||
responses = []
|
||||
for _, node in self.nodes.items():
|
||||
url = f'{node["url"]}/put?key={key}'
|
||||
response = requests.post(url, json={'value': value})
|
||||
if response.status_code == 200:
|
||||
responses.append(response.content)
|
||||
|
||||
if len(responses) == 0:
|
||||
return {'error': 'No nodes available.'}, 500
|
||||
|
||||
if len(responses) > len(self.nodes) / 2:
|
||||
return responses[0], 200
|
||||
|
||||
return jsonify({'error': 'Not enough responses for quorum write.', 'responses': responses}), 300
|
||||
|
||||
def get(self, key):
|
||||
non_ok_responses = dict()
|
||||
values = defaultdict(lambda: 0)
|
||||
for node_name, node in self.nodes.items():
|
||||
url = f'{node["url"]}/get?key={key}'
|
||||
response = requests.get(url)
|
||||
if response.status_code == 200:
|
||||
values[response.content] += 1
|
||||
else:
|
||||
non_ok_responses[node_name] = response.json()
|
||||
|
||||
for value, cnt in values.items():
|
||||
if cnt > len(self.nodes) / 2:
|
||||
return value, 200
|
||||
|
||||
if len(values) > 0:
|
||||
values_for_response = []
|
||||
for value, cnt in values.items():
|
||||
values_for_response.append({'value': json.loads(value)['value'], 'count': cnt})
|
||||
return jsonify({'error': 'Got multiple values, none of them has enough responses for quorum', 'values': values_for_response}), 300
|
||||
|
||||
return jsonify({'error': 'None of the nodes responded with 200.', 'responses': non_ok_responses}), 500
|
||||
|
||||
|
||||
@dataclass
|
||||
class DataBlock:
|
||||
index: Union[int, None]
|
||||
data: bytes
|
||||
|
||||
|
||||
def raid3_split(data_blocks: int, value: str) -> list[DataBlock]:
|
||||
bytes_value = value.encode()
|
||||
L = int(math.ceil(len(bytes_value) / data_blocks))
|
||||
split: list[bytes] = [
|
||||
bytes_value[i:i + L]
|
||||
for i in range(0, len(bytes_value), L)
|
||||
]
|
||||
res = []
|
||||
for i, block in enumerate(split):
|
||||
data_bytes: bytes = block + (L - len(block)) * b'\x00'
|
||||
res.append(DataBlock(i, data_bytes))
|
||||
|
||||
return res
|
||||
|
||||
|
||||
def raid3_parity(blocks: list[DataBlock]) -> DataBlock:
|
||||
parity = len(blocks[0].data) * b'\x00'
|
||||
for block in blocks:
|
||||
if len(block.data) != len(parity):
|
||||
raise ValueError('All blocks must have the same length.')
|
||||
parity = bytes(a ^ b for a, b in zip(parity, block.data))
|
||||
return DataBlock(None, parity)
|
||||
|
||||
|
||||
def raid3_join(blocks: list[DataBlock]) -> str:
|
||||
return (b''.join(block.data for block in blocks[:-1])).rstrip(b'\x00').decode()
|
||||
|
||||
|
||||
def raid3_recover(blocks: list[Union[DataBlock, None]], error_index: int) -> str:
|
||||
if error_index == len(blocks) - 1:
|
||||
return raid3_join(blocks)
|
||||
|
||||
recovered = None
|
||||
for block in blocks:
|
||||
if recovered is None and block is not None:
|
||||
recovered = len(block.data) * b'\x00'
|
||||
if block is None:
|
||||
continue
|
||||
recovered = bytes(a ^ b for a, b in zip(recovered, block.data))
|
||||
|
||||
blocks[error_index] = DataBlock(error_index, recovered)
|
||||
return raid3_join(blocks)
|
||||
|
||||
|
||||
class RAID3Proxy(Proxy):
|
||||
def __init__(self, nodes):
|
||||
if len(nodes) < 3:
|
||||
raise ValueError('RAID3 requires at least 3 nodes.')
|
||||
if len(nodes) > 255:
|
||||
raise ValueError('This implementation does not support more than 255 nodes.')
|
||||
super().__init__(nodes)
|
||||
self._divisor = len(self.nodes) - 1
|
||||
|
||||
def put(self, key, value):
|
||||
if len(value) < self._divisor:
|
||||
return {'error': f'Too small body for RAID3 algorithm with {self._divisor} blocks.'}, 500
|
||||
|
||||
blocks = raid3_split(self._divisor, value)
|
||||
parity = raid3_parity(blocks)
|
||||
|
||||
for node, block in zip(self.nodes.values(), blocks + [parity]):
|
||||
value_dict = {
|
||||
'index': block.index,
|
||||
'value': base64.b64encode(block.data).decode('utf-8'),
|
||||
}
|
||||
|
||||
url = f'{node["url"]}/put?key={key}'
|
||||
response = requests.post(url, json={'value': json.dumps(value_dict)})
|
||||
if response.status_code != 200:
|
||||
return {'error': f'Error while writing to node {node["url"]}.'}, 500
|
||||
|
||||
return jsonify({'message': 'Key-value pair stored successfully on all nodes.'}), 200
|
||||
|
||||
def get(self, key):
|
||||
blocks = []
|
||||
errors = []
|
||||
error_index = None
|
||||
for node in self.nodes.values():
|
||||
url = f'{node["url"]}/get?key={key}'
|
||||
try:
|
||||
response = requests.get(url)
|
||||
if response.status_code == 200:
|
||||
value_dict = json.loads(response.json()['value'])
|
||||
blocks.append(DataBlock(value_dict['index'], base64.b64decode(value_dict['value'])))
|
||||
else:
|
||||
error_index = len(blocks)
|
||||
blocks.append(None)
|
||||
errors.append(response)
|
||||
if len(errors) > 1:
|
||||
return {'error': f'Got too many errors while reading key \'{key}\''}, 500
|
||||
except ConnectionError as e:
|
||||
error_index = len(blocks)
|
||||
blocks.append(None)
|
||||
errors.append(e)
|
||||
if len(errors) > 1:
|
||||
return {'error': f'Got too many errors while reading key \'{key}\''}, 500
|
||||
|
||||
if error_index is None:
|
||||
value = raid3_join(blocks)
|
||||
else:
|
||||
value = raid3_recover(blocks, error_index)
|
||||
|
||||
return jsonify({'value': value}), 200
|
||||
@@ -0,0 +1,20 @@
|
||||
from .strategies import raid3_join, raid3_split
|
||||
|
||||
|
||||
def test_raid3_split():
|
||||
value = 'Hello, world!' # len = 13
|
||||
blocks = raid3_split(2, value) # 2 data blocks of 7 bytes, 1 parity block
|
||||
assert len(blocks) == 2
|
||||
|
||||
assert blocks[0].index == 0
|
||||
assert blocks[0].data == b'Hello, '
|
||||
assert blocks[1].index == 1
|
||||
assert blocks[1].data == b'world!\x00'
|
||||
|
||||
|
||||
def test_raid3_join_utf():
|
||||
value = 'съешь ещё этих мягких французских булок'
|
||||
blocks = raid3_split(4, value)
|
||||
assert len(blocks) == 4
|
||||
joined = raid3_join(blocks + [None])
|
||||
assert joined == value
|
||||
@@ -0,0 +1,124 @@
|
||||
# Шардирование на примере простого key-value-хранилища
|
||||
|
||||
Этот пример показывает, как разделить данные между несколькими узлами. В стенде работают три простых
|
||||
key-value-хранилища: каждое принимает ключ и значение и сохраняет их в памяти. Перед ними находится прокси
|
||||
с единым API. Клиент обращается только к прокси, а тот определяет владельца ключа и пересылает запрос
|
||||
нужному хранилищу.
|
||||
|
||||
В основном режиме прокси вычисляет `SHA-256(key) mod N` и направляет запись и чтение владельцу ключа.
|
||||
В конце также кратко рассмотрен учебный режим репликации.
|
||||
|
||||
## API
|
||||
|
||||
- `POST /put?key=<key>` записывает значение из JSON-поля `value`.
|
||||
- `GET /get?key=<key>` возвращает значение или `404`, если ключа нет на выбранном узле.
|
||||
- `GET /state` показывает содержимое всех хранилищ.
|
||||
|
||||
## Запуск
|
||||
|
||||
Из корня репозитория:
|
||||
|
||||
```bash
|
||||
cd materials/08-scaling/seminar/kv
|
||||
docker compose up -d --build
|
||||
docker compose ps
|
||||
```
|
||||
|
||||
Прокси будет доступен на порту 5005. Хранилища будут доступны на портах 5001, 5002 и 5003.
|
||||
|
||||
## Запись и чтение
|
||||
|
||||
Записать одно значение через прокси:
|
||||
|
||||
```bash
|
||||
curl -sS -X POST 'http://localhost:5005/put?key=key-0' \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d '{"value":"value-0"}'
|
||||
```
|
||||
|
||||
Прочитать его:
|
||||
|
||||
```bash
|
||||
curl -sS 'http://localhost:5005/get?key=key-0'
|
||||
```
|
||||
|
||||
Записать `N` значений с ключами от `key-0` до `key-(N-1)`:
|
||||
|
||||
```bash
|
||||
N=20
|
||||
|
||||
for ((i = 0; i < N; i++)); do
|
||||
curl -sS -X POST "http://localhost:5005/put?key=key-$i" \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d "{\"value\":\"value-$i\"}"
|
||||
echo
|
||||
done
|
||||
```
|
||||
|
||||
## Проверка распределения
|
||||
|
||||
Получить состояние всех узлов через прокси:
|
||||
|
||||
```bash
|
||||
open 'http://localhost:5005/state'
|
||||
```
|
||||
|
||||
Состояние отдельных хранилищ:
|
||||
|
||||
```bash
|
||||
curl -sS 'http://localhost:5001/state'
|
||||
curl -sS 'http://localhost:5002/state'
|
||||
```
|
||||
|
||||
Для текущей конфигурации `key-0` находится на третьем узле:
|
||||
|
||||
```bash
|
||||
curl -i 'http://localhost:5003/get?key=key-0' # 200
|
||||
curl -i 'http://localhost:5002/get?key=key-0' # 404
|
||||
```
|
||||
|
||||
## Изменение конфигурации
|
||||
|
||||
Состав узлов задаётся в `proxy/compose_config.yml`:
|
||||
|
||||
```yaml
|
||||
mode: sharding
|
||||
nodes:
|
||||
- name: node1
|
||||
url: http://kv1:5000
|
||||
- name: node2
|
||||
url: http://kv2:5000
|
||||
- name: node3
|
||||
url: http://kv3:5000
|
||||
```
|
||||
|
||||
Конфигурация подключена к контейнеру как файл только для чтения. После её изменения достаточно
|
||||
перезапустить прокси:
|
||||
|
||||
```bash
|
||||
docker compose restart proxy
|
||||
```
|
||||
|
||||
Хранилища при этом не перезапускаются, и данные в их памяти сохраняются.
|
||||
|
||||
## Режим репликации
|
||||
|
||||
В режиме `replication` прокси записывает каждое значение во все хранилища. При чтении он опрашивает
|
||||
узлы по очереди и возвращает первый успешный ответ. В отличие от шардирования, данные не разделяются,
|
||||
а копируются.
|
||||
|
||||
Чтобы включить этот режим, изменить первую строку в `proxy/compose_config.yml`:
|
||||
|
||||
```yaml
|
||||
mode: replication
|
||||
```
|
||||
|
||||
Затем перезапустить только прокси и повторить команды записи и чтения:
|
||||
|
||||
```bash
|
||||
docker compose restart proxy
|
||||
open 'http://localhost:5005/state'
|
||||
```
|
||||
|
||||
После записи новый ключ должен появиться на всех трёх узлах. Это упрощённая учебная реализация:
|
||||
она не гарантирует согласованность при частичных записях и не обрабатывает отказы как промышленная система.
|
||||
@@ -0,0 +1,9 @@
|
||||
FROM python:3.9-slim
|
||||
|
||||
WORKDIR /app
|
||||
COPY requirements.txt .
|
||||
RUN pip install -r requirements.txt
|
||||
|
||||
COPY . .
|
||||
|
||||
ENTRYPOINT [ "python", "main.py"]
|
||||
@@ -0,0 +1,64 @@
|
||||
import argparse
|
||||
from flask import Flask, jsonify, request
|
||||
|
||||
app = Flask(__name__)
|
||||
data = {}
|
||||
|
||||
|
||||
@app.route('/put', methods=['POST'])
|
||||
def put_handler():
|
||||
key = request.args.get('key')
|
||||
value = request.get_json().get('value')
|
||||
|
||||
if key is None or value is None:
|
||||
return jsonify({'error': 'Please provide key and value.'}), 400
|
||||
|
||||
data[key] = value
|
||||
return jsonify({'message': 'Key-value pair stored successfully.'}), 200
|
||||
|
||||
|
||||
@app.route('/get', methods=['GET'])
|
||||
def get_handler():
|
||||
key = request.args.get('key')
|
||||
|
||||
if key is None:
|
||||
return jsonify({'error': 'Please provide key.'}), 400
|
||||
|
||||
value = data.get(key)
|
||||
|
||||
if value is None:
|
||||
return jsonify({'error': 'Key not found.', 'key': key}), 404
|
||||
|
||||
return jsonify({'value': value}), 200
|
||||
|
||||
|
||||
@app.route('/state', methods=['GET'])
|
||||
def state_handler():
|
||||
table = '<table>'
|
||||
table += f'<tr><th colspan="2" style="font-size: 24px; font-weight: bold; font-family: monospace; padding-bottom: 10px;">Data on {app.name}</th></tr>'
|
||||
table += '<tr><th style="font-weight: bold; text-align: left; font-family: monospace; padding-bottom: 15px;">Key</th><th style="font-family: monospace; text-align: left; padding-bottom: 15px; padding-left: 20px;">Value</th></tr>'
|
||||
|
||||
for key, value in data.items():
|
||||
value = value[:64] + f'...[total of {len(value)} bytes]' if len(value) > 100 else value
|
||||
table += f'<tr><td style="font-weight: bold; font-family: monospace;">{key}</td><td style="font-family: monospace; text-align: left; padding-left: 20px;">{value}</td></tr>'
|
||||
|
||||
table += '</table>'
|
||||
return table
|
||||
|
||||
|
||||
@app.route('/health', methods=['GET'])
|
||||
def health_handler():
|
||||
return f'Hi from {app.name}'
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = argparse.ArgumentParser(description='Flask app with hostname as app name')
|
||||
parser.add_argument('-n', '--hostname', type=str, help='Hostname of the server')
|
||||
parser.add_argument('-p', '--port', type=int, help='Port of the server')
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.hostname:
|
||||
app.name = args.hostname
|
||||
|
||||
|
||||
app.run('0.0.0.0', args.port)
|
||||
@@ -0,0 +1,3 @@
|
||||
Flask==2.3.3
|
||||
requests==2.31.0
|
||||
pyyaml==6.0.1
|
||||
Reference in new issue
Block a user