Add week 8 materials
This commit is contained in:
osukhoroslov committed
1 parent
a987b82834
commit
0bf71ed374
38 files changed
+2908
No files matched your search
@@ -0,0 +1,9 @@
|
||||
FROM python:3.9-slim
|
||||
|
||||
WORKDIR /app
|
||||
COPY requirements.txt .
|
||||
RUN pip install -r requirements.txt
|
||||
|
||||
COPY . .
|
||||
|
||||
ENTRYPOINT [ "python", "main.py"]
|
||||
Whitespace-only changes.
@@ -0,0 +1,8 @@
|
||||
mode: sharding
|
||||
nodes:
|
||||
- name: node1
|
||||
url: http://kv1:5000
|
||||
- name: node2
|
||||
url: http://kv2:5000
|
||||
- name: node3
|
||||
url: http://kv3:5000
|
||||
@@ -0,0 +1,76 @@
|
||||
import argparse
|
||||
import requests
|
||||
from strategies import (
|
||||
Proxy, ShardingProxy,
|
||||
ReplicatingProxy, ReplicatingQuorumProxy,
|
||||
RAID3Proxy
|
||||
)
|
||||
import yaml
|
||||
from flask import Flask, jsonify, request
|
||||
|
||||
app = Flask(__name__)
|
||||
|
||||
proxy: Proxy = None
|
||||
|
||||
|
||||
@app.route('/put', methods=['POST'])
|
||||
def put_handler():
|
||||
key = request.args.get('key')
|
||||
value = request.get_json().get('value')
|
||||
|
||||
if key is None or value is None:
|
||||
return jsonify({'error': 'Please provide key and value.'}), 400
|
||||
|
||||
return proxy.put(key, value)
|
||||
|
||||
@app.route('/get', methods=['GET'])
|
||||
def get_handler():
|
||||
key = request.args.get('key')
|
||||
|
||||
if key is None:
|
||||
return jsonify({'error': 'Please provide key.'}), 400
|
||||
|
||||
return proxy.get(key)
|
||||
|
||||
|
||||
@app.route('/state', methods=['GET'])
|
||||
def state_handler():
|
||||
results = ""
|
||||
for _, node in proxy.nodes.items():
|
||||
url = f'{node["url"]}/state'
|
||||
response = requests.get(url)
|
||||
results += f'<h1 style="font-family: monospace;">Response from {url}</h1>{response.content.decode("utf-8")}<hr>'
|
||||
|
||||
return f"<html><body>{results}</body></html>", 200
|
||||
|
||||
|
||||
def load_proxy_from_config(config_path):
|
||||
with open(config_path, 'r') as file:
|
||||
config = yaml.safe_load(file)
|
||||
|
||||
mode = config.get('mode', '')
|
||||
nodes = {}
|
||||
for node in config['nodes']:
|
||||
name = node['name']
|
||||
url = node['url']
|
||||
nodes[name] = {'url': url}
|
||||
if mode == 'sharding':
|
||||
return ShardingProxy(nodes)
|
||||
elif mode == 'replication':
|
||||
return ReplicatingProxy(nodes)
|
||||
elif mode == 'replication-quorum':
|
||||
return ReplicatingQuorumProxy(nodes)
|
||||
elif mode == 'replication-raid3':
|
||||
return RAID3Proxy(nodes)
|
||||
raise ValueError('Invalid mode, allowed values: sharding, replication, replication-quorum')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = argparse.ArgumentParser(description='Flask app acting as a proxy for key-value nodes')
|
||||
parser.add_argument('-c', '--config', type=str, required=True, help='Path to the YAML config file')
|
||||
parser.add_argument('-p', '--port', type=int, help='Port of the server')
|
||||
args = parser.parse_args()
|
||||
|
||||
proxy = load_proxy_from_config(args.config)
|
||||
|
||||
app.run('0.0.0.0', args.port)
|
||||
@@ -0,0 +1,3 @@
|
||||
Flask==2.3.3
|
||||
requests==2.31.0
|
||||
pyyaml==6.0.1
|
||||
@@ -0,0 +1,244 @@
|
||||
from dataclasses import dataclass
|
||||
import json
|
||||
from collections import defaultdict
|
||||
import math
|
||||
from typing import Union
|
||||
from flask import jsonify
|
||||
import requests
|
||||
from abc import ABC, abstractmethod
|
||||
import base64
|
||||
|
||||
from requests.exceptions import ConnectionError
|
||||
|
||||
import hashlib
|
||||
|
||||
|
||||
class Proxy(ABC):
|
||||
def __init__(self, nodes):
|
||||
self.nodes = nodes
|
||||
|
||||
@abstractmethod
|
||||
def put(self, key, value):
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def get(self, key):
|
||||
pass
|
||||
|
||||
|
||||
class ShardingProxy(Proxy):
|
||||
def __init__(self, nodes):
|
||||
super().__init__(nodes)
|
||||
|
||||
def put(self, key, value):
|
||||
node = self.get_node(key)
|
||||
if node is None:
|
||||
return {'error': 'No nodes available.'}, 500
|
||||
|
||||
url = f'{node["url"]}/put?key={key}'
|
||||
response = requests.post(url, json={'value': value})
|
||||
|
||||
return response.content, response.status_code
|
||||
|
||||
def get(self, key):
|
||||
node = self.get_node(key)
|
||||
if node is None:
|
||||
return {'error': 'No nodes available.'}, 500
|
||||
|
||||
url = f'{node["url"]}/get?key={key}'
|
||||
response = requests.get(url)
|
||||
|
||||
return response.content, response.status_code
|
||||
|
||||
def get_node(self, key):
|
||||
if len(self.nodes) == 0:
|
||||
return None
|
||||
|
||||
hash_val = hashlib.sha256(key.encode()).hexdigest()
|
||||
node_index = int(hash_val, 16) % len(self.nodes)
|
||||
return self.nodes[list(self.nodes.keys())[node_index]]
|
||||
|
||||
|
||||
class ReplicatingProxy(Proxy):
|
||||
def __init__(self, nodes):
|
||||
super().__init__(nodes)
|
||||
|
||||
def put(self, key, value):
|
||||
responses = []
|
||||
for _, node in self.nodes.items():
|
||||
url = f'{node["url"]}/put?key={key}'
|
||||
response = requests.post(url, json={'value': value})
|
||||
if response.status_code == 200:
|
||||
responses.append(response.content)
|
||||
|
||||
if len(responses) == 0:
|
||||
return {'error': 'No nodes available.'}, 500
|
||||
|
||||
return responses[0], 200
|
||||
|
||||
def get(self, key):
|
||||
non_ok_responses = dict()
|
||||
for node_name, node in self.nodes.items():
|
||||
url = f'{node["url"]}/get?key={key}'
|
||||
response = requests.get(url)
|
||||
if response.status_code == 200:
|
||||
return response.content, 200
|
||||
else:
|
||||
non_ok_responses[node_name] = response.json()
|
||||
|
||||
return jsonify({'error': 'None of the nodes responded with 200.', 'responses': non_ok_responses}), 500
|
||||
|
||||
|
||||
class ReplicatingQuorumProxy(Proxy):
|
||||
def __init__(self, nodes):
|
||||
super().__init__(nodes)
|
||||
|
||||
def put(self, key, value):
|
||||
responses = []
|
||||
for _, node in self.nodes.items():
|
||||
url = f'{node["url"]}/put?key={key}'
|
||||
response = requests.post(url, json={'value': value})
|
||||
if response.status_code == 200:
|
||||
responses.append(response.content)
|
||||
|
||||
if len(responses) == 0:
|
||||
return {'error': 'No nodes available.'}, 500
|
||||
|
||||
if len(responses) > len(self.nodes) / 2:
|
||||
return responses[0], 200
|
||||
|
||||
return jsonify({'error': 'Not enough responses for quorum write.', 'responses': responses}), 300
|
||||
|
||||
def get(self, key):
|
||||
non_ok_responses = dict()
|
||||
values = defaultdict(lambda: 0)
|
||||
for node_name, node in self.nodes.items():
|
||||
url = f'{node["url"]}/get?key={key}'
|
||||
response = requests.get(url)
|
||||
if response.status_code == 200:
|
||||
values[response.content] += 1
|
||||
else:
|
||||
non_ok_responses[node_name] = response.json()
|
||||
|
||||
for value, cnt in values.items():
|
||||
if cnt > len(self.nodes) / 2:
|
||||
return value, 200
|
||||
|
||||
if len(values) > 0:
|
||||
values_for_response = []
|
||||
for value, cnt in values.items():
|
||||
values_for_response.append({'value': json.loads(value)['value'], 'count': cnt})
|
||||
return jsonify({'error': 'Got multiple values, none of them has enough responses for quorum', 'values': values_for_response}), 300
|
||||
|
||||
return jsonify({'error': 'None of the nodes responded with 200.', 'responses': non_ok_responses}), 500
|
||||
|
||||
|
||||
@dataclass
|
||||
class DataBlock:
|
||||
index: Union[int, None]
|
||||
data: bytes
|
||||
|
||||
|
||||
def raid3_split(data_blocks: int, value: str) -> list[DataBlock]:
|
||||
bytes_value = value.encode()
|
||||
L = int(math.ceil(len(bytes_value) / data_blocks))
|
||||
split: list[bytes] = [
|
||||
bytes_value[i:i + L]
|
||||
for i in range(0, len(bytes_value), L)
|
||||
]
|
||||
res = []
|
||||
for i, block in enumerate(split):
|
||||
data_bytes: bytes = block + (L - len(block)) * b'\x00'
|
||||
res.append(DataBlock(i, data_bytes))
|
||||
|
||||
return res
|
||||
|
||||
|
||||
def raid3_parity(blocks: list[DataBlock]) -> DataBlock:
|
||||
parity = len(blocks[0].data) * b'\x00'
|
||||
for block in blocks:
|
||||
if len(block.data) != len(parity):
|
||||
raise ValueError('All blocks must have the same length.')
|
||||
parity = bytes(a ^ b for a, b in zip(parity, block.data))
|
||||
return DataBlock(None, parity)
|
||||
|
||||
|
||||
def raid3_join(blocks: list[DataBlock]) -> str:
|
||||
return (b''.join(block.data for block in blocks[:-1])).rstrip(b'\x00').decode()
|
||||
|
||||
|
||||
def raid3_recover(blocks: list[Union[DataBlock, None]], error_index: int) -> str:
|
||||
if error_index == len(blocks) - 1:
|
||||
return raid3_join(blocks)
|
||||
|
||||
recovered = None
|
||||
for block in blocks:
|
||||
if recovered is None and block is not None:
|
||||
recovered = len(block.data) * b'\x00'
|
||||
if block is None:
|
||||
continue
|
||||
recovered = bytes(a ^ b for a, b in zip(recovered, block.data))
|
||||
|
||||
blocks[error_index] = DataBlock(error_index, recovered)
|
||||
return raid3_join(blocks)
|
||||
|
||||
|
||||
class RAID3Proxy(Proxy):
|
||||
def __init__(self, nodes):
|
||||
if len(nodes) < 3:
|
||||
raise ValueError('RAID3 requires at least 3 nodes.')
|
||||
if len(nodes) > 255:
|
||||
raise ValueError('This implementation does not support more than 255 nodes.')
|
||||
super().__init__(nodes)
|
||||
self._divisor = len(self.nodes) - 1
|
||||
|
||||
def put(self, key, value):
|
||||
if len(value) < self._divisor:
|
||||
return {'error': f'Too small body for RAID3 algorithm with {self._divisor} blocks.'}, 500
|
||||
|
||||
blocks = raid3_split(self._divisor, value)
|
||||
parity = raid3_parity(blocks)
|
||||
|
||||
for node, block in zip(self.nodes.values(), blocks + [parity]):
|
||||
value_dict = {
|
||||
'index': block.index,
|
||||
'value': base64.b64encode(block.data).decode('utf-8'),
|
||||
}
|
||||
|
||||
url = f'{node["url"]}/put?key={key}'
|
||||
response = requests.post(url, json={'value': json.dumps(value_dict)})
|
||||
if response.status_code != 200:
|
||||
return {'error': f'Error while writing to node {node["url"]}.'}, 500
|
||||
|
||||
return jsonify({'message': 'Key-value pair stored successfully on all nodes.'}), 200
|
||||
|
||||
def get(self, key):
|
||||
blocks = []
|
||||
errors = []
|
||||
error_index = None
|
||||
for node in self.nodes.values():
|
||||
url = f'{node["url"]}/get?key={key}'
|
||||
try:
|
||||
response = requests.get(url)
|
||||
if response.status_code == 200:
|
||||
value_dict = json.loads(response.json()['value'])
|
||||
blocks.append(DataBlock(value_dict['index'], base64.b64decode(value_dict['value'])))
|
||||
else:
|
||||
error_index = len(blocks)
|
||||
blocks.append(None)
|
||||
errors.append(response)
|
||||
if len(errors) > 1:
|
||||
return {'error': f'Got too many errors while reading key \'{key}\''}, 500
|
||||
except ConnectionError as e:
|
||||
error_index = len(blocks)
|
||||
blocks.append(None)
|
||||
errors.append(e)
|
||||
if len(errors) > 1:
|
||||
return {'error': f'Got too many errors while reading key \'{key}\''}, 500
|
||||
|
||||
if error_index is None:
|
||||
value = raid3_join(blocks)
|
||||
else:
|
||||
value = raid3_recover(blocks, error_index)
|
||||
|
||||
return jsonify({'value': value}), 200
|
||||
@@ -0,0 +1,20 @@
|
||||
from .strategies import raid3_join, raid3_split
|
||||
|
||||
|
||||
def test_raid3_split():
|
||||
value = 'Hello, world!' # len = 13
|
||||
blocks = raid3_split(2, value) # 2 data blocks of 7 bytes, 1 parity block
|
||||
assert len(blocks) == 2
|
||||
|
||||
assert blocks[0].index == 0
|
||||
assert blocks[0].data == b'Hello, '
|
||||
assert blocks[1].index == 1
|
||||
assert blocks[1].data == b'world!\x00'
|
||||
|
||||
|
||||
def test_raid3_join_utf():
|
||||
value = 'съешь ещё этих мягких французских булок'
|
||||
blocks = raid3_split(4, value)
|
||||
assert len(blocks) == 4
|
||||
joined = raid3_join(blocks + [None])
|
||||
assert joined == value
|
||||
Reference in new issue
Block a user