course/hse-2026
1
Fork 1
Code Issues Pull Requests Actions Packages Activity

Add HW 3

Browse Source
This commit is contained in:
osukhoroslov
2026-09-24 21:30:37 +03:00
parent 0a1eb51bf8
commit 6210b46c1f
35 changed files with 4012 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
homework/03-http-server/tests/.dockerignore
+7
View File
@@ -0,0 +1,7 @@
target
**/target
**/__pycache__
**/.pytest_cache
*.pyc
tmp/
tests.log
homework/03-http-server/tests/.gitignore
+1
View File
@@ -0,0 +1 @@
tmp
homework/03-http-server/tests/Dockerfile
+18
View File
@@ -0,0 +1,18 @@
# syntax=docker/dockerfile:1
FROM docker:dind
RUN apk add --no-cache bash ca-certificates go wget
WORKDIR /hw
COPY go.mod go.sum tests/
RUN --mount=type=cache,id=distsys-course-go-mod,target=/go/pkg/mod,sharing=locked \
cd tests && go mod download
COPY . tests
RUN --mount=type=cache,id=distsys-course-go-mod,target=/go/pkg/mod,sharing=locked \
--mount=type=cache,id=distsys-course-go-build,target=/root/.cache/go-build,sharing=locked \
cd tests && go test -c -o /usr/local/bin/hw3test .
COPY entrypoint.sh /usr/local/bin/
COPY verify-zram-scratch.sh /usr/local/bin/
RUN chmod +x /usr/local/bin/entrypoint.sh /usr/local/bin/verify-zram-scratch.sh
ENTRYPOINT ["timeout", "-k", "10", "600", "entrypoint.sh"]
homework/03-http-server/tests/compare.go
+124
View File
@@ -0,0 +1,124 @@
package hw3test
import (
"bytes"
"fmt"
"github.com/stretchr/testify/require"
"io"
"math/rand"
"os"
"path"
)
// RequireFileContent requires that the contents of the file match the contents of the reader.
func RequireFileContent(t *TC, workdir, p string, f *EnvFile) {
fullpath := path.Join(workdir, p)
file, err := os.Open(fullpath)
require.NoError(t, err, "failed to open file %s, expected it exists on disk", p)
defer file.Close()
stat, err := file.Stat()
require.NoError(t, err, "failed to stat file %s", p)
require.Equal(t, f.Size, stat.Size(), "file %s has wrong size", p)
require.NoError(t, CompareFileContent(t, file, f), "file %s has wrong content", p)
}
// CompareFileContent compares first f.Size bytes of the reader with generated content.
func CompareFileContent(t *TC, r io.Reader, f *EnvFile) error {
actual := make([]byte, 64*1024)
expected := make([]byte, len(actual))
var generator io.Reader
if f.TextOnly {
generator = f.Open()
} else {
generator = newComparisonReader(f.GenSeed)
}
for offset := int64(0); offset < f.Size; {
count := int64(len(actual))
if remaining := f.Size - offset; remaining < count {
count = remaining
}
if _, err := io.ReadFull(r, actual[:count]); err != nil {
return fmt.Errorf("unexpected file error at position %d: %w", offset, err)
}
if _, err := io.ReadFull(generator, expected[:count]); err != nil {
return fmt.Errorf("unexpected generator error at position %d: %w", offset, err)
}
if !bytes.Equal(actual[:count], expected[:count]) {
for i := int64(0); i < count; i++ {
if actual[i] != expected[i] {
return fmt.Errorf("position %d, expected byte %d, got %d", offset+i, expected[i], actual[i])
}
}
}
offset += count
}
return nil
}
// comparisonReader emits the same bytes as rand.New(source).Read, seven bytes
// from each Int63 value. It fills complete seven-byte groups without the
// per-byte branch in math/rand.Read. Request bodies still use EnvFile.Open.
type comparisonReader struct {
source rand.Source
value uint64
remaining uint8
}
func newComparisonReader(seed int64) *comparisonReader {
return &comparisonReader{source: rand.NewSource(seed)}
}
func (r *comparisonReader) Read(p []byte) (int, error) {
n := len(p)
for len(p) > 0 && r.remaining > 0 {
p[0] = byte(r.value)
r.value >>= 8
r.remaining--
p = p[1:]
}
for len(p) >= 7 {
value := uint64(r.source.Int63())
p[0] = byte(value)
p[1] = byte(value >> 8)
p[2] = byte(value >> 16)
p[3] = byte(value >> 24)
p[4] = byte(value >> 32)
p[5] = byte(value >> 40)
p[6] = byte(value >> 48)
p = p[7:]
}
if len(p) > 0 {
r.value = uint64(r.source.Int63())
r.remaining = 7
for i := range p {
p[i] = byte(r.value)
r.value >>= 8
r.remaining--
}
}
return n, nil
}
// RequireDir ensures that the directory exists on disk.
func RequireDir(t *TC, workdir string, p string, dir *EnvDir) {
fullpath := path.Join(workdir, p)
stat, err := os.Stat(fullpath)
require.NoError(t, err, "failed to stat directory %s, expected it exists on disk", p)
require.True(t, stat.IsDir(), "expected %s to be a directory", p)
}
// RequireNotExists ensures that the file/directory does not exist on disk.
func RequireNotExists(t *TC, workdir string, p string) {
fullpath := path.Join(workdir, p)
_, err := os.Stat(fullpath)
require.True(t, os.IsNotExist(err), "expected %s to not exist on disk", p)
}
// RequireExists ensures that the file/directory exists on disk.
func RequireExists(t *TC, workdir string, p string) {
fullpath := path.Join(workdir, p)
_, err := os.Stat(fullpath)
require.NoError(t, err, "failed to stat %s, expected it exists on disk", p)
}
homework/03-http-server/tests/compare_reader_test.go
+60
View File
@@ -0,0 +1,60 @@
package hw3test
import (
"bytes"
"io"
"math/rand"
"testing"
)
func TestComparisonReaderMatchesEnvFile(t *testing.T) {
for _, seed := range []int64{-7824, 0, 1, 7824, 9223372036854775807} {
for _, chunkSize := range []int{1, 6, 7, 8, 13, 64 * 1024, 131071} {
const total = 1024*1024 + 19
expected := make([]byte, total)
actual := make([]byte, total)
_, err := io.ReadFull(rand.New(rand.NewSource(seed)), expected)
if err != nil {
t.Fatal(err)
}
reader := newComparisonReader(seed)
for offset := 0; offset < total; {
end := offset + chunkSize
if end > total {
end = total
}
if _, err := io.ReadFull(reader, actual[offset:end]); err != nil {
t.Fatal(err)
}
offset = end
}
if !bytes.Equal(actual, expected) {
t.Fatalf("different bytes for seed %d and chunk size %d", seed, chunkSize)
}
}
}
}
func BenchmarkComparisonReaders(b *testing.B) {
const size = 128 * 1024 * 1024
buffer := make([]byte, 64*1024)
for _, candidate := range []struct {
name string
open func() io.Reader
}{
{"math/rand", func() io.Reader { return rand.New(rand.NewSource(7824)) }},
{"comparison", func() io.Reader { return newComparisonReader(7824) }},
} {
b.Run(candidate.name, func(b *testing.B) {
b.SetBytes(size)
for i := 0; i < b.N; i++ {
reader := candidate.open()
for remaining := size; remaining > 0; remaining -= len(buffer) {
if _, err := io.ReadFull(reader, buffer); err != nil {
b.Fatal(err)
}
}
}
})
}
}
homework/03-http-server/tests/contract.go
+301
View File
@@ -0,0 +1,301 @@
package hw3test
import (
"bufio"
"bytes"
"compress/gzip"
"crypto/sha256"
"fmt"
"io"
"math/rand"
"net"
"net/http"
"net/url"
"os"
"path/filepath"
"reflect"
"strings"
"time"
"github.com/stretchr/testify/require"
)
// Contract cases use a small separate tree, so snapshots never scan G6/G7 files.
func RunContractTests(t *TC, runner Runner, tmpRoot, group string) {
t.RunByName("contract", func(t *TC) {
workdir, err := os.MkdirTemp(tmpRoot, "contract-")
require.NoError(t, err)
defer os.RemoveAll(workdir)
require.NoError(t, os.Chmod(workdir, 0755))
require.NoError(t, os.MkdirAll(filepath.Join(workdir, "dir", "nested"), 0755))
ascii := []byte("hello\r\nworld\n")
binary := []byte{0, 255, 13, 10, 13, 10, 128, 1}
if group == "G1" || group == "G3" {
binary = ascii
}
for name, body := range map[string][]byte{
"alpha.txt": ascii, "binary": binary, "empty": {},
".hidden": []byte("hidden"), "dir/nested/child": []byte("child"),
} {
require.NoError(t, os.WriteFile(filepath.Join(workdir, filepath.FromSlash(name)), body, 0644))
}
port, err := GetFreePort()
require.NoError(t, err)
opts := RunOpts{Port: port, WorkingDirectory: workdir, ListenAddr: "0.0.0.0", ServerDomain: "localhost"}
opts.GenerateRunConfig(t, rand.New(rand.NewSource(9103)), &EnvGen{AllowEnv: true})
stop, err := runner.Run(t, opts)
require.NoError(t, err)
defer stop()
require.NoError(t, WaitForServer(t, opts))
extra := group == "G5" || group == "G7"
request := func(method, path, headers string, body []byte) string {
return fmt.Sprintf("%s %s HTTP/1.1\r\nhOsT: LOCALHOST\r\ncOnTeNt-LeNgTh: %d\r\n%s\r\n%s", method, path, len(body), headers, body)
}
run := func(name, raw string, split bool, codes []int, expected []byte, unchanged bool) {
t.RunByName(name, func(t *TC) {
before, err := snapshotTree(workdir)
require.NoError(t, err)
parts := [][]byte{[]byte(raw)}
if split {
// Split inside a header and inside CRLFCRLF; coalesce its end with body bytes.
boundary := strings.Index(raw, "\r\n\r\n")
parts = [][]byte{[]byte(raw[:9]), []byte(raw[9 : boundary+3]), []byte(raw[boundary+3:])}
}
resp, body := contractExchange(t, opts, parts)
require.Contains(t, codes, resp.StatusCode)
if resp.StatusCode >= 400 {
require.NotEmpty(t, bytes.TrimSpace(body), "error explanation is empty")
}
if extra {
require.NotEmpty(t, resp.Header.Get("Server"))
if len(body) > 0 {
mediaType, err := parseContentType(resp.Header.Get("Content-Type"))
require.NoError(t, err)
if methodIsDirectoryGet(raw) && resp.StatusCode == 200 {
require.Contains(t, []string{"text/plain", "text/html"}, mediaType)
}
}
}
if expected != nil {
require.Empty(t, resp.Header.Get("Content-Encoding"))
require.Equal(t, expected, body)
}
if unchanged {
after, err := snapshotTree(workdir)
require.NoError(t, err)
require.NoError(t, compareSnapshots(before, after))
}
})
}
if group == "G4" || group == "G5" || group == "G6" || group == "G7" {
run("post-root-file", request("POST", "/", "", binary), false, []int{409}, nil, true)
run("post-root-dir", request("POST", "/", "Create-Directory: True\r\n", nil), false, []int{409}, nil, true)
run("put-root", request("PUT", "/", "", binary), false, []int{409}, nil, true)
for _, value := range []string{"absent", "False", "True"} {
header := ""
if value != "absent" {
header = "Remove-Directory: " + value + "\r\n"
}
run("delete-root-"+value, request("DELETE", "/", header, nil), false, []int{403}, nil, true)
}
if extra {
raw := strings.Replace(request("DELETE", "/", "Remove-Directory: True\r\n", nil), "LOCALHOST", "wrong.example", 1)
run("wrong-host-delete-root", raw, false, []int{400}, nil, true)
}
}
switch group {
case "G1":
run("split-headers", request("GET", "/alpha.txt", "", nil), true, []int{200}, ascii, true)
// No Content-Length is also valid for an empty request.
run("no-request-body", "GET /alpha.txt HTTP/1.1\r\nHost: localhost\r\n\r\n", false, []int{200}, ascii, true)
run("empty-file", request("GET", "/empty", "", nil), false, []int{200}, []byte{}, true)
case "G2":
run("binary-body", request("GET", "/binary", "", nil), true, []int{200}, binary, true)
case "G3":
run("missing", request("GET", "/missing", "", nil), false, []int{404}, nil, true)
t.RunByName("root-listing", func(t *TC) {
resp, body := contractExchange(t, opts, [][]byte{[]byte(request("GET", "/", "", nil))})
require.Equal(t, 200, resp.StatusCode)
require.Empty(t, resp.Header.Get("Content-Encoding"))
for _, name := range []string{"alpha.txt", "binary", "empty", ".hidden", "dir"} {
require.Contains(t, string(body), name)
}
})
case "G4", "G5":
for _, tc := range []struct {
name, method, path, headers string
code int
}{
{"post-existing-file", "POST", "/alpha.txt", "", 409},
{"post-existing-dir", "POST", "/dir", "Create-Directory: True\r\n", 409},
{"post-missing-parent", "POST", "/missing/new", "", 404},
{"post-dir-missing-parent", "POST", "/missing/new", "Create-Directory: True\r\n", 404},
{"post-file-as-parent", "POST", "/alpha.txt/child", "", 404},
{"put-missing", "PUT", "/missing", "", 404},
{"put-dir", "PUT", "/dir", "", 409},
{"delete-missing", "DELETE", "/missing", "", 404},
{"delete-dir-absent", "DELETE", "/dir", "", 406},
{"delete-dir-false", "DELETE", "/dir", "rEmOvE-dIrEcToRy: False\r\n", 406},
} {
body := []byte(nil)
if tc.method == "PUT" || (tc.method == "POST" && tc.headers == "") {
body = binary
}
run(tc.name, request(tc.method, tc.path, tc.headers, body), false, []int{tc.code}, nil, true)
}
run("create-file-false", request("POST", "/new", "cReAtE-dIrEcToRy: False\r\n", binary), true, []int{200, 201}, nil, false)
run("read-created", request("GET", "/new", "", nil), false, []int{200}, binary, true)
run("create-empty", request("POST", "/new-empty", "", nil), false, []int{200, 201}, nil, false)
run("read-empty", request("GET", "/new-empty", "", nil), false, []int{200}, []byte{}, true)
run("replace-shorter", request("PUT", "/alpha.txt", "", []byte("x")), true, []int{200, 204}, nil, false)
run("read-shorter", request("GET", "/alpha.txt", "", nil), false, []int{200}, []byte("x"), true)
run("replace-empty", request("PUT", "/alpha.txt", "", nil), false, []int{200, 204}, nil, false)
run("read-replaced-empty", request("GET", "/alpha.txt", "", nil), false, []int{200}, []byte{}, true)
run("create-dir", request("POST", "/created-dir", "cReAtE-dIrEcToRy: True\r\n", nil), false, []int{200, 201}, nil, false)
t.RunByName("created-dir-on-disk", func(t *TC) {
entries, err := os.ReadDir(filepath.Join(workdir, "created-dir"))
require.NoError(t, err)
require.Empty(t, entries)
})
run("recursive-delete", request("DELETE", "/dir", "rEmOvE-dIrEcToRy: True\r\n", nil), false, []int{200}, nil, false)
t.RunByName("deleted-tree-on-disk", func(t *TC) {
_, err := os.Stat(filepath.Join(workdir, "dir"))
require.True(t, os.IsNotExist(err), "directory must be removed")
})
if extra {
for _, method := range []string{"GET", "POST", "PUT", "DELETE"} {
raw := strings.Replace(request(method, "/alpha.txt", "", []byte("changed")), "LOCALHOST", "wrong.example", 1)
run("wrong-host-"+method, raw, false, []int{400}, nil, true)
}
}
case "G7":
for _, path := range []string{"/binary", "/dir", "/empty"} {
t.RunByName("gzip-"+strings.TrimPrefix(path, "/"), func(t *TC) {
before, err := snapshotTree(workdir)
require.NoError(t, err)
resp, body := contractExchange(t, opts, [][]byte{[]byte(request("GET", path, "aCcEpT-eNcOdInG: gzip\r\n", nil))})
require.Equal(t, 200, resp.StatusCode)
require.Equal(t, "gzip", resp.Header.Get("Content-Encoding"))
require.NotEmpty(t, resp.Header.Get("Server"))
mediaType, err := parseContentType(resp.Header.Get("Content-Type"))
require.NoError(t, err)
gz, err := gzip.NewReader(bytes.NewReader(body))
require.NoError(t, err)
decoded, err := io.ReadAll(gz)
require.NoError(t, err)
require.NoError(t, gz.Close())
if path == "/binary" {
require.Equal(t, binary, decoded)
}
if path == "/empty" {
require.Empty(t, decoded)
}
if path == "/dir" {
require.Contains(t, string(decoded), "nested")
require.Contains(t, []string{"text/plain", "text/html"}, mediaType)
}
after, err := snapshotTree(workdir)
require.NoError(t, err)
require.NoError(t, compareSnapshots(before, after))
})
}
}
})
}
func methodIsDirectoryGet(raw string) bool { return strings.HasPrefix(raw, "GET / HTTP/") }
// The client leaves its sending side open: a server reading to EOF must time out.
func contractExchange(t *TC, opts RunOpts, parts [][]byte) (*http.Response, []byte) {
req, err := http.ReadRequest(bufio.NewReader(bytes.NewReader(bytes.Join(parts, nil))))
require.NoError(t, err)
defer req.Body.Close()
address, err := url.Parse(opts.Address())
require.NoError(t, err)
conn, err := net.DialTimeout("tcp", address.Host, 10*time.Second)
require.NoError(t, err)
defer conn.Close()
require.NoError(t, conn.SetDeadline(time.Now().Add(10*time.Second)))
for i, part := range parts {
_, err := io.Copy(conn, bytes.NewReader(part))
require.NoError(t, err)
if i+1 < len(parts) {
time.Sleep(20 * time.Millisecond)
}
}
resp, body, err := readContractResponse(bufio.NewReader(conn), req.ContentLength > 0)
require.NoError(t, err)
return resp, body
}
func readContractResponse(reader *bufio.Reader, requestHasBody bool) (*http.Response, []byte, error) {
resp, err := http.ReadResponse(reader, nil)
if err != nil {
return nil, nil, err
}
defer resp.Body.Close()
if err := validateResponseFraming(resp); err != nil {
return resp, nil, err
}
body, err := io.ReadAll(resp.Body)
if err != nil {
return resp, nil, err
}
if resp.StatusCode != 204 && int64(len(body)) != resp.ContentLength {
return resp, nil, fmt.Errorf("response length mismatch")
}
if err := checkResponseEnd(reader, requestHasBody); err != nil {
return resp, nil, err
}
return resp, body, nil
}
type treeEntry struct {
Mode os.FileMode
Size int64
Digest [32]byte
}
func snapshotTree(root string) (map[string]treeEntry, error) {
result := make(map[string]treeEntry)
err := filepath.Walk(root, func(p string, info os.FileInfo, err error) error {
if err != nil {
return err
}
rel, err := filepath.Rel(root, p)
if err != nil {
return err
}
entry := treeEntry{Mode: info.Mode()}
if info.Mode().IsRegular() {
entry.Size = info.Size()
f, err := os.Open(p)
if err != nil {
return err
}
h := sha256.New()
_, copyErr := io.Copy(h, f)
closeErr := f.Close()
if copyErr != nil {
return copyErr
}
if closeErr != nil {
return closeErr
}
copy(entry.Digest[:], h.Sum(nil))
}
result[rel] = entry
return nil
})
return result, err
}
func compareSnapshots(before, after map[string]treeEntry) error {
if !reflect.DeepEqual(before, after) {
return fmt.Errorf("request changed the file system unexpectedly")
}
return nil
}
homework/03-http-server/tests/contract_test.go
+144
View File
@@ -0,0 +1,144 @@
package hw3test
import (
"bufio"
"errors"
"io"
"net"
"os"
"path/filepath"
"strings"
"testing"
)
func TestContractResponseRejectsBadFraming(t *testing.T) {
for _, tc := range []struct {
name, response string
valid bool
}{
{"ok", "HTTP/1.1 200 OK\r\nConnection: close\r\nContent-Length: 4\r\n\r\ndata", true},
{"short", "HTTP/1.1 200 OK\r\nConnection: close\r\nContent-Length: 5\r\n\r\ndata", false},
{"long", "HTTP/1.1 200 OK\r\nConnection: close\r\nContent-Length: 3\r\n\r\ndata", false},
{"no-close", "HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n", false},
{"no-length", "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", false},
{"204", "HTTP/1.1 204 No Content\r\nConnection: close\r\n\r\n", true},
{"204-length", "HTTP/1.1 204 No Content\r\nConnection: close\r\nContent-Length: 0\r\n\r\n", false},
{"204-body", "HTTP/1.1 204 No Content\r\nConnection: close\r\n\r\nx", false},
{"chunked", "HTTP/1.1 200 OK\r\nConnection: close\r\nTransfer-Encoding: chunked\r\n\r\n0\r\n\r\n", false},
} {
t.Run(tc.name, func(t *testing.T) {
_, _, err := readContractResponse(bufio.NewReader(strings.NewReader(tc.response)), false)
if (err == nil) != tc.valid {
t.Fatalf("valid=%v, error=%v", tc.valid, err)
}
})
}
}
type resetReader struct{}
func (resetReader) Read([]byte) (int, error) {
return 0, &net.OpError{Op: "read", Net: "tcp", Err: errors.New("connection reset")}
}
func TestContractResponseAfterEarlyRejection(t *testing.T) {
for _, tc := range []struct {
name, body string
requestHasBody bool
valid bool
}{
{"complete error then reset", "error", true, true},
{"reset without request body", "error", false, false},
{"truncated error", "err", true, false},
{"extra response byte", "error!", true, false},
} {
t.Run(tc.name, func(t *testing.T) {
wire := "HTTP/1.1 409 Conflict\r\nConnection: close\r\nContent-Length: 5\r\n\r\n" + tc.body
reader := bufio.NewReader(io.MultiReader(strings.NewReader(wire), resetReader{}))
_, _, err := readContractResponse(reader, tc.requestHasBody)
if (err == nil) != tc.valid {
t.Fatalf("valid=%v, error=%v", tc.valid, err)
}
})
}
}
func TestSnapshotDetectsErrorSideEffects(t *testing.T) {
for _, mutation := range []string{"overwrite", "create", "delete", "mkdir"} {
t.Run(mutation, func(t *testing.T) {
root := t.TempDir()
file := filepath.Join(root, "file")
if err := os.WriteFile(file, []byte("before"), 0644); err != nil {
t.Fatal(err)
}
before, err := snapshotTree(root)
if err != nil {
t.Fatal(err)
}
after, err := snapshotTree(root)
if err != nil {
t.Fatal(err)
}
if err := compareSnapshots(before, after); err != nil {
t.Fatal(err)
}
switch mutation {
case "overwrite":
err = os.WriteFile(file, []byte("after!"), 0644)
case "create":
err = os.WriteFile(filepath.Join(root, "new"), nil, 0644)
case "delete":
err = os.Remove(file)
case "mkdir":
err = os.Mkdir(filepath.Join(root, "new"), 0755)
}
if err != nil {
t.Fatal(err)
}
after, err = snapshotTree(root)
if err != nil {
t.Fatal(err)
}
if err := compareSnapshots(before, after); err == nil {
t.Fatal("side effect was accepted")
}
})
}
}
func TestMissingPathActionsAndRoot(t *testing.T) {
env := &Env{RootDir: &EnvDir{Listing: map[string]EnvNode{"file": &EnvFile{}, "dir": &EnvDir{Listing: map[string]EnvNode{}}}}}
opts := &RunOpts{ServerDomain: "localhost"}
for _, tc := range []struct {
method, path string
status int
}{
{"POST", "missing/new", 404}, {"POST", "file/new", 404},
{"POST", "dir", 409}, {"PUT", "missing", 404}, {"PUT", "dir", 409}, {"DELETE", "missing", 404},
{"POST", "/", 409}, {"PUT", "/", 409}, {"DELETE", "/", 403},
} {
q := Query{Method: tc.method, Path: tc.path, HostHeader: "LOCALHOST"}
action, ok := q.Action(env, opts).(HttpErrorAction)
if !ok || action.Status != tc.status {
t.Errorf("%s %s: got %#v", tc.method, tc.path, q.Action(env, opts))
}
}
q := Query{Method: "GET", Path: "", HostHeader: "LOCALHOST"}
if _, ok := q.Action(env, opts).(GetDirAction); !ok {
t.Fatal("root GET is not a listing")
}
for _, root := range []string{"", "/"} {
for _, remove := range []bool{false, true} {
q := Query{Method: "DELETE", Path: root, HostHeader: "localhost", RemoveDirectory: remove}
action, ok := q.Action(env, opts).(HttpErrorAction)
if !ok || action.Status != 403 {
t.Fatalf("root DELETE with RemoveDirectory=%v: got %#v", remove, action)
}
q.HostHeader = "wrong.example"
action, ok = q.Action(env, opts).(HttpErrorAction)
if !ok || action.Status != 400 {
t.Fatalf("Host must be checked before root DELETE: got %#v", action)
}
}
}
}
homework/03-http-server/tests/entrypoint.sh
+128
View File
@@ -0,0 +1,128 @@
#!/bin/bash
set -u
set -o pipefail
fatal() {
printf 'harness: %s\n' "$*" >&2
exit 1
}
wait_for_docker() {
attempts=30
while [ "$attempts" -gt 0 ]; do
docker info >/dev/null 2>&1 && return 0
attempts=$((attempts - 1))
sleep 1
done
return 1
}
cleanup() {
docker ps -q --filter ancestor=hw3img | while IFS= read -r container_id; do
[ -n "$container_id" ] || continue
docker rm --force "$container_id" >/dev/null 2>&1 || true
done
}
run_tests() {
local arg
local -a original_args=("$@")
local -a test_args=(-docker -test.timeout=10m)
while (($#)); do
arg="$1"
shift
case "$arg" in
-run|-count|-timeout|-parallel|-skip)
if (($# == 0)); then
fatal "$arg requires a value"
fi
test_args+=("-test.${arg#-}" "$1")
shift
;;
-run=*|-count=*|-timeout=*|-parallel=*|-skip=*)
test_args+=("-test.${arg#-}")
;;
-v|-short|-failfast|-v=*|-short=*|-failfast=*)
test_args+=("-test.${arg#-}")
;;
-test.*|-docker)
test_args+=("$arg")
;;
*)
# Keep Go's flag handling for less common test and build options.
go test --docker -timeout 10m "${original_args[@]}"
return $?
;;
esac
done
/usr/local/bin/hw3test "${test_args[@]}"
}
validate_score_file() {
score_lines="$(grep -c '^SCORE:' "$1" || true)"
valid_score_lines="$(grep -Ec '^SCORE: [0-9]+([.][0-9]{1,2})?$' "$1" || true)"
[ "$score_lines" -eq 1 ] && [ "$valid_score_lines" -eq 1 ] \
|| fatal "test command must report exactly one valid SCORE line"
}
configure_registry_proxy() {
proxy_url="${DISTSYS_REGISTRY_PROXY_URL:-}"
ca_file="/run/distsys-registry-proxy/ca.crt"
[ -n "$proxy_url" ] || return 1
printf '%s' "$proxy_url" | grep -Eq '^http://[A-Za-z0-9.-]+(:[0-9]+)?$' || {
printf 'warning: invalid registry proxy URL; using direct Docker pulls\n' >&2
return 1
}
[ -f "$ca_file" ] && [ ! -L "$ca_file" ] || {
printf 'warning: registry proxy CA is unavailable; using direct Docker pulls\n' >&2
return 1
}
downloaded_ca="$(mktemp)" || return 1
if ! wget -q -T 5 -O "$downloaded_ca" "${proxy_url}/ca.crt" \
|| ! cmp -s "$ca_file" "$downloaded_ca"; then
rm -f "$downloaded_ca"
printf 'warning: registry proxy is unavailable or its CA differs; using direct Docker pulls\n' >&2
return 1
fi
rm -f "$downloaded_ca"
cp "$ca_file" /usr/local/share/ca-certificates/distsys-registry-proxy.crt
update-ca-certificates >/dev/null
proxy_no_proxy='localhost,127.0.0.1,::1,10.0.0.0/8,172.16.0.0/12,192.168.0.0/16'
export HTTP_PROXY="$proxy_url" HTTPS_PROXY="$proxy_url" NO_PROXY="$proxy_no_proxy"
export http_proxy="$proxy_url" https_proxy="$proxy_url" no_proxy="$proxy_no_proxy"
}
configure_registry_proxy || unset HTTP_PROXY HTTPS_PROXY NO_PROXY http_proxy https_proxy no_proxy
if [ "${HW3_REQUIRE_ZRAM:-0}" = 1 ]; then
/usr/local/bin/verify-zram-scratch.sh || fatal "HW3_REQUIRE_ZRAM=1 but /hw/tests/tmp is not mounted from a zram ext4 device"
fi
# Start docker daemon
trap cleanup EXIT
dockerd-entrypoint.sh --storage-driver=overlay2 >/var/log/dockerd-entrypoint.log 2>&1 &
if ! wait_for_docker; then
tail -n 200 /var/log/dockerd-entrypoint.log >&2 || true
fatal "inner Docker daemon did not become ready"
fi
# Build server image
docker build solution -t hw3img || fatal "could not build solution image"
# Run tests
cd tests
export NO_COLOR=1
test_status=0
run_tests "$@" 2>&1 | tee tests.log || test_status=$?
validate_score_file tests.log
# Go reports a non-zero status when some grading groups fail. A valid score is
# the grading result; absence of a score remains an infrastructure failure.
if [ "$test_status" -ne 0 ]; then
printf 'harness: grading tests exited with status %s after reporting a score\n' \
"$test_status" >&2
fi
homework/03-http-server/tests/env.go
+233
View File
@@ -0,0 +1,233 @@
package hw3test
import (
"fmt"
"io"
"math/rand"
"os"
"path"
"path/filepath"
"sort"
"strings"
)
// Env contains information about all files in the test environment.
// Can be written to disk.
type Env struct {
RootDir *EnvDir
}
// Lookup returns the node for the given path.
func (e *Env) Lookup(p string) (*EnvDir, EnvNode) {
p = path.Clean(strings.TrimPrefix(p, "/"))
if p == "." {
return nil, e.RootDir
}
dir, file := path.Split(p)
dirs := strings.Split(dir, "/")
dirs = dirs[:len(dirs)-1]
parent := e.RootDir
for _, d := range dirs {
nxt, ok := parent.Listing[d]
if !ok {
return nil, nil
}
parent, ok = nxt.(*EnvDir)
if !ok {
// not a directory
return nil, nil
}
}
nxt := parent.Listing[file]
return parent, nxt
}
// Clone returns deep clone of an Env.
func (e *Env) Clone() *Env {
return &Env{
RootDir: e.RootDir.Clone().(*EnvDir),
}
}
// EnvNode is a file/dir.
type EnvNode interface {
// WriteToDisk persists node and its children to disk.
WriteToDisk(path string) error
// Stats aggregates stats of the node and its children.
Stats(path string, stats *Stats)
// Clone creates a deep copy of the node.
Clone() EnvNode
}
// EnvDir is a virtual directory, that can be written to disk.
type EnvDir struct {
Listing map[string]EnvNode
Depth int
}
func (d *EnvDir) WriteToDisk(p string) error {
err := os.Mkdir(p, 0777)
if err != nil {
return err
}
for name, writter := range d.Listing {
nxt := path.Join(p, name)
err := writter.WriteToDisk(nxt)
if err != nil {
return err
}
}
return nil
}
// WriteToDiskSelected keeps the size and directory structure of every initial
// file, but only materializes contents for paths that a query may read.
func (d *EnvDir) WriteToDiskSelected(p string, needed map[string]bool) error {
return d.writeToDiskSelected(p, "", needed)
}
func (d *EnvDir) writeToDiskSelected(p, relative string, needed map[string]bool) error {
if err := os.Mkdir(p, 0777); err != nil {
return err
}
for name, node := range d.Listing {
childPath := filepath.Join(p, name)
childRelative := path.Join(relative, name)
switch child := node.(type) {
case *EnvDir:
if err := child.writeToDiskSelected(childPath, childRelative, needed); err != nil {
return err
}
case *EnvFile:
if needed[childRelative] {
if err := child.WriteToDisk(childPath); err != nil {
return err
}
} else if err := child.WriteSparseToDisk(childPath); err != nil {
return err
}
default:
return fmt.Errorf("unsupported environment node %T", child)
}
}
return nil
}
func (d *EnvDir) Stats(p string, stats *Stats) {
stats.Dirs++
stats.DirPaths = append(stats.DirPaths, p)
for name, node := range d.Listing {
node.Stats(path.Join(p, name), stats)
}
}
func (d *EnvDir) Clone() EnvNode {
newDir := &EnvDir{
Listing: map[string]EnvNode{},
Depth: d.Depth,
}
for name, node := range d.Listing {
newDir.Listing[name] = node.Clone()
}
return newDir
}
func (d *EnvDir) CreateDir(name string) (dir *EnvDir, exist bool) {
_, exist = d.Listing[name]
if exist {
return nil, exist
}
newDir := &EnvDir{
Listing: map[string]EnvNode{},
Depth: d.Depth + 1,
}
d.Listing[name] = newDir
return newDir, false
}
// EnvFile is a virtual file, that can be written to disk.
type EnvFile struct {
GenSeed int64
Size int64
TextOnly bool
}
func (f *EnvFile) Open() io.Reader {
var gen io.Reader = rand.New(rand.NewSource(f.GenSeed))
if f.TextOnly {
gen = &TextReader{gen}
}
gen = io.LimitReader(gen, f.Size)
return gen
}
func (f *EnvFile) WriteToDisk(p string) error {
file, err := os.OpenFile(p, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0666)
if err != nil {
return err
}
defer file.Close()
reader := f.Open()
_, err = io.CopyBuffer(file, reader, make([]byte, 256*1024))
return err
}
func (f *EnvFile) WriteSparseToDisk(p string) error {
file, err := os.OpenFile(p, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0666)
if err != nil {
return err
}
defer file.Close()
return file.Truncate(f.Size)
}
func (f *EnvFile) Stats(p string, stats *Stats) {
stats.Files++
stats.Size += f.Size
stats.FilePaths = append(stats.FilePaths, p)
}
func (f *EnvFile) Clone() EnvNode {
return &EnvFile{
GenSeed: f.GenSeed,
Size: f.Size,
TextOnly: f.TextOnly,
}
}
// Stats is a helper for listing all files/dirs in environment.
type Stats struct {
// Number of directories in the environment.
Files int
// Count of directories in the environment.
Dirs int
// Size of all files in the environment.
Size int64
// FilePaths to all files in the environment.
FilePaths []string
// DirPaths to all directories in the environment.
DirPaths []string
}
// Normalize will fix stats to be deterministic and don't contain root dir.
func (s *Stats) Normalize() {
sort.Strings(s.FilePaths)
sort.Strings(s.DirPaths)
if len(s.DirPaths) > 0 && s.DirPaths[0] == "" {
s.DirPaths = s.DirPaths[1:]
}
}
homework/03-http-server/tests/env_gen.go
+73
View File
@@ -0,0 +1,73 @@
package hw3test
import "math/rand"
// EnvGen contains config for generating test environment.
// Test environment is a directory with files and subdirectories.
type EnvGen struct {
// Generated file tree will not contain subdirectories deeper than this depth.
MaxDepth int
// Generated file tree will not contain more than this number of subdirs.
MaxDirs int
// Generated file tree will not contain more than this number of files.
MaxFiles int
// Generated files will be text files.
TextOnly bool
// Maximum file size in KB
MaxFileSizeKB int
// TempDirectory will contain subdirectory for every new environment.
TempDirectory string
// SparseUnusedFiles avoids writing initial file contents that no query can read.
SparseUnusedFiles bool
// Allow to use env for configuration.
AllowEnv bool
// Filename generator.
FilenameGen func(r *rand.Rand) string
}
// GenerateFile returns file with random contents.
func (g *EnvGen) GenerateFile(r *rand.Rand) *EnvFile {
return &EnvFile{
GenSeed: r.Int63(),
Size: r.Int63n(1024 * int64(g.MaxFileSizeKB)),
TextOnly: g.TextOnly,
}
}
// Generate generates whole file tree.
func (g *EnvGen) Generate(seed int64) (*Env, error) {
r := rand.New(rand.NewSource(seed))
root := &EnvDir{Listing: map[string]EnvNode{}}
dirs := []*EnvDir{root}
for i := 0; i < g.MaxDirs; i++ {
parent := dirs[r.Intn(len(dirs))]
if parent.Depth >= g.MaxDepth {
continue
}
name := g.FilenameGen(r)
newDir, exist := parent.CreateDir(name)
if exist {
continue
}
dirs = append(dirs, newDir)
}
for i := 0; i < g.MaxFiles; i++ {
parent := dirs[r.Intn(len(dirs))]
name := g.FilenameGen(r)
_, exist := parent.Listing[name]
if exist {
continue
}
parent.Listing[name] = g.GenerateFile(r)
}
return &Env{
RootDir: root,
}, nil
}
homework/03-http-server/tests/env_selected_test.go
+44
View File
@@ -0,0 +1,44 @@
package hw3test
import (
"bytes"
"io"
"os"
"path/filepath"
"testing"
)
func TestWriteToDiskSelected(t *testing.T) {
selected := &EnvFile{GenSeed: 42, Size: 4096}
unused := &EnvFile{GenSeed: 43, Size: 4096}
root := &EnvDir{Listing: map[string]EnvNode{
"nested": &EnvDir{Listing: map[string]EnvNode{
"selected": selected,
"unused": unused,
}},
}}
location := filepath.Join(t.TempDir(), "environment")
if err := root.WriteToDiskSelected(location, map[string]bool{"nested/selected": true}); err != nil {
t.Fatal(err)
}
gotSelected, err := os.ReadFile(filepath.Join(location, "nested", "selected"))
if err != nil {
t.Fatal(err)
}
wantSelected, err := io.ReadAll(selected.Open())
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(gotSelected, wantSelected) {
t.Fatal("selected file contents differ from generated contents")
}
gotUnused, err := os.ReadFile(filepath.Join(location, "nested", "unused"))
if err != nil {
t.Fatal(err)
}
if len(gotUnused) != int(unused.Size) || !bytes.Equal(gotUnused, make([]byte, unused.Size)) {
t.Fatal("unused file must retain its size without generated contents")
}
}
homework/03-http-server/tests/framing.go
+49
View File
@@ -0,0 +1,49 @@
package hw3test
import (
"fmt"
"net"
"net/http"
"strings"
"time"
)
// validateResponseFraming checks headers before reading the response body.
func validateResponseFraming(resp *http.Response) error {
// net/http removes Connection: close from Header and records it in Close.
// With HTTP/1.1 and explicit framing, Close reflects that header.
closeFound := resp.ProtoMajor == 1 && resp.ProtoMinor == 1 && resp.Close
for _, value := range resp.Header.Values("Connection") {
for _, token := range strings.Split(value, ",") {
closeFound = closeFound || strings.EqualFold(strings.TrimSpace(token), "close")
}
}
if !closeFound {
return fmt.Errorf("expected Connection: close")
}
if len(resp.TransferEncoding) != 0 || len(resp.Header.Values("Transfer-Encoding")) != 0 {
return fmt.Errorf("Transfer-Encoding is not supported")
}
lengths := resp.Header.Values("Content-Length")
if resp.StatusCode == http.StatusNoContent {
if len(lengths) != 0 {
return fmt.Errorf("204 must not contain Content-Length")
}
} else if len(lengths) != 1 || resp.ContentLength < 0 {
return fmt.Errorf("expected exactly one valid Content-Length")
}
return nil
}
// Bound a stalled peer, without imposing a total duration on large transfers.
type idleDeadlineReader struct {
net.Conn
Timeout time.Duration
}
func (r *idleDeadlineReader) Read(p []byte) (int, error) {
if err := r.Conn.SetReadDeadline(time.Now().Add(r.Timeout)); err != nil {
return 0, err
}
return r.Conn.Read(p)
}
homework/03-http-server/tests/gen_utils.go
+49
View File
@@ -0,0 +1,49 @@
package hw3test
import (
"io"
"math/rand"
)
func SimpleFilenameGenerator(maxLen int) func(r *rand.Rand) string {
return func(r *rand.Rand) string {
return GenFilename(r, maxLen)
}
}
func GenFilename(r *rand.Rand, maxLen int) string {
// TODO: better filename generation, right now it's only uppercase english letters
n := r.Intn(maxLen) + 1
b := make([]byte, n)
for i := 0; i < n; i++ {
b[i] = byte(r.Intn(26) + 65)
}
return string(b)
}
// TextReader is io.Reader filtering out non-text characters.
type TextReader struct {
r io.Reader
}
func (t *TextReader) Read(p []byte) (n int, err error) {
for {
n, err = t.r.Read(p)
if n == 0 || err != nil {
return
}
m := 0
for i := 0; i < n; i++ {
// p[i] is a plain text character
if p[i] == '\n' || (p[i] >= 32 && p[i] <= 126) {
p[m] = p[i]
m++
}
}
if m == 0 {
continue
}
return m, nil
}
}
homework/03-http-server/tests/go.mod
+16
View File
@@ -0,0 +1,16 @@
module hw3test
go 1.18
require (
github.com/stretchr/testify v1.8.0
go.uber.org/zap v1.23.0
)
require (
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
go.uber.org/atomic v1.10.0 // indirect
go.uber.org/multierr v1.8.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)
homework/03-http-server/tests/go.sum
+28
View File
@@ -0,0 +1,28 @@
github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ=
go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI=
go.uber.org/multierr v1.8.0 h1:dg6GjLku4EH+249NNmoIciG9N/jURbDG+pFlTkhzIC8=
go.uber.org/multierr v1.8.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak=
go.uber.org/zap v1.23.0 h1:OjGQ5KQDEUawVHxNwQgPpiypGHOxo2mNZsOqTak4fFY=
go.uber.org/zap v1.23.0/go.mod h1:D+nX8jyLsMHMYrln8A0rJjFt/T/9/bGgIhAqxv5URuY=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
homework/03-http-server/tests/hw_test.go
+309
View File
@@ -0,0 +1,309 @@
package hw3test
import (
"flag"
"fmt"
"os"
"path/filepath"
"testing"
"text/template"
"github.com/stretchr/testify/require"
"go.uber.org/zap"
)
var (
useDocker bool
)
func init() {
flag.BoolVar(&useDocker, "docker", false, "Run solution in Docker")
}
func TestHW(tt *testing.T) {
if !flag.Parsed() {
flag.Parse()
}
t := NewTestContext(tt)
Info(t, "Starting tests")
// loading commandline args template from disk, to run solution with it
launchTmpl := os.Getenv("LAUNCH_TMPL")
if launchTmpl == "" {
launchTmpl = "./launch.tmpl"
if useDocker {
launchTmpl = "./launch_docker.tmpl"
}
}
tmplContent, err := os.ReadFile(launchTmpl)
require.NoError(t, err, "failed to read template file")
launchTemplate := template.Must(template.New("launch").Parse(string(tmplContent)))
runner := NewCmdRunner(launchTemplate, useDocker)
workdir, err := os.Getwd()
require.NoError(t, err, "failed to get current directory")
tmpRoot := filepath.Join(workdir, "tmp")
require.NoError(t, os.MkdirAll(tmpRoot, 0777), "failed to create tmp dir")
workspace, err := filepath.EvalSymlinks(workdir)
require.NoError(t, err, "failed to resolve test directory")
tmpRoot, err = filepath.EvalSymlinks(tmpRoot)
require.NoError(t, err, "failed to resolve tmp dir")
rel, err := filepath.Rel(workspace, tmpRoot)
require.NoError(t, err, "failed to verify tmp dir")
require.Equal(t, "tmp", rel, "tmp dir must stay inside the test directory")
tmpDir, err := os.MkdirTemp(tmpRoot, "run-")
require.NoError(t, err, "failed to create run-specific tmp dir")
require.NoError(t, os.Chmod(tmpDir, 0755), "failed to make tmp dir accessible to the server")
t.Cleanup(func() {
resolved, err := filepath.EvalSymlinks(tmpDir)
if os.IsNotExist(err) {
return
}
require.NoError(t, err, "failed to resolve run-specific tmp dir")
rel, err := filepath.Rel(tmpRoot, resolved)
require.NoError(t, err, "failed to verify run-specific tmp dir")
require.Equal(t, filepath.Base(tmpDir), rel, "run-specific tmp dir must stay inside tmp dir")
require.NoError(t, os.RemoveAll(tmpDir), "failed to remove run-specific tmp dir")
})
score := 0
type RunResult struct {
Name string
Scored int
Max int
}
runResults := []RunResult{}
runGroup := func(name string, points int, f func(t *TC)) {
t.RunByName(name, func(t *TC) {
Info(t, "Starting tests group", zap.Int("points", points))
t.Cleanup(func() {
r := RunResult{
Name: name,
Max: points,
}
ok := !t.Failed()
if ok {
score += points
r.Scored = points
Info(t, "Tests group passed", zap.String("name", name), zap.Int("score", score))
} else {
Warn(t, "Tests group failed", zap.String("name", name), zap.Int("score", score))
}
runResults = append(runResults, r)
})
f(t)
})
}
textEnv := &EnvGen{
MaxDepth: 1,
MaxDirs: 3,
MaxFiles: 5,
TextOnly: true,
MaxFileSizeKB: 64,
TempDirectory: tmpDir,
FilenameGen: SimpleFilenameGenerator(8),
AllowEnv: false,
}
binaryEnv := &EnvGen{
MaxDepth: 4,
MaxDirs: 16,
MaxFiles: 25,
TextOnly: false,
MaxFileSizeKB: 1024,
TempDirectory: tmpDir,
FilenameGen: SimpleFilenameGenerator(16),
AllowEnv: true,
}
largeEnv := &EnvGen{
MaxDepth: 1,
MaxDirs: 6,
MaxFiles: 6,
TextOnly: false,
MaxFileSizeKB: 192 * 1024, // 192 MB
TempDirectory: tmpDir,
SparseUnusedFiles: true,
FilenameGen: SimpleFilenameGenerator(16),
AllowEnv: true,
}
// Simple GET queries for existing text files, 3 points.
runGroup("G1", 3, func(t *TC) {
RunContractTests(t, runner, tmpDir, "G1")
RunTestEmptyWorkDir(t, 42, runner)
env := textEnv
queries := &QueriesGen{
Count: 20,
GetFile: true,
GetFileNoErrors: true,
GetDirectory: false,
Post: false,
Put: false,
Delete: false,
Compression: false,
AllHeaders: false,
}
RunTests(t, 1337, runner, env, queries)
RunTests(t, 1338, runner, env, queries)
RunTests(t, 1339, runner, env, queries)
})
// Simple GET queries for existing binary files, 1 point.
runGroup("G2", 1, func(t *TC) {
RunContractTests(t, runner, tmpDir, "G2")
env := binaryEnv
queries := &QueriesGen{
Count: 40,
GetFile: true,
GetFileNoErrors: true,
GetDirectory: false,
Post: false,
Put: false,
Delete: false,
Compression: false,
AllHeaders: false,
}
RunTests(t, 93, runner, env, queries)
RunTests(t, 2945, runner, env, queries)
RunTests(t, 3110, runner, env, queries)
})
// Any GET queries, 1 point.
runGroup("G3", 1, func(t *TC) {
RunContractTests(t, runner, tmpDir, "G3")
env := textEnv
queries := &QueriesGen{
Count: 30,
GetFile: true,
GetFileNoErrors: false,
GetDirectory: true,
Post: false,
Put: false,
Delete: false,
Compression: false,
AllHeaders: false,
}
RunTests(t, 5311, runner, env, queries)
RunTests(t, 2863, runner, env, queries)
RunTests(t, 6712, runner, env, queries)
RunTests(t, 7233, runner, env, queries)
RunTests(t, 7067, runner, env, queries)
RunTests(t, 3930, runner, env, queries)
})
// Simple file server, 2 points.
runGroup("G4", 2, func(t *TC) {
RunContractTests(t, runner, tmpDir, "G4")
env := binaryEnv
queries := &QueriesGen{
Count: 30,
GetFile: true,
GetDirectory: true,
Post: true,
Put: true,
Delete: true,
Compression: false,
AllHeaders: false,
}
RunTests(t, 3152, runner, env, queries)
RunTests(t, 2929, runner, env, queries)
RunTests(t, 6554, runner, env, queries)
RunTests(t, 1388, runner, env, queries)
RunTests(t, 1672, runner, env, queries)
RunTests(t, 1769, runner, env, queries)
})
// Extra headers, 1 point.
runGroup("G5", 1, func(t *TC) {
RunContractTests(t, runner, tmpDir, "G5")
env := binaryEnv
queries := &QueriesGen{
Count: 30,
GetFile: true,
GetDirectory: true,
Post: true,
Put: true,
Delete: true,
Compression: false,
AllHeaders: true,
}
RunTests(t, 7942, runner, env, queries)
RunTests(t, 1479, runner, env, queries)
RunTests(t, 3324, runner, env, queries)
RunTests(t, 6519, runner, env, queries)
RunTests(t, 3746, runner, env, queries)
RunTests(t, 1961, runner, env, queries)
})
// Large files, 1 point.
runGroup("G6", 1, func(t *TC) {
RunContractTests(t, runner, tmpDir, "G6")
env := largeEnv
queries := &QueriesGen{
Count: 25,
GetFile: true,
GetDirectory: true,
LargePuts: &LargePutPolicy{
RunSeed: 3793,
LargeSizes: []int64{160 << 20, 191 << 20},
OtherMaxBytes: 8 << 20,
},
Post: true,
Put: true,
Delete: true,
Compression: false,
AllHeaders: false,
}
RunTests(t, 7824, runner, env, queries)
RunTests(t, 1671, runner, env, queries)
RunTests(t, 3793, runner, env, queries)
RunTests(t, 272, runner, env, queries)
RunTests(t, 2715, runner, env, queries)
RunTests(t, 1436, runner, env, queries)
})
// Large files and gzip, 1 point.
runGroup("G7", 1, func(t *TC) {
RunContractTests(t, runner, tmpDir, "G7")
env := largeEnv
queries := &QueriesGen{
Count: 30,
GetFile: true,
GetDirectory: true,
LargePuts: &LargePutPolicy{
RunSeed: 3224,
LargeSizes: []int64{160 << 20, 191 << 20},
OtherMaxBytes: 8 << 20,
},
Post: true,
Put: true,
Delete: true,
Compression: true,
AllHeaders: true,
}
RunTests(t, 3224, runner, env, queries)
RunTests(t, 7507, runner, env, queries)
RunTests(t, 4172, runner, env, queries)
RunTests(t, 7777, runner, env, queries)
RunTests(t, 6666, runner, env, queries)
RunTests(t, 6094, runner, env, queries)
RunTests(t, 6442, runner, env, queries)
})
for _, r := range runResults {
Info(t, fmt.Sprintf("Score for group [%s]: %d / %d", r.Name, r.Scored, r.Max))
}
Info(t, "Tests finished", zap.Int("score", score))
fmt.Println("==================================================================================")
fmt.Printf("SCORE: %d\n", score)
fmt.Println("==================================================================================")
}
homework/03-http-server/tests/large_put_policy_test.go
+119
View File
@@ -0,0 +1,119 @@
package hw3test
import (
"math/rand"
"reflect"
"testing"
)
func TestLargePutPlansKeepBoundaryWritesAndQueries(t *testing.T) {
tests := []struct {
name string
runSeeds []int64
count int
largeSeed int64
allHeaders bool
compression bool
}{
{"G6", []int64{7824, 1671, 3793, 272, 2715, 1436}, 25, 3793, false, false},
{"G7", []int64{3224, 7507, 4172, 7777, 6666, 6094, 6442}, 30, 3224, true, true},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
context := NewTestContext(t)
envGen := &EnvGen{
MaxDepth: 1,
MaxDirs: 6,
MaxFiles: 6,
MaxFileSizeKB: 192 * 1024,
FilenameGen: SimpleFilenameGenerator(16),
}
policy := &LargePutPolicy{
RunSeed: test.largeSeed,
LargeSizes: []int64{160 << 20, 191 << 20},
OtherMaxBytes: 8 << 20,
}
options := RunOpts{ServerDomain: "localhost"}
largePosts := 0
largeGets := 0
largeGzipGets := 0
for _, runSeed := range test.runSeeds {
env, err := envGen.Generate(runSeed)
if err != nil {
t.Fatal(err)
}
r := rand.New(rand.NewSource(runSeed))
seeds := make([]int64, test.count)
for i := range seeds {
seeds[i] = r.Int63n(100000)
}
generator := &QueriesGen{
Count: test.count, GetFile: true, GetDirectory: true,
Post: true, Put: true, Delete: true,
Compression: test.compression, AllHeaders: test.allHeaders,
LargePuts: policy,
}
queries := generator.Generate(context, env, envGen, seeds, options, runSeed)
generator.LargePuts = nil
baseline := generator.Generate(context, env, envGen, seeds, options, runSeed)
if len(queries) != len(baseline) {
t.Fatalf("run %d changed query count", runSeed)
}
state := env.Clone()
var largePutSizes []int64
for i, query := range queries {
if query.Method == "PUT" {
baseline[i].FileContent.Size = query.FileContent.Size
}
if !reflect.DeepEqual(query, baseline[i]) {
t.Fatalf("run %d changed query %d beyond PUT body size", runSeed, i)
}
action := query.Action(state, &options)
if query.Path == "" && query.Method != "GET" {
t.Fatal("generated a modifying request to the working directory")
}
if get, ok := action.(GetFileAction); ok && get.File.Size > 128<<20 {
largeGets++
if get.Compression {
largeGzipGets++
}
}
if query.Method == "PUT" {
if query.FileContent.Size > 128<<20 {
if _, ok := action.(ReplaceFileAction); !ok {
t.Fatalf("run %d has a large PUT that cannot update a file", runSeed)
}
largePutSizes = append(largePutSizes, query.FileContent.Size)
} else if query.FileContent.Size > policy.OtherMaxBytes {
t.Fatalf("run %d has an unbounded PUT", runSeed)
}
}
if query.Method == "POST" && query.FileContent != nil && query.FileContent.Size > 128<<20 {
if _, ok := action.(CreateFileAction); ok {
largePosts++
}
}
if action != nil {
action.ApplyEnv(context, state)
}
}
if runSeed == test.largeSeed {
if !reflect.DeepEqual(largePutSizes, policy.LargeSizes) {
t.Fatalf("run %d large PUT sizes: got %v, want %v", runSeed, largePutSizes, policy.LargeSizes)
}
} else if len(largePutSizes) != 0 {
t.Fatalf("run %d has unexpected large PUTs", runSeed)
}
}
if largePosts == 0 {
t.Fatal("no successful POST larger than the solution memory limit")
}
if largeGets == 0 {
t.Fatal("no successful GET larger than the solution memory limit")
}
if test.compression && largeGzipGets == 0 {
t.Fatal("no gzip GET larger than the solution memory limit")
}
})
}
}
homework/03-http-server/tests/launch.tmpl
+2
View File
@@ -0,0 +1,2 @@
{{- /*gotype: hw3test.RunOpts */ -}}
python3 ../solution/server.py {{.CommandLineArgs}}
homework/03-http-server/tests/launch_docker.tmpl
+2
View File
@@ -0,0 +1,2 @@
{{- /*gotype: hw3test.RunOpts */ -}}
docker run --memory=128m --memory-swap=128m --memory-swappiness=0 --rm {{.DockerVolumeArgs}} {{.DockerPortArgs}} {{.DockerEnvArgs}} hw3img {{.DockerCommandLineArgs}}
homework/03-http-server/tests/logs.go
+94
View File
@@ -0,0 +1,94 @@
package hw3test
import (
"fmt"
"go.uber.org/zap"
"go.uber.org/zap/zapcore"
"os"
"sync"
)
func logLevelFromEnv() zap.AtomicLevel {
levelStr := os.Getenv("LOG_LEVEL")
if levelStr == "" {
return zap.NewAtomicLevelAt(zap.DebugLevel)
}
level, err := zap.ParseAtomicLevel(levelStr)
if err != nil {
panic(fmt.Errorf("failed to parse log level: %w", err))
}
return level
}
var global *zap.Logger
var globalOnce sync.Once
func Global() *zap.Logger {
if global == nil {
globalOnce.Do(func() {
encoder := zapcore.EncoderConfig{
// Keys can be anything except the empty string.
TimeKey: "",
LevelKey: "L",
NameKey: "_",
CallerKey: "C",
FunctionKey: "F",
MessageKey: "M",
StacktraceKey: "S",
LineEnding: zapcore.DefaultLineEnding,
EncodeLevel: zapcore.CapitalColorLevelEncoder,
EncodeTime: zapcore.ISO8601TimeEncoder,
EncodeDuration: zapcore.StringDurationEncoder,
EncodeCaller: zapcore.ShortCallerEncoder,
}
cfg := zap.Config{
Level: logLevelFromEnv(),
Development: true,
DisableCaller: true,
DisableStacktrace: true,
Encoding: "console",
EncoderConfig: encoder,
OutputPaths: []string{"stderr"},
ErrorOutputPaths: []string{"stderr"},
}
options := []zap.Option{
zap.AddCallerSkip(1),
}
var err error
global, err = cfg.Build(options...)
if err != nil {
panic(fmt.Errorf("failed to initialize global logger: %w", err))
}
})
}
return global
}
func Logger(t *TC) *zap.Logger {
logger := Global()
logger = logger.Named(t.Name())
return logger
}
func Debug(t *TC, msg string, args ...zap.Field) {
Logger(t).Debug(msg, args...)
_ = Logger(t).Sync()
}
func Info(t *TC, msg string, args ...zap.Field) {
Logger(t).Info(msg, args...)
_ = Logger(t).Sync()
}
func Warn(t *TC, msg string, args ...zap.Field) {
Logger(t).Warn(msg, args...)
_ = Logger(t).Sync()
}
func Error(t *TC, msg string, args ...zap.Field) {
Logger(t).Error(msg, args...)
_ = Logger(t).Sync()
}
homework/03-http-server/tests/query.go
+424
View File
@@ -0,0 +1,424 @@
package hw3test
import (
"compress/gzip"
"fmt"
"github.com/stretchr/testify/require"
"io"
"mime"
"net/http"
"os"
"path"
"strings"
)
func parseContentType(value string) (string, error) {
mediaType, _, err := mime.ParseMediaType(value)
if err != nil {
return "", err
}
if !strings.Contains(mediaType, "/") {
return "", fmt.Errorf("content type %q has no subtype", value)
}
return mediaType, nil
}
// Query represents a query to the solution server.
type Query struct {
// Seed is used as a query ID.
Seed int64
// Method of a request, e.g. "GET", "POST", "PUT", "DELETE".
Method string
// Simple file path, splitted by slashes. E.g. "foo/bar/baz".
Path string
// If true, will use `Accept-Encoding` for GET query.
Gzip bool
// If true, will pass `Create-Directory: True` in POST query.
CreateDirectory bool
// If true, will pass `Remove-Directory: True` in DELETE query.
RemoveDirectory bool
// Pass the Host header.
HostHeader string
// Verify all server headers: `Content-Length`, `Content-Type`, `Server`.
VerifyHeaders bool
// FileContent when creating a file.
FileContent *EnvFile
}
func (q *Query) CreateRequest(t *TC, queryURL string) *http.Request {
var body io.Reader
if q.FileContent != nil {
body = q.FileContent.Open()
if q.FileContent.Size == 0 {
body = http.NoBody
}
}
req, err := http.NewRequest(q.Method, queryURL, body)
require.NoError(t, err, "failed to create go request")
if q.FileContent != nil {
req.ContentLength = q.FileContent.Size
}
if q.CreateDirectory {
req.Header.Set("Create-Directory", "True")
}
if q.RemoveDirectory {
req.Header.Set("Remove-Directory", "True")
}
if q.HostHeader != "" {
req.Host = q.HostHeader
}
if q.Gzip {
req.Header.Set("Accept-Encoding", "gzip")
}
return req
}
func (q *Query) CommonValidate(t *TC, r *http.Request, resp *http.Response) {
require.NoError(t, validateResponseFraming(resp))
if q.VerifyHeaders {
require.NotEmpty(t, resp.Header.Get("Server"), "missing Server header")
}
}
type Action interface {
// VerifyBefore verifies file content on disk before sending query to the server.
VerifyBefore(t *TC, workdir string)
// VerifyAfter verifies file content on disk after sending query to the server.
VerifyAfter(t *TC, workdir string)
// ApplyEnv applies the action to the environment.
ApplyEnv(t *TC, env *Env) (changed bool)
// VerifyResponse verifies the response from the server.
VerifyResponse(t *TC, r *http.Request, resp *http.Response)
}
// HttpErrorAction is an Action, which expect specific status code in response.
type HttpErrorAction struct {
Status int
}
func (h HttpErrorAction) VerifyBefore(t *TC, workdir string) {
return
}
func (h HttpErrorAction) VerifyAfter(t *TC, workdir string) {
return
}
func (h HttpErrorAction) ApplyEnv(t *TC, env *Env) bool {
return false
}
func (h HttpErrorAction) VerifyResponse(t *TC, r *http.Request, resp *http.Response) {
require.Equal(t, h.Status, resp.StatusCode, "expected http status code")
require.Positive(t, resp.ContentLength, "error response must contain an explanation")
}
// GetFileAction is an Action, which returns file content in response.
type GetFileAction struct {
Path string
File *EnvFile
Compression bool
VerifyHeaders bool
}
func (g GetFileAction) VerifyBefore(t *TC, workdir string) {
RequireFileContent(t, workdir, g.Path, g.File)
}
func (g GetFileAction) VerifyAfter(t *TC, workdir string) {
RequireFileContent(t, workdir, g.Path, g.File)
}
func (g GetFileAction) ApplyEnv(t *TC, env *Env) bool {
return false
}
func (g GetFileAction) VerifyResponse(t *TC, r *http.Request, resp *http.Response) {
require.Equal(t, 200, resp.StatusCode, "expected OK")
var body io.Reader = resp.Body
if g.Compression {
require.Equal(t, "gzip", resp.Header.Get("Content-Encoding"), "expected gzip response")
gz, err := gzip.NewReader(resp.Body)
require.NoError(t, err, "expected valid gzip response")
defer gz.Close()
body = gz
} else {
require.Empty(t, resp.Header.Get("Content-Encoding"), "unexpected content encoding")
require.Equal(t, g.File.Size, resp.ContentLength, "expected content length")
}
if g.VerifyHeaders && g.File.Size > 0 {
_, err := parseContentType(resp.Header.Get("Content-Type"))
require.NoError(t, err, "expected valid content type")
}
require.NoError(t, CompareFileContent(t, body, g.File), "file content mismatch")
remaining, err := io.Copy(io.Discard, body)
require.NoError(t, err, "failed to read remaining response body")
require.Zero(t, remaining, "response contains extra file content")
}
// GetDirAction is an Action, which returns directory listing in response.
type GetDirAction struct {
Path string
Dir *EnvDir
Compression bool
VerifyHeaders bool
}
func (g GetDirAction) VerifyBefore(t *TC, workdir string) {
RequireDir(t, workdir, g.Path, g.Dir)
}
func (g GetDirAction) VerifyAfter(t *TC, workdir string) {
RequireDir(t, workdir, g.Path, g.Dir)
}
func (g GetDirAction) ApplyEnv(t *TC, env *Env) bool {
return false
}
func (g GetDirAction) VerifyResponse(t *TC, r *http.Request, resp *http.Response) {
require.Equal(t, 200, resp.StatusCode, "expected OK")
var body io.Reader = resp.Body
if g.Compression {
require.Equal(t, "gzip", resp.Header.Get("Content-Encoding"), "expected gzip response")
gz, err := gzip.NewReader(resp.Body)
require.NoError(t, err, "expected valid gzip response")
defer gz.Close()
body = gz
} else {
require.Empty(t, resp.Header.Get("Content-Encoding"), "unexpected content encoding")
}
content, err := io.ReadAll(body)
require.NoError(t, err, "expected response to be read without errors")
if g.VerifyHeaders && len(content) > 0 {
mediaType, err := parseContentType(resp.Header.Get("Content-Type"))
require.NoError(t, err, "expected valid content type")
require.Contains(t, []string{"text/plain", "text/html"}, mediaType, "expected directory listing content type")
}
strlist := string(content)
for name := range g.Dir.Listing {
require.Contains(t, strlist, name, "expected dir listing to contain child %s", name)
}
}
// CreateDirAction is an Action, which creates a directory.
type CreateDirAction struct {
Path string
Name string
Parent *EnvDir
PathOnDisk string
}
func (c CreateDirAction) VerifyBefore(t *TC, workdir string) {
RequireNotExists(t, workdir, c.Path)
}
func (c CreateDirAction) VerifyAfter(t *TC, workdir string) {
RequireDir(t, workdir, c.Path, c.Parent)
entries, err := os.ReadDir(c.PathOnDisk)
require.NoError(t, err, "expected to read dir %s without errors", c.PathOnDisk)
require.Empty(t, entries, "expected dir %s to be empty", c.PathOnDisk)
}
func (c CreateDirAction) ApplyEnv(t *TC, env *Env) bool {
_, exist := c.Parent.CreateDir(c.Name)
require.False(t, exist)
return true
}
func (c CreateDirAction) VerifyResponse(t *TC, r *http.Request, resp *http.Response) {
require.Contains(t, []int{http.StatusOK, http.StatusCreated}, resp.StatusCode, "expected successful creation")
}
// CreateFileAction is an Action, which creates a file.
type CreateFileAction struct {
Path string
Name string
Parent *EnvDir
Content *EnvFile
}
func (c CreateFileAction) VerifyBefore(t *TC, workdir string) {
RequireNotExists(t, workdir, c.Path)
}
func (c CreateFileAction) VerifyAfter(t *TC, workdir string) {
RequireFileContent(t, workdir, c.Path, c.Content)
}
func (c CreateFileAction) ApplyEnv(t *TC, env *Env) bool {
c.Parent.Listing[c.Name] = c.Content
return true
}
func (c CreateFileAction) VerifyResponse(t *TC, r *http.Request, resp *http.Response) {
require.Contains(t, []int{http.StatusOK, http.StatusCreated}, resp.StatusCode, "expected successful creation")
}
// ReplaceFileAction is an Action, which replaces file content.
type ReplaceFileAction struct {
Path string
Name string
Parent *EnvDir
OldContent *EnvFile
NewContent *EnvFile
}
func (r ReplaceFileAction) VerifyBefore(t *TC, workdir string) {
RequireFileContent(t, workdir, r.Path, r.OldContent)
}
func (r ReplaceFileAction) VerifyAfter(t *TC, workdir string) {
RequireFileContent(t, workdir, r.Path, r.NewContent)
}
func (r ReplaceFileAction) ApplyEnv(t *TC, env *Env) bool {
r.Parent.Listing[r.Name] = r.NewContent
return true
}
func (r ReplaceFileAction) VerifyResponse(t *TC, req *http.Request, resp *http.Response) {
require.Contains(t, []int{http.StatusOK, http.StatusNoContent}, resp.StatusCode, "expected successful update")
}
// DeleteAction is an Action, which deletes a file or directory.
type DeleteAction struct {
Path string
Parent *EnvDir
Name string
}
func (d DeleteAction) VerifyBefore(t *TC, workdir string) {
RequireExists(t, workdir, d.Path)
}
func (d DeleteAction) VerifyAfter(t *TC, workdir string) {
RequireNotExists(t, workdir, d.Path)
}
func (d DeleteAction) ApplyEnv(t *TC, env *Env) bool {
delete(d.Parent.Listing, d.Name)
return true
}
func (d DeleteAction) VerifyResponse(t *TC, r *http.Request, resp *http.Response) {
require.Equal(t, 200, resp.StatusCode, "expected OK")
}
// Action returns action for the query, or nil if query shouldn't do anything.
func (q *Query) Action(env *Env, opts *RunOpts) Action {
if !strings.EqualFold(q.HostHeader, opts.ServerDomain) {
return HttpErrorAction{
Status: 400,
}
}
parent, child := env.Lookup(q.Path)
switch q.Method {
case "GET":
if child == nil {
return HttpErrorAction{
Status: 404,
}
}
if f, ok := child.(*EnvFile); ok {
return GetFileAction{
Path: q.Path,
File: f,
Compression: q.Gzip,
VerifyHeaders: q.VerifyHeaders,
}
}
if d, ok := child.(*EnvDir); ok {
return GetDirAction{
Path: q.Path,
Dir: d,
Compression: q.Gzip,
VerifyHeaders: q.VerifyHeaders,
}
}
// shouldn't get here
return nil
case "POST":
if child != nil {
return HttpErrorAction{
Status: 409,
}
}
if parent == nil {
return HttpErrorAction{Status: http.StatusNotFound}
}
_, name := path.Split(q.Path)
if q.CreateDirectory {
return CreateDirAction{
Path: q.Path,
Name: name,
Parent: parent,
PathOnDisk: path.Join(opts.WorkingDirectory, q.Path),
}
}
return CreateFileAction{
Path: q.Path,
Name: name,
Parent: parent,
Content: q.FileContent,
}
case "PUT":
if child == nil {
return HttpErrorAction{Status: http.StatusNotFound}
}
f, ok := child.(*EnvFile)
if !ok {
return HttpErrorAction{
Status: 409,
}
}
_, name := path.Split(q.Path)
return ReplaceFileAction{
Path: q.Path,
Name: name,
Parent: parent,
OldContent: f,
NewContent: q.FileContent,
}
case "DELETE":
if child == env.RootDir {
return HttpErrorAction{Status: http.StatusForbidden}
}
if child == nil {
return HttpErrorAction{Status: http.StatusNotFound}
}
_, ok := child.(*EnvDir)
if ok && !q.RemoveDirectory {
return HttpErrorAction{
Status: 406,
}
}
_, name := path.Split(q.Path)
return DeleteAction{
Path: q.Path,
Parent: parent,
Name: name,
}
}
return nil
}
homework/03-http-server/tests/query_gen.go
+146
View File
@@ -0,0 +1,146 @@
package hw3test
import (
"math/rand"
"strings"
"github.com/stretchr/testify/require"
)
// LargePutPolicy keeps a small number of guaranteed successful writes above
// the solution's memory limit, while bounding the remaining PUT bodies.
type LargePutPolicy struct {
RunSeed int64
LargeSizes []int64
OtherMaxBytes int64
}
// QueriesGen contains config for generating queries.
type QueriesGen struct {
// Number of queries to generate.
Count int
GetFile bool // Allow GET requests for files.
GetFileNoErrors bool // Disable requests for non-existing files.
GetDirectory bool // Allow GET requests for directories.
Compression bool // Allow compression in GET requests.
Post bool // Allow POST requests.
Put bool // Allow PUT requests.
Delete bool // Allow DELETE requests.
AllHeaders bool // Allow extra headers.
LargePuts *LargePutPolicy
}
func (g *QueriesGen) Generate(t *TC, env *Env, gen *EnvGen, seeds []int64, opts RunOpts, runSeed int64) []Query {
env = env.Clone()
largePuts := 0
stats := &Stats{}
env.RootDir.Stats("", stats)
stats.Normalize()
var methods []string
if g.GetFile || g.GetDirectory {
methods = append(methods, "GET")
}
if g.Post {
methods = append(methods, "POST")
}
if g.Put {
methods = append(methods, "PUT")
}
if g.Delete {
methods = append(methods, "DELETE")
}
var queries []Query
for _, seed := range seeds {
r := rand.New(rand.NewSource(seed))
method := methods[r.Intn(len(methods))]
var genPaths []string
if len(stats.DirPaths) > 0 && !(method == "GET" && !g.GetDirectory) {
genPaths = append(genPaths, stats.DirPaths[r.Intn(len(stats.DirPaths))])
genPaths = append(genPaths, stats.DirPaths[r.Intn(len(stats.DirPaths))])
}
if len(stats.FilePaths) > 0 && !(method == "GET" && !g.GetFile) {
genPaths = append(genPaths, stats.FilePaths[r.Intn(len(stats.FilePaths))])
genPaths = append(genPaths, stats.FilePaths[r.Intn(len(stats.FilePaths))])
}
if !(method == "GET" && g.GetFileNoErrors) {
// TODO: better non-existing path generator
randomPath := gen.FilenameGen(r) + "/" + gen.FilenameGen(r)
genPaths = append(genPaths, randomPath)
if len(stats.DirPaths) > 0 {
randomDir := stats.DirPaths[r.Intn(len(stats.DirPaths))]
num := 2
if method == "POST" {
num = 4
}
for i := 0; i < num; i++ {
genPaths = append(genPaths, randomDir+"/"+gen.FilenameGen(r))
}
}
}
if len(genPaths) == 0 {
continue
}
hostHeader := opts.ServerDomain
if g.AllHeaders && r.Intn(5) == 0 {
// to get 400
hostHeader = "hse.ru"
}
path := strings.TrimPrefix(genPaths[r.Intn(len(genPaths))], "/")
if path == "" && method != "GET" {
// Keep the existing large-file scenarios and RNG sequence.
// Requests targeting the root are covered on a small tree in contract.go.
path = "MISSING_ROOT_TARGET"
}
gzipRequested := g.Compression && r.Intn(2) == 1
query := Query{
Seed: seed,
Method: method,
Path: path,
Gzip: method == "GET" && gzipRequested,
CreateDirectory: method == "POST" && (r.Intn(2) == 1),
RemoveDirectory: method == "DELETE" && (r.Intn(2) == 1),
HostHeader: hostHeader,
VerifyHeaders: g.AllHeaders,
}
if (method == "POST" && !query.CreateDirectory) || method == "PUT" {
query.FileContent = gen.GenerateFile(r)
}
action := query.Action(env, &opts)
if query.Method == "PUT" && g.LargePuts != nil {
policy := g.LargePuts
if _, ok := action.(ReplaceFileAction); ok && runSeed == policy.RunSeed && largePuts < len(policy.LargeSizes) {
query.FileContent.Size = policy.LargeSizes[largePuts]
largePuts++
} else if query.FileContent.Size > policy.OtherMaxBytes {
query.FileContent.Size = policy.OtherMaxBytes
}
}
queries = append(queries, query)
if action != nil {
changed := action.ApplyEnv(t, env)
if changed {
stats = &Stats{}
env.RootDir.Stats("", stats)
stats.Normalize()
}
}
}
if g.LargePuts != nil && runSeed == g.LargePuts.RunSeed {
require.Equal(t, len(g.LargePuts.LargeSizes), largePuts, "not enough successful PUT queries for large-file coverage")
}
return queries
}
homework/03-http-server/tests/readiness_test.go
+56
View File
@@ -0,0 +1,56 @@
package hw3test
import (
"net"
"net/http"
"net/http/httptest"
"testing"
)
func TestWaitForServerUsesConfiguredHost(t *testing.T) {
t.Setenv("SOLUTION_HOST", "127.0.0.1")
for _, tc := range []struct {
domain string
status int
}{
{"localhost", http.StatusOK},
{"files.example.com", http.StatusNotFound},
{"redirect.example.com", http.StatusFound},
} {
t.Run(tc.domain, func(t *testing.T) {
type probe struct{ method, target, host string }
received := make(chan probe, 16)
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
select {
case received <- probe{r.Method, r.RequestURI, r.Host}:
default:
}
if tc.status == http.StatusFound {
w.Header().Set("Location", "/redirected")
}
w.WriteHeader(tc.status)
}))
defer server.Close()
opts := RunOpts{
Port: server.Listener.Addr().(*net.TCPAddr).Port,
ServerDomain: tc.domain,
}
if err := WaitForServer(NewTestContext(t), opts); err != nil {
t.Fatal(err)
}
select {
case got := <-received:
if want := (probe{"GET", "/", tc.domain}); got != want {
t.Fatalf("readiness request = %+v, want %+v", got, want)
}
default:
t.Fatal("server was marked ready without receiving a request")
}
select {
case extra := <-received:
t.Fatalf("unexpected extra readiness request: %+v", extra)
default:
}
})
}
}
homework/03-http-server/tests/response_validation_test.go
+269
View File
@@ -0,0 +1,269 @@
package hw3test
import (
"bufio"
"bytes"
"compress/gzip"
"errors"
"io"
"net"
"net/http"
"strconv"
"testing"
"time"
)
func TestResponseEndChecksWireBytes(t *testing.T) {
for _, tc := range []struct {
name string
response string
valid bool
}{
{"valid body", "HTTP/1.1 200 OK\r\nContent-Length: 4\r\n\r\ndata", true},
{"extra body bytes", "HTTP/1.1 200 OK\r\nContent-Length: 4\r\n\r\ndataextra", false},
{"valid empty body", "HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n", true},
{"body after zero length", "HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\nextra", false},
{"valid no content", "HTTP/1.1 204 No Content\r\n\r\n", true},
{"body after no content", "HTTP/1.1 204 No Content\r\n\r\nextra", false},
} {
t.Run(tc.name, func(t *testing.T) {
listener, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
defer listener.Close()
go func() {
server, err := listener.Accept()
if err != nil {
return
}
defer server.Close()
_, _ = io.WriteString(server, tc.response)
}()
client, err := net.Dial("tcp", listener.Addr().String())
if err != nil {
t.Fatal(err)
}
defer client.Close()
reader := bufio.NewReader(client)
response, err := http.ReadResponse(reader, nil)
if err != nil {
t.Fatal(err)
}
defer response.Body.Close()
if _, err := io.Copy(io.Discard, response.Body); err != nil {
t.Fatal(err)
}
if err := checkResponseEnd(reader, false); (err == nil) != tc.valid {
t.Fatalf("valid=%v, checkResponseEnd error: %v", tc.valid, err)
}
})
}
}
func TestParseContentTypeRequiresSubtype(t *testing.T) {
for _, tc := range []struct {
value string
valid bool
}{
{"application/octet-stream", true},
{"text/plain; charset=utf-8", true},
{"nonsense", false},
{"text/", false},
{"", false},
} {
_, err := parseContentType(tc.value)
if (err == nil) != tc.valid {
t.Errorf("content type %q: valid=%v, error=%v", tc.value, tc.valid, err)
}
}
}
func TestRunQueryAcceptsEarlyErrorResponse(t *testing.T) {
listener, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
defer listener.Close()
go func() {
server, err := listener.Accept()
if err != nil {
return
}
defer server.Close()
reader := bufio.NewReader(server)
for {
line, err := reader.ReadString('\n')
if err != nil || line == "\r\n" {
break
}
}
_, _ = io.WriteString(server, "HTTP/1.1 400 Bad Request\r\nConnection: close\r\nContent-Length: 5\r\n\r\nerror")
}()
t.Setenv("SOLUTION_HOST", "127.0.0.1")
RunQuery(NewTestContext(t), &Env{}, t.TempDir(), RunOpts{
Port: listener.Addr().(*net.TCPAddr).Port,
ServerDomain: "localhost",
}, Query{
Method: "PUT",
Path: "item",
HostHeader: "other.example",
FileContent: &EnvFile{GenSeed: 42, Size: 8 * 1024 * 1024},
})
}
func TestRunQueryWaitsForLargeRequestBeforeCloseTimeout(t *testing.T) {
listener, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
defer listener.Close()
done := make(chan error, 1)
go func() {
server, err := listener.Accept()
if err != nil {
done <- err
return
}
defer server.Close()
request, err := http.ReadRequest(bufio.NewReader(server))
if err != nil {
done <- err
return
}
if _, err := io.WriteString(server, "HTTP/1.1 400 Bad Request\r\nConnection: close\r\nContent-Length: 5\r\n\r\nerror"); err != nil {
done <- err
return
}
time.Sleep(2500 * time.Millisecond)
_, err = io.Copy(io.Discard, request.Body)
done <- err
}()
t.Setenv("SOLUTION_HOST", "127.0.0.1")
RunQuery(NewTestContext(t), &Env{}, t.TempDir(), RunOpts{
Port: listener.Addr().(*net.TCPAddr).Port,
ServerDomain: "localhost",
}, Query{
Method: "POST",
Path: "item",
HostHeader: "other.example",
FileContent: &EnvFile{GenSeed: 42, Size: 160 << 20},
})
if err := <-done; err != nil {
t.Fatal(err)
}
}
func TestIdleDeadlineWriter(t *testing.T) {
t.Run("stalled write", func(t *testing.T) {
client, server := net.Pipe()
defer client.Close()
defer server.Close()
writer := &idleDeadlineWriter{Conn: client, Timeout: 50 * time.Millisecond}
_, err := writer.Write([]byte("x"))
var netErr net.Error
if !errors.As(err, &netErr) || !netErr.Timeout() {
t.Fatalf("expected write timeout, got %v", err)
}
})
t.Run("request body timeout", func(t *testing.T) {
client, server := net.Pipe()
defer client.Close()
defer server.Close()
release := make(chan struct{})
defer close(release)
go func() {
_, _ = http.ReadRequest(bufio.NewReader(server))
<-release
}()
writer := &idleDeadlineWriter{Conn: client, Timeout: 50 * time.Millisecond}
request := (&Query{Method: "POST", FileContent: &EnvFile{GenSeed: 42, Size: 1 << 20}}).
CreateRequest(NewTestContext(t), "http://localhost/item")
if err := request.Write(writer); err == nil {
t.Fatal("expected request write to time out")
}
var netErr net.Error
if !errors.As(writer.lastError, &netErr) || !netErr.Timeout() {
t.Fatalf("expected underlying write timeout, got %v", writer.lastError)
}
})
}
func TestCompressedGetResponseValidation(t *testing.T) {
file := &EnvFile{GenSeed: 42, Size: 1024}
directory := &EnvDir{Listing: map[string]EnvNode{"item": file}}
for _, tc := range []struct {
name string
body io.Reader
validate func(*TC, *http.Response)
}{
{
name: "file",
body: file.Open(),
validate: func(t *TC, resp *http.Response) {
GetFileAction{File: file, Compression: true, VerifyHeaders: true}.VerifyResponse(t, nil, resp)
},
},
{
name: "directory",
body: bytes.NewBufferString("item\n"),
validate: func(t *TC, resp *http.Response) {
GetDirAction{Dir: directory, Compression: true, VerifyHeaders: true}.VerifyResponse(t, nil, resp)
},
},
} {
t.Run(tc.name, func(tt *testing.T) {
testContext := NewTestContext(tt)
var encoded bytes.Buffer
writer := gzip.NewWriter(&encoded)
_, err := io.Copy(writer, tc.body)
if err != nil {
tt.Fatal(err)
}
if err := writer.Close(); err != nil {
tt.Fatal(err)
}
resp := &http.Response{
StatusCode: http.StatusOK,
ContentLength: int64(encoded.Len()),
Header: http.Header{
"Content-Length": {strconv.Itoa(encoded.Len())},
"Content-Encoding": {"gzip"},
"Content-Type": {"text/plain"},
"Server": {"test-server"},
"Connection": {"close"},
},
Body: io.NopCloser(bytes.NewReader(encoded.Bytes())),
}
defer resp.Body.Close()
q := Query{Gzip: true, VerifyHeaders: true}
req := q.CreateRequest(testContext, "http://localhost/item")
if got := req.Header.Get("Accept-Encoding"); got != "gzip" {
tt.Fatalf("expected Accept-Encoding: gzip, got %q", got)
}
q.CommonValidate(testContext, req, resp)
tc.validate(testContext, resp)
})
}
}
func TestCompareFileContentBlocks(t *testing.T) {
file := &EnvFile{GenSeed: 17, Size: 64*1024 + 17}
context := NewTestContext(t)
if err := CompareFileContent(context, file.Open(), file); err != nil {
t.Fatalf("valid file was rejected: %v", err)
}
if err := CompareFileContent(context, io.LimitReader(file.Open(), file.Size-1), file); err == nil {
t.Fatal("truncated file was accepted")
}
content, err := io.ReadAll(file.Open())
if err != nil {
t.Fatal(err)
}
content[len(content)-1] ^= 1
if err := CompareFileContent(context, bytes.NewReader(content), file); err == nil {
t.Fatal("changed final byte was accepted")
}
}
homework/03-http-server/tests/runner.go
+303
View File
@@ -0,0 +1,303 @@
package hw3test
import (
"bytes"
"errors"
"fmt"
"io"
"math/rand"
"os"
"os/exec"
"path/filepath"
"strings"
"text/template"
"unicode"
"go.uber.org/zap"
)
// RunOpts contains command line arguments for the solution.
// Those are passed to the template.
type RunOpts struct {
// Solution options
Port int
WorkingDirectory string
ListenAddr string // Host
ServerDomain string
// Full run config that will be provided to the solution.
CommandLineArgs string
Env []string
// Full run config if solution will be running in docker.
DockerCommandLineArgs string
DockerEnvArgs string
DockerPortArgs string
DockerVolumeArgs string
// Hack to get exitcode of the solution.
ExitCode chan int
}
func (o *RunOpts) Address() string {
host := os.Getenv("SOLUTION_HOST")
if host == "" {
host = "localhost"
}
return fmt.Sprintf("http://%s:%d", host, o.Port)
}
func (o *RunOpts) GenerateRunConfig(t *TC, r *rand.Rand, gen *EnvGen) {
o.CommandLineArgs, o.Env = o.BuildConfig(r, gen)
dirsInDocker := []string{
"/files",
"/files0",
"/files1",
"/files2",
"/files3",
}
dirInDocker := dirsInDocker[r.Intn(len(dirsInDocker))]
if o.WorkingDirectory == "" {
dirInDocker = ""
} else {
o.DockerVolumeArgs = fmt.Sprintf(`-v "%s:%s"`, o.WorkingDirectory, dirInDocker)
}
o.DockerPortArgs = fmt.Sprintf("-p %d:%d", o.Port, o.Port)
dockerArgs, dockerEnv := RunOpts{
ListenAddr: o.ListenAddr,
Port: o.Port,
WorkingDirectory: dirInDocker,
ServerDomain: o.ServerDomain,
}.BuildConfig(r, gen)
o.DockerEnvArgs = ""
for _, env := range dockerEnv {
o.DockerEnvArgs += "--env \"" + env + "\" "
}
o.DockerCommandLineArgs = dockerArgs
}
// BuildConfig uses ListenAddr, Port, WorkingDirectory, ServerDomain.
func (o RunOpts) BuildConfig(r *rand.Rand, gen *EnvGen) (args string, env []string) {
if o.ListenAddr == "0.0.0.0" && r.Intn(2) == 1 {
// can omit default value
} else if gen.AllowEnv && r.Intn(3) == 1 {
// use env
env = append(env, fmt.Sprintf("SERVER_HOST=%s", o.ListenAddr))
} else if o.ListenAddr != "" {
// use plain cmdline args
args += fmt.Sprintf(" \"--host=%s\"", o.ListenAddr)
if r.Intn(2) == 1 {
// pass dummy env
env = append(env, fmt.Sprintf("SERVER_HOST=%s", "8.8.8.8"))
}
}
if o.Port == 8080 && r.Intn(2) == 1 {
// can omit default value
} else if gen.AllowEnv && r.Intn(3) == 1 {
// use env
env = append(env, fmt.Sprintf("SERVER_PORT=%d", o.Port))
} else {
// use plain cmdline args
args += fmt.Sprintf(" \"--port=%d\"", o.Port)
if r.Intn(2) == 1 {
// pass dummy env
env = append(env, fmt.Sprintf("SERVER_PORT=%d", 80))
}
}
if o.WorkingDirectory == "" && r.Intn(2) == 1 {
// can omit default value
} else if gen.AllowEnv && r.Intn(3) == 1 {
// use env
env = append(env, fmt.Sprintf("SERVER_WORKING_DIRECTORY=%s", o.WorkingDirectory))
} else if o.WorkingDirectory != "" {
// use plain cmdline args
args += fmt.Sprintf(" \"--working-directory=%s\"", o.WorkingDirectory)
if r.Intn(2) == 1 {
// pass dummy env
env = append(env, fmt.Sprintf("SERVER_WORKING_DIRECTORY=%s", "/"))
}
}
if o.ServerDomain == "localhost" && r.Intn(2) == 1 {
// can omit default value
} else if gen.AllowEnv && r.Intn(3) == 1 {
// use env
env = append(env, fmt.Sprintf("SERVER_DOMAIN=%s", o.ServerDomain))
} else if o.ServerDomain != "" {
// use plain cmdline args
args += fmt.Sprintf(" \"--server-domain=%s\"", o.ServerDomain)
if r.Intn(2) == 1 {
// pass dummy env
env = append(env, fmt.Sprintf("SERVER_DOMAIN=%s", "example.com"))
}
}
return args, env
}
// Runner is a helper for running HTTP server solution.
type Runner interface {
// Run the solution with the given options.
// Returns a function that can be used to stop the solution.
Run(t *TC, opts RunOpts) (stop func(), err error)
}
// CmdRunner runs command based on template.
type CmdRunner struct {
tmpl template.Template
useDocker bool
}
func NewCmdRunner(tmpl *template.Template, useDocker bool) *CmdRunner {
return &CmdRunner{
tmpl: *tmpl,
useDocker: useDocker,
}
}
// splitCommand preserves quoted arguments without invoking a shell.
func splitCommand(command string) ([]string, error) {
var args []string
var arg strings.Builder
quoted := false
started := false
for _, char := range command {
switch {
case char == '"':
quoted = !quoted
started = true
case unicode.IsSpace(char) && !quoted:
if started {
args = append(args, arg.String())
arg.Reset()
started = false
}
default:
arg.WriteRune(char)
started = true
}
}
if quoted {
return nil, fmt.Errorf("unclosed quote in launch command")
}
if started {
args = append(args, arg.String())
}
if len(args) == 0 {
return nil, fmt.Errorf("empty launch command")
}
return args, nil
}
func (r *CmdRunner) Run(t *TC, opts RunOpts) (stop func(), err error) {
var b bytes.Buffer
err = r.tmpl.Execute(&b, opts)
if err != nil {
return nil, fmt.Errorf("failed to execute template: %v", err)
}
envOpts := opts.Env
if r.useDocker {
envOpts = nil
}
cmdParts, err := splitCommand(b.String())
if err != nil {
return nil, err
}
var cidDir, cidFile string
if r.useDocker {
if len(cmdParts) < 2 || !strings.EqualFold(filepath.Base(cmdParts[0]), "docker") || cmdParts[1] != "run" {
return nil, fmt.Errorf("Docker launch template must start with docker run")
}
cidDir, err = os.MkdirTemp("", "hw3-container-")
if err != nil {
return nil, fmt.Errorf("failed to create container ID directory: %w", err)
}
cidFile = filepath.Join(cidDir, "id")
cmdParts = append(append([]string{}, cmdParts[:2]...), append([]string{"--cidfile", cidFile}, cmdParts[2:]...)...)
}
Info(t, "Running command", zap.Strings("command", cmdParts), zap.Strings("env", envOpts))
cmd := exec.Command(cmdParts[0], cmdParts[1:]...)
cmd.Env = append(os.Environ(), envOpts...)
cmd.Stdout = NewProxyWriter(os.Stderr)
cmd.Stderr = NewProxyWriter(os.Stderr)
err = cmd.Start()
if err != nil {
if cidDir != "" {
_ = os.Remove(cidDir)
}
return nil, fmt.Errorf("failed to run command: %v", err)
}
go func() {
err := cmd.Wait()
if err != nil && err.Error() != "signal: killed" && err.Error() != "exit status 1" {
Warn(t, "Command finished with error", zap.Error(err))
}
if opts.ExitCode != nil {
if e, ok := err.(*exec.ExitError); ok {
opts.ExitCode <- e.ExitCode()
} else {
opts.ExitCode <- 0
}
}
}()
return func() {
if cmd.Process != nil {
if err := cmd.Process.Kill(); err != nil && !errors.Is(err, os.ErrProcessDone) {
Error(t, "Failed to kill command", zap.Error(err))
}
}
if r.useDocker {
id, err := os.ReadFile(cidFile)
if err == nil && len(strings.TrimSpace(string(id))) > 0 {
output, removeErr := exec.Command("docker", "rm", "-f", strings.TrimSpace(string(id))).CombinedOutput()
if removeErr != nil && !strings.Contains(string(output), "No such container") && !strings.Contains(string(output), "is already in progress") {
Error(t, "Failed to remove Docker container", zap.Error(removeErr), zap.ByteString("output", output))
}
} else if err != nil && !errors.Is(err, os.ErrNotExist) {
Error(t, "Failed to read container ID", zap.Error(err))
}
_ = os.Remove(cidFile)
_ = os.Remove(cidDir)
}
}, nil
}
// ProxyWriter is used to forward solution output to standard output.
type ProxyWriter struct {
w io.Writer
disable bool
}
func NewProxyWriter(w io.Writer) *ProxyWriter {
return &ProxyWriter{
w: w,
disable: boolFromEnv("DISABLE_SOLUTION_OUTPUT", false),
}
}
func (w *ProxyWriter) Write(b []byte) (n int, err error) {
if w.disable {
return len(b), nil
}
// TODO: if there will be sync problems, we can take a global lock
// and read until \n, then flush and apply color
return w.w.Write(b)
}
homework/03-http-server/tests/runner_test.go
+38
View File
@@ -0,0 +1,38 @@
package hw3test
import (
"reflect"
"testing"
)
func TestSplitCommand(t *testing.T) {
for _, test := range []struct {
name string
command string
want []string
wantErr bool
}{
{
name: "working directory with spaces",
command: `python3 "../solution with spaces/server.py" "--working-directory=C:\Users\student name\files"`,
want: []string{"python3", "../solution with spaces/server.py", `--working-directory=C:\Users\student name\files`},
},
{
name: "Docker volume with spaces",
command: `docker run -v "C:\Users\student name\files:/files" hw3img`,
want: []string{"docker", "run", "-v", `C:\Users\student name\files:/files`, "hw3img"},
},
{name: "empty command", command: " ", wantErr: true},
{name: "unclosed quote", command: `python3 "unterminated`, wantErr: true},
} {
t.Run(test.name, func(t *testing.T) {
got, err := splitCommand(test.command)
if (err != nil) != test.wantErr {
t.Fatalf("splitCommand(%q) error = %v, wantErr %v", test.command, err, test.wantErr)
}
if !reflect.DeepEqual(got, test.want) {
t.Fatalf("splitCommand(%q) = %#v, want %#v", test.command, got, test.want)
}
})
}
}
homework/03-http-server/tests/test_starter.py
+92
View File
@@ -0,0 +1,92 @@
"""Run from the assignment directory: python -m unittest discover -s tests -p test_starter.py."""
import importlib.util
import os
from pathlib import Path
import socket
import subprocess
import sys
import tempfile
import time
import unittest
from unittest.mock import patch
from click.testing import CliRunner
STARTER = Path(__file__).resolve().parents[1] / "solution" / "server.py"
spec = importlib.util.spec_from_file_location("hw3_starter", STARTER)
starter = importlib.util.module_from_spec(spec)
sys.modules[spec.name] = starter
spec.loader.exec_module(starter)
ENV_NAMES = ("SERVER_HOST", "SERVER_PORT", "SERVER_DOMAIN", "SERVER_WORKING_DIRECTORY")
class StarterTests(unittest.TestCase):
def run_config(self, args, env):
clean_env = {k: v for k, v in os.environ.items() if k not in ENV_NAMES}
clean_env.update(env)
with patch.dict(os.environ, clean_env, clear=True), patch.object(starter.socket, "socket") as factory, patch.object(starter, "HTTPServer") as server:
factory.return_value.accept.side_effect = OSError("stop after bind")
result = CliRunner().invoke(starter.main, args)
return result, factory, server
def test_defaults(self):
result, factory, server = self.run_config(["--working-directory", str(STARTER.parent)], {})
self.assertEqual(result.exit_code, 0, result.output)
factory.return_value.bind.assert_called_once_with(("0.0.0.0", 8080))
self.assertEqual(server.call_args.args[2:], ("localhost", STARTER.parent))
def test_environment(self):
env = dict(zip(ENV_NAMES, ("127.0.0.1", "9090", "example.com", str(STARTER.parent))))
result, factory, server = self.run_config([], env)
self.assertEqual(result.exit_code, 0, result.output)
factory.return_value.bind.assert_called_once_with(("127.0.0.1", 9090))
self.assertEqual(server.call_args.args[2:], ("example.com", STARTER.parent))
def test_cli_overrides_environment(self):
env = dict(zip(ENV_NAMES, ("8.8.8.8", "80", "wrong.example", "wrong-directory")))
result, factory, server = self.run_config(
["--host", "127.0.0.1", "--port", "9091", "--server-domain", "example.com",
"--working-directory", str(STARTER.parent)], env)
self.assertEqual(result.exit_code, 0, result.output)
factory.return_value.bind.assert_called_once_with(("127.0.0.1", 9091))
self.assertEqual(server.call_args.args[2:], ("example.com", STARTER.parent))
def test_missing_directory(self):
for args, env in [([], {}), ([], {"SERVER_WORKING_DIRECTORY": ""}),
(["--working-directory", ""], {"SERVER_WORKING_DIRECTORY": str(STARTER.parent)})]:
with self.subTest(args=args, env=env):
result, factory, _ = self.run_config(args, env)
self.assertEqual(result.exit_code, 1, result.output)
factory.assert_not_called()
def test_tcp_startup(self):
with socket.socket() as probe:
probe.bind(("127.0.0.1", 0))
port = probe.getsockname()[1]
with tempfile.TemporaryDirectory() as workdir:
process = subprocess.Popen(
[sys.executable, str(STARTER), "--host", "127.0.0.1", "--port", str(port),
"--working-directory", workdir],
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
try:
deadline = time.monotonic() + 5
while True:
self.assertIsNone(process.poll(), "starter exited before accepting a connection")
try:
client = socket.create_connection(("127.0.0.1", port), timeout=0.5)
break
except OSError:
if time.monotonic() >= deadline:
self.fail("starter did not bind within 5 seconds")
time.sleep(0.05)
with client:
client.sendall(b"GET / HTTP/1.1\r\n")
# The starter has no HTTP handler yet, but its TCP lifecycle works.
self.assertEqual(client.recv(1), b"")
finally:
process.terminate()
process.wait(timeout=5)
if __name__ == "__main__":
unittest.main()
homework/03-http-server/tests/test_utils.go
+103
View File
@@ -0,0 +1,103 @@
package hw3test
import (
"context"
"fmt"
"net"
"os"
"strconv"
"strings"
"testing"
)
// TC implements *testing.T and context.Context at the same time.
// Also has some helper methods.
type TC struct {
*testing.T
ctx context.Context
}
func NewTestContext(t *testing.T) *TC {
return &TC{
ctx: context.Background(),
T: t,
}
}
func (tc *TC) Errorf(format string, args ...any) {
str := fmt.Sprintf(format, args...)
if strings.HasPrefix(str, "\n\tError Trace:") {
_, after, ok := strings.Cut(str, "\tError:")
if ok {
str = "\n\tError:" + after
}
}
Error(tc, str)
tc.T.Errorf(str)
}
func (tc *TC) RunByName(name string, f func(*TC)) bool {
return tc.Run(name, func(t *testing.T) {
f(tc.derive(t))
})
}
func (tc *TC) RunBySeed(seed int64, f func(*TC)) bool {
return tc.Run(strconv.FormatInt(seed, 10), func(t *testing.T) {
f(tc.derive(t))
})
}
func (tc *TC) Done() <-chan struct{} {
return tc.ctx.Done()
}
func (tc *TC) Err() error {
return tc.ctx.Err()
}
func (tc *TC) Value(key any) any {
return tc.ctx.Value(key)
}
// derive copies *TC and replaces *testing.T
func (tc TC) derive(t *testing.T) *TC {
tc.T = t
return &tc
}
// Get bool from envvars
func boolFromEnv(key string, def bool) bool {
value := os.Getenv(key)
if value == "" {
return def
}
value = strings.ToLower(value)
switch value {
case "t":
return true
case "true":
return true
case "y":
return true
case "yes":
return true
}
return false
}
// GetFreePort asks the kernel for a free open port that is ready to use.
func GetFreePort() (int, error) {
addr, err := net.ResolveTCPAddr("tcp", "localhost:0")
if err != nil {
return 0, err
}
l, err := net.ListenTCP("tcp", addr)
if err != nil {
return 0, err
}
defer l.Close()
return l.Addr().(*net.TCPAddr).Port, nil
}
homework/03-http-server/tests/tester.go
+308
View File
@@ -0,0 +1,308 @@
package hw3test
import (
"bufio"
"context"
"errors"
"fmt"
"io"
"math/rand"
"net"
"net/http"
"os"
"path"
"path/filepath"
"time"
"github.com/stretchr/testify/require"
"go.uber.org/zap"
)
// RunTestEmptyWorkDir will check that server does exit(1) if working directory is empty.
func RunTestEmptyWorkDir(t *TC, seed int64, runner Runner) {
t.RunBySeed(seed, func(t *TC) {
r := rand.New(rand.NewSource(seed))
port, err := GetFreePort()
require.NoError(t, err, "failed to get free port for the server")
runOpts := RunOpts{
Port: port,
WorkingDirectory: "",
ServerDomain: "localhost",
ListenAddr: "0.0.0.0",
ExitCode: make(chan int),
}
// Start the solution.
runOpts.GenerateRunConfig(t, r, &EnvGen{})
stop, err := runner.Run(t, runOpts)
require.NoError(t, err, "failed to start solution")
defer stop()
select {
case <-time.After(time.Second * 10):
require.FailNow(t, "Server didn't exit(1) in 10 seconds")
case ec := <-runOpts.ExitCode:
require.Equal(t, 1, ec, "Server exited with wrong code")
}
})
}
func RunTests(t *TC, seed int64, runner Runner, envGen *EnvGen, queriesGen *QueriesGen) {
failed := !t.RunBySeed(seed, func(t *TC) {
env, err := envGen.Generate(seed)
require.NoError(t, err, "failed to generate environment")
r := rand.New(rand.NewSource(seed))
queriesSeeds := make([]int64, queriesGen.Count)
for i := range queriesSeeds {
queriesSeeds[i] = r.Int63n(100000)
}
envDirName := GenFilename(r, 16)
envDir := filepath.Join(envGen.TempDirectory, envDirName)
runOpts := RunOpts{
WorkingDirectory: envDir,
ServerDomain: "localhost",
ListenAddr: "0.0.0.0",
}
if queriesGen.AllHeaders {
possibleDomains := []string{
"localhost",
"cs.hse.ru",
"example.com",
"z0r.de",
"distsys-course.homework.net",
}
runOpts.ServerDomain = possibleDomains[r.Intn(len(possibleDomains))]
}
queries := queriesGen.Generate(t, env, envGen, queriesSeeds, runOpts, seed)
if envGen.SparseUnusedFiles {
needed := make(map[string]bool)
for _, query := range queries {
if query.Method == http.MethodGet || query.Method == http.MethodPut {
needed[path.Clean(query.Path)] = true
}
}
err = env.RootDir.WriteToDiskSelected(envDir, needed)
} else {
err = env.RootDir.WriteToDisk(envDir)
}
require.NoError(t, err, "failed to write environment to disk")
defer os.RemoveAll(envDir)
port, err := GetFreePort()
require.NoError(t, err, "failed to get free port for the server")
runOpts.Port = port
// Start the solution.
runOpts.GenerateRunConfig(t, r, envGen)
stop, err := runner.Run(t, runOpts)
require.NoError(t, err, "failed to start solution")
defer stop()
// Await server to bind to port.
err = WaitForServer(t, runOpts)
require.NoError(t, err, "failed to wait for server")
// Run the queries.
for i, query := range queries {
query := query
ok := t.RunBySeed(query.Seed, func(t *TC) {
RunQuery(t, env, envDir, runOpts, query)
})
shouldAbort := !ok
if shouldAbort {
Warn(t, "Skipping next queries because of the failed query", zap.Int("skipped", len(queries)-1-i), zap.String("failed", fmt.Sprintf("%s/%v", t.Name(), query.Seed)))
break
}
}
})
shouldAbort := failed
if shouldAbort {
Warn(t, "Skipping next tests in a group because last test has failed")
t.FailNow()
}
}
func WaitForServer(t *TC, opts RunOpts) error {
client := &http.Client{
CheckRedirect: func(*http.Request, []*http.Request) error {
return http.ErrUseLastResponse
},
}
deadline := time.Now().Add(10 * time.Second)
var lastErr error
for attempt := 0; time.Until(deadline) > 0; attempt++ {
probeTimeout := time.Until(deadline)
if probeTimeout > time.Second {
probeTimeout = time.Second
}
ctx, cancel := context.WithTimeout(t, probeTimeout)
req, err := http.NewRequestWithContext(ctx, "GET", opts.Address(), nil)
if err != nil {
cancel()
return fmt.Errorf("failed to create context request: %w", err)
}
req.Host = opts.ServerDomain
resp, err := client.Do(req)
if err == nil {
if resp != nil {
resp.Body.Close()
}
cancel()
return nil
}
cancel()
lastErr = err
if attempt == 0 || attempt%10 == 0 {
Debug(t, "Waiting for server startup", zap.Int("attempt", attempt), zap.Error(err))
}
if remaining := time.Until(deadline); remaining > 0 {
if remaining > 100*time.Millisecond {
remaining = 100 * time.Millisecond
}
time.Sleep(remaining)
}
}
Warn(t, "Server didn't get up in time, aborting", zap.Error(lastErr))
return fmt.Errorf("server did not start in 10 seconds: %w", lastErr)
}
// RunQuery runs a single query. Server address and its configuration is taken from runOpts.
// Initial environment is described by env, but it may be changed with queries. Actual
// environment is located in envDir. Query describes the query itself.
func RunQuery(t *TC, env *Env, envDir string, opts RunOpts, query Query) {
queryURL := fmt.Sprintf("%s/%s", opts.Address(), query.Path)
Debug(
t,
"Sending query",
zap.String("method", query.Method),
zap.String("path", query.Path),
zap.String("url", queryURL),
)
action := query.Action(env, &opts)
if action != nil {
action.VerifyBefore(t, envDir)
}
req := query.CreateRequest(t, queryURL)
req.Close = true
conn, err := net.DialTimeout("tcp", req.URL.Host, 10*time.Second)
require.NoError(t, err, "failed to connect to server")
defer conn.Close()
writeDone := make(chan error, 1)
writer := &idleDeadlineWriter{Conn: conn, Timeout: 15 * time.Second}
go func() { writeDone <- req.Write(writer) }()
reader := bufio.NewReader(&idleDeadlineReader{Conn: conn, Timeout: 30 * time.Second})
resp, err := http.ReadResponse(reader, req)
require.NoError(t, err, "failed to run query on server")
defer resp.Body.Close()
body := &countingBody{ReadCloser: resp.Body}
resp.Body = body
query.CommonValidate(t, req, resp)
if action != nil {
action.VerifyResponse(t, req, resp)
}
_, err = io.Copy(io.Discard, resp.Body)
require.NoError(t, err, "failed to read response body")
if resp.StatusCode != http.StatusNoContent {
require.Equal(t, resp.ContentLength, body.Size, "Content-Length differs from response body size")
}
endDone := make(chan error, 1)
go func() { endDone <- checkResponseEnd(reader, req.Body != nil) }()
var endErr, writeErr error
writeCompleted := false
select {
case endErr = <-endDone:
// The server closed the connection before the request writer finished.
case writeErr = <-writeDone:
writeCompleted = true
var netErr net.Error
// http.Request.Write wraps body write errors without preserving Unwrap.
// Inspect the original socket error captured by the writer instead.
if errors.As(writer.lastError, &netErr) && netErr.Timeout() {
require.NoError(t, writer.lastError, "request upload stalled")
}
// A server may finish reading a large request after sending its response.
// Start the close timeout only after the request writer has finished.
closeTimeout := 2 * time.Second
if req.Body != nil {
closeTimeout = 10 * time.Second
}
require.NoError(t, conn.SetReadDeadline(time.Now().Add(closeTimeout)))
endErr = <-endDone
}
require.NoError(t, endErr, "response contains bytes beyond Content-Length or connection was not closed")
// The server may reply without consuming the whole request body. In that
// case the response and the file-system checks determine correctness.
if req.Body == nil {
if !writeCompleted {
writeErr = <-writeDone
}
require.NoError(t, writeErr, "failed to send query to server")
}
if query.VerifyHeaders && body.Size > 0 {
_, err = parseContentType(resp.Header.Get("Content-Type"))
require.NoError(t, err, "expected valid content type")
}
if action != nil {
action.VerifyAfter(t, envDir)
action.ApplyEnv(t, env)
}
}
// idleDeadlineWriter bounds pauses in sending a request without limiting the
// total time for a large request that continues to make progress.
type idleDeadlineWriter struct {
net.Conn
Timeout time.Duration
lastError error
}
func (w *idleDeadlineWriter) Write(p []byte) (int, error) {
if err := w.Conn.SetWriteDeadline(time.Now().Add(w.Timeout)); err != nil {
w.lastError = err
return 0, err
}
n, err := w.Conn.Write(p)
if err != nil {
w.lastError = err
}
return n, err
}
func checkResponseEnd(reader *bufio.Reader, requestHasBody bool) error {
_, err := reader.ReadByte()
if err == io.EOF {
return nil
}
// Closing a connection with an unread request body can produce a TCP reset
// after a complete error response. Its error code differs by platform.
var netErr *net.OpError
if requestHasBody && errors.As(err, &netErr) && netErr.Op == "read" && !netErr.Timeout() {
return nil
}
if err != nil {
return err
}
return fmt.Errorf("unexpected byte after response body")
}
type countingBody struct {
io.ReadCloser
Size int64
}
func (b *countingBody) Read(p []byte) (int, error) {
n, err := b.ReadCloser.Read(p)
b.Size += int64(n)
return n, err
}
homework/03-http-server/tests/verify-zram-scratch.sh
+15
View File
@@ -0,0 +1,15 @@
#!/bin/sh
set -eu
# Docker exposes the filesystem source after the separator in mountinfo.
# A bind mount whose host source is an ext4 filesystem on zram retains that
# source; a plain host directory on the HDD must not pass this check.
awk '
$5 == "/hw/tests/tmp" {
for (i = 6; i <= NF - 2; i++) {
if ($i == "-" && $(i + 1) == "ext4" && $(i + 2) ~ /^\/dev\/zram[0-9]+$/)
found = 1
}
}
END { exit !found }
' /proc/self/mountinfo