Seregon/zftpd

Zero-copy FTP/HTTP Daemon compatible with all POSIX systems

C/11.0 KB/No license
tools/qa/ftp_roundtrip.sh
raw
zftpd / tools / qa / ftp_roundtrip.sh
1#!/usr/bin/env bash
2set -euo pipefail
3 
4for arg in "$@"; do
5 case "$arg" in
6 *=*) export "$arg" ;;
7 esac
8done
9 
10ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
11BUILD_TYPE="${BUILD_TYPE:-debug}"
12TARGET="${TARGET:-macos}"
13FTP_PORT="${FTP_PORT:-21234}"
14PARALLEL="${PARALLEL:-4}"
15FILE_COUNT="${FILE_COUNT:-8}"
16FILE_SIZE_BYTES="${FILE_SIZE_BYTES:-33554432}"
17USE_APPE="${USE_APPE:-0}"
18USE_REST="${USE_REST:-0}"
19 
20if [[ "$TARGET" == "macos" ]]; then
21 if [[ -x "$ROOT_DIR/build/$TARGET/$BUILD_TYPE/zftpd" ]]; then
22 FTPD_BIN="$ROOT_DIR/build/$TARGET/$BUILD_TYPE/zftpd"
23 elif [[ -x "$ROOT_DIR/build/$TARGET/$BUILD_TYPE/ftpd" ]]; then
24 FTPD_BIN="$ROOT_DIR/build/$TARGET/$BUILD_TYPE/ftpd"
25 else
26 FTPD_BIN="$(ls -1t "$ROOT_DIR/build/$TARGET/$BUILD_TYPE"/zftpd-macos-*-v* 2>/dev/null | head -n 1 || true)"
27 fi
28else
29 if [[ -x "$ROOT_DIR/build/$TARGET/$BUILD_TYPE/zftpd.elf" ]]; then
30 FTPD_BIN="$ROOT_DIR/build/$TARGET/$BUILD_TYPE/zftpd.elf"
31 elif [[ -x "$ROOT_DIR/build/$TARGET/$BUILD_TYPE/ftpd.elf" ]]; then
32 FTPD_BIN="$ROOT_DIR/build/$TARGET/$BUILD_TYPE/ftpd.elf"
33 else
34 FTPD_BIN="$(ls -1t "$ROOT_DIR/build/$TARGET/$BUILD_TYPE"/zftpd-"$TARGET"-v*.elf 2>/dev/null | head -n 1 || true)"
35 fi
36fi
37 
38if [[ ! -x "$FTPD_BIN" ]]; then
39 echo "error: server binary not found for TARGET=$TARGET BUILD_TYPE=$BUILD_TYPE" >&2
40 exit 1
41fi
42 
43if command -v md5sum >/dev/null 2>&1; then
44 md5cmd() { md5sum "$1" | awk '{print $1}'; }
45elif command -v md5 >/dev/null 2>&1; then
46 md5cmd() { md5 -q "$1"; }
47else
48 md5cmd() { python3 - "$1" <<'PY'
49import hashlib,sys
50p=sys.argv[1]
51h=hashlib.md5()
52with open(p,'rb') as f:
53 for b in iter(lambda:f.read(1024*1024), b''):
54 h.update(b)
55print(h.hexdigest())
56PY
57 }
58fi
59 
60WORK="$(mktemp -d "${TMPDIR:-/tmp}/zftpd-qa.XXXXXXXX")"
61cleanup() {
62 if [[ -n "${FTPD_PID:-}" ]]; then
63 kill "$FTPD_PID" 2>/dev/null || true
64 wait "$FTPD_PID" 2>/dev/null || true
65 fi
66 if [[ "${KEEP_WORK:-0}" != "1" ]]; then
67 rm -rf "$WORK"
68 else
69 echo "KEEP_WORK=1 workdir=$WORK"
70 fi
71}
72trap cleanup EXIT
73 
74pushd "$ROOT_DIR" >/dev/null
75"$FTPD_BIN" -p "$FTP_PORT" >"$WORK/ftpd.log" 2>&1 &
76FTPD_PID=$!
77popd >/dev/null
78 
79sleep 0.2
80if ! kill -0 "$FTPD_PID" 2>/dev/null; then
81 echo "error: ftpd exited early" >&2
82 sed -n '1,200p' "$WORK/ftpd.log" >&2 || true
83 exit 2
84fi
85 
86python3 - "$FTP_PORT" <<'PY'
87import socket,sys,time
88port=int(sys.argv[1])
89deadline=time.time()+5
90while time.time()<deadline:
91 s=socket.socket()
92 s.settimeout(0.3)
93 try:
94 s.connect(("127.0.0.1", port))
95 s.close()
96 sys.exit(0)
97 except Exception:
98 time.sleep(0.1)
99 finally:
100 try: s.close()
101 except Exception: pass
102print("port not ready", port, file=sys.stderr)
103sys.exit(1)
104PY
105 
106python3 - "$WORK" "$FTP_PORT" "$PARALLEL" "$FILE_COUNT" "$FILE_SIZE_BYTES" "$USE_APPE" "$USE_REST" <<'PY'
107import os, sys, threading, time
108from ftplib import FTP, error_perm
109 
110work=sys.argv[1]
111port=int(sys.argv[2])
112parallel=int(sys.argv[3])
113file_count=int(sys.argv[4])
114file_size=int(sys.argv[5])
115use_appe=int(sys.argv[6])!=0
116use_rest=int(sys.argv[7])!=0
117 
118host="127.0.0.1"
119remote_dir="qa"
120 
121def randfile(path, n):
122 with open(path, "wb") as f:
123 left=n
124 while left>0:
125 chunk=min(left, 1024*1024)
126 f.write(os.urandom(chunk))
127 left-=chunk
128 
129def ftp_mkdir_p(ftp, path):
130 parts=[p for p in path.split("/") if p]
131 cur=""
132 for p in parts:
133 cur = p if not cur else (cur + "/" + p)
134 try:
135 ftp.mkd(cur)
136 except error_perm:
137 pass
138 
139def upload_download_one(i):
140 src=os.path.join(work, f"src_{i}.bin")
141 dst=os.path.join(work, f"dst_{i}.bin")
142 randfile(src, file_size)
143 
144 ftp=FTP()
145 ftp.connect(host, port, timeout=30)
146 ftp.login("anonymous","x")
147 ftp.voidcmd("TYPE I")
148 ftp_mkdir_p(ftp, remote_dir)
149 remote_path=f"{remote_dir}/file_{i}.bin"
150 
151 with open(src, "rb") as f:
152 if use_appe:
153 try:
154 ftp.delete(remote_path)
155 except Exception:
156 pass
157 ftp.storbinary("STOR "+remote_path, f, blocksize=64*1024)
158 f.seek(0)
159 ftp.storbinary("APPE "+remote_path, f, blocksize=64*1024)
160 else:
161 ftp.storbinary("STOR "+remote_path, f, blocksize=64*1024)
162 
163 with open(dst, "wb") as out:
164 ftp.retrbinary("RETR "+remote_path, out.write, blocksize=64*1024)
165 
166 if use_rest:
167 off=max(1, file_size//2)
168 rest_out=os.path.join(work, f"rest_{i}.bin")
169 with open(rest_out, "wb") as out:
170 ftp.retrbinary("RETR "+remote_path, out.write, blocksize=64*1024, rest=off)
171 with open(src, "rb") as fsrc, open(rest_out, "rb") as frest:
172 fsrc.seek(off)
173 if fsrc.read(1024*1024) != frest.read(1024*1024):
174 raise RuntimeError("REST mismatch")
175 
176 ftp.quit()
177 return src, dst
178 
179jobs=list(range(file_count))
180lock=threading.Lock()
181results=[]
182errors=[]
183 
184def worker():
185 while True:
186 with lock:
187 if not jobs:
188 return
189 i=jobs.pop()
190 try:
191 results.append(upload_download_one(i))
192 except Exception as e:
193 errors.append((i, repr(e)))
194 
195threads=[threading.Thread(target=worker) for _ in range(parallel)]
196for t in threads: t.start()
197for t in threads: t.join()
198 
199if errors:
200 print("errors:", errors)
201 sys.exit(2)
202 
203print("ok", len(results))
204PY
205 
206ok=0
207for ((i=0;i<FILE_COUNT;i++)); do
208 SRC="$WORK/src_${i}.bin"
209 DST="$WORK/dst_${i}.bin"
210 if [[ ! -f "$SRC" || ! -f "$DST" ]]; then
211 echo "error: missing $SRC or $DST" >&2
212 exit 3
213 fi
214 A="$(md5cmd "$SRC")"
215 B="$(md5cmd "$DST")"
216 if [[ "$A" != "$B" ]]; then
217 echo "error: checksum mismatch for $i ($A != $B)" >&2
218 exit 4
219 fi
220 ok=$((ok+1))
221done
222 
223echo "ftp_roundtrip_ok files=$ok parallel=$PARALLEL size=$FILE_SIZE_BYTES port=$FTP_PORT"
224