test_08_caddy.py (11774B)
1 #!/usr/bin/env python3 2 # -*- coding: utf-8 -*- 3 #*************************************************************************** 4 # _ _ ____ _ 5 # Project ___| | | | _ \| | 6 # / __| | | | |_) | | 7 # | (__| |_| | _ <| |___ 8 # \___|\___/|_| \_\_____| 9 # 10 # Copyright (C) Daniel Stenberg, <daniel@haxx.se>, et al. 11 # 12 # This software is licensed as described in the file COPYING, which 13 # you should have received as part of this distribution. The terms 14 # are also available at https://curl.se/docs/copyright.html. 15 # 16 # You may opt to use, copy, modify, merge, publish, distribute and/or sell 17 # copies of the Software, and permit persons to whom the Software is 18 # furnished to do so, under the terms of the COPYING file. 19 # 20 # This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY 21 # KIND, either express or implied. 22 # 23 # SPDX-License-Identifier: curl 24 # 25 ########################################################################### 26 # 27 import difflib 28 import filecmp 29 import logging 30 import os 31 import re 32 import pytest 33 34 from testenv import Env, CurlClient, Caddy, LocalClient 35 36 37 log = logging.getLogger(__name__) 38 39 40 @pytest.mark.skipif(condition=not Env.has_caddy(), reason="missing caddy") 41 @pytest.mark.skipif(condition=not Env.have_ssl_curl(), reason="curl without SSL") 42 class TestCaddy: 43 44 @pytest.fixture(autouse=True, scope='class') 45 def caddy(self, env): 46 caddy = Caddy(env=env) 47 assert caddy.initial_start() 48 yield caddy 49 caddy.stop() 50 51 def _make_docs_file(self, docs_dir: str, fname: str, fsize: int): 52 fpath = os.path.join(docs_dir, fname) 53 data1k = 1024*'x' 54 flen = 0 55 with open(fpath, 'w') as fd: 56 while flen < fsize: 57 fd.write(data1k) 58 flen += len(data1k) 59 return flen 60 61 @pytest.fixture(autouse=True, scope='class') 62 def _class_scope(self, env, caddy): 63 self._make_docs_file(docs_dir=caddy.docs_dir, fname='data10k.data', fsize=10*1024) 64 self._make_docs_file(docs_dir=caddy.docs_dir, fname='data1.data', fsize=1024*1024) 65 self._make_docs_file(docs_dir=caddy.docs_dir, fname='data5.data', fsize=5*1024*1024) 66 self._make_docs_file(docs_dir=caddy.docs_dir, fname='data10.data', fsize=10*1024*1024) 67 self._make_docs_file(docs_dir=caddy.docs_dir, fname='data100.data', fsize=100*1024*1024) 68 env.make_data_file(indir=env.gen_dir, fname="data-10m", fsize=10*1024*1024) 69 70 # download 1 file 71 @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3']) 72 def test_08_01_download_1(self, env: Env, caddy: Caddy, proto): 73 if proto == 'h3' and not env.have_h3_curl(): 74 pytest.skip("h3 not supported in curl") 75 if proto == 'h3' and env.curl_uses_lib('msh3'): 76 pytest.skip("msh3 itself crashes") 77 curl = CurlClient(env=env) 78 url = f'https://{env.domain1}:{caddy.port}/data.json' 79 r = curl.http_download(urls=[url], alpn_proto=proto) 80 r.check_response(count=1, http_status=200) 81 82 # download 1MB files sequentially 83 @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3']) 84 def test_08_02_download_1mb_sequential(self, env: Env, caddy: Caddy, proto): 85 if proto == 'h3' and not env.have_h3_curl(): 86 pytest.skip("h3 not supported in curl") 87 if proto == 'h3' and env.curl_uses_lib('msh3'): 88 pytest.skip("msh3 itself crashes") 89 count = 50 90 curl = CurlClient(env=env) 91 urln = f'https://{env.domain1}:{caddy.port}/data1.data?[0-{count-1}]' 92 r = curl.http_download(urls=[urln], alpn_proto=proto) 93 r.check_response(count=count, http_status=200, connect_count=1) 94 95 # download 1MB files parallel 96 @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3']) 97 def test_08_03_download_1mb_parallel(self, env: Env, caddy: Caddy, proto): 98 if proto == 'h3' and not env.have_h3_curl(): 99 pytest.skip("h3 not supported in curl") 100 if proto == 'h3' and env.curl_uses_lib('msh3'): 101 pytest.skip("msh3 itself crashes") 102 count = 20 103 curl = CurlClient(env=env) 104 urln = f'https://{env.domain1}:{caddy.port}/data1.data?[0-{count-1}]' 105 r = curl.http_download(urls=[urln], alpn_proto=proto, extra_args=[ 106 '--parallel' 107 ]) 108 r.check_response(count=count, http_status=200) 109 if proto == 'http/1.1': 110 # http/1.1 parallel transfers will open multiple connections 111 assert r.total_connects > 1, r.dump_logs() 112 else: 113 assert r.total_connects == 1, r.dump_logs() 114 115 # download 5MB files sequentially 116 @pytest.mark.skipif(condition=Env().slow_network, reason="not suitable for slow network tests") 117 @pytest.mark.skipif(condition=Env().ci_run, reason="not suitable for CI runs") 118 @pytest.mark.parametrize("proto", ['h2', 'h3']) 119 def test_08_04a_download_10mb_sequential(self, env: Env, caddy: Caddy, proto): 120 if proto == 'h3' and not env.have_h3_curl(): 121 pytest.skip("h3 not supported in curl") 122 if proto == 'h3' and env.curl_uses_lib('msh3'): 123 pytest.skip("msh3 itself crashes") 124 count = 40 125 curl = CurlClient(env=env) 126 urln = f'https://{env.domain1}:{caddy.port}/data5.data?[0-{count-1}]' 127 r = curl.http_download(urls=[urln], alpn_proto=proto) 128 r.check_response(count=count, http_status=200, connect_count=1) 129 130 # download 10MB files sequentially 131 @pytest.mark.skipif(condition=Env().slow_network, reason="not suitable for slow network tests") 132 @pytest.mark.skipif(condition=Env().ci_run, reason="not suitable for CI runs") 133 @pytest.mark.parametrize("proto", ['h2', 'h3']) 134 def test_08_04b_download_10mb_sequential(self, env: Env, caddy: Caddy, proto): 135 if proto == 'h3' and not env.have_h3_curl(): 136 pytest.skip("h3 not supported in curl") 137 if proto == 'h3' and env.curl_uses_lib('msh3'): 138 pytest.skip("msh3 itself crashes") 139 count = 20 140 curl = CurlClient(env=env) 141 urln = f'https://{env.domain1}:{caddy.port}/data10.data?[0-{count-1}]' 142 r = curl.http_download(urls=[urln], alpn_proto=proto) 143 r.check_response(count=count, http_status=200, connect_count=1) 144 145 # download 10MB files parallel 146 @pytest.mark.skipif(condition=Env().slow_network, reason="not suitable for slow network tests") 147 @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3']) 148 @pytest.mark.skipif(condition=Env().ci_run, reason="not suitable for CI runs") 149 def test_08_05_download_1mb_parallel(self, env: Env, caddy: Caddy, proto): 150 if proto == 'h3' and not env.have_h3_curl(): 151 pytest.skip("h3 not supported in curl") 152 if proto == 'h3' and env.curl_uses_lib('msh3'): 153 pytest.skip("msh3 itself crashes") 154 if proto == 'http/1.1' and env.curl_uses_lib('mbedtls'): 155 pytest.skip("mbedtls 3.6.0 fails on 50 connections with: " 156 "ssl_handshake returned: (-0x7F00) SSL - Memory allocation failed") 157 count = 50 158 curl = CurlClient(env=env) 159 urln = f'https://{env.domain1}:{caddy.port}/data10.data?[0-{count-1}]' 160 r = curl.http_download(urls=[urln], alpn_proto=proto, extra_args=[ 161 '--parallel' 162 ]) 163 r.check_response(count=count, http_status=200) 164 if proto == 'http/1.1': 165 # http/1.1 parallel transfers will open multiple connections 166 assert r.total_connects > 1, r.dump_logs() 167 else: 168 assert r.total_connects == 1, r.dump_logs() 169 170 # post data parallel, check that they were echoed 171 @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3']) 172 def test_08_06_post_parallel(self, env: Env, httpd, caddy, proto): 173 if proto == 'h3' and not env.have_h3(): 174 pytest.skip("h3 not supported") 175 if proto == 'h3' and env.curl_uses_lib('msh3'): 176 pytest.skip("msh3 stalls here") 177 # limit since we use a separate connection in h1 178 count = 20 179 data = '0123456789' 180 curl = CurlClient(env=env) 181 url = f'https://{env.domain2}:{caddy.port}/curltest/echo?id=[0-{count-1}]' 182 r = curl.http_upload(urls=[url], data=data, alpn_proto=proto, 183 extra_args=['--parallel']) 184 r.check_stats(count=count, http_status=200, exitcode=0) 185 for i in range(count): 186 respdata = open(curl.response_file(i)).readlines() 187 assert respdata == [data] 188 189 # put large file, check that they length were echoed 190 @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3']) 191 def test_08_07_put_large(self, env: Env, httpd, caddy, proto): 192 if proto == 'h3' and not env.have_h3(): 193 pytest.skip("h3 not supported") 194 if proto == 'h3' and env.curl_uses_lib('msh3'): 195 pytest.skip("msh3 stalls here") 196 # limit since we use a separate connection in h1< 197 count = 1 198 fdata = os.path.join(env.gen_dir, 'data-10m') 199 curl = CurlClient(env=env) 200 url = f'https://{env.domain2}:{caddy.port}/curltest/put?id=[0-{count-1}]' 201 r = curl.http_put(urls=[url], fdata=fdata, alpn_proto=proto) 202 exp_data = [f'{os.path.getsize(fdata)}'] 203 r.check_response(count=count, http_status=200) 204 for i in range(count): 205 respdata = open(curl.response_file(i)).readlines() 206 assert respdata == exp_data 207 208 @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3']) 209 def test_08_08_earlydata(self, env: Env, httpd, caddy, proto): 210 if not env.curl_can_early_data(): 211 pytest.skip('TLS earlydata not implemented') 212 if proto == 'h3' and \ 213 (not env.have_h3() or not env.curl_can_h3_early_data()): 214 pytest.skip("h3 not supported") 215 count = 2 216 docname = 'data10k.data' 217 url = f'https://{env.domain1}:{caddy.port}/{docname}' 218 client = LocalClient(name='hx_download', env=env) 219 if not client.exists(): 220 pytest.skip(f'example client not built: {client.name}') 221 r = client.run(args=[ 222 '-n', f'{count}', 223 '-e', # use TLS earlydata 224 '-f', # forbid reuse of connections 225 '-r', f'{env.domain1}:{caddy.port}:127.0.0.1', 226 '-V', proto, url 227 ]) 228 r.check_exit_code(0) 229 srcfile = os.path.join(caddy.docs_dir, docname) 230 self.check_downloads(client, srcfile, count) 231 earlydata = {} 232 for line in r.trace_lines: 233 m = re.match(r'^\[t-(\d+)] EarlyData: (-?\d+)', line) 234 if m: 235 earlydata[int(m.group(1))] = int(m.group(2)) 236 assert earlydata[0] == 0, f'{earlydata}' 237 if proto == 'h3': 238 assert earlydata[1] == 113, f'{earlydata}' 239 else: 240 # Caddy does not support early data on TCP 241 assert earlydata[1] == 0, f'{earlydata}' 242 243 def check_downloads(self, client, srcfile: str, count: int, 244 complete: bool = True): 245 for i in range(count): 246 dfile = client.download_file(i) 247 assert os.path.exists(dfile) 248 if complete and not filecmp.cmp(srcfile, dfile, shallow=False): 249 diff = "".join(difflib.unified_diff(a=open(srcfile).readlines(), 250 b=open(dfile).readlines(), 251 fromfile=srcfile, 252 tofile=dfile, 253 n=1)) 254 assert False, f'download {dfile} differs:\n{diff}'