test_02_download.py (35302B)
1 #!/usr/bin/env python3 2 # -*- coding: utf-8 -*- 3 #*************************************************************************** 4 # _ _ ____ _ 5 # Project ___| | | | _ \| | 6 # / __| | | | |_) | | 7 # | (__| |_| | _ <| |___ 8 # \___|\___/|_| \_\_____| 9 # 10 # Copyright (C) Daniel Stenberg, <daniel@haxx.se>, et al. 11 # 12 # This software is licensed as described in the file COPYING, which 13 # you should have received as part of this distribution. The terms 14 # are also available at https://curl.se/docs/copyright.html. 15 # 16 # You may opt to use, copy, modify, merge, publish, distribute and/or sell 17 # copies of the Software, and permit persons to whom the Software is 18 # furnished to do so, under the terms of the COPYING file. 19 # 20 # This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY 21 # KIND, either express or implied. 22 # 23 # SPDX-License-Identifier: curl 24 # 25 ########################################################################### 26 # 27 import difflib 28 import filecmp 29 import logging 30 import math 31 import os 32 import re 33 import sys 34 from datetime import timedelta 35 import pytest 36 37 from testenv import Env, CurlClient, LocalClient 38 39 40 log = logging.getLogger(__name__) 41 42 43 class TestDownload: 44 45 @pytest.fixture(autouse=True, scope='class') 46 def _class_scope(self, env, httpd): 47 indir = httpd.docs_dir 48 env.make_data_file(indir=indir, fname="data-10k", fsize=10*1024) 49 env.make_data_file(indir=indir, fname="data-100k", fsize=100*1024) 50 env.make_data_file(indir=indir, fname="data-1m", fsize=1024*1024) 51 env.make_data_file(indir=indir, fname="data-10m", fsize=10*1024*1024) 52 env.make_data_file(indir=indir, fname="data-50m", fsize=50*1024*1024) 53 env.make_data_gzipbomb(indir=indir, fname="bomb-100m.txt", fsize=100*1024*1024) 54 55 # download 1 file 56 @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3']) 57 def test_02_01_download_1(self, env: Env, httpd, nghttpx, proto): 58 if proto == 'h3' and not env.have_h3(): 59 pytest.skip("h3 not supported") 60 curl = CurlClient(env=env) 61 url = f'https://{env.authority_for(env.domain1, proto)}/data.json' 62 r = curl.http_download(urls=[url], alpn_proto=proto) 63 r.check_response(http_status=200) 64 65 # download 2 files 66 @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3']) 67 def test_02_02_download_2(self, env: Env, httpd, nghttpx, proto): 68 if proto == 'h3' and not env.have_h3(): 69 pytest.skip("h3 not supported") 70 curl = CurlClient(env=env) 71 url = f'https://{env.authority_for(env.domain1, proto)}/data.json?[0-1]' 72 r = curl.http_download(urls=[url], alpn_proto=proto) 73 r.check_response(http_status=200, count=2) 74 75 # download 100 files sequentially 76 @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3']) 77 def test_02_03_download_sequential(self, env: Env, httpd, nghttpx, proto): 78 if proto == 'h3' and not env.have_h3(): 79 pytest.skip("h3 not supported") 80 if (proto == 'http/1.1' or proto == 'h2') and env.curl_uses_lib('mbedtls') and \ 81 sys.platform.startswith('darwin') and env.ci_run: 82 pytest.skip('mbedtls 3.6.3 fails this test on macOS CI runners') 83 count = 10 84 curl = CurlClient(env=env) 85 urln = f'https://{env.authority_for(env.domain1, proto)}/data.json?[0-{count-1}]' 86 r = curl.http_download(urls=[urln], alpn_proto=proto) 87 r.check_response(http_status=200, count=count, connect_count=1) 88 89 # download 100 files parallel 90 @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3']) 91 def test_02_04_download_parallel(self, env: Env, httpd, nghttpx, proto): 92 if proto == 'h3' and not env.have_h3(): 93 pytest.skip("h3 not supported") 94 if proto == 'h2' and env.curl_uses_lib('mbedtls') and \ 95 sys.platform.startswith('darwin') and env.ci_run: 96 pytest.skip('mbedtls 3.6.3 fails this test on macOS CI runners') 97 count = 10 98 max_parallel = 5 99 curl = CurlClient(env=env) 100 urln = f'https://{env.authority_for(env.domain1, proto)}/data.json?[0-{count-1}]' 101 r = curl.http_download(urls=[urln], alpn_proto=proto, extra_args=[ 102 '--parallel', '--parallel-max', f'{max_parallel}' 103 ]) 104 r.check_response(http_status=200, count=count) 105 if proto == 'http/1.1': 106 # http/1.1 parallel transfers will open multiple connections 107 assert r.total_connects > 1, r.dump_logs() 108 else: 109 # http2 parallel transfers will use one connection (common limit is 100) 110 assert r.total_connects == 1, r.dump_logs() 111 112 # download 500 files sequential 113 @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3']) 114 def test_02_05_download_many_sequential(self, env: Env, httpd, nghttpx, proto): 115 if proto == 'h3' and not env.have_h3(): 116 pytest.skip("h3 not supported") 117 if proto == 'h3' and env.curl_uses_lib('msh3'): 118 pytest.skip("msh3 shaky here") 119 if proto == 'h2' and env.curl_uses_lib('mbedtls') and \ 120 sys.platform.startswith('darwin') and env.ci_run: 121 pytest.skip('mbedtls 3.6.3 fails this test on macOS CI runners') 122 count = 200 123 curl = CurlClient(env=env) 124 urln = f'https://{env.authority_for(env.domain1, proto)}/data.json?[0-{count-1}]' 125 r = curl.http_download(urls=[urln], alpn_proto=proto) 126 r.check_response(http_status=200, count=count) 127 if proto == 'http/1.1': 128 # http/1.1 parallel transfers will open multiple connections 129 assert r.total_connects > 1, r.dump_logs() 130 else: 131 # http2 parallel transfers will use one connection (common limit is 100) 132 assert r.total_connects == 1, r.dump_logs() 133 134 # download 500 files parallel 135 @pytest.mark.parametrize("proto", ['h2', 'h3']) 136 def test_02_06_download_many_parallel(self, env: Env, httpd, nghttpx, proto): 137 if proto == 'h3' and not env.have_h3(): 138 pytest.skip("h3 not supported") 139 if proto == 'h2' and env.curl_uses_lib('mbedtls') and \ 140 sys.platform.startswith('darwin') and env.ci_run: 141 pytest.skip('mbedtls 3.6.3 fails this test on macOS CI runners') 142 count = 200 143 max_parallel = 50 144 curl = CurlClient(env=env) 145 urln = f'https://{env.authority_for(env.domain1, proto)}/data.json?[000-{count-1}]' 146 r = curl.http_download(urls=[urln], alpn_proto=proto, extra_args=[ 147 '--parallel', '--parallel-max', f'{max_parallel}' 148 ]) 149 r.check_response(http_status=200, count=count, connect_count=1) 150 151 # download files parallel, check connection reuse/multiplex 152 @pytest.mark.parametrize("proto", ['h2', 'h3']) 153 def test_02_07_download_reuse(self, env: Env, httpd, nghttpx, proto): 154 if proto == 'h3' and not env.have_h3(): 155 pytest.skip("h3 not supported") 156 count = 200 157 curl = CurlClient(env=env) 158 urln = f'https://{env.authority_for(env.domain1, proto)}/data.json?[0-{count-1}]' 159 r = curl.http_download(urls=[urln], alpn_proto=proto, 160 with_stats=True, extra_args=[ 161 '--parallel', '--parallel-max', '200' 162 ]) 163 r.check_response(http_status=200, count=count) 164 # should have used at most 2 connections only (test servers allow 100 req/conn) 165 # it may be just 1 on slow systems where request are answered faster than 166 # curl can exhaust the capacity or if curl runs with address-sanitizer speed 167 assert r.total_connects <= 2, "h2 should use fewer connections here" 168 169 # download files parallel with http/1.1, check connection not reused 170 @pytest.mark.parametrize("proto", ['http/1.1']) 171 def test_02_07b_download_reuse(self, env: Env, httpd, nghttpx, proto): 172 count = 6 173 curl = CurlClient(env=env) 174 urln = f'https://{env.authority_for(env.domain1, proto)}/data.json?[0-{count-1}]' 175 r = curl.http_download(urls=[urln], alpn_proto=proto, 176 with_stats=True, extra_args=[ 177 '--parallel' 178 ]) 179 r.check_response(count=count, http_status=200) 180 # http/1.1 should have used count connections 181 assert r.total_connects == count, "http/1.1 should use this many connections" 182 183 @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3']) 184 def test_02_08_1MB_serial(self, env: Env, httpd, nghttpx, proto): 185 if proto == 'h3' and not env.have_h3(): 186 pytest.skip("h3 not supported") 187 count = 5 188 urln = f'https://{env.authority_for(env.domain1, proto)}/data-1m?[0-{count-1}]' 189 curl = CurlClient(env=env) 190 r = curl.http_download(urls=[urln], alpn_proto=proto) 191 r.check_response(count=count, http_status=200) 192 193 @pytest.mark.parametrize("proto", ['h2', 'h3']) 194 def test_02_09_1MB_parallel(self, env: Env, httpd, nghttpx, proto): 195 if proto == 'h3' and not env.have_h3(): 196 pytest.skip("h3 not supported") 197 count = 5 198 urln = f'https://{env.authority_for(env.domain1, proto)}/data-1m?[0-{count-1}]' 199 curl = CurlClient(env=env) 200 r = curl.http_download(urls=[urln], alpn_proto=proto, extra_args=[ 201 '--parallel' 202 ]) 203 r.check_response(count=count, http_status=200) 204 205 @pytest.mark.skipif(condition=Env().slow_network, reason="not suitable for slow network tests") 206 @pytest.mark.skipif(condition=Env().ci_run, reason="not suitable for CI runs") 207 @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3']) 208 def test_02_10_10MB_serial(self, env: Env, httpd, nghttpx, proto): 209 if proto == 'h3' and not env.have_h3(): 210 pytest.skip("h3 not supported") 211 count = 3 212 urln = f'https://{env.authority_for(env.domain1, proto)}/data-10m?[0-{count-1}]' 213 curl = CurlClient(env=env) 214 r = curl.http_download(urls=[urln], alpn_proto=proto) 215 r.check_response(count=count, http_status=200) 216 217 @pytest.mark.skipif(condition=Env().slow_network, reason="not suitable for slow network tests") 218 @pytest.mark.skipif(condition=Env().ci_run, reason="not suitable for CI runs") 219 @pytest.mark.parametrize("proto", ['h2', 'h3']) 220 def test_02_11_10MB_parallel(self, env: Env, httpd, nghttpx, proto): 221 if proto == 'h3' and not env.have_h3(): 222 pytest.skip("h3 not supported") 223 if proto == 'h3' and env.curl_uses_lib('msh3'): 224 pytest.skip("msh3 stalls here") 225 count = 3 226 urln = f'https://{env.authority_for(env.domain1, proto)}/data-10m?[0-{count-1}]' 227 curl = CurlClient(env=env) 228 r = curl.http_download(urls=[urln], alpn_proto=proto, extra_args=[ 229 '--parallel' 230 ]) 231 r.check_response(count=count, http_status=200) 232 233 @pytest.mark.parametrize("proto", ['h2', 'h3']) 234 def test_02_12_head_serial_https(self, env: Env, httpd, nghttpx, proto): 235 if proto == 'h3' and not env.have_h3(): 236 pytest.skip("h3 not supported") 237 count = 5 238 urln = f'https://{env.authority_for(env.domain1, proto)}/data-10m?[0-{count-1}]' 239 curl = CurlClient(env=env) 240 r = curl.http_download(urls=[urln], alpn_proto=proto, extra_args=[ 241 '--head' 242 ]) 243 r.check_response(count=count, http_status=200) 244 245 @pytest.mark.parametrize("proto", ['h2']) 246 def test_02_13_head_serial_h2c(self, env: Env, httpd, nghttpx, proto): 247 if proto == 'h3' and not env.have_h3(): 248 pytest.skip("h3 not supported") 249 count = 5 250 urln = f'http://{env.domain1}:{env.http_port}/data-10m?[0-{count-1}]' 251 curl = CurlClient(env=env) 252 r = curl.http_download(urls=[urln], alpn_proto=proto, extra_args=[ 253 '--head', '--http2-prior-knowledge', '--fail-early' 254 ]) 255 r.check_response(count=count, http_status=200) 256 257 @pytest.mark.parametrize("proto", ['h2', 'h3']) 258 def test_02_14_not_found(self, env: Env, httpd, nghttpx, proto): 259 if proto == 'h3' and not env.have_h3(): 260 pytest.skip("h3 not supported") 261 if proto == 'h3' and env.curl_uses_lib('msh3'): 262 pytest.skip("msh3 stalls here") 263 count = 5 264 urln = f'https://{env.authority_for(env.domain1, proto)}/not-found?[0-{count-1}]' 265 curl = CurlClient(env=env) 266 r = curl.http_download(urls=[urln], alpn_proto=proto, extra_args=[ 267 '--parallel' 268 ]) 269 r.check_stats(count=count, http_status=404, exitcode=0, 270 remote_port=env.port_for(alpn_proto=proto), 271 remote_ip='127.0.0.1') 272 273 @pytest.mark.parametrize("proto", ['h2', 'h3']) 274 def test_02_15_fail_not_found(self, env: Env, httpd, nghttpx, proto): 275 if proto == 'h3' and not env.have_h3(): 276 pytest.skip("h3 not supported") 277 if proto == 'h3' and env.curl_uses_lib('msh3'): 278 pytest.skip("msh3 stalls here") 279 count = 5 280 urln = f'https://{env.authority_for(env.domain1, proto)}/not-found?[0-{count-1}]' 281 curl = CurlClient(env=env) 282 r = curl.http_download(urls=[urln], alpn_proto=proto, extra_args=[ 283 '--fail' 284 ]) 285 r.check_stats(count=count, http_status=404, exitcode=22, 286 remote_port=env.port_for(alpn_proto=proto), 287 remote_ip='127.0.0.1') 288 289 @pytest.mark.skipif(condition=Env().slow_network, reason="not suitable for slow network tests") 290 def test_02_20_h2_small_frames(self, env: Env, httpd, configures_httpd): 291 # Test case to reproduce content corruption as observed in 292 # https://github.com/curl/curl/issues/10525 293 # To reliably reproduce, we need an Apache httpd that supports 294 # setting smaller frame sizes. This is not released yet, we 295 # test if it works and back out if not. 296 httpd.set_extra_config(env.domain1, lines=[ 297 'H2MaxDataFrameLen 1024', 298 ]) 299 if not httpd.reload_if_config_changed(): 300 pytest.skip('H2MaxDataFrameLen not supported') 301 # ok, make 100 downloads with 2 parallel running and they 302 # are expected to stumble into the issue when using `lib/http2.c` 303 # from curl 7.88.0 304 count = 5 305 urln = f'https://{env.authority_for(env.domain1, "h2")}/data-1m?[0-{count-1}]' 306 curl = CurlClient(env=env) 307 r = curl.http_download(urls=[urln], alpn_proto="h2", extra_args=[ 308 '--parallel', '--parallel-max', '2' 309 ]) 310 r.check_response(count=count, http_status=200) 311 srcfile = os.path.join(httpd.docs_dir, 'data-1m') 312 self.check_downloads(curl, srcfile, count) 313 314 # download serial via lib client, pause/resume at different offsets 315 @pytest.mark.parametrize("pause_offset", [0, 10*1024, 100*1023, 640000]) 316 @pytest.mark.parametrize("proto", ['http/1.1', 'h3']) 317 def test_02_21_lib_serial(self, env: Env, httpd, nghttpx, proto, pause_offset): 318 if proto == 'h3' and not env.have_h3(): 319 pytest.skip("h3 not supported") 320 count = 2 321 docname = 'data-10m' 322 url = f'https://localhost:{env.https_port}/{docname}' 323 client = LocalClient(name='hx_download', env=env) 324 if not client.exists(): 325 pytest.skip(f'example client not built: {client.name}') 326 r = client.run(args=[ 327 '-n', f'{count}', '-P', f'{pause_offset}', '-V', proto, url 328 ]) 329 r.check_exit_code(0) 330 srcfile = os.path.join(httpd.docs_dir, docname) 331 self.check_downloads(client, srcfile, count) 332 333 # h2 download parallel via lib client, pause/resume at different offsets 334 # debug-override stream window size to reproduce #16955 335 @pytest.mark.parametrize("pause_offset", [0, 10*1024, 100*1023, 640000]) 336 @pytest.mark.parametrize("swin_max", [0, 10*1024]) 337 def test_02_21_h2_lib_serial(self, env: Env, httpd, pause_offset, swin_max): 338 proto = 'h2' 339 count = 2 340 docname = 'data-10m' 341 url = f'https://localhost:{env.https_port}/{docname}' 342 run_env = os.environ.copy() 343 run_env['CURL_DEBUG'] = 'multi,http/2' 344 if swin_max > 0: 345 run_env['CURL_H2_STREAM_WIN_MAX'] = f'{swin_max}' 346 client = LocalClient(name='hx_download', env=env, run_env=run_env) 347 if not client.exists(): 348 pytest.skip(f'example client not built: {client.name}') 349 r = client.run(args=[ 350 '-n', f'{count}', '-P', f'{pause_offset}', '-V', proto, url 351 ]) 352 r.check_exit_code(0) 353 srcfile = os.path.join(httpd.docs_dir, docname) 354 self.check_downloads(client, srcfile, count) 355 356 # download via lib client, several at a time, pause/resume 357 @pytest.mark.parametrize("pause_offset", [100*1023]) 358 @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3']) 359 def test_02_22_lib_parallel_resume(self, env: Env, httpd, nghttpx, proto, pause_offset): 360 if proto == 'h3' and not env.have_h3(): 361 pytest.skip("h3 not supported") 362 count = 2 363 max_parallel = 5 364 docname = 'data-10m' 365 url = f'https://localhost:{env.https_port}/{docname}' 366 client = LocalClient(name='hx_download', env=env) 367 if not client.exists(): 368 pytest.skip(f'example client not built: {client.name}') 369 r = client.run(args=[ 370 '-n', f'{count}', '-m', f'{max_parallel}', 371 '-P', f'{pause_offset}', '-V', proto, url 372 ]) 373 r.check_exit_code(0) 374 srcfile = os.path.join(httpd.docs_dir, docname) 375 self.check_downloads(client, srcfile, count) 376 377 # download, several at a time, pause and abort paused 378 @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3']) 379 def test_02_23a_lib_abort_paused(self, env: Env, httpd, nghttpx, proto): 380 if proto == 'h3' and not env.have_h3(): 381 pytest.skip("h3 not supported") 382 if proto == 'h3' and env.curl_uses_ossl_quic(): 383 pytest.skip('OpenSSL QUIC fails here') 384 if proto == 'h3' and env.ci_run and env.curl_uses_lib('quiche'): 385 pytest.skip("fails in CI, but works locally for unknown reasons") 386 count = 10 387 max_parallel = 5 388 if proto in ['h2', 'h3']: 389 pause_offset = 64 * 1024 390 else: 391 pause_offset = 12 * 1024 392 docname = 'data-1m' 393 url = f'https://localhost:{env.https_port}/{docname}' 394 client = LocalClient(name='hx_download', env=env) 395 if not client.exists(): 396 pytest.skip(f'example client not built: {client.name}') 397 r = client.run(args=[ 398 '-n', f'{count}', '-m', f'{max_parallel}', '-a', 399 '-P', f'{pause_offset}', '-V', proto, url 400 ]) 401 r.check_exit_code(0) 402 srcfile = os.path.join(httpd.docs_dir, docname) 403 # downloads should be there, but not necessarily complete 404 self.check_downloads(client, srcfile, count, complete=False) 405 406 # download, several at a time, abort after n bytes 407 @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3']) 408 def test_02_23b_lib_abort_offset(self, env: Env, httpd, nghttpx, proto): 409 if proto == 'h3' and not env.have_h3(): 410 pytest.skip("h3 not supported") 411 if proto == 'h3' and env.curl_uses_ossl_quic(): 412 pytest.skip('OpenSSL QUIC fails here') 413 if proto == 'h3' and env.ci_run and env.curl_uses_lib('quiche'): 414 pytest.skip("fails in CI, but works locally for unknown reasons") 415 count = 10 416 max_parallel = 5 417 if proto in ['h2', 'h3']: 418 abort_offset = 64 * 1024 419 else: 420 abort_offset = 12 * 1024 421 docname = 'data-1m' 422 url = f'https://localhost:{env.https_port}/{docname}' 423 client = LocalClient(name='hx_download', env=env) 424 if not client.exists(): 425 pytest.skip(f'example client not built: {client.name}') 426 r = client.run(args=[ 427 '-n', f'{count}', '-m', f'{max_parallel}', '-a', 428 '-A', f'{abort_offset}', '-V', proto, url 429 ]) 430 r.check_exit_code(42) # CURLE_ABORTED_BY_CALLBACK 431 srcfile = os.path.join(httpd.docs_dir, docname) 432 # downloads should be there, but not necessarily complete 433 self.check_downloads(client, srcfile, count, complete=False) 434 435 # download, several at a time, abort after n bytes 436 @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3']) 437 def test_02_23c_lib_fail_offset(self, env: Env, httpd, nghttpx, proto): 438 if proto == 'h3' and not env.have_h3(): 439 pytest.skip("h3 not supported") 440 if proto == 'h3' and env.curl_uses_ossl_quic(): 441 pytest.skip('OpenSSL QUIC fails here') 442 if proto == 'h3' and env.ci_run and env.curl_uses_lib('quiche'): 443 pytest.skip("fails in CI, but works locally for unknown reasons") 444 count = 10 445 max_parallel = 5 446 if proto in ['h2', 'h3']: 447 fail_offset = 64 * 1024 448 else: 449 fail_offset = 12 * 1024 450 docname = 'data-1m' 451 url = f'https://localhost:{env.https_port}/{docname}' 452 client = LocalClient(name='hx_download', env=env) 453 if not client.exists(): 454 pytest.skip(f'example client not built: {client.name}') 455 r = client.run(args=[ 456 '-n', f'{count}', '-m', f'{max_parallel}', '-a', 457 '-F', f'{fail_offset}', '-V', proto, url 458 ]) 459 r.check_exit_code(23) # CURLE_WRITE_ERROR 460 srcfile = os.path.join(httpd.docs_dir, docname) 461 # downloads should be there, but not necessarily complete 462 self.check_downloads(client, srcfile, count, complete=False) 463 464 # speed limited download 465 @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3']) 466 def test_02_24_speed_limit(self, env: Env, httpd, nghttpx, proto): 467 if proto == 'h3' and not env.have_h3(): 468 pytest.skip("h3 not supported") 469 count = 1 470 url = f'https://{env.authority_for(env.domain1, proto)}/data-1m' 471 curl = CurlClient(env=env) 472 speed_limit = 384 * 1024 473 min_duration = math.floor((1024 * 1024)/speed_limit) 474 r = curl.http_download(urls=[url], alpn_proto=proto, extra_args=[ 475 '--limit-rate', f'{speed_limit}' 476 ]) 477 r.check_response(count=count, http_status=200) 478 assert r.duration > timedelta(seconds=min_duration), \ 479 f'rate limited transfer should take more than {min_duration}s, '\ 480 f'not {r.duration}' 481 482 # make extreme parallel h2 upgrades, check invalid conn reuse 483 # before protocol switch has happened 484 def test_02_25_h2_upgrade_x(self, env: Env, httpd): 485 url = f'http://localhost:{env.http_port}/data-100k' 486 client = LocalClient(name='h2_upgrade_extreme', env=env, timeout=15) 487 if not client.exists(): 488 pytest.skip(f'example client not built: {client.name}') 489 r = client.run(args=[url]) 490 assert r.exit_code == 0, f'{client.dump_logs()}' 491 492 # Special client that tests TLS session reuse in parallel transfers 493 # TODO: just uses a single connection for h2/h3. Not sure how to prevent that 494 @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3']) 495 def test_02_26_session_shared_reuse(self, env: Env, proto, httpd, nghttpx): 496 url = f'https://{env.authority_for(env.domain1, proto)}/data-100k' 497 client = LocalClient(name='tls_session_reuse', env=env) 498 if not client.exists(): 499 pytest.skip(f'example client not built: {client.name}') 500 r = client.run(args=[proto, url]) 501 r.check_exit_code(0) 502 503 # test on paused transfers, based on issue #11982 504 @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3']) 505 def test_02_27a_paused_no_cl(self, env: Env, httpd, nghttpx, proto): 506 url = f'https://{env.authority_for(env.domain1, proto)}' \ 507 '/curltest/tweak/?&chunks=6&chunk_size=8000' 508 client = LocalClient(env=env, name='h2_pausing') 509 r = client.run(args=['-V', proto, url]) 510 r.check_exit_code(0) 511 512 # test on paused transfers, based on issue #11982 513 @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3']) 514 def test_02_27b_paused_no_cl(self, env: Env, httpd, nghttpx, proto): 515 url = f'https://{env.authority_for(env.domain1, proto)}' \ 516 '/curltest/tweak/?error=502' 517 client = LocalClient(env=env, name='h2_pausing') 518 r = client.run(args=['-V', proto, url]) 519 r.check_exit_code(0) 520 521 # test on paused transfers, based on issue #11982 522 @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3']) 523 def test_02_27c_paused_no_cl(self, env: Env, httpd, nghttpx, proto): 524 url = f'https://{env.authority_for(env.domain1, proto)}' \ 525 '/curltest/tweak/?status=200&chunks=1&chunk_size=100' 526 client = LocalClient(env=env, name='h2_pausing') 527 r = client.run(args=['-V', proto, url]) 528 r.check_exit_code(0) 529 530 @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3']) 531 def test_02_28_get_compressed(self, env: Env, httpd, nghttpx, proto): 532 if proto == 'h3' and not env.have_h3(): 533 pytest.skip("h3 not supported") 534 count = 1 535 urln = f'https://{env.authority_for(env.domain1brotli, proto)}/data-100k?[0-{count-1}]' 536 curl = CurlClient(env=env) 537 r = curl.http_download(urls=[urln], alpn_proto=proto, extra_args=[ 538 '--compressed' 539 ]) 540 r.check_exit_code(code=0) 541 r.check_response(count=count, http_status=200) 542 543 def check_downloads(self, client, srcfile: str, count: int, 544 complete: bool = True): 545 for i in range(count): 546 dfile = client.download_file(i) 547 assert os.path.exists(dfile) 548 if complete and not filecmp.cmp(srcfile, dfile, shallow=False): 549 diff = "".join(difflib.unified_diff(a=open(srcfile).readlines(), 550 b=open(dfile).readlines(), 551 fromfile=srcfile, 552 tofile=dfile, 553 n=1)) 554 assert False, f'download {dfile} differs:\n{diff}' 555 556 # download via lib client, 1 at a time, pause/resume at different offsets 557 @pytest.mark.parametrize("pause_offset", [0, 10*1024, 100*1023, 640000]) 558 @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3']) 559 def test_02_29_h2_lib_serial(self, env: Env, httpd, nghttpx, proto, pause_offset): 560 count = 2 561 docname = 'data-10m' 562 url = f'https://localhost:{env.https_port}/{docname}' 563 client = LocalClient(name='hx_download', env=env) 564 if not client.exists(): 565 pytest.skip(f'example client not built: {client.name}') 566 r = client.run(args=[ 567 '-n', f'{count}', '-P', f'{pause_offset}', '-V', proto, url 568 ]) 569 r.check_exit_code(0) 570 srcfile = os.path.join(httpd.docs_dir, docname) 571 self.check_downloads(client, srcfile, count) 572 573 # download parallel with prior knowledge 574 def test_02_30_parallel_prior_knowledge(self, env: Env, httpd): 575 count = 3 576 curl = CurlClient(env=env) 577 urln = f'http://{env.domain1}:{env.http_port}/data.json?[0-{count-1}]' 578 r = curl.http_download(urls=[urln], extra_args=[ 579 '--parallel', '--http2-prior-knowledge' 580 ]) 581 r.check_response(http_status=200, count=count) 582 assert r.total_connects == 1, r.dump_logs() 583 584 # download parallel with h2 "Upgrade:" 585 def test_02_31_parallel_upgrade(self, env: Env, httpd, nghttpx): 586 count = 3 587 curl = CurlClient(env=env) 588 urln = f'http://{env.domain1}:{env.http_port}/data.json?[0-{count-1}]' 589 r = curl.http_download(urls=[urln], extra_args=[ 590 '--parallel', '--http2' 591 ]) 592 r.check_response(http_status=200, count=count) 593 # we see up to 3 connections, because Apache wants to serve only a single 594 # request via Upgrade: and then closes the connection. But if a new 595 # request comes in time, it might still get served. 596 assert r.total_connects <= 3, r.dump_logs() 597 598 # nghttpx is the only server we have that supports TLS early data 599 @pytest.mark.skipif(condition=not Env.have_nghttpx(), reason="no nghttpx") 600 @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3']) 601 def test_02_32_earlydata(self, env: Env, httpd, nghttpx, proto): 602 if not env.curl_can_early_data(): 603 pytest.skip('TLS earlydata not implemented') 604 if proto == 'h3' and \ 605 (not env.have_h3() or not env.curl_can_h3_early_data()): 606 pytest.skip("h3 not supported") 607 if proto != 'h3' and sys.platform.startswith('darwin') and env.ci_run: 608 pytest.skip('failing on macOS CI runners') 609 count = 2 610 docname = 'data-10k' 611 # we want this test to always connect to nghttpx, since it is 612 # the only server we have that supports TLS earlydata 613 port = env.port_for(proto) 614 if proto != 'h3': 615 port = env.nghttpx_https_port 616 url = f'https://{env.domain1}:{port}/{docname}' 617 client = LocalClient(name='hx_download', env=env) 618 if not client.exists(): 619 pytest.skip(f'example client not built: {client.name}') 620 r = client.run(args=[ 621 '-n', f'{count}', 622 '-e', # use TLS earlydata 623 '-f', # forbid reuse of connections 624 '-r', f'{env.domain1}:{port}:127.0.0.1', 625 '-V', proto, url 626 ]) 627 r.check_exit_code(0) 628 srcfile = os.path.join(httpd.docs_dir, docname) 629 self.check_downloads(client, srcfile, count) 630 # check that TLS earlydata worked as expected 631 earlydata = {} 632 reused_session = False 633 for line in r.trace_lines: 634 m = re.match(r'^\[t-(\d+)] EarlyData: (-?\d+)', line) 635 if m: 636 earlydata[int(m.group(1))] = int(m.group(2)) 637 continue 638 if re.match(r'\[1-1] \* SSL reusing session.*', line): 639 reused_session = True 640 assert reused_session, 'session was not reused for 2nd transfer' 641 assert earlydata[0] == 0, f'{earlydata}' 642 if proto == 'http/1.1': 643 assert earlydata[1] == 111, f'{earlydata}' 644 elif proto == 'h2': 645 assert earlydata[1] == 127, f'{earlydata}' 646 elif proto == 'h3': 647 assert earlydata[1] == 109, f'{earlydata}' 648 649 @pytest.mark.parametrize("proto", ['http/1.1', 'h2']) 650 @pytest.mark.parametrize("max_host_conns", [0, 1, 5]) 651 def test_02_33_max_host_conns(self, env: Env, httpd, nghttpx, proto, max_host_conns): 652 if not env.curl_is_debug(): 653 pytest.skip('only works for curl debug builds') 654 if proto == 'h3' and not env.have_h3(): 655 pytest.skip("h3 not supported") 656 count = 50 657 max_parallel = 50 658 docname = 'data-10k' 659 port = env.port_for(proto) 660 url = f'https://{env.domain1}:{port}/{docname}' 661 run_env = os.environ.copy() 662 run_env['CURL_DEBUG'] = 'multi' 663 client = LocalClient(name='hx_download', env=env, run_env=run_env) 664 if not client.exists(): 665 pytest.skip(f'example client not built: {client.name}') 666 r = client.run(args=[ 667 '-n', f'{count}', 668 '-m', f'{max_parallel}', 669 '-x', # always use a fresh connection 670 '-M', str(max_host_conns), # limit conns per host 671 '-r', f'{env.domain1}:{port}:127.0.0.1', 672 '-V', proto, url 673 ]) 674 r.check_exit_code(0) 675 srcfile = os.path.join(httpd.docs_dir, docname) 676 self.check_downloads(client, srcfile, count) 677 if max_host_conns > 0: 678 matched_lines = 0 679 for line in r.trace_lines: 680 m = re.match(r'.*The cache now contains (\d+) members.*', line) 681 if m: 682 matched_lines += 1 683 n = int(m.group(1)) 684 assert n <= max_host_conns 685 assert matched_lines > 0 686 687 @pytest.mark.parametrize("proto", ['http/1.1', 'h2']) 688 @pytest.mark.parametrize("max_total_conns", [0, 1, 5]) 689 def test_02_34_max_total_conns(self, env: Env, httpd, nghttpx, proto, max_total_conns): 690 if not env.curl_is_debug(): 691 pytest.skip('only works for curl debug builds') 692 if proto == 'h3' and not env.have_h3(): 693 pytest.skip("h3 not supported") 694 count = 50 695 max_parallel = 50 696 docname = 'data-10k' 697 port = env.port_for(proto) 698 url = f'https://{env.domain1}:{port}/{docname}' 699 run_env = os.environ.copy() 700 run_env['CURL_DEBUG'] = 'multi' 701 client = LocalClient(name='hx_download', env=env, run_env=run_env) 702 if not client.exists(): 703 pytest.skip(f'example client not built: {client.name}') 704 r = client.run(args=[ 705 '-n', f'{count}', 706 '-m', f'{max_parallel}', 707 '-x', # always use a fresh connection 708 '-T', str(max_total_conns), # limit total connections 709 '-r', f'{env.domain1}:{port}:127.0.0.1', 710 '-V', proto, url 711 ]) 712 r.check_exit_code(0) 713 srcfile = os.path.join(httpd.docs_dir, docname) 714 self.check_downloads(client, srcfile, count) 715 if max_total_conns > 0: 716 matched_lines = 0 717 for line in r.trace_lines: 718 m = re.match(r'.*The cache now contains (\d+) members.*', line) 719 if m: 720 matched_lines += 1 721 n = int(m.group(1)) 722 assert n <= max_total_conns 723 assert matched_lines > 0 724 725 # 2 parallel transers, pause and resume. Load a 100 MB zip bomb from 726 # the server with "Content-Encoding: gzip" that gets exloded during 727 # response writing to the client. Client pauses after 1MB unzipped data 728 # and causes buffers to fill while the server sends more response 729 # data. 730 # * http/1.1: not much buffering is done as curl does no longer 731 # serve the connections that are paused 732 # * h2/h3: server continues sending what the stream window allows and 733 # since the one connection involved unpaused transfers, data continues 734 # to be received, requiring buffering. 735 @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3']) 736 def test_02_35_pause_bomb(self, env: Env, httpd, nghttpx, proto): 737 if proto == 'h3' and not env.have_h3(): 738 pytest.skip("h3 not supported") 739 count = 2 740 pause_offset = 1024 * 1024 741 docname = 'bomb-100m.txt.var' 742 url = f'https://localhost:{env.https_port}/{docname}' 743 client = LocalClient(name='hx_download', env=env) 744 if not client.exists(): 745 pytest.skip(f'example client not built: {client.name}') 746 r = client.run(args=[ 747 '-n', f'{count}', '-m', f'{count}', 748 '-P', f'{pause_offset}', '-V', proto, url 749 ]) 750 r.check_exit_code(0)