test_07_upload.py (38491B)
1 #!/usr/bin/env python3 2 # -*- coding: utf-8 -*- 3 #*************************************************************************** 4 # _ _ ____ _ 5 # Project ___| | | | _ \| | 6 # / __| | | | |_) | | 7 # | (__| |_| | _ <| |___ 8 # \___|\___/|_| \_\_____| 9 # 10 # Copyright (C) Daniel Stenberg, <daniel@haxx.se>, et al. 11 # 12 # This software is licensed as described in the file COPYING, which 13 # you should have received as part of this distribution. The terms 14 # are also available at https://curl.se/docs/copyright.html. 15 # 16 # You may opt to use, copy, modify, merge, publish, distribute and/or sell 17 # copies of the Software, and permit persons to whom the Software is 18 # furnished to do so, under the terms of the COPYING file. 19 # 20 # This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY 21 # KIND, either express or implied. 22 # 23 # SPDX-License-Identifier: curl 24 # 25 ########################################################################### 26 # 27 import difflib 28 import filecmp 29 import logging 30 import os 31 import re 32 import sys 33 import pytest 34 from typing import List, Union 35 36 from testenv import Env, CurlClient, LocalClient, ExecResult 37 38 39 log = logging.getLogger(__name__) 40 41 42 class TestUpload: 43 44 @pytest.fixture(autouse=True, scope='class') 45 def _class_scope(self, env, httpd, nghttpx): 46 env.make_data_file(indir=env.gen_dir, fname="data-10k", fsize=10*1024) 47 env.make_data_file(indir=env.gen_dir, fname="data-63k", fsize=63*1024) 48 env.make_data_file(indir=env.gen_dir, fname="data-64k", fsize=64*1024) 49 env.make_data_file(indir=env.gen_dir, fname="data-100k", fsize=100*1024) 50 env.make_data_file(indir=env.gen_dir, fname="data-1m+", fsize=(1024*1024)+1) 51 env.make_data_file(indir=env.gen_dir, fname="data-10m", fsize=10*1024*1024) 52 53 # upload small data, check that this is what was echoed 54 @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3']) 55 def test_07_01_upload_1_small(self, env: Env, httpd, nghttpx, proto): 56 if proto == 'h3' and not env.have_h3(): 57 pytest.skip("h3 not supported") 58 if proto == 'h3' and env.curl_uses_lib('msh3'): 59 pytest.skip("msh3 fails here") 60 data = '0123456789' 61 curl = CurlClient(env=env) 62 url = f'https://{env.authority_for(env.domain1, proto)}/curltest/echo?id=[0-0]' 63 r = curl.http_upload(urls=[url], data=data, alpn_proto=proto) 64 r.check_stats(count=1, http_status=200, exitcode=0) 65 respdata = open(curl.response_file(0)).readlines() 66 assert respdata == [data] 67 68 # upload large data, check that this is what was echoed 69 @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3']) 70 def test_07_02_upload_1_large(self, env: Env, httpd, nghttpx, proto): 71 if proto == 'h3' and not env.have_h3(): 72 pytest.skip("h3 not supported") 73 if proto == 'h3' and env.curl_uses_lib('msh3'): 74 pytest.skip("msh3 fails here") 75 fdata = os.path.join(env.gen_dir, 'data-100k') 76 curl = CurlClient(env=env) 77 url = f'https://{env.authority_for(env.domain1, proto)}/curltest/echo?id=[0-0]' 78 r = curl.http_upload(urls=[url], data=f'@{fdata}', alpn_proto=proto) 79 r.check_stats(count=1, http_status=200, exitcode=0) 80 indata = open(fdata).readlines() 81 respdata = open(curl.response_file(0)).readlines() 82 assert respdata == indata 83 84 # upload data sequentially, check that they were echoed 85 @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3']) 86 def test_07_10_upload_sequential(self, env: Env, httpd, nghttpx, proto): 87 if proto == 'h3' and not env.have_h3(): 88 pytest.skip("h3 not supported") 89 if proto == 'h3' and env.curl_uses_lib('msh3'): 90 pytest.skip("msh3 stalls here") 91 count = 20 92 data = '0123456789' 93 curl = CurlClient(env=env) 94 url = f'https://{env.authority_for(env.domain1, proto)}/curltest/echo?id=[0-{count-1}]' 95 r = curl.http_upload(urls=[url], data=data, alpn_proto=proto) 96 r.check_stats(count=count, http_status=200, exitcode=0) 97 for i in range(count): 98 respdata = open(curl.response_file(i)).readlines() 99 assert respdata == [data] 100 101 # upload data parallel, check that they were echoed 102 @pytest.mark.parametrize("proto", ['h2', 'h3']) 103 def test_07_11_upload_parallel(self, env: Env, httpd, nghttpx, proto): 104 if proto == 'h3' and not env.have_h3(): 105 pytest.skip("h3 not supported") 106 if proto == 'h3' and env.curl_uses_lib('msh3'): 107 pytest.skip("msh3 stalls here") 108 # limit since we use a separate connection in h1 109 count = 20 110 data = '0123456789' 111 curl = CurlClient(env=env) 112 url = f'https://{env.authority_for(env.domain1, proto)}/curltest/echo?id=[0-{count-1}]' 113 r = curl.http_upload(urls=[url], data=data, alpn_proto=proto, 114 extra_args=['--parallel']) 115 r.check_stats(count=count, http_status=200, exitcode=0) 116 for i in range(count): 117 respdata = open(curl.response_file(i)).readlines() 118 assert respdata == [data] 119 120 # upload large data sequentially, check that this is what was echoed 121 @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3']) 122 def test_07_12_upload_seq_large(self, env: Env, httpd, nghttpx, proto): 123 if proto == 'h3' and not env.have_h3(): 124 pytest.skip("h3 not supported") 125 if proto == 'h3' and env.curl_uses_lib('msh3'): 126 pytest.skip("msh3 stalls here") 127 fdata = os.path.join(env.gen_dir, 'data-100k') 128 count = 10 129 curl = CurlClient(env=env) 130 url = f'https://{env.authority_for(env.domain1, proto)}/curltest/echo?id=[0-{count-1}]' 131 r = curl.http_upload(urls=[url], data=f'@{fdata}', alpn_proto=proto) 132 r.check_response(count=count, http_status=200) 133 indata = open(fdata).readlines() 134 r.check_stats(count=count, http_status=200, exitcode=0) 135 for i in range(count): 136 respdata = open(curl.response_file(i)).readlines() 137 assert respdata == indata 138 139 # upload very large data sequentially, check that this is what was echoed 140 @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3']) 141 def test_07_13_upload_seq_large(self, env: Env, httpd, nghttpx, proto): 142 if proto == 'h3' and not env.have_h3(): 143 pytest.skip("h3 not supported") 144 if proto == 'h3' and env.curl_uses_lib('msh3'): 145 pytest.skip("msh3 stalls here") 146 fdata = os.path.join(env.gen_dir, 'data-10m') 147 count = 2 148 curl = CurlClient(env=env) 149 url = f'https://{env.authority_for(env.domain1, proto)}/curltest/echo?id=[0-{count-1}]' 150 r = curl.http_upload(urls=[url], data=f'@{fdata}', alpn_proto=proto) 151 r.check_stats(count=count, http_status=200, exitcode=0) 152 indata = open(fdata).readlines() 153 for i in range(count): 154 respdata = open(curl.response_file(i)).readlines() 155 assert respdata == indata 156 157 # upload from stdin, issue #14870 158 @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3']) 159 @pytest.mark.parametrize("indata", [ 160 '', '1', '123\n456andsomething\n\n' 161 ]) 162 def test_07_14_upload_stdin(self, env: Env, httpd, nghttpx, proto, indata): 163 if proto == 'h3' and not env.have_h3(): 164 pytest.skip("h3 not supported") 165 if proto == 'h3' and env.curl_uses_lib('msh3'): 166 pytest.skip("msh3 stalls here") 167 count = 1 168 curl = CurlClient(env=env) 169 url = f'https://{env.authority_for(env.domain1, proto)}/curltest/put?id=[0-{count-1}]' 170 r = curl.http_put(urls=[url], data=indata, alpn_proto=proto) 171 r.check_stats(count=count, http_status=200, exitcode=0) 172 for i in range(count): 173 respdata = open(curl.response_file(i)).readlines() 174 assert respdata == [f'{len(indata)}'] 175 176 @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3']) 177 def test_07_15_hx_put(self, env: Env, httpd, nghttpx, proto): 178 if proto == 'h3' and not env.have_h3(): 179 pytest.skip("h3 not supported") 180 count = 2 181 upload_size = 128*1024 182 url = f'https://localhost:{env.https_port}/curltest/put' 183 client = LocalClient(name='hx_upload', env=env) 184 if not client.exists(): 185 pytest.skip(f'example client not built: {client.name}') 186 r = client.run(args=[ 187 '-n', f'{count}', '-S', f'{upload_size}', '-V', proto, url 188 ]) 189 r.check_exit_code(0) 190 self.check_downloads(client, r, [f"{upload_size}"], count) 191 192 @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3']) 193 def test_07_16_hx_put_reuse(self, env: Env, httpd, nghttpx, proto): 194 if proto == 'h3' and not env.have_h3(): 195 pytest.skip("h3 not supported") 196 count = 2 197 upload_size = 128*1024 198 url = f'https://localhost:{env.https_port}/curltest/put' 199 client = LocalClient(name='hx_upload', env=env) 200 if not client.exists(): 201 pytest.skip(f'example client not built: {client.name}') 202 r = client.run(args=[ 203 '-n', f'{count}', '-S', f'{upload_size}', '-R', '-V', proto, url 204 ]) 205 r.check_exit_code(0) 206 self.check_downloads(client, r, [f"{upload_size}"], count) 207 208 @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3']) 209 def test_07_17_hx_post_reuse(self, env: Env, httpd, nghttpx, proto): 210 if proto == 'h3' and not env.have_h3(): 211 pytest.skip("h3 not supported") 212 count = 2 213 upload_size = 128*1024 214 url = f'https://localhost:{env.https_port}/curltest/echo' 215 client = LocalClient(name='hx_upload', env=env) 216 if not client.exists(): 217 pytest.skip(f'example client not built: {client.name}') 218 r = client.run(args=[ 219 '-n', f'{count}', '-M', 'POST', '-S', f'{upload_size}', '-R', '-V', proto, url 220 ]) 221 r.check_exit_code(0) 222 self.check_downloads(client, r, ["x" * upload_size], count) 223 224 # upload data parallel, check that they were echoed 225 @pytest.mark.parametrize("proto", ['h2', 'h3']) 226 def test_07_20_upload_parallel(self, env: Env, httpd, nghttpx, proto): 227 if proto == 'h3' and not env.have_h3(): 228 pytest.skip("h3 not supported") 229 if proto == 'h3' and env.curl_uses_lib('msh3'): 230 pytest.skip("msh3 stalls here") 231 # limit since we use a separate connection in h1 232 count = 10 233 data = '0123456789' 234 curl = CurlClient(env=env) 235 url = f'https://{env.authority_for(env.domain1, proto)}/curltest/echo?id=[0-{count-1}]' 236 r = curl.http_upload(urls=[url], data=data, alpn_proto=proto, 237 extra_args=['--parallel']) 238 r.check_stats(count=count, http_status=200, exitcode=0) 239 for i in range(count): 240 respdata = open(curl.response_file(i)).readlines() 241 assert respdata == [data] 242 243 # upload large data parallel, check that this is what was echoed 244 @pytest.mark.parametrize("proto", ['h2', 'h3']) 245 def test_07_21_upload_parallel_large(self, env: Env, httpd, nghttpx, proto): 246 if proto == 'h3' and not env.have_h3(): 247 pytest.skip("h3 not supported") 248 if proto == 'h3' and env.curl_uses_lib('msh3'): 249 pytest.skip("msh3 stalls here") 250 fdata = os.path.join(env.gen_dir, 'data-100k') 251 # limit since we use a separate connection in h1 252 count = 10 253 curl = CurlClient(env=env) 254 url = f'https://{env.authority_for(env.domain1, proto)}/curltest/echo?id=[0-{count-1}]' 255 r = curl.http_upload(urls=[url], data=f'@{fdata}', alpn_proto=proto, 256 extra_args=['--parallel']) 257 r.check_response(count=count, http_status=200) 258 self.check_download(r, count, fdata, curl) 259 260 # upload large data parallel to a URL that denies uploads 261 @pytest.mark.parametrize("proto", ['h2', 'h3']) 262 def test_07_22_upload_parallel_fail(self, env: Env, httpd, nghttpx, proto): 263 if proto == 'h3' and not env.have_h3(): 264 pytest.skip("h3 not supported") 265 if proto == 'h3' and env.curl_uses_lib('msh3'): 266 pytest.skip("msh3 stalls here") 267 fdata = os.path.join(env.gen_dir, 'data-10m') 268 count = 20 269 curl = CurlClient(env=env) 270 url = f'https://{env.authority_for(env.domain1, proto)}'\ 271 f'/curltest/tweak?status=400&delay=5ms&chunks=1&body_error=reset&id=[0-{count-1}]' 272 r = curl.http_upload(urls=[url], data=f'@{fdata}', alpn_proto=proto, 273 extra_args=['--parallel']) 274 # depending on timing and protocol, we might get CURLE_PARTIAL_FILE or 275 # CURLE_SEND_ERROR or CURLE_HTTP3 or CURLE_HTTP2_STREAM 276 r.check_stats(count=count, exitcode=[18, 55, 92, 95]) 277 278 # PUT 100k 279 @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3']) 280 def test_07_30_put_100k(self, env: Env, httpd, nghttpx, proto): 281 if proto == 'h3' and not env.have_h3(): 282 pytest.skip("h3 not supported") 283 if proto == 'h3' and env.curl_uses_lib('msh3'): 284 pytest.skip("msh3 fails here") 285 fdata = os.path.join(env.gen_dir, 'data-100k') 286 count = 1 287 curl = CurlClient(env=env) 288 url = f'https://{env.authority_for(env.domain1, proto)}/curltest/put?id=[0-{count-1}]' 289 r = curl.http_put(urls=[url], fdata=fdata, alpn_proto=proto, 290 extra_args=['--parallel']) 291 r.check_stats(count=count, http_status=200, exitcode=0) 292 exp_data = [f'{os.path.getsize(fdata)}'] 293 r.check_response(count=count, http_status=200) 294 for i in range(count): 295 respdata = open(curl.response_file(i)).readlines() 296 assert respdata == exp_data 297 298 # PUT 10m 299 @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3']) 300 def test_07_31_put_10m(self, env: Env, httpd, nghttpx, proto): 301 if proto == 'h3' and not env.have_h3(): 302 pytest.skip("h3 not supported") 303 if proto == 'h3' and env.curl_uses_lib('msh3'): 304 pytest.skip("msh3 fails here") 305 fdata = os.path.join(env.gen_dir, 'data-10m') 306 count = 1 307 curl = CurlClient(env=env) 308 url = f'https://{env.authority_for(env.domain1, proto)}/curltest/put?id=[0-{count-1}]&chunk_delay=2ms' 309 r = curl.http_put(urls=[url], fdata=fdata, alpn_proto=proto, 310 extra_args=['--parallel']) 311 r.check_stats(count=count, http_status=200, exitcode=0) 312 exp_data = [f'{os.path.getsize(fdata)}'] 313 r.check_response(count=count, http_status=200) 314 for i in range(count): 315 respdata = open(curl.response_file(i)).readlines() 316 assert respdata == exp_data 317 318 # issue #10591 319 @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3']) 320 def test_07_32_issue_10591(self, env: Env, httpd, nghttpx, proto): 321 if proto == 'h3' and not env.have_h3(): 322 pytest.skip("h3 not supported") 323 if proto == 'h3' and env.curl_uses_lib('msh3'): 324 pytest.skip("msh3 fails here") 325 fdata = os.path.join(env.gen_dir, 'data-10m') 326 count = 1 327 curl = CurlClient(env=env) 328 url = f'https://{env.authority_for(env.domain1, proto)}/curltest/put?id=[0-{count-1}]' 329 r = curl.http_put(urls=[url], fdata=fdata, alpn_proto=proto) 330 r.check_stats(count=count, http_status=200, exitcode=0) 331 332 # issue #11157, upload that is 404'ed by server, needs to terminate 333 # correctly and not time out on sending 334 def test_07_33_issue_11157a(self, env: Env, httpd, nghttpx): 335 proto = 'h2' 336 fdata = os.path.join(env.gen_dir, 'data-10m') 337 # send a POST to our PUT handler which will send immediately a 404 back 338 url = f'https://{env.authority_for(env.domain1, proto)}/curltest/put' 339 curl = CurlClient(env=env) 340 r = curl.run_direct(with_stats=True, args=[ 341 '--resolve', f'{env.authority_for(env.domain1, proto)}:127.0.0.1', 342 '--cacert', env.ca.cert_file, 343 '--request', 'POST', 344 '--max-time', '5', '-v', 345 '--url', url, 346 '--form', 'idList=12345678', 347 '--form', 'pos=top', 348 '--form', 'name=mr_test', 349 '--form', f'fileSource=@{fdata};type=application/pdf', 350 ]) 351 assert r.exit_code == 0, f'{r}' 352 r.check_stats(1, 404) 353 354 # issue #11157, send upload that is slowly read in 355 def test_07_33_issue_11157b(self, env: Env, httpd, nghttpx): 356 proto = 'h2' 357 fdata = os.path.join(env.gen_dir, 'data-10m') 358 # tell our test PUT handler to read the upload more slowly, so 359 # that the send buffering and transfer loop needs to wait 360 url = f'https://{env.authority_for(env.domain1, proto)}/curltest/put?chunk_delay=2ms' 361 curl = CurlClient(env=env) 362 r = curl.run_direct(with_stats=True, args=[ 363 '--verbose', '--trace-config', 'ids,time', 364 '--resolve', f'{env.authority_for(env.domain1, proto)}:127.0.0.1', 365 '--cacert', env.ca.cert_file, 366 '--request', 'PUT', 367 '--max-time', '10', '-v', 368 '--url', url, 369 '--form', 'idList=12345678', 370 '--form', 'pos=top', 371 '--form', 'name=mr_test', 372 '--form', f'fileSource=@{fdata};type=application/pdf', 373 ]) 374 assert r.exit_code == 0, r.dump_logs() 375 r.check_stats(1, 200) 376 377 def test_07_34_issue_11194(self, env: Env, httpd, nghttpx): 378 proto = 'h2' 379 # tell our test PUT handler to read the upload more slowly, so 380 # that the send buffering and transfer loop needs to wait 381 fdata = os.path.join(env.gen_dir, 'data-100k') 382 url = f'https://{env.authority_for(env.domain1, proto)}/curltest/put' 383 curl = CurlClient(env=env) 384 r = curl.run_direct(with_stats=True, args=[ 385 '--verbose', '--trace-config', 'ids,time', 386 '--resolve', f'{env.authority_for(env.domain1, proto)}:127.0.0.1', 387 '--cacert', env.ca.cert_file, 388 '--request', 'PUT', 389 '--digest', '--user', 'test:test', 390 '--data-binary', f'@{fdata}', 391 '--url', url, 392 ]) 393 assert r.exit_code == 0, r.dump_logs() 394 r.check_stats(1, 200) 395 396 # upload large data on a h1 to h2 upgrade 397 def test_07_35_h1_h2_upgrade_upload(self, env: Env, httpd, nghttpx): 398 fdata = os.path.join(env.gen_dir, 'data-100k') 399 curl = CurlClient(env=env) 400 url = f'http://{env.domain1}:{env.http_port}/curltest/echo?id=[0-0]' 401 r = curl.http_upload(urls=[url], data=f'@{fdata}', extra_args=[ 402 '--http2' 403 ]) 404 r.check_response(count=1, http_status=200) 405 # apache does not Upgrade on request with a body 406 assert r.stats[0]['http_version'] == '1.1', f'{r}' 407 indata = open(fdata).readlines() 408 respdata = open(curl.response_file(0)).readlines() 409 assert respdata == indata 410 411 # upload to a 301,302,303 response 412 @pytest.mark.parametrize("redir", ['301', '302', '303']) 413 @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3']) 414 def test_07_36_upload_30x(self, env: Env, httpd, nghttpx, redir, proto): 415 if proto == 'h3' and not env.have_h3(): 416 pytest.skip("h3 not supported") 417 if proto == 'h3' and env.curl_uses_ossl_quic(): 418 pytest.skip("OpenSSL's own QUIC is flaky here") 419 if proto == 'h3' and env.curl_uses_lib('msh3'): 420 pytest.skip("msh3 fails here") 421 data = '0123456789' * 10 422 curl = CurlClient(env=env) 423 url = f'https://{env.authority_for(env.domain1, proto)}/curltest/echo{redir}?id=[0-0]' 424 r = curl.http_upload(urls=[url], data=data, alpn_proto=proto, extra_args=[ 425 '-L', '--trace-config', 'http/2,http/3' 426 ]) 427 r.check_response(count=1, http_status=200) 428 respdata = open(curl.response_file(0)).readlines() 429 assert respdata == [] # was transformed to a GET 430 431 # upload to a 307 response 432 @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3']) 433 def test_07_37_upload_307(self, env: Env, httpd, nghttpx, proto): 434 if proto == 'h3' and not env.have_h3(): 435 pytest.skip("h3 not supported") 436 if proto == 'h3' and env.curl_uses_ossl_quic(): 437 pytest.skip("OpenSSL's own QUIC is flaky here") 438 if proto == 'h3' and env.curl_uses_lib('msh3'): 439 pytest.skip("msh3 fails here") 440 data = '0123456789' * 10 441 curl = CurlClient(env=env) 442 url = f'https://{env.authority_for(env.domain1, proto)}/curltest/echo307?id=[0-0]' 443 r = curl.http_upload(urls=[url], data=data, alpn_proto=proto, extra_args=[ 444 '-L', '--trace-config', 'http/2,http/3' 445 ]) 446 r.check_response(count=1, http_status=200) 447 respdata = open(curl.response_file(0)).readlines() 448 assert respdata == [data] # was POST again 449 450 # POST form data, yet another code path in transfer 451 @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3']) 452 def test_07_38_form_small(self, env: Env, httpd, nghttpx, proto): 453 if proto == 'h3' and not env.have_h3(): 454 pytest.skip("h3 not supported") 455 if proto == 'h3' and env.curl_uses_lib('msh3'): 456 pytest.skip("msh3 fails here") 457 curl = CurlClient(env=env) 458 url = f'https://{env.authority_for(env.domain1, proto)}/curltest/echo?id=[0-0]' 459 r = curl.http_form(urls=[url], alpn_proto=proto, form={ 460 'name1': 'value1', 461 }) 462 r.check_stats(count=1, http_status=200, exitcode=0) 463 464 # POST data urlencoded, small enough to be sent with request headers 465 @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3']) 466 def test_07_39_post_urlenc_small(self, env: Env, httpd, nghttpx, proto): 467 if proto == 'h3' and not env.have_h3(): 468 pytest.skip("h3 not supported") 469 if proto == 'h3' and env.curl_uses_lib('msh3'): 470 pytest.skip("msh3 fails here") 471 fdata = os.path.join(env.gen_dir, 'data-63k') 472 curl = CurlClient(env=env) 473 url = f'https://{env.authority_for(env.domain1, proto)}/curltest/echo?id=[0-0]' 474 r = curl.http_upload(urls=[url], data=f'@{fdata}', alpn_proto=proto, extra_args=[ 475 '--trace-config', 'http/2,http/3' 476 ]) 477 r.check_stats(count=1, http_status=200, exitcode=0) 478 indata = open(fdata).readlines() 479 respdata = open(curl.response_file(0)).readlines() 480 assert respdata == indata 481 482 # POST data urlencoded, large enough to be sent separate from request headers 483 @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3']) 484 def test_07_40_post_urlenc_large(self, env: Env, httpd, nghttpx, proto): 485 if proto == 'h3' and not env.have_h3(): 486 pytest.skip("h3 not supported") 487 if proto == 'h3' and env.curl_uses_lib('msh3'): 488 pytest.skip("msh3 fails here") 489 fdata = os.path.join(env.gen_dir, 'data-64k') 490 curl = CurlClient(env=env) 491 url = f'https://{env.authority_for(env.domain1, proto)}/curltest/echo?id=[0-0]' 492 r = curl.http_upload(urls=[url], data=f'@{fdata}', alpn_proto=proto, extra_args=[ 493 '--trace-config', 'http/2,http/3' 494 ]) 495 r.check_stats(count=1, http_status=200, exitcode=0) 496 indata = open(fdata).readlines() 497 respdata = open(curl.response_file(0)).readlines() 498 assert respdata == indata 499 500 # POST data urlencoded, small enough to be sent with request headers 501 # and request headers are so large that the first send is larger 502 # than our default upload buffer length (64KB). 503 # Unfixed, this will fail when run with CURL_DBG_SOCK_WBLOCK=80 most 504 # of the time 505 @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3']) 506 def test_07_41_post_urlenc_small(self, env: Env, httpd, nghttpx, proto): 507 if proto == 'h3' and not env.have_h3(): 508 pytest.skip("h3 not supported") 509 if proto == 'h3' and env.curl_uses_lib('msh3'): 510 pytest.skip("msh3 fails here") 511 if proto == 'h3' and env.curl_uses_lib('quiche'): 512 pytest.skip("quiche has CWND issues with large requests") 513 fdata = os.path.join(env.gen_dir, 'data-63k') 514 curl = CurlClient(env=env) 515 extra_args = ['--trace-config', 'http/2,http/3'] 516 # add enough headers so that the first send chunk is > 64KB 517 for i in range(63): 518 extra_args.extend(['-H', f'x{i:02d}: {"y"*1019}']) 519 url = f'https://{env.authority_for(env.domain1, proto)}/curltest/echo?id=[0-0]' 520 r = curl.http_upload(urls=[url], data=f'@{fdata}', alpn_proto=proto, extra_args=extra_args) 521 r.check_stats(count=1, http_status=200, exitcode=0) 522 indata = open(fdata).readlines() 523 respdata = open(curl.response_file(0)).readlines() 524 assert respdata == indata 525 526 def check_download(self, r: ExecResult, count: int, srcfile: Union[str, os.PathLike], curl: CurlClient): 527 for i in range(count): 528 dfile = curl.download_file(i) 529 assert os.path.exists(dfile), f'download {dfile} missing\n{r.dump_logs()}' 530 if not filecmp.cmp(srcfile, dfile, shallow=False): 531 diff = "".join(difflib.unified_diff(a=open(srcfile).readlines(), 532 b=open(dfile).readlines(), 533 fromfile=srcfile, 534 tofile=dfile, 535 n=1)) 536 assert False, f'download {dfile} differs:\n{diff}\n{r.dump_logs()}' 537 538 # upload data, pause, let connection die with an incomplete response 539 # issues #11769 #13260 540 @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3']) 541 def test_07_42a_upload_disconnect(self, env: Env, httpd, nghttpx, proto): 542 if proto == 'h3' and not env.have_h3(): 543 pytest.skip("h3 not supported") 544 if proto == 'h3' and env.curl_uses_lib('msh3'): 545 pytest.skip("msh3 fails here") 546 client = LocalClient(name='upload_pausing', env=env, timeout=60) 547 if not client.exists(): 548 pytest.skip(f'example client not built: {client.name}') 549 url = f'https://{env.authority_for(env.domain1, proto)}/curltest/echo?id=[0-0]&die_after=0' 550 r = client.run(['-V', proto, url]) 551 if r.exit_code == 18: # PARTIAL_FILE is always ok 552 pass 553 elif proto == 'h2': 554 # CURLE_HTTP2, CURLE_HTTP2_STREAM 555 assert r.exit_code in [16, 92], f'unexpected exit code\n{r.dump_logs()}' 556 elif proto == 'h3': 557 r.check_exit_code(95) # CURLE_HTTP3 also ok 558 else: 559 r.check_exit_code(18) # will fail as it should 560 561 # upload data, pause, let connection die without any response at all 562 @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3']) 563 def test_07_42b_upload_disconnect(self, env: Env, httpd, nghttpx, proto): 564 if proto == 'h3' and not env.have_h3(): 565 pytest.skip("h3 not supported") 566 if proto == 'h3' and env.curl_uses_lib('msh3'): 567 pytest.skip("msh3 fails here") 568 client = LocalClient(name='upload_pausing', env=env, timeout=60) 569 if not client.exists(): 570 pytest.skip(f'example client not built: {client.name}') 571 url = f'https://{env.authority_for(env.domain1, proto)}/curltest/echo?id=0&just_die=1' 572 r = client.run(['-V', proto, url]) 573 exp_code = 52 # GOT_NOTHING 574 if proto == 'h2' or proto == 'h3': 575 exp_code = 0 # we get a 500 from the server 576 r.check_exit_code(exp_code) # GOT_NOTHING 577 578 # upload data, pause, let connection die after 100 continue 579 @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3']) 580 def test_07_42c_upload_disconnect(self, env: Env, httpd, nghttpx, proto): 581 if proto == 'h3' and not env.have_h3(): 582 pytest.skip("h3 not supported") 583 if proto == 'h3' and env.curl_uses_lib('msh3'): 584 pytest.skip("msh3 fails here") 585 client = LocalClient(name='upload_pausing', env=env, timeout=60) 586 if not client.exists(): 587 pytest.skip(f'example client not built: {client.name}') 588 url = f'https://{env.authority_for(env.domain1, proto)}/curltest/echo?id=0&die_after_100=1' 589 r = client.run(['-V', proto, url]) 590 exp_code = 52 # GOT_NOTHING 591 if proto == 'h2' or proto == 'h3': 592 exp_code = 0 # we get a 500 from the server 593 r.check_exit_code(exp_code) # GOT_NOTHING 594 595 @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3']) 596 def test_07_43_upload_denied(self, env: Env, httpd, nghttpx, proto): 597 if proto == 'h3' and not env.have_h3(): 598 pytest.skip("h3 not supported") 599 if proto == 'h3' and env.curl_uses_ossl_quic(): 600 pytest.skip("openssl-quic is flaky in filed PUTs") 601 if proto == 'h3' and env.curl_uses_lib('msh3'): 602 pytest.skip("msh3 fails here") 603 fdata = os.path.join(env.gen_dir, 'data-10m') 604 count = 1 605 max_upload = 128 * 1024 606 curl = CurlClient(env=env) 607 url = f'https://{env.authority_for(env.domain1, proto)}/curltest/put?'\ 608 f'id=[0-{count-1}]&max_upload={max_upload}' 609 r = curl.http_put(urls=[url], fdata=fdata, alpn_proto=proto, 610 extra_args=['--trace-config', 'all']) 611 r.check_stats(count=count, http_status=413, exitcode=0) 612 613 @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3']) 614 @pytest.mark.parametrize("httpcode", [301, 302, 307, 308]) 615 def test_07_44_put_redir(self, env: Env, httpd, nghttpx, proto, httpcode): 616 if proto == 'h3' and not env.have_h3(): 617 pytest.skip("h3 not supported") 618 count = 1 619 upload_size = 128*1024 620 url = f'https://localhost:{env.https_port}/curltest/put-redir-{httpcode}' 621 client = LocalClient(name='hx_upload', env=env) 622 if not client.exists(): 623 pytest.skip(f'example client not built: {client.name}') 624 r = client.run(args=[ 625 '-n', f'{count}', '-l', '-S', f'{upload_size}', '-V', proto, url 626 ]) 627 r.check_exit_code(0) 628 results = [int(m.group(1)) for line in r.trace_lines 629 if (m := re.match(r'.* FINISHED, result=(\d+), response=(\d+)', line))] 630 httpcodes = [int(m.group(2)) for line in r.trace_lines 631 if (m := re.match(r'.* FINISHED, result=(\d+), response=(\d+)', line))] 632 if httpcode == 308: 633 assert results[0] == 65, f'{r}' # could not rewind input 634 else: 635 assert httpcodes[0] == httpcode, f'{r}' 636 637 # speed limited on put handler 638 @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3']) 639 def test_07_50_put_speed_limit(self, env: Env, httpd, nghttpx, proto): 640 if proto == 'h3' and not env.have_h3(): 641 pytest.skip("h3 not supported") 642 count = 1 643 fdata = os.path.join(env.gen_dir, 'data-100k') 644 up_len = 100 * 1024 645 speed_limit = 50 * 1024 646 curl = CurlClient(env=env) 647 url = f'https://{env.authority_for(env.domain1, proto)}/curltest/put?id=[0-0]' 648 r = curl.http_put(urls=[url], fdata=fdata, alpn_proto=proto, 649 with_headers=True, extra_args=[ 650 '--limit-rate', f'{speed_limit}' 651 ]) 652 r.check_response(count=count, http_status=200) 653 assert r.responses[0]['header']['received-length'] == f'{up_len}', f'{r.responses[0]}' 654 up_speed = r.stats[0]['speed_upload'] 655 assert (speed_limit * 0.5) <= up_speed <= (speed_limit * 1.5), f'{r.stats[0]}' 656 657 # speed limited on echo handler 658 @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3']) 659 def test_07_51_echo_speed_limit(self, env: Env, httpd, nghttpx, proto): 660 if proto == 'h3' and not env.have_h3(): 661 pytest.skip("h3 not supported") 662 count = 1 663 fdata = os.path.join(env.gen_dir, 'data-100k') 664 speed_limit = 50 * 1024 665 curl = CurlClient(env=env) 666 url = f'https://{env.authority_for(env.domain1, proto)}/curltest/echo?id=[0-0]' 667 r = curl.http_upload(urls=[url], data=f'@{fdata}', alpn_proto=proto, 668 with_headers=True, extra_args=[ 669 '--limit-rate', f'{speed_limit}' 670 ]) 671 r.check_response(count=count, http_status=200) 672 up_speed = r.stats[0]['speed_upload'] 673 assert (speed_limit * 0.5) <= up_speed <= (speed_limit * 1.5), f'{r.stats[0]}' 674 675 # upload larger data, triggering "Expect: 100-continue" code paths 676 @pytest.mark.parametrize("proto", ['http/1.1']) 677 def test_07_60_upload_exp100(self, env: Env, httpd, nghttpx, proto): 678 fdata = os.path.join(env.gen_dir, 'data-1m+') 679 read_delay = 1 680 curl = CurlClient(env=env) 681 url = f'https://{env.authority_for(env.domain1, proto)}/curltest/put?id=[0-0]'\ 682 f'&read_delay={read_delay}s' 683 r = curl.http_put(urls=[url], fdata=fdata, alpn_proto=proto, extra_args=[ 684 '--expect100-timeout', f'{read_delay+1}' 685 ]) 686 r.check_stats(count=1, http_status=200, exitcode=0) 687 688 # upload larger data, triggering "Expect: 100-continue" code paths 689 @pytest.mark.parametrize("proto", ['http/1.1']) 690 def test_07_61_upload_exp100_timeout(self, env: Env, httpd, nghttpx, proto): 691 fdata = os.path.join(env.gen_dir, 'data-1m+') 692 read_delay = 2 693 curl = CurlClient(env=env) 694 url = f'https://{env.authority_for(env.domain1, proto)}/curltest/put?id=[0-0]'\ 695 f'&read_delay={read_delay}s' 696 r = curl.http_put(urls=[url], fdata=fdata, alpn_proto=proto, extra_args=[ 697 '--expect100-timeout', f'{read_delay-1}' 698 ]) 699 r.check_stats(count=1, http_status=200, exitcode=0) 700 701 # issue #15688 when posting a form and cr_mime_read() is called with 702 # length < 4, we did not progress 703 @pytest.mark.parametrize("proto", ['http/1.1']) 704 def test_07_62_upload_issue_15688(self, env: Env, httpd, proto): 705 # this length leads to (including multipart formatting) to a 706 # client reader invocation with length 1. 707 upload_len = 196169 708 fname = f'data-{upload_len}' 709 env.make_data_file(indir=env.gen_dir, fname=fname, fsize=upload_len) 710 fdata = os.path.join(env.gen_dir, fname) 711 curl = CurlClient(env=env) 712 url = f'https://{env.authority_for(env.domain1, proto)}/curltest/echo?id=[0-0]' 713 r = curl.http_form(urls=[url], form={ 714 'file': f'@{fdata}', 715 }, alpn_proto=proto, extra_args=[ 716 '--max-time', '10' 717 ]) 718 r.check_stats(count=1, http_status=200, exitcode=0) 719 720 # nghttpx is the only server we have that supports TLS early data and 721 # has a limit of 16k it announces 722 @pytest.mark.skipif(condition=not Env.have_nghttpx(), reason="no nghttpx") 723 @pytest.mark.parametrize("proto,upload_size,exp_early", [ 724 pytest.param('http/1.1', 100, 203, id='h1-small-body'), 725 pytest.param('http/1.1', 10*1024, 10345, id='h1-medium-body'), 726 pytest.param('http/1.1', 32*1024, 16384, id='h1-limited-body'), 727 pytest.param('h2', 10*1024, 10378, id='h2-medium-body'), 728 pytest.param('h2', 32*1024, 16384, id='h2-limited-body'), 729 pytest.param('h3', 1024, 1126, id='h3-small-body'), 730 pytest.param('h3', 1024 * 1024, 131177, id='h3-limited-body'), 731 # h3: limited+body (long app data). The 0RTT size is limited by 732 # our sendbuf size of 128K. 733 ]) 734 def test_07_70_put_earlydata(self, env: Env, httpd, nghttpx, proto, upload_size, exp_early): 735 if not env.curl_can_early_data(): 736 pytest.skip('TLS earlydata not implemented') 737 if proto == 'h3' and \ 738 (not env.have_h3() or not env.curl_can_h3_early_data()): 739 pytest.skip("h3 not supported") 740 if proto != 'h3' and sys.platform.startswith('darwin') and env.ci_run: 741 pytest.skip('failing on macOS CI runners') 742 count = 2 743 # we want this test to always connect to nghttpx, since it is 744 # the only server we have that supports TLS earlydata 745 port = env.port_for(proto) 746 if proto != 'h3': 747 port = env.nghttpx_https_port 748 url = f'https://{env.domain1}:{port}/curltest/put' 749 client = LocalClient(name='hx_upload', env=env) 750 if not client.exists(): 751 pytest.skip(f'example client not built: {client.name}') 752 r = client.run(args=[ 753 '-n', f'{count}', 754 '-e', # use TLS earlydata 755 '-f', # forbid reuse of connections 756 '-l', # announce upload length, no 'Expect: 100' 757 '-S', f'{upload_size}', 758 '-r', f'{env.domain1}:{port}:127.0.0.1', 759 '-V', proto, url 760 ]) 761 r.check_exit_code(0) 762 self.check_downloads(client, r, [f"{upload_size}"], count) 763 earlydata = {} 764 for line in r.trace_lines: 765 m = re.match(r'^\[t-(\d+)] EarlyData: (-?\d+)', line) 766 if m: 767 earlydata[int(m.group(1))] = int(m.group(2)) 768 assert earlydata[0] == 0, f'{earlydata}\n{r.dump_logs()}' 769 # depending on cpu load, curl might not upload as much before 770 # the handshake starts and early data stops. 771 assert 0 < earlydata[1] <= exp_early, f'{earlydata}\n{r.dump_logs()}' 772 773 def check_downloads(self, client, r, source: List[str], count: int, 774 complete: bool = True): 775 for i in range(count): 776 dfile = client.download_file(i) 777 assert os.path.exists(dfile), f'download {dfile} missing\n{r.dump_logs()}' 778 if complete: 779 diff = "".join(difflib.unified_diff(a=source, 780 b=open(dfile).readlines(), 781 fromfile='-', 782 tofile=dfile, 783 n=1)) 784 assert not diff, f'download {dfile} differs:\n{diff}\n{r.dump_logs()}'