hx_download.c (13389B)
1 /*************************************************************************** 2 * _ _ ____ _ 3 * Project ___| | | | _ \| | 4 * / __| | | | |_) | | 5 * | (__| |_| | _ <| |___ 6 * \___|\___/|_| \_\_____| 7 * 8 * Copyright (C) Daniel Stenberg, <daniel@haxx.se>, et al. 9 * 10 * This software is licensed as described in the file COPYING, which 11 * you should have received as part of this distribution. The terms 12 * are also available at https://curl.se/docs/copyright.html. 13 * 14 * You may opt to use, copy, modify, merge, publish, distribute and/or sell 15 * copies of the Software, and permit persons to whom the Software is 16 * furnished to do so, under the terms of the COPYING file. 17 * 18 * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY 19 * KIND, either express or implied. 20 * 21 * SPDX-License-Identifier: curl 22 * 23 ***************************************************************************/ 24 #include "first.h" 25 26 static int verbose_d = 1; 27 28 struct transfer_d { 29 int idx; 30 CURL *easy; 31 char filename[128]; 32 FILE *out; 33 curl_off_t recv_size; 34 curl_off_t fail_at; 35 curl_off_t pause_at; 36 curl_off_t abort_at; 37 int started; 38 int paused; 39 int resumed; 40 int done; 41 CURLcode result; 42 }; 43 44 static size_t transfer_count_d = 1; 45 static struct transfer_d *transfer_d; 46 static int forbid_reuse_d = 0; 47 48 static struct transfer_d *get_transfer_for_easy_d(CURL *easy) 49 { 50 size_t i; 51 for(i = 0; i < transfer_count_d; ++i) { 52 if(easy == transfer_d[i].easy) 53 return &transfer_d[i]; 54 } 55 return NULL; 56 } 57 58 static size_t my_write_d_cb(char *buf, size_t nitems, size_t buflen, 59 void *userdata) 60 { 61 struct transfer_d *t = userdata; 62 size_t blen = (nitems * buflen); 63 size_t nwritten; 64 65 curl_mfprintf(stderr, "[t-%d] RECV %ld bytes, total=%ld, pause_at=%ld\n", 66 t->idx, (long)blen, (long)t->recv_size, (long)t->pause_at); 67 if(!t->out) { 68 curl_msnprintf(t->filename, sizeof(t->filename)-1, "download_%u.data", 69 t->idx); 70 t->out = fopen(t->filename, "wb"); 71 if(!t->out) 72 return 0; 73 } 74 75 if(!t->resumed && 76 t->recv_size < t->pause_at && 77 ((t->recv_size + (curl_off_t)blen) >= t->pause_at)) { 78 curl_mfprintf(stderr, "[t-%d] PAUSE\n", t->idx); 79 t->paused = 1; 80 return CURL_WRITEFUNC_PAUSE; 81 } 82 83 nwritten = fwrite(buf, nitems, buflen, t->out); 84 if(nwritten < blen) { 85 curl_mfprintf(stderr, "[t-%d] write failure\n", t->idx); 86 return 0; 87 } 88 t->recv_size += (curl_off_t)nwritten; 89 if(t->fail_at > 0 && t->recv_size >= t->fail_at) { 90 curl_mfprintf(stderr, "[t-%d] FAIL by write callback at %ld bytes\n", 91 t->idx, (long)t->recv_size); 92 return CURL_WRITEFUNC_ERROR; 93 } 94 95 return (size_t)nwritten; 96 } 97 98 static int my_progress_d_cb(void *userdata, 99 curl_off_t dltotal, curl_off_t dlnow, 100 curl_off_t ultotal, curl_off_t ulnow) 101 { 102 struct transfer_d *t = userdata; 103 (void)ultotal; 104 (void)ulnow; 105 (void)dltotal; 106 if(t->abort_at > 0 && dlnow >= t->abort_at) { 107 curl_mfprintf(stderr, "[t-%d] ABORT by progress_cb at %ld bytes\n", 108 t->idx, (long)dlnow); 109 return 1; 110 } 111 return 0; 112 } 113 114 static int setup_hx_download(CURL *hnd, const char *url, struct transfer_d *t, 115 long http_version, struct curl_slist *host, 116 CURLSH *share, int use_earlydata, 117 int fresh_connect) 118 { 119 curl_easy_setopt(hnd, CURLOPT_SHARE, share); 120 curl_easy_setopt(hnd, CURLOPT_URL, url); 121 curl_easy_setopt(hnd, CURLOPT_HTTP_VERSION, http_version); 122 curl_easy_setopt(hnd, CURLOPT_SSL_VERIFYPEER, 0L); 123 curl_easy_setopt(hnd, CURLOPT_SSL_VERIFYHOST, 0L); 124 curl_easy_setopt(hnd, CURLOPT_ACCEPT_ENCODING, ""); 125 curl_easy_setopt(hnd, CURLOPT_BUFFERSIZE, (long)(128 * 1024)); 126 curl_easy_setopt(hnd, CURLOPT_WRITEFUNCTION, my_write_d_cb); 127 curl_easy_setopt(hnd, CURLOPT_WRITEDATA, t); 128 curl_easy_setopt(hnd, CURLOPT_NOPROGRESS, 0L); 129 curl_easy_setopt(hnd, CURLOPT_XFERINFOFUNCTION, my_progress_d_cb); 130 curl_easy_setopt(hnd, CURLOPT_XFERINFODATA, t); 131 if(use_earlydata) 132 curl_easy_setopt(hnd, CURLOPT_SSL_OPTIONS, (long)CURLSSLOPT_EARLYDATA); 133 if(forbid_reuse_d) 134 curl_easy_setopt(hnd, CURLOPT_FORBID_REUSE, 1L); 135 if(host) 136 curl_easy_setopt(hnd, CURLOPT_RESOLVE, host); 137 if(fresh_connect) 138 curl_easy_setopt(hnd, CURLOPT_FRESH_CONNECT, 1L); 139 140 /* please be verbose */ 141 if(verbose_d) { 142 curl_easy_setopt(hnd, CURLOPT_VERBOSE, 1L); 143 curl_easy_setopt(hnd, CURLOPT_DEBUGFUNCTION, debug_cb); 144 } 145 146 /* wait for pipe connection to confirm */ 147 curl_easy_setopt(hnd, CURLOPT_PIPEWAIT, 1L); 148 149 return 0; /* all is good */ 150 } 151 152 static void usage_hx_download(const char *msg) 153 { 154 if(msg) 155 curl_mfprintf(stderr, "%s\n", msg); 156 curl_mfprintf(stderr, 157 "usage: [options] url\n" 158 " download a url with following options:\n" 159 " -a abort paused transfer\n" 160 " -m number max parallel downloads\n" 161 " -e use TLS early data when possible\n" 162 " -f forbid connection reuse\n" 163 " -n number total downloads\n"); 164 curl_mfprintf(stderr, 165 " -A number abort transfer after `number` response bytes\n" 166 " -F number fail writing response after `number` response bytes\n" 167 " -M number max concurrent connections to a host\n" 168 " -P number pause transfer after `number` response bytes\n" 169 " -r <host>:<port>:<addr> resolve information\n" 170 " -T number max concurrent connections total\n" 171 " -V http_version (http/1.1, h2, h3) http version to use\n" 172 ); 173 } 174 175 /* 176 * Download a file over HTTP/2, take care of server push. 177 */ 178 static int test_hx_download(int argc, char *argv[]) 179 { 180 CURLM *multi_handle; 181 struct CURLMsg *m; 182 CURLSH *share; 183 const char *url; 184 size_t i, n, max_parallel = 1; 185 size_t active_transfers; 186 size_t pause_offset = 0; 187 size_t abort_offset = 0; 188 size_t fail_offset = 0; 189 int abort_paused = 0, use_earlydata = 0; 190 struct transfer_d *t; 191 int http_version = CURL_HTTP_VERSION_2_0; 192 int ch; 193 struct curl_slist *host = NULL; 194 char *resolve = NULL; 195 size_t max_host_conns = 0; 196 size_t max_total_conns = 0; 197 int fresh_connect = 0; 198 int result = 0; 199 200 while((ch = cgetopt(argc, argv, "aefhm:n:xA:F:M:P:r:T:V:")) != -1) { 201 switch(ch) { 202 case 'h': 203 usage_hx_download(NULL); 204 result = 2; 205 goto cleanup; 206 case 'a': 207 abort_paused = 1; 208 break; 209 case 'e': 210 use_earlydata = 1; 211 break; 212 case 'f': 213 forbid_reuse_d = 1; 214 break; 215 case 'm': 216 max_parallel = (size_t)strtol(coptarg, NULL, 10); 217 break; 218 case 'n': 219 transfer_count_d = (size_t)strtol(coptarg, NULL, 10); 220 break; 221 case 'x': 222 fresh_connect = 1; 223 break; 224 case 'A': 225 abort_offset = (size_t)strtol(coptarg, NULL, 10); 226 break; 227 case 'F': 228 fail_offset = (size_t)strtol(coptarg, NULL, 10); 229 break; 230 case 'M': 231 max_host_conns = (size_t)strtol(coptarg, NULL, 10); 232 break; 233 case 'P': 234 pause_offset = (size_t)strtol(coptarg, NULL, 10); 235 break; 236 case 'r': 237 free(resolve); 238 resolve = strdup(coptarg); 239 break; 240 case 'T': 241 max_total_conns = (size_t)strtol(coptarg, NULL, 10); 242 break; 243 case 'V': { 244 if(!strcmp("http/1.1", coptarg)) 245 http_version = CURL_HTTP_VERSION_1_1; 246 else if(!strcmp("h2", coptarg)) 247 http_version = CURL_HTTP_VERSION_2_0; 248 else if(!strcmp("h3", coptarg)) 249 http_version = CURL_HTTP_VERSION_3ONLY; 250 else { 251 usage_hx_download("invalid http version"); 252 result = 1; 253 goto cleanup; 254 } 255 break; 256 } 257 default: 258 usage_hx_download("invalid option"); 259 result = 1; 260 goto cleanup; 261 } 262 } 263 argc -= coptind; 264 argv += coptind; 265 266 curl_global_init(CURL_GLOBAL_DEFAULT); 267 curl_global_trace("ids,time,http/2,http/3"); 268 269 if(argc != 1) { 270 usage_hx_download("not enough arguments"); 271 result = 2; 272 goto cleanup; 273 } 274 url = argv[0]; 275 276 if(resolve) 277 host = curl_slist_append(NULL, resolve); 278 279 share = curl_share_init(); 280 if(!share) { 281 curl_mfprintf(stderr, "error allocating share\n"); 282 result = 1; 283 goto cleanup; 284 } 285 curl_share_setopt(share, CURLSHOPT_SHARE, CURL_LOCK_DATA_COOKIE); 286 curl_share_setopt(share, CURLSHOPT_SHARE, CURL_LOCK_DATA_DNS); 287 curl_share_setopt(share, CURLSHOPT_SHARE, CURL_LOCK_DATA_SSL_SESSION); 288 /* curl_share_setopt(share, CURLSHOPT_SHARE, CURL_LOCK_DATA_CONNECT); */ 289 curl_share_setopt(share, CURLSHOPT_SHARE, CURL_LOCK_DATA_PSL); 290 curl_share_setopt(share, CURLSHOPT_SHARE, CURL_LOCK_DATA_HSTS); 291 292 transfer_d = calloc(transfer_count_d, sizeof(*transfer_d)); 293 if(!transfer_d) { 294 curl_mfprintf(stderr, "error allocating transfer structs\n"); 295 result = 1; 296 goto cleanup; 297 } 298 299 multi_handle = curl_multi_init(); 300 curl_multi_setopt(multi_handle, CURLMOPT_PIPELINING, CURLPIPE_MULTIPLEX); 301 curl_multi_setopt(multi_handle, CURLMOPT_MAX_TOTAL_CONNECTIONS, 302 (long)max_total_conns); 303 curl_multi_setopt(multi_handle, CURLMOPT_MAX_HOST_CONNECTIONS, 304 (long)max_host_conns); 305 306 active_transfers = 0; 307 for(i = 0; i < transfer_count_d; ++i) { 308 t = &transfer_d[i]; 309 t->idx = (int)i; 310 t->abort_at = (curl_off_t)abort_offset; 311 t->fail_at = (curl_off_t)fail_offset; 312 t->pause_at = (curl_off_t)pause_offset; 313 } 314 315 n = (max_parallel < transfer_count_d) ? max_parallel : transfer_count_d; 316 for(i = 0; i < n; ++i) { 317 t = &transfer_d[i]; 318 t->easy = curl_easy_init(); 319 if(!t->easy || 320 setup_hx_download(t->easy, url, t, http_version, host, share, 321 use_earlydata, fresh_connect)) { 322 curl_mfprintf(stderr, "[t-%d] FAILED setup\n", (int)i); 323 result = 1; 324 goto cleanup; 325 } 326 curl_multi_add_handle(multi_handle, t->easy); 327 t->started = 1; 328 ++active_transfers; 329 curl_mfprintf(stderr, "[t-%d] STARTED\n", t->idx); 330 } 331 332 do { 333 int still_running; /* keep number of running handles */ 334 CURLMcode mc = curl_multi_perform(multi_handle, &still_running); 335 336 if(still_running) { 337 /* wait for activity, timeout or "nothing" */ 338 mc = curl_multi_poll(multi_handle, NULL, 0, 1000, NULL); 339 } 340 341 if(mc) 342 break; 343 344 do { 345 int msgq = 0; 346 m = curl_multi_info_read(multi_handle, &msgq); 347 if(m && (m->msg == CURLMSG_DONE)) { 348 CURL *e = m->easy_handle; 349 --active_transfers; 350 curl_multi_remove_handle(multi_handle, e); 351 t = get_transfer_for_easy_d(e); 352 if(t) { 353 t->done = 1; 354 t->result = m->data.result; 355 curl_mfprintf(stderr, "[t-%d] FINISHED with result %d\n", 356 t->idx, t->result); 357 if(use_earlydata) { 358 curl_off_t sent; 359 curl_easy_getinfo(e, CURLINFO_EARLYDATA_SENT_T, &sent); 360 curl_mfprintf(stderr, "[t-%d] EarlyData: %ld\n", t->idx, 361 (long)sent); 362 } 363 } 364 else { 365 curl_easy_cleanup(e); 366 curl_mfprintf(stderr, "unknown FINISHED???\n"); 367 } 368 } 369 370 /* nothing happening, maintenance */ 371 if(abort_paused) { 372 /* abort paused transfers */ 373 for(i = 0; i < transfer_count_d; ++i) { 374 t = &transfer_d[i]; 375 if(!t->done && t->paused && t->easy) { 376 curl_multi_remove_handle(multi_handle, t->easy); 377 t->done = 1; 378 active_transfers--; 379 curl_mfprintf(stderr, "[t-%d] ABORTED\n", t->idx); 380 } 381 } 382 } 383 else { 384 /* resume one paused transfer */ 385 for(i = 0; i < transfer_count_d; ++i) { 386 t = &transfer_d[i]; 387 if(!t->done && t->paused) { 388 t->resumed = 1; 389 t->paused = 0; 390 curl_easy_pause(t->easy, CURLPAUSE_CONT); 391 curl_mfprintf(stderr, "[t-%d] RESUMED\n", t->idx); 392 break; 393 } 394 } 395 } 396 397 while(active_transfers < max_parallel) { 398 for(i = 0; i < transfer_count_d; ++i) { 399 t = &transfer_d[i]; 400 if(!t->started) { 401 t->easy = curl_easy_init(); 402 if(!t->easy || 403 setup_hx_download(t->easy, url, t, http_version, host, share, 404 use_earlydata, fresh_connect)) { 405 curl_mfprintf(stderr, "[t-%d] FAILED setup\n", (int)i); 406 result = 1; 407 goto cleanup; 408 } 409 curl_multi_add_handle(multi_handle, t->easy); 410 t->started = 1; 411 ++active_transfers; 412 curl_mfprintf(stderr, "[t-%d] STARTED\n", t->idx); 413 break; 414 } 415 } 416 /* all started */ 417 if(i == transfer_count_d) 418 break; 419 } 420 } while(m); 421 422 } while(active_transfers); /* as long as we have transfers going */ 423 424 curl_multi_cleanup(multi_handle); 425 426 for(i = 0; i < transfer_count_d; ++i) { 427 t = &transfer_d[i]; 428 if(t->out) { 429 fclose(t->out); 430 t->out = NULL; 431 } 432 if(t->easy) { 433 curl_easy_cleanup(t->easy); 434 t->easy = NULL; 435 } 436 if(t->result) 437 result = t->result; 438 } 439 free(transfer_d); 440 441 curl_share_cleanup(share); 442 curl_slist_free_all(host); 443 cleanup: 444 free(resolve); 445 446 return result; 447 }