scorecard.py (35110B)
1 #!/usr/bin/env python3 2 # -*- coding: utf-8 -*- 3 #*************************************************************************** 4 # _ _ ____ _ 5 # Project ___| | | | _ \| | 6 # / __| | | | |_) | | 7 # | (__| |_| | _ <| |___ 8 # \___|\___/|_| \_\_____| 9 # 10 # Copyright (C) Daniel Stenberg, <daniel@haxx.se>, et al. 11 # 12 # This software is licensed as described in the file COPYING, which 13 # you should have received as part of this distribution. The terms 14 # are also available at https://curl.se/docs/copyright.html. 15 # 16 # You may opt to use, copy, modify, merge, publish, distribute and/or sell 17 # copies of the Software, and permit persons to whom the Software is 18 # furnished to do so, under the terms of the COPYING file. 19 # 20 # This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY 21 # KIND, either express or implied. 22 # 23 # SPDX-License-Identifier: curl 24 # 25 ########################################################################### 26 # 27 import argparse 28 import datetime 29 import json 30 import logging 31 import os 32 import re 33 import sys 34 from statistics import mean 35 from typing import Dict, Any, Optional, List 36 37 from testenv import Env, Httpd, CurlClient, Caddy, ExecResult, NghttpxQuic, RunProfile 38 39 log = logging.getLogger(__name__) 40 41 42 class ScoreCardError(Exception): 43 pass 44 45 46 class Card: 47 @classmethod 48 def fmt_ms(cls, tval): 49 return f'{int(tval*1000)} ms' if tval >= 0 else '--' 50 51 @classmethod 52 def fmt_size(cls, val): 53 if val >= (1024*1024*1024): 54 return f'{val / (1024*1024*1024):0.000f}GB' 55 elif val >= (1024 * 1024): 56 return f'{val / (1024*1024):0.000f}MB' 57 elif val >= 1024: 58 return f'{val / 1024:0.000f}KB' 59 else: 60 return f'{val:0.000f}B' 61 62 @classmethod 63 def fmt_mbs(cls, val): 64 return f'{val/(1024*1024):0.000f} MB/s' if val >= 0 else '--' 65 66 @classmethod 67 def fmt_reqs(cls, val): 68 return f'{val:0.000f} r/s' if val >= 0 else '--' 69 70 @classmethod 71 def mk_mbs_cell(cls, samples, profiles, errors): 72 val = mean(samples) if len(samples) else -1 73 cell = { 74 'val': val, 75 'sval': Card.fmt_mbs(val) if val >= 0 else '--', 76 } 77 if len(profiles): 78 cell['stats'] = RunProfile.AverageStats(profiles) 79 if len(errors): 80 cell['errors'] = errors 81 return cell 82 83 @classmethod 84 def mk_reqs_cell(cls, samples, profiles, errors): 85 val = mean(samples) if len(samples) else -1 86 cell = { 87 'val': val, 88 'sval': Card.fmt_reqs(val) if val >= 0 else '--', 89 } 90 if len(profiles): 91 cell['stats'] = RunProfile.AverageStats(profiles) 92 if len(errors): 93 cell['errors'] = errors 94 return cell 95 96 @classmethod 97 def parse_size(cls, s): 98 m = re.match(r'(\d+)(mb|kb|gb)?', s, re.IGNORECASE) 99 if m is None: 100 raise Exception(f'unrecognized size: {s}') 101 size = int(m.group(1)) 102 if not m.group(2): 103 pass 104 elif m.group(2).lower() == 'kb': 105 size *= 1024 106 elif m.group(2).lower() == 'mb': 107 size *= 1024 * 1024 108 elif m.group(2).lower() == 'gb': 109 size *= 1024 * 1024 * 1024 110 return size 111 112 @classmethod 113 def print_score(cls, score): 114 print(f'Scorecard curl, protocol {score["meta"]["protocol"]} ' 115 f'via {score["meta"]["implementation"]}/' 116 f'{score["meta"]["implementation_version"]}') 117 print(f'Date: {score["meta"]["date"]}') 118 if 'curl_V' in score["meta"]: 119 print(f'Version: {score["meta"]["curl_V"]}') 120 if 'curl_features' in score["meta"]: 121 print(f'Features: {score["meta"]["curl_features"]}') 122 print(f'Samples Size: {score["meta"]["samples"]}') 123 if 'handshakes' in score: 124 print(f'{"Handshakes":<24} {"ipv4":25} {"ipv6":28}') 125 print(f' {"Host":<17} {"Connect":>12} {"Handshake":>12} ' 126 f'{"Connect":>12} {"Handshake":>12} {"Errors":<20}') 127 for key, val in score["handshakes"].items(): 128 print(f' {key:<17} {Card.fmt_ms(val["ipv4-connect"]):>12} ' 129 f'{Card.fmt_ms(val["ipv4-handshake"]):>12} ' 130 f'{Card.fmt_ms(val["ipv6-connect"]):>12} ' 131 f'{Card.fmt_ms(val["ipv6-handshake"]):>12} ' 132 f'{"/".join(val["ipv4-errors"] + val["ipv6-errors"]):<20}' 133 ) 134 for name in ['downloads', 'uploads', 'requests']: 135 if name in score: 136 Card.print_score_table(score[name]) 137 138 @classmethod 139 def print_score_table(cls, score): 140 cols = score['cols'] 141 rows = score['rows'] 142 colw = [] 143 statw = 13 144 errors = [] 145 col_has_stats = [] 146 for idx, col in enumerate(cols): 147 cellw = max([len(r[idx]["sval"]) for r in rows]) 148 colw.append(max(cellw, len(col))) 149 col_has_stats.append(False) 150 for row in rows: 151 if 'stats' in row[idx]: 152 col_has_stats[idx] = True 153 break 154 if 'title' in score['meta']: 155 print(score['meta']['title']) 156 for idx, col in enumerate(cols): 157 if col_has_stats[idx]: 158 print(f' {col:>{colw[idx]}} {"[cpu/rss]":<{statw}}', end='') 159 else: 160 print(f' {col:>{colw[idx]}}', end='') 161 print('') 162 for row in rows: 163 for idx, cell in enumerate(row): 164 print(f' {cell["sval"]:>{colw[idx]}}', end='') 165 if col_has_stats[idx]: 166 if 'stats' in cell: 167 s = f'[{cell["stats"]["cpu"]:>.1f}%' \ 168 f'/{Card.fmt_size(cell["stats"]["rss"])}]' 169 else: 170 s = '' 171 print(f' {s:<{statw}}', end='') 172 if 'errors' in cell: 173 errors.extend(cell['errors']) 174 print('') 175 if len(errors): 176 print(f'Errors: {errors}') 177 178 179 class ScoreRunner: 180 181 def __init__(self, env: Env, 182 protocol: str, 183 server_descr: str, 184 server_port: int, 185 verbose: int, 186 curl_verbose: int, 187 download_parallel: int = 0, 188 server_addr: Optional[str] = None, 189 with_dtrace: bool = False, 190 with_flame: bool = False): 191 self.verbose = verbose 192 self.env = env 193 self.protocol = protocol 194 self.server_descr = server_descr 195 self.server_addr = server_addr 196 self.server_port = server_port 197 self._silent_curl = not curl_verbose 198 self._download_parallel = download_parallel 199 self._with_dtrace = with_dtrace 200 self._with_flame = with_flame 201 202 def info(self, msg): 203 if self.verbose > 0: 204 sys.stderr.write(msg) 205 sys.stderr.flush() 206 207 def mk_curl_client(self): 208 return CurlClient(env=self.env, silent=self._silent_curl, 209 server_addr=self.server_addr, 210 with_dtrace=self._with_dtrace, 211 with_flame=self._with_flame) 212 213 def handshakes(self) -> Dict[str, Any]: 214 props = {} 215 sample_size = 5 216 self.info('TLS Handshake\n') 217 for authority in [ 218 'curl.se', 'google.com', 'cloudflare.com', 'nghttp2.org' 219 ]: 220 self.info(f' {authority}...') 221 props[authority] = {} 222 for ipv in ['ipv4', 'ipv6']: 223 self.info(f'{ipv}...') 224 c_samples = [] 225 hs_samples = [] 226 errors = [] 227 for _ in range(sample_size): 228 curl = self.mk_curl_client() 229 args = [ 230 '--http3-only' if self.protocol == 'h3' else '--http2', 231 f'--{ipv}', f'https://{authority}/' 232 ] 233 r = curl.run_direct(args=args, with_stats=True) 234 if r.exit_code == 0 and len(r.stats) == 1: 235 c_samples.append(r.stats[0]['time_connect']) 236 hs_samples.append(r.stats[0]['time_appconnect']) 237 else: 238 errors.append(f'exit={r.exit_code}') 239 props[authority][f'{ipv}-connect'] = mean(c_samples) \ 240 if len(c_samples) else -1 241 props[authority][f'{ipv}-handshake'] = mean(hs_samples) \ 242 if len(hs_samples) else -1 243 props[authority][f'{ipv}-errors'] = errors 244 self.info('ok.\n') 245 return props 246 247 def _make_docs_file(self, docs_dir: str, fname: str, fsize: int): 248 fpath = os.path.join(docs_dir, fname) 249 data1k = 1024*'x' 250 flen = 0 251 with open(fpath, 'w') as fd: 252 while flen < fsize: 253 fd.write(data1k) 254 flen += len(data1k) 255 return fpath 256 257 def setup_resources(self, server_docs: str, 258 downloads: Optional[List[int]] = None): 259 if downloads is not None: 260 for fsize in downloads: 261 label = Card.fmt_size(fsize) 262 fname = f'score{label}.data' 263 self._make_docs_file(docs_dir=server_docs, 264 fname=fname, fsize=fsize) 265 self._make_docs_file(docs_dir=server_docs, 266 fname='reqs10.data', fsize=10*1024) 267 268 def _check_downloads(self, r: ExecResult, count: int): 269 error = '' 270 if r.exit_code != 0: 271 error += f'exit={r.exit_code} ' 272 if r.exit_code != 0 or len(r.stats) != count: 273 error += f'stats={len(r.stats)}/{count} ' 274 fails = [s for s in r.stats if s['response_code'] != 200] 275 if len(fails) > 0: 276 error += f'{len(fails)} failed' 277 return error if len(error) > 0 else None 278 279 def dl_single(self, url: str, nsamples: int = 1): 280 count = 1 281 samples = [] 282 errors = [] 283 profiles = [] 284 self.info('single...') 285 for _ in range(nsamples): 286 curl = self.mk_curl_client() 287 r = curl.http_download(urls=[url], alpn_proto=self.protocol, 288 no_save=True, with_headers=False, 289 with_profile=True) 290 err = self._check_downloads(r, count) 291 if err: 292 errors.append(err) 293 else: 294 total_size = sum([s['size_download'] for s in r.stats]) 295 samples.append(total_size / r.duration.total_seconds()) 296 profiles.append(r.profile) 297 return Card.mk_mbs_cell(samples, profiles, errors) 298 299 def dl_serial(self, url: str, count: int, nsamples: int = 1): 300 samples = [] 301 errors = [] 302 profiles = [] 303 url = f'{url}?[0-{count - 1}]' 304 self.info('serial...') 305 for _ in range(nsamples): 306 curl = self.mk_curl_client() 307 r = curl.http_download(urls=[url], alpn_proto=self.protocol, 308 no_save=True, 309 with_headers=False, with_profile=True) 310 err = self._check_downloads(r, count) 311 if err: 312 errors.append(err) 313 else: 314 total_size = sum([s['size_download'] for s in r.stats]) 315 samples.append(total_size / r.duration.total_seconds()) 316 profiles.append(r.profile) 317 return Card.mk_mbs_cell(samples, profiles, errors) 318 319 def dl_parallel(self, url: str, count: int, nsamples: int = 1): 320 samples = [] 321 errors = [] 322 profiles = [] 323 max_parallel = self._download_parallel if self._download_parallel > 0 else count 324 url = f'{url}?[0-{count - 1}]' 325 self.info('parallel...') 326 for _ in range(nsamples): 327 curl = self.mk_curl_client() 328 r = curl.http_download(urls=[url], alpn_proto=self.protocol, 329 no_save=True, 330 with_headers=False, 331 with_profile=True, 332 extra_args=[ 333 '--parallel', 334 '--parallel-max', str(max_parallel) 335 ]) 336 err = self._check_downloads(r, count) 337 if err: 338 errors.append(err) 339 else: 340 total_size = sum([s['size_download'] for s in r.stats]) 341 samples.append(total_size / r.duration.total_seconds()) 342 profiles.append(r.profile) 343 return Card.mk_mbs_cell(samples, profiles, errors) 344 345 def downloads(self, count: int, fsizes: List[int], meta: Dict[str, Any]) -> Dict[str, Any]: 346 nsamples = meta['samples'] 347 max_parallel = self._download_parallel if self._download_parallel > 0 else count 348 cols = ['size'] 349 if not self._download_parallel: 350 cols.append('single') 351 if count > 1: 352 cols.append(f'serial({count})') 353 if count > 1: 354 cols.append(f'parallel({count}x{max_parallel})') 355 rows = [] 356 for fsize in fsizes: 357 row = [{ 358 'val': fsize, 359 'sval': Card.fmt_size(fsize) 360 }] 361 self.info(f'{row[0]["sval"]} downloads...') 362 url = f'https://{self.env.domain1}:{self.server_port}/score{row[0]["sval"]}.data' 363 if 'single' in cols: 364 row.append(self.dl_single(url=url, nsamples=nsamples)) 365 if count > 1: 366 if 'single' in cols: 367 row.append(self.dl_serial(url=url, count=count, nsamples=nsamples)) 368 row.append(self.dl_parallel(url=url, count=count, nsamples=nsamples)) 369 rows.append(row) 370 self.info('done.\n') 371 return { 372 'meta': { 373 'title': f'Downloads from {meta["server"]}', 374 'count': count, 375 'max-parallel': max_parallel, 376 }, 377 'cols': cols, 378 'rows': rows, 379 } 380 381 def _check_uploads(self, r: ExecResult, count: int): 382 error = '' 383 if r.exit_code != 0: 384 error += f'exit={r.exit_code} ' 385 if r.exit_code != 0 or len(r.stats) != count: 386 error += f'stats={len(r.stats)}/{count} ' 387 fails = [s for s in r.stats if s['response_code'] != 200] 388 if len(fails) > 0: 389 error += f'{len(fails)} failed' 390 for f in fails: 391 error += f'[{f["response_code"]}]' 392 return error if len(error) > 0 else None 393 394 def ul_single(self, url: str, fpath: str, nsamples: int = 1): 395 samples = [] 396 errors = [] 397 profiles = [] 398 self.info('single...') 399 for _ in range(nsamples): 400 curl = self.mk_curl_client() 401 r = curl.http_put(urls=[url], fdata=fpath, alpn_proto=self.protocol, 402 with_headers=False, with_profile=True) 403 err = self._check_uploads(r, 1) 404 if err: 405 errors.append(err) 406 else: 407 total_size = sum([s['size_upload'] for s in r.stats]) 408 samples.append(total_size / r.duration.total_seconds()) 409 profiles.append(r.profile) 410 return Card.mk_mbs_cell(samples, profiles, errors) 411 412 def ul_serial(self, url: str, fpath: str, count: int, nsamples: int = 1): 413 samples = [] 414 errors = [] 415 profiles = [] 416 url = f'{url}?id=[0-{count - 1}]' 417 self.info('serial...') 418 for _ in range(nsamples): 419 curl = self.mk_curl_client() 420 r = curl.http_put(urls=[url], fdata=fpath, alpn_proto=self.protocol, 421 with_headers=False, with_profile=True) 422 err = self._check_uploads(r, count) 423 if err: 424 errors.append(err) 425 else: 426 total_size = sum([s['size_upload'] for s in r.stats]) 427 samples.append(total_size / r.duration.total_seconds()) 428 profiles.append(r.profile) 429 return Card.mk_mbs_cell(samples, profiles, errors) 430 431 def ul_parallel(self, url: str, fpath: str, count: int, nsamples: int = 1): 432 samples = [] 433 errors = [] 434 profiles = [] 435 max_parallel = self._download_parallel if self._download_parallel > 0 else count 436 url = f'{url}?id=[0-{count - 1}]' 437 self.info('parallel...') 438 for _ in range(nsamples): 439 curl = self.mk_curl_client() 440 r = curl.http_put(urls=[url], fdata=fpath, alpn_proto=self.protocol, 441 with_headers=False, with_profile=True, 442 extra_args=[ 443 '--parallel', 444 '--parallel-max', str(max_parallel) 445 ]) 446 err = self._check_uploads(r, count) 447 if err: 448 errors.append(err) 449 else: 450 total_size = sum([s['size_upload'] for s in r.stats]) 451 samples.append(total_size / r.duration.total_seconds()) 452 profiles.append(r.profile) 453 return Card.mk_mbs_cell(samples, profiles, errors) 454 455 def uploads(self, count: int, fsizes: List[int], meta: Dict[str, Any]) -> Dict[str, Any]: 456 nsamples = meta['samples'] 457 max_parallel = self._download_parallel if self._download_parallel > 0 else count 458 url = f'https://{self.env.domain2}:{self.server_port}/curltest/put' 459 cols = ['size', 'single'] 460 if count > 1: 461 cols.append(f'serial({count})') 462 cols.append(f'parallel({count}x{max_parallel})') 463 rows = [] 464 for fsize in fsizes: 465 row = [{ 466 'val': fsize, 467 'sval': Card.fmt_size(fsize) 468 }] 469 fname = f'upload{row[0]["sval"]}.data' 470 fpath = self._make_docs_file(docs_dir=self.env.gen_dir, 471 fname=fname, fsize=fsize) 472 473 self.info(f'{row[0]["sval"]} uploads...') 474 row.append(self.ul_single(url=url, fpath=fpath, nsamples=nsamples)) 475 if count > 1: 476 row.append(self.ul_serial(url=url, fpath=fpath, count=count, nsamples=nsamples)) 477 row.append(self.ul_parallel(url=url, fpath=fpath, count=count, nsamples=nsamples)) 478 rows.append(row) 479 self.info('done.\n') 480 return { 481 'meta': { 482 'title': f'Uploads to {meta["server"]}', 483 'count': count, 484 'max-parallel': max_parallel, 485 }, 486 'cols': cols, 487 'rows': rows, 488 } 489 490 def do_requests(self, url: str, count: int, max_parallel: int = 1, nsamples: int = 1): 491 samples = [] 492 errors = [] 493 profiles = [] 494 url = f'{url}?[0-{count - 1}]' 495 extra_args = [ 496 '-w', '%{response_code},\\n', 497 ] 498 if max_parallel > 1: 499 extra_args.extend([ 500 '--parallel', '--parallel-max', str(max_parallel) 501 ]) 502 self.info(f'{max_parallel}...') 503 for _ in range(nsamples): 504 curl = self.mk_curl_client() 505 r = curl.http_download(urls=[url], alpn_proto=self.protocol, no_save=True, 506 with_headers=False, with_profile=True, 507 with_stats=False, extra_args=extra_args) 508 if r.exit_code != 0: 509 errors.append(f'exit={r.exit_code}') 510 else: 511 samples.append(count / r.duration.total_seconds()) 512 non_200s = 0 513 for line in r.stdout.splitlines(): 514 if not line.startswith('200,'): 515 non_200s += 1 516 if non_200s > 0: 517 errors.append(f'responses != 200: {non_200s}') 518 profiles.append(r.profile) 519 return Card.mk_reqs_cell(samples, profiles, errors) 520 521 def requests(self, count: int, meta: Dict[str, Any]) -> Dict[str, Any]: 522 url = f'https://{self.env.domain1}:{self.server_port}/reqs10.data' 523 fsize = 10*1024 524 cols = ['size', 'total'] 525 rows = [] 526 mparallel = meta['request_parallels'] 527 cols.extend([f'{mp} max' for mp in mparallel]) 528 row = [{ 529 'val': fsize, 530 'sval': Card.fmt_size(fsize) 531 },{ 532 'val': count, 533 'sval': f'{count}', 534 }] 535 self.info('requests, max parallel...') 536 row.extend([self.do_requests(url=url, count=count, 537 max_parallel=mp, nsamples=meta["samples"]) 538 for mp in mparallel]) 539 rows.append(row) 540 self.info('done.\n') 541 return { 542 'meta': { 543 'title': f'Requests in parallel to {meta["server"]}', 544 'count': count, 545 }, 546 'cols': cols, 547 'rows': rows, 548 } 549 550 def score(self, 551 handshakes: bool = True, 552 downloads: Optional[List[int]] = None, 553 download_count: int = 50, 554 uploads: Optional[List[int]] = None, 555 upload_count: int = 50, 556 req_count=5000, 557 request_parallels=None, 558 nsamples: int = 1, 559 requests: bool = True): 560 self.info(f"scoring {self.protocol} against {self.server_descr}\n") 561 562 score = { 563 'meta': { 564 'curl_version': self.env.curl_version(), 565 'curl_V': self.env.curl_fullname(), 566 'curl_features': self.env.curl_features_string(), 567 'os': self.env.curl_os(), 568 'server': self.server_descr, 569 'samples': nsamples, 570 'date': f'{datetime.datetime.now(tz=datetime.timezone.utc).isoformat()}', 571 } 572 } 573 if self.protocol == 'h3': 574 score['meta']['protocol'] = 'h3' 575 if not self.env.have_h3_curl(): 576 raise ScoreCardError('curl does not support HTTP/3') 577 for lib in ['ngtcp2', 'quiche', 'msh3', 'nghttp3']: 578 if self.env.curl_uses_lib(lib): 579 score['meta']['implementation'] = lib 580 break 581 elif self.protocol == 'h2': 582 score['meta']['protocol'] = 'h2' 583 if not self.env.have_h2_curl(): 584 raise ScoreCardError('curl does not support HTTP/2') 585 for lib in ['nghttp2']: 586 if self.env.curl_uses_lib(lib): 587 score['meta']['implementation'] = lib 588 break 589 elif self.protocol == 'h1' or self.protocol == 'http/1.1': 590 score['meta']['protocol'] = 'http/1.1' 591 score['meta']['implementation'] = 'native' 592 else: 593 raise ScoreCardError(f"unknown protocol: {self.protocol}") 594 595 if 'implementation' not in score['meta']: 596 raise ScoreCardError('did not recognized protocol lib') 597 score['meta']['implementation_version'] = Env.curl_lib_version(score['meta']['implementation']) 598 599 if handshakes: 600 score['handshakes'] = self.handshakes() 601 if downloads and len(downloads) > 0: 602 score['downloads'] = self.downloads(count=download_count, 603 fsizes=downloads, 604 meta=score['meta']) 605 if uploads and len(uploads) > 0: 606 score['uploads'] = self.uploads(count=upload_count, 607 fsizes=uploads, 608 meta=score['meta']) 609 if requests: 610 if request_parallels is None: 611 request_parallels = [1, 6, 25, 50, 100, 300] 612 score['meta']['request_parallels'] = request_parallels 613 score['requests'] = self.requests(count=req_count, meta=score['meta']) 614 return score 615 616 617 def run_score(args, protocol): 618 if protocol not in ['http/1.1', 'h1', 'h2', 'h3']: 619 sys.stderr.write(f'ERROR: protocol "{protocol}" not known to scorecard\n') 620 sys.exit(1) 621 if protocol == 'h1': 622 protocol = 'http/1.1' 623 624 handshakes = True 625 downloads = [1024 * 1024, 10 * 1024 * 1024, 100 * 1024 * 1024] 626 if args.download_sizes is not None: 627 downloads = [] 628 for x in args.download_sizes: 629 downloads.extend([Card.parse_size(s) for s in x.split(',')]) 630 631 uploads = [1024 * 1024, 10 * 1024 * 1024, 100 * 1024 * 1024] 632 if args.upload_sizes is not None: 633 uploads = [] 634 for x in args.upload_sizes: 635 uploads.extend([Card.parse_size(s) for s in x.split(',')]) 636 637 requests = True 638 request_parallels = None 639 if args.request_parallels: 640 request_parallels = [] 641 for x in args.request_parallels: 642 request_parallels.extend([int(s) for s in x.split(',')]) 643 644 645 if args.downloads or args.uploads or args.requests or args.handshakes: 646 handshakes = args.handshakes 647 if not args.downloads: 648 downloads = None 649 if not args.uploads: 650 uploads = None 651 requests = args.requests 652 653 test_httpd = protocol != 'h3' 654 test_caddy = protocol == 'h3' 655 if args.caddy or args.httpd: 656 test_caddy = args.caddy 657 test_httpd = args.httpd 658 659 rv = 0 660 env = Env() 661 env.setup() 662 env.test_timeout = None 663 httpd = None 664 nghttpx = None 665 caddy = None 666 try: 667 cards = [] 668 669 if args.remote: 670 m = re.match(r'^(.+):(\d+)$', args.remote) 671 if m is None: 672 raise ScoreCardError(f'unable to parse ip:port from --remote {args.remote}') 673 test_httpd = False 674 test_caddy = False 675 remote_addr = m.group(1) 676 remote_port = int(m.group(2)) 677 card = ScoreRunner(env=env, 678 protocol=protocol, 679 server_descr=f'Server at {args.remote}', 680 server_addr=remote_addr, 681 server_port=remote_port, 682 verbose=args.verbose, 683 curl_verbose=args.curl_verbose, 684 download_parallel=args.download_parallel, 685 with_dtrace=args.dtrace, 686 with_flame=args.flame) 687 cards.append(card) 688 689 if test_httpd: 690 httpd = Httpd(env=env) 691 assert httpd.exists(), \ 692 f'httpd not found: {env.httpd}' 693 httpd.clear_logs() 694 server_docs = httpd.docs_dir 695 assert httpd.initial_start() 696 if protocol == 'h3': 697 nghttpx = NghttpxQuic(env=env) 698 nghttpx.clear_logs() 699 assert nghttpx.initial_start() 700 server_descr = f'nghttpx: https:{env.h3_port} [backend httpd/{env.httpd_version()}]' 701 server_port = env.h3_port 702 else: 703 server_descr = f'httpd/{env.httpd_version()}' 704 server_port = env.https_port 705 card = ScoreRunner(env=env, 706 protocol=protocol, 707 server_descr=server_descr, 708 server_port=server_port, 709 verbose=args.verbose, curl_verbose=args.curl_verbose, 710 download_parallel=args.download_parallel, 711 with_dtrace=args.dtrace, 712 with_flame=args.flame) 713 card.setup_resources(server_docs, downloads) 714 cards.append(card) 715 716 if test_caddy and env.caddy: 717 backend = '' 718 if uploads and httpd is None: 719 backend = f' [backend httpd: {env.httpd_version()}]' 720 httpd = Httpd(env=env) 721 assert httpd.exists(), \ 722 f'httpd not found: {env.httpd}' 723 httpd.clear_logs() 724 assert httpd.initial_start() 725 caddy = Caddy(env=env) 726 caddy.clear_logs() 727 assert caddy.initial_start() 728 server_descr = f'Caddy/{env.caddy_version()} {backend}' 729 server_port = caddy.port 730 server_docs = caddy.docs_dir 731 card = ScoreRunner(env=env, 732 protocol=protocol, 733 server_descr=server_descr, 734 server_port=server_port, 735 verbose=args.verbose, curl_verbose=args.curl_verbose, 736 download_parallel=args.download_parallel, 737 with_dtrace=args.dtrace) 738 card.setup_resources(server_docs, downloads) 739 cards.append(card) 740 741 if args.start_only: 742 print('started servers:') 743 for card in cards: 744 print(f'{card.server_descr}') 745 sys.stderr.write('press [RETURN] to finish') 746 sys.stderr.flush() 747 sys.stdin.readline() 748 else: 749 for card in cards: 750 score = card.score(handshakes=handshakes, 751 downloads=downloads, 752 download_count=args.download_count, 753 uploads=uploads, 754 upload_count=args.upload_count, 755 req_count=args.request_count, 756 requests=requests, 757 request_parallels=request_parallels, 758 nsamples=args.samples) 759 if args.json: 760 print(json.JSONEncoder(indent=2).encode(score)) 761 else: 762 Card.print_score(score) 763 764 except ScoreCardError as ex: 765 sys.stderr.write(f"ERROR: {ex}\n") 766 rv = 1 767 except KeyboardInterrupt: 768 log.warning("aborted") 769 rv = 1 770 finally: 771 if caddy: 772 caddy.stop() 773 if nghttpx: 774 nghttpx.stop(wait_dead=False) 775 if httpd: 776 httpd.stop() 777 return rv 778 779 780 def print_file(filename): 781 if not os.path.exists(filename): 782 sys.stderr.write(f"ERROR: file does not exist {filename}\n") 783 return 1 784 with open(filename) as file: 785 data = json.load(file) 786 Card.print_score(data) 787 return 0 788 789 790 def main(): 791 parser = argparse.ArgumentParser(prog='scorecard', description=""" 792 Run a range of tests to give a scorecard for a HTTP protocol 793 'h3' or 'h2' implementation in curl. 794 """) 795 parser.add_argument("-v", "--verbose", action='count', default=1, 796 help="log more output on stderr") 797 parser.add_argument("-j", "--json", action='store_true', 798 default=False, help="print json instead of text") 799 parser.add_argument("--samples", action='store', type=int, metavar='number', 800 default=1, help="how many sample runs to make") 801 parser.add_argument("--httpd", action='store_true', default=False, 802 help="evaluate httpd server only") 803 parser.add_argument("--caddy", action='store_true', default=False, 804 help="evaluate caddy server only") 805 parser.add_argument("--curl-verbose", action='store_true', 806 default=False, help="run curl with `-v`") 807 parser.add_argument("--print", type=str, default=None, metavar='filename', 808 help="print the results from a JSON file") 809 parser.add_argument("protocol", default=None, nargs='?', 810 help="Name of protocol to score") 811 parser.add_argument("--start-only", action='store_true', default=False, 812 help="only start the servers") 813 parser.add_argument("--remote", action='store', type=str, 814 default=None, help="score against the remote server at <ip>:<port>") 815 parser.add_argument("--dtrace", action='store_true', 816 default = False, help="produce dtrace of curl") 817 parser.add_argument("--flame", action='store_true', 818 default = False, help="produce a flame graph on curl, implies --dtrace") 819 820 parser.add_argument("-H", "--handshakes", action='store_true', 821 default=False, help="evaluate handshakes only") 822 823 parser.add_argument("-d", "--downloads", action='store_true', 824 default=False, help="evaluate downloads") 825 parser.add_argument("--download-sizes", action='append', type=str, 826 metavar='numberlist', 827 default=None, help="evaluate download size") 828 parser.add_argument("--download-count", action='store', type=int, 829 metavar='number', 830 default=50, help="perform that many downloads") 831 parser.add_argument("--download-parallel", action='store', type=int, 832 metavar='number', default=0, 833 help="perform that many downloads in parallel (default all)") 834 835 parser.add_argument("-u", "--uploads", action='store_true', 836 default=False, help="evaluate uploads") 837 parser.add_argument("--upload-sizes", action='append', type=str, 838 metavar='numberlist', 839 default=None, help="evaluate upload size") 840 parser.add_argument("--upload-count", action='store', type=int, 841 metavar='number', default=50, 842 help="perform that many uploads") 843 844 parser.add_argument("-r", "--requests", action='store_true', 845 default=False, help="evaluate requests") 846 parser.add_argument("--request-count", action='store', type=int, 847 metavar='number', 848 default=5000, help="perform that many requests") 849 parser.add_argument("--request-parallels", action='append', type=str, 850 metavar='numberlist', 851 default=None, help="evaluate request with these max-parallel numbers") 852 args = parser.parse_args() 853 854 if args.verbose > 0: 855 console = logging.StreamHandler() 856 console.setLevel(logging.INFO) 857 console.setFormatter(logging.Formatter(logging.BASIC_FORMAT)) 858 logging.getLogger('').addHandler(console) 859 860 if args.print: 861 rv = print_file(args.print) 862 elif not args.protocol: 863 parser.print_usage() 864 rv = 1 865 else: 866 rv = run_score(args, args.protocol) 867 868 sys.exit(rv) 869 870 871 if __name__ == "__main__": 872 main()