httpd.py (24182B)
1 #!/usr/bin/env python3 2 # -*- coding: utf-8 -*- 3 #*************************************************************************** 4 # _ _ ____ _ 5 # Project ___| | | | _ \| | 6 # / __| | | | |_) | | 7 # | (__| |_| | _ <| |___ 8 # \___|\___/|_| \_\_____| 9 # 10 # Copyright (C) Daniel Stenberg, <daniel@haxx.se>, et al. 11 # 12 # This software is licensed as described in the file COPYING, which 13 # you should have received as part of this distribution. The terms 14 # are also available at https://curl.se/docs/copyright.html. 15 # 16 # You may opt to use, copy, modify, merge, publish, distribute and/or sell 17 # copies of the Software, and permit persons to whom the Software is 18 # furnished to do so, under the terms of the COPYING file. 19 # 20 # This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY 21 # KIND, either express or implied. 22 # 23 # SPDX-License-Identifier: curl 24 # 25 ########################################################################### 26 # 27 import inspect 28 import logging 29 import os 30 import shutil 31 import socket 32 import subprocess 33 from datetime import timedelta, datetime 34 from json import JSONEncoder 35 import time 36 from typing import List, Union, Optional, Dict 37 import copy 38 39 from .curl import CurlClient, ExecResult 40 from .env import Env 41 from .ports import alloc_ports_and_do 42 43 log = logging.getLogger(__name__) 44 45 46 class Httpd: 47 48 MODULES = [ 49 'log_config', 'logio', 'unixd', 'version', 'watchdog', 50 'authn_core', 'authn_file', 51 'authz_user', 'authz_core', 'authz_host', 52 'auth_basic', 'auth_digest', 53 'alias', 'env', 'filter', 'headers', 'mime', 'setenvif', 'negotiation', 54 'socache_shmcb', 55 'rewrite', 'http2', 'ssl', 'proxy', 'proxy_http', 'proxy_connect', 56 'brotli', 57 'mpm_event', 58 ] 59 COMMON_MODULES_DIRS = [ 60 '/usr/lib/apache2/modules', # debian 61 '/usr/libexec/apache2/', # macos 62 ] 63 64 MOD_CURLTEST = None 65 66 PORT_SPECS = { 67 'http': socket.SOCK_STREAM, 68 'https': socket.SOCK_STREAM, 69 'https-tcp-only': socket.SOCK_STREAM, 70 'proxy': socket.SOCK_STREAM, 71 'proxys': socket.SOCK_STREAM, 72 } 73 74 def __init__(self, env: Env): 75 self.env = env 76 self._apache_dir = os.path.join(env.gen_dir, 'apache') 77 self._run_dir = os.path.join(self._apache_dir, 'run') 78 self._lock_dir = os.path.join(self._apache_dir, 'locks') 79 self._docs_dir = os.path.join(self._apache_dir, 'docs') 80 self._conf_dir = os.path.join(self._apache_dir, 'conf') 81 self._conf_file = os.path.join(self._conf_dir, 'test.conf') 82 self._logs_dir = os.path.join(self._apache_dir, 'logs') 83 self._error_log = os.path.join(self._logs_dir, 'error_log') 84 self._tmp_dir = os.path.join(self._apache_dir, 'tmp') 85 self._basic_passwords = os.path.join(self._conf_dir, 'basic.passwords') 86 self._digest_passwords = os.path.join(self._conf_dir, 'digest.passwords') 87 self._mods_dir = None 88 self._auth_digest = True 89 self._proxy_auth_basic = False 90 # name used to lookup credentials for env.domain1 91 self._domain1_cred_name = env.domain1 92 self._extra_configs = {} 93 self._loaded_extra_configs = None 94 self._loaded_proxy_auth = None 95 self._loaded_domain1_cred_name = None 96 assert env.apxs 97 p = subprocess.run(args=[env.apxs, '-q', 'libexecdir'], 98 capture_output=True, text=True) 99 if p.returncode != 0: 100 raise Exception(f'{env.apxs} failed to query libexecdir: {p}') 101 self._mods_dir = p.stdout.strip() 102 if self._mods_dir is None: 103 raise Exception('apache modules dir cannot be found') 104 if not os.path.exists(self._mods_dir): 105 raise Exception(f'apache modules dir does not exist: {self._mods_dir}') 106 self._maybe_running = False 107 self.ports = {} 108 self._rmf(self._error_log) 109 self._init_curltest() 110 111 @property 112 def docs_dir(self): 113 return self._docs_dir 114 115 def clear_logs(self): 116 self._rmf(self._error_log) 117 118 def exists(self): 119 return os.path.exists(self.env.httpd) 120 121 def set_extra_config(self, domain: str, lines: Optional[Union[str, List[str]]]): 122 if lines is None: 123 self._extra_configs.pop(domain, None) 124 else: 125 self._extra_configs[domain] = lines 126 127 def reset_config(self): 128 self._extra_configs = {} 129 self.set_proxy_auth(False) 130 self._domain1_cred_name = self.env.domain1 131 132 def set_proxy_auth(self, active: bool): 133 self._proxy_auth_basic = active 134 135 def set_domain1_cred_name(self, name): 136 self._domain1_cred_name = name 137 138 def _run(self, args, intext=''): 139 env = os.environ.copy() 140 env['APACHE_RUN_DIR'] = self._run_dir 141 env['APACHE_RUN_USER'] = os.environ['USER'] 142 env['APACHE_LOCK_DIR'] = self._lock_dir 143 env['APACHE_CONFDIR'] = self._apache_dir 144 p = subprocess.run(args, stderr=subprocess.PIPE, stdout=subprocess.PIPE, 145 cwd=self.env.gen_dir, 146 input=intext.encode() if intext else None, 147 env=env) 148 start = datetime.now() 149 return ExecResult(args=args, exit_code=p.returncode, 150 stdout=p.stdout.decode().splitlines(), 151 stderr=p.stderr.decode().splitlines(), 152 duration=datetime.now() - start) 153 154 def _cmd_httpd(self, cmd: str): 155 args = [self.env.httpd, 156 "-d", self._apache_dir, 157 "-f", self._conf_file, 158 "-k", cmd] 159 return self._run(args=args) 160 161 def initial_start(self): 162 163 def startup(ports: Dict[str, int]) -> bool: 164 self.ports.update(ports) 165 if self.start(): 166 self.env.update_ports(ports) 167 return True 168 self.stop() 169 self.ports.clear() 170 return False 171 172 return alloc_ports_and_do(Httpd.PORT_SPECS, startup, 173 self.env.gen_root, max_tries=3) 174 175 def start(self): 176 # assure ports are allocated 177 for key, _ in Httpd.PORT_SPECS.items(): 178 assert self.ports[key] is not None 179 if self._maybe_running: 180 self.stop() 181 self._write_config() 182 with open(self._error_log, 'a') as fd: 183 fd.write('start of server\n') 184 with open(os.path.join(self._apache_dir, 'xxx'), 'a') as fd: 185 fd.write('start of server\n') 186 r = self._cmd_httpd('start') 187 if r.exit_code != 0 or len(r.stderr): 188 log.error(f'failed to start httpd: {r}') 189 self.stop() 190 return False 191 self._loaded_extra_configs = copy.deepcopy(self._extra_configs) 192 self._loaded_proxy_auth = self._proxy_auth_basic 193 return self.wait_live(timeout=timedelta(seconds=Env.SERVER_TIMEOUT)) 194 195 def stop(self): 196 r = self._cmd_httpd('stop') 197 self._loaded_extra_configs = None 198 self._loaded_proxy_auth = None 199 if r.exit_code == 0: 200 return self.wait_dead(timeout=timedelta(seconds=Env.SERVER_TIMEOUT)) 201 log.fatal(f'stopping httpd failed: {r}') 202 return r.exit_code == 0 203 204 def reload(self): 205 self._write_config() 206 r = self._cmd_httpd("graceful") 207 if r.exit_code != 0: 208 log.error(f'failed to reload httpd: {r}') 209 return False 210 self._loaded_extra_configs = None 211 self._loaded_proxy_auth = None 212 if r.exit_code != 0: 213 log.error(f'failed to reload httpd: {r}') 214 self._loaded_extra_configs = copy.deepcopy(self._extra_configs) 215 self._loaded_proxy_auth = self._proxy_auth_basic 216 return self.wait_live(timeout=timedelta(seconds=Env.SERVER_TIMEOUT)) 217 218 def reload_if_config_changed(self): 219 if self._maybe_running and \ 220 self._loaded_extra_configs == self._extra_configs and \ 221 self._loaded_proxy_auth == self._proxy_auth_basic and \ 222 self._loaded_domain1_cred_name == self._domain1_cred_name: 223 return True 224 return self.reload() 225 226 def wait_dead(self, timeout: timedelta): 227 curl = CurlClient(env=self.env, run_dir=self._tmp_dir) 228 try_until = datetime.now() + timeout 229 while datetime.now() < try_until: 230 r = curl.http_get(url=f'http://{self.env.domain1}:{self.ports["http"]}/') 231 if r.exit_code != 0: 232 self._maybe_running = False 233 return True 234 time.sleep(.1) 235 log.debug(f"Server still responding after {timeout}") 236 return False 237 238 def wait_live(self, timeout: timedelta): 239 curl = CurlClient(env=self.env, run_dir=self._tmp_dir, 240 timeout=timeout.total_seconds()) 241 try_until = datetime.now() + timeout 242 while datetime.now() < try_until: 243 r = curl.http_get(url=f'http://{self.env.domain1}:{self.ports["http"]}/') 244 if r.exit_code == 0: 245 self._maybe_running = True 246 return True 247 time.sleep(.1) 248 log.error(f"Server still not responding after {timeout}") 249 return False 250 251 def _rmf(self, path): 252 if os.path.exists(path): 253 return os.remove(path) 254 255 def _mkpath(self, path): 256 if not os.path.exists(path): 257 return os.makedirs(path) 258 259 def _write_config(self): 260 domain1 = self.env.domain1 261 domain1brotli = self.env.domain1brotli 262 creds1 = self.env.get_credentials(self._domain1_cred_name) 263 assert creds1 # convince pytype this isn't None 264 self._loaded_domain1_cred_name = self._domain1_cred_name 265 domain2 = self.env.domain2 266 creds2 = self.env.get_credentials(domain2) 267 assert creds2 # convince pytype this isn't None 268 exp_domain = self.env.expired_domain 269 exp_creds = self.env.get_credentials(exp_domain) 270 assert exp_creds # convince pytype this isn't None 271 proxy_domain = self.env.proxy_domain 272 proxy_creds = self.env.get_credentials(proxy_domain) 273 assert proxy_creds # convince pytype this isn't None 274 self._mkpath(self._conf_dir) 275 self._mkpath(self._docs_dir) 276 self._mkpath(self._logs_dir) 277 self._mkpath(self._tmp_dir) 278 self._mkpath(os.path.join(self._docs_dir, 'two')) 279 with open(os.path.join(self._docs_dir, 'data.json'), 'w') as fd: 280 data = { 281 'server': f'{domain1}', 282 } 283 fd.write(JSONEncoder().encode(data)) 284 with open(os.path.join(self._docs_dir, 'two/data.json'), 'w') as fd: 285 data = { 286 'server': f'{domain2}', 287 } 288 fd.write(JSONEncoder().encode(data)) 289 if self._proxy_auth_basic: 290 with open(self._basic_passwords, 'w') as fd: 291 fd.write('proxy:$apr1$FQfeInbs$WQZbODJlVg60j0ogEIlTW/\n') 292 if self._auth_digest: 293 with open(self._digest_passwords, 'w') as fd: 294 fd.write('test:restricted area:57123e269fd73d71ae0656594e938e2f\n') 295 self._mkpath(os.path.join(self.docs_dir, 'restricted/digest')) 296 with open(os.path.join(self.docs_dir, 'restricted/digest/data.json'), 'w') as fd: 297 fd.write('{"area":"digest"}\n') 298 with open(self._conf_file, 'w') as fd: 299 for m in self.MODULES: 300 if os.path.exists(os.path.join(self._mods_dir, f'mod_{m}.so')): 301 fd.write(f'LoadModule {m}_module "{self._mods_dir}/mod_{m}.so"\n') 302 if Httpd.MOD_CURLTEST is not None: 303 fd.write(f'LoadModule curltest_module "{Httpd.MOD_CURLTEST}"\n') 304 conf = [ # base server config 305 f'ServerRoot "{self._apache_dir}"', 306 'DefaultRuntimeDir logs', 307 'PidFile httpd.pid', 308 f'ServerName {self.env.tld}', 309 f'ErrorLog {self._error_log}', 310 f'LogLevel {self._get_log_level()}', 311 'StartServers 4', 312 'ReadBufferSize 16000', 313 'H2MinWorkers 16', 314 'H2MaxWorkers 256', 315 f'TypesConfig "{self._conf_dir}/mime.types', 316 'SSLSessionCache "shmcb:ssl_gcache_data(32000)"', 317 'AddEncoding x-gzip .gz .tgz .gzip', 318 'AddHandler type-map .var', 319 ] 320 conf.extend([f'Listen {port}' for _, port in self.ports.items()]) 321 322 if 'base' in self._extra_configs: 323 conf.extend(self._extra_configs['base']) 324 conf.extend([ # plain http host for domain1 325 f'<VirtualHost *:{self.ports["http"]}>', 326 f' ServerName {domain1}', 327 ' ServerAlias localhost', 328 f' DocumentRoot "{self._docs_dir}"', 329 ' Protocols h2c http/1.1', 330 ' H2Direct on', 331 ]) 332 conf.extend(self._curltest_conf(domain1)) 333 conf.extend([ 334 '</VirtualHost>', 335 '', 336 ]) 337 conf.extend([ # https host for domain1, h1 + h2 338 f'<VirtualHost *:{self.ports["https"]}>', 339 f' ServerName {domain1}', 340 ' ServerAlias localhost', 341 ' Protocols h2 http/1.1', 342 ' SSLEngine on', 343 f' SSLCertificateFile {creds1.cert_file}', 344 f' SSLCertificateKeyFile {creds1.pkey_file}', 345 f' DocumentRoot "{self._docs_dir}"', 346 ]) 347 conf.extend(self._curltest_conf(domain1)) 348 if domain1 in self._extra_configs: 349 conf.extend(self._extra_configs[domain1]) 350 conf.extend([ 351 '</VirtualHost>', 352 '', 353 ]) 354 conf.extend([ # https host for domain1, h1 + h2, tcp only 355 f'<VirtualHost *:{self.ports["https-tcp-only"]}>', 356 f' ServerName {domain1}', 357 ' ServerAlias localhost', 358 ' Protocols h2 http/1.1', 359 ' SSLEngine on', 360 f' SSLCertificateFile {creds1.cert_file}', 361 f' SSLCertificateKeyFile {creds1.pkey_file}', 362 f' DocumentRoot "{self._docs_dir}"', 363 ]) 364 conf.extend(self._curltest_conf(domain1)) 365 if domain1 in self._extra_configs: 366 conf.extend(self._extra_configs[domain1]) 367 conf.extend([ 368 '</VirtualHost>', 369 '', 370 ]) 371 # Alternate to domain1 with BROTLI compression 372 conf.extend([ # https host for domain1, h1 + h2 373 f'<VirtualHost *:{self.ports["https"]}>', 374 f' ServerName {domain1brotli}', 375 ' Protocols h2 http/1.1', 376 ' SSLEngine on', 377 f' SSLCertificateFile {creds1.cert_file}', 378 f' SSLCertificateKeyFile {creds1.pkey_file}', 379 f' DocumentRoot "{self._docs_dir}"', 380 ' SetOutputFilter BROTLI_COMPRESS', 381 ]) 382 conf.extend(self._curltest_conf(domain1)) 383 if domain1 in self._extra_configs: 384 conf.extend(self._extra_configs[domain1]) 385 conf.extend([ 386 '</VirtualHost>', 387 '', 388 ]) 389 conf.extend([ # plain http host for domain2 390 f'<VirtualHost *:{self.ports["http"]}>', 391 f' ServerName {domain2}', 392 ' ServerAlias localhost', 393 f' DocumentRoot "{self._docs_dir}"', 394 ' Protocols h2c http/1.1', 395 ]) 396 conf.extend(self._curltest_conf(domain2)) 397 conf.extend([ 398 '</VirtualHost>', 399 '', 400 ]) 401 self._mkpath(os.path.join(self._docs_dir, 'two')) 402 conf.extend([ # https host for domain2, no h2 403 f'<VirtualHost *:{self.ports["https"]}>', 404 f' ServerName {domain2}', 405 ' Protocols http/1.1', 406 ' SSLEngine on', 407 f' SSLCertificateFile {creds2.cert_file}', 408 f' SSLCertificateKeyFile {creds2.pkey_file}', 409 f' DocumentRoot "{self._docs_dir}/two"', 410 ]) 411 conf.extend(self._curltest_conf(domain2)) 412 if domain2 in self._extra_configs: 413 conf.extend(self._extra_configs[domain2]) 414 conf.extend([ 415 '</VirtualHost>', 416 '', 417 ]) 418 conf.extend([ # https host for domain2, no h2, tcp only 419 f'<VirtualHost *:{self.ports["https-tcp-only"]}>', 420 f' ServerName {domain2}', 421 ' Protocols http/1.1', 422 ' SSLEngine on', 423 f' SSLCertificateFile {creds2.cert_file}', 424 f' SSLCertificateKeyFile {creds2.pkey_file}', 425 f' DocumentRoot "{self._docs_dir}/two"', 426 ]) 427 conf.extend(self._curltest_conf(domain2)) 428 if domain2 in self._extra_configs: 429 conf.extend(self._extra_configs[domain2]) 430 conf.extend([ 431 '</VirtualHost>', 432 '', 433 ]) 434 self._mkpath(os.path.join(self._docs_dir, 'expired')) 435 conf.extend([ # https host for expired domain 436 f'<VirtualHost *:{self.ports["https"]}>', 437 f' ServerName {exp_domain}', 438 ' Protocols h2 http/1.1', 439 ' SSLEngine on', 440 f' SSLCertificateFile {exp_creds.cert_file}', 441 f' SSLCertificateKeyFile {exp_creds.pkey_file}', 442 f' DocumentRoot "{self._docs_dir}/expired"', 443 ]) 444 conf.extend(self._curltest_conf(exp_domain)) 445 if exp_domain in self._extra_configs: 446 conf.extend(self._extra_configs[exp_domain]) 447 conf.extend([ 448 '</VirtualHost>', 449 '', 450 ]) 451 conf.extend([ # http forward proxy 452 f'<VirtualHost *:{self.ports["proxy"]}>', 453 f' ServerName {proxy_domain}', 454 ' Protocols h2c http/1.1', 455 ' ProxyRequests On', 456 ' H2ProxyRequests On', 457 ' ProxyVia On', 458 f' AllowCONNECT {self.ports["http"]} {self.ports["https"]}', 459 ]) 460 conf.extend(self._get_proxy_conf()) 461 conf.extend([ 462 '</VirtualHost>', 463 '', 464 ]) 465 conf.extend([ # https forward proxy 466 f'<VirtualHost *:{self.ports["proxys"]}>', 467 f' ServerName {proxy_domain}', 468 ' Protocols h2 http/1.1', 469 ' SSLEngine on', 470 f' SSLCertificateFile {proxy_creds.cert_file}', 471 f' SSLCertificateKeyFile {proxy_creds.pkey_file}', 472 ' ProxyRequests On', 473 ' H2ProxyRequests On', 474 ' ProxyVia On', 475 f' AllowCONNECT {self.ports["http"]} {self.ports["https"]}', 476 ]) 477 conf.extend(self._get_proxy_conf()) 478 conf.extend([ 479 '</VirtualHost>', 480 '', 481 ]) 482 483 fd.write("\n".join(conf)) 484 with open(os.path.join(self._conf_dir, 'mime.types'), 'w') as fd: 485 fd.write("\n".join([ 486 'text/plain txt', 487 'text/html html', 488 'application/json json', 489 'application/x-gzip gzip', 490 'application/x-gzip gz', 491 '' 492 ])) 493 494 def _get_proxy_conf(self): 495 if self._proxy_auth_basic: 496 return [ 497 ' <Proxy "*">', 498 ' AuthType Basic', 499 ' AuthName "Restricted Proxy"', 500 ' AuthBasicProvider file', 501 f' AuthUserFile "{self._basic_passwords}"', 502 ' Require user proxy', 503 ' </Proxy>', 504 ] 505 else: 506 return [ 507 ' <Proxy "*">', 508 ' Require ip 127.0.0.1', 509 ' </Proxy>', 510 ] 511 512 def _get_log_level(self): 513 if self.env.verbose > 3: 514 return 'trace2' 515 if self.env.verbose > 2: 516 return 'trace1' 517 if self.env.verbose > 1: 518 return 'debug' 519 return 'info' 520 521 def _curltest_conf(self, servername) -> List[str]: 522 lines = [] 523 if Httpd.MOD_CURLTEST is not None: 524 lines.extend([ 525 ' Redirect 302 /data.json.302 /data.json', 526 ' Redirect 301 /curltest/echo301 /curltest/echo', 527 ' Redirect 302 /curltest/echo302 /curltest/echo', 528 ' Redirect 303 /curltest/echo303 /curltest/echo', 529 ' Redirect 307 /curltest/echo307 /curltest/echo', 530 ' <Location /curltest/sslinfo>', 531 ' SSLOptions StdEnvVars', 532 ' SetHandler curltest-sslinfo', 533 ' </Location>', 534 ' <Location /curltest/echo>', 535 ' SetHandler curltest-echo', 536 ' </Location>', 537 ' <Location /curltest/put>', 538 ' SetHandler curltest-put', 539 ' </Location>', 540 ' <Location /curltest/tweak>', 541 ' SetHandler curltest-tweak', 542 ' </Location>', 543 ' Redirect 302 /tweak /curltest/tweak', 544 ' <Location /curltest/1_1>', 545 ' SetHandler curltest-1_1-required', 546 ' </Location>', 547 ' <Location /curltest/shutdown_unclean>', 548 ' SetHandler curltest-tweak', 549 ' SetEnv force-response-1.0 1', 550 ' </Location>', 551 ' SetEnvIf Request_URI "/shutdown_unclean" ssl-unclean=1', 552 ' RewriteEngine on', 553 ' RewriteRule "^/curltest/put-redir-301$" "/curltest/put" [R=301]', 554 ' RewriteRule "^/curltest/put-redir-302$" "/curltest/put" [R=302]', 555 ' RewriteRule "^/curltest/put-redir-307$" "/curltest/put" [R=307]', 556 ' RewriteRule "^/curltest/put-redir-308$" "/curltest/put" [R=308]', 557 ]) 558 if self._auth_digest: 559 lines.extend([ 560 f' <Directory {self.docs_dir}/restricted/digest>', 561 ' AuthType Digest', 562 ' AuthName "restricted area"', 563 f' AuthDigestDomain "https://{servername}"', 564 ' AuthBasicProvider file', 565 f' AuthUserFile "{self._digest_passwords}"', 566 ' Require valid-user', 567 ' </Directory>', 568 569 ]) 570 return lines 571 572 def _init_curltest(self): 573 if Httpd.MOD_CURLTEST is not None: 574 return 575 local_dir = os.path.dirname(inspect.getfile(Httpd)) 576 out_dir = os.path.join(self.env.gen_dir, 'mod_curltest') 577 out_source = os.path.join(out_dir, 'mod_curltest.c') 578 if not os.path.exists(out_dir): 579 os.mkdir(out_dir) 580 if not os.path.exists(out_source): 581 shutil.copy(os.path.join(local_dir, 'mod_curltest/mod_curltest.c'), out_source) 582 p = subprocess.run([ 583 self.env.apxs, '-c', out_source 584 ], capture_output=True, cwd=out_dir) 585 rv = p.returncode 586 if rv != 0: 587 log.error(f"compiling mod_curltest failed: {p.stderr}") 588 raise Exception(f"compiling mod_curltest failed: {p.stderr}") 589 Httpd.MOD_CURLTEST = os.path.join(out_dir, '.libs/mod_curltest.so')