talerconfig.py (20982B)
1 # (C) 2016, 2019 Taler Systems SA 2 # 3 # This library is free software; you can redistribute it and/or 4 # modify it under the terms of the GNU Lesser General Public 5 # License as published by the Free Software Foundation; either 6 # version 3 of the License, or (at your option) any later 7 # version. 8 # 9 # This library is distributed in the hope that it will be useful, 10 # but WITHOUT ANY WARRANTY; without even the implied warranty of 11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 # GNU Lesser General Public License for more details. 13 # 14 # You should have received a copy of the GNU Lesser General Public 15 # License along with this library; if not, write to the Free 16 # Software Foundation, Inc., 51 Franklin Street, Fifth Floor, 17 # Boston, MA 02110-1301 USA 18 # 19 # @author Florian Dold 20 # @author Marcello Stanisci 21 # @brief Parse GNUnet-style configurations in pure Python 22 23 import logging 24 import collections 25 import os 26 import weakref 27 import sys 28 import re 29 from typing import Callable, Any 30 31 LOGGER = logging.getLogger(__name__) 32 33 __all__ = ["TalerConfig"] 34 35 36 ## 37 # Exception class for a any configuration error. 38 class ConfigurationError(Exception): 39 pass 40 41 42 ## 43 # Exception class for malformed strings having with parameter 44 # expansion. 45 class ExpansionSyntaxError(Exception): 46 pass 47 48 49 ## 50 # Do shell-style parameter expansion. 51 # Supported syntax: 52 # - ${X} 53 # - ${X:-Y} 54 # - $X 55 # 56 # @param var entire config value that might contain a parameter 57 # to expand. 58 # @param getter function that is in charge of returning _some_ 59 # value to be used in place of the parameter to expand. 60 # Typically, the replacement is searched first under the 61 # PATHS section of the current configuration, or (if not 62 # found) in the environment. 63 # 64 # @return the expanded config value. 65 def expand(var: str, getter: Callable[[str], str]) -> str: 66 pos = 0 67 result = "" 68 while pos != -1: 69 start = var.find("$", pos) 70 if start == -1: 71 break 72 if var[start:].startswith("${"): 73 balance = 1 74 end = start + 2 75 while balance > 0 and end < len(var): 76 balance += {"{": 1, "}": -1}.get(var[end], 0) 77 end += 1 78 if balance != 0: 79 raise ExpansionSyntaxError("unbalanced parentheses") 80 piece = var[start + 2 : end - 1] 81 if piece.find(":-") > 0: 82 varname, alt = piece.split(":-", 1) 83 replace = getter(varname) 84 if replace is None: 85 replace = expand(alt, getter) 86 else: 87 varname = piece 88 replace = getter(varname) 89 if replace is None: 90 replace = var[start:end] 91 else: 92 end = start + 2 93 while end < len(var) and var[start + 1 : end + 1].isalnum(): 94 end += 1 95 varname = var[start + 1 : end] 96 replace = getter(varname) 97 if replace is None: 98 replace = var[start:end] 99 result = result + replace 100 pos = end 101 102 return result + var[pos:] 103 104 105 ## 106 # A configuration entry. 107 class Entry: 108 109 ## 110 # Init constructor. 111 # 112 # @param self the object itself. 113 # @param config reference to a configuration object - FIXME 114 # define "configuration object". 115 # @param section name of the config section where this entry 116 # got defined. 117 # @param option name of the config option associated with this 118 # entry. 119 # @param kwargs keyword arguments that hold the value / filename 120 # / line number of this current option. 121 def __init__(self, config, section: str, option: str, **kwargs) -> None: 122 self.value = kwargs.get("value") 123 self.filename = kwargs.get("filename") 124 self.lineno = kwargs.get("lineno") 125 self.section = section 126 self.option = option 127 self.config = weakref.ref(config) 128 129 ## 130 # XML representation of this entry. 131 # 132 # @param self the object itself. 133 # @return XML string holding all the relevant information 134 # for this entry. 135 def __repr__(self) -> str: 136 return "<Entry section=%s, option=%s, value=%s>" % ( 137 self.section, 138 self.option, 139 repr(self.value), 140 ) 141 142 ## 143 # Return the value for this entry, as is. 144 # 145 # @param self the object itself. 146 # @return the config value. 147 def __str__(self) -> Any: 148 return self.value 149 150 ## 151 # Return entry value, accepting defaults. 152 # 153 # @param self the object itself 154 # @param default default value to return if none was found. 155 # @param required indicate whether the value was required or not. 156 # If the value was required, but was not found, an exception 157 # is found. 158 # @param warn if True, outputs a warning message if the value was 159 # not found -- regardless of it being required or not. 160 # @return the value, or the given @a default, if not found. 161 def value_string(self, default=None, required=False, warn=False) -> str: 162 if required and self.value is None: 163 print( 164 "Missing required option '%s' in section '%s'" 165 % (self.option.upper(), self.section.upper()) 166 ) 167 sys.exit(1) 168 169 if self.value is None: 170 if warn: 171 if default is not None: 172 LOGGER.warning( 173 "Configuration is missing option '%s' in section '%s',\ 174 falling back to '%s'", 175 self.option, 176 self.section, 177 default, 178 ) 179 else: 180 LOGGER.warning( 181 "Configuration ** is missing option '%s' in section '%s'", 182 self.option.upper(), 183 self.section.upper(), 184 ) 185 return default 186 return self.value 187 188 ## 189 # Return entry value as a _int_. Raise exception if the 190 # value is not convertible to a integer. 191 # 192 # @param self the object itself 193 # @param default currently ignored. 194 # @param required currently ignored. 195 # @param warn currently ignored. 196 # @return the value, or the given @a default, if not found. 197 def value_int(self, default=None, required=False, warn=False) -> int: 198 value = self.value_string(default, required, warn) 199 if value is None: 200 return None 201 try: 202 return int(value) 203 except ValueError: 204 print( 205 "Expected number for option '%s' in section '%s'" 206 % (self.option.upper(), self.section.upper()) 207 ) 208 sys.exit(1) 209 210 ## 211 # Fetch value to substitute to expansion variables. 212 # 213 # @param self the object itself. 214 # @param key the value's name to lookup. 215 # @return the value, if found, None otherwise. 216 def _getsubst(self, key: str) -> Any: 217 value = self.config()["paths"][key].value 218 if value is not None: 219 return value 220 value = os.environ.get(key) 221 if value is not None: 222 return value 223 return None 224 225 ## 226 # Fetch the config value that should be a filename, 227 # taking care of invoking the variable-expansion logic first. 228 # 229 # @param self the object itself. 230 # @param default currently ignored. 231 # @param required currently ignored. 232 # @param warn currently ignored. 233 # @return the (expanded) filename. 234 def value_filename(self, default=None, required=False, warn=False) -> str: 235 value = self.value_string(default, required, warn) 236 if value is None: 237 return None 238 return expand(value, self._getsubst) 239 240 ## 241 # Give the filename and line number of this config entry. 242 # 243 # @param self this object. 244 # @return <filename>:<linenumber>, or "<unknown>" if one 245 # is not known. 246 def location(self) -> str: 247 if self.filename is None or self.lineno is None: 248 return "<unknown>" 249 return "%s:%s" % (self.filename, self.lineno) 250 251 252 ## 253 # Represent a section by inheriting from 'defaultdict'. 254 class OptionDict(collections.defaultdict): 255 256 ## 257 # Init constructor. 258 # 259 # @param self the object itself 260 # @param config the "config" object -- typically a @a TalerConfig instance. 261 # @param section_name the section name to assign to this object. 262 def __init__(self, config, section_name: str) -> None: 263 self.config = weakref.ref(config) 264 self.section_name = section_name 265 super().__init__() 266 267 ## 268 # Logic to run when a non-existent key is dereferenced. 269 # Just create and return a empty config @a Entry. Note 270 # that the freshly created entry will nonetheless put 271 # under the accessed key (that *does* become existent 272 # afterwards). 273 # 274 # @param self the object itself. 275 # @param key the key attempted to be accessed. 276 # @return the no-value entry. 277 def __missing__(self, key: str) -> Entry: 278 entry = Entry(self.config(), self.section_name, key) 279 self[key] = entry 280 return entry 281 282 ## 283 # Attempt to fetch one value from the object. 284 # 285 # @param self the object itself. 286 # @param chunk the key (?) that is tried to access. 287 # @return the object, if it exists, or a freshly created 288 # (empty) one, if it doesn't exist. 289 def __getitem__(self, chunk: str) -> Entry: 290 return super().__getitem__(chunk.lower()) 291 292 ## 293 # Set one value into the object. 294 # 295 # @param self the object itself. 296 # @param chunk key under which the value is going to be set. 297 # @param value value to set the @a chunk to. 298 def __setitem__(self, chunk: str, value: Entry) -> None: 299 super().__setitem__(chunk.lower(), value) 300 301 302 ## 303 # Collection of all the (@a OptionDict) sections. 304 class SectionDict(collections.defaultdict): 305 306 ## 307 # Automatically invoked when a missing section is 308 # dereferenced. It creates the missing - empty - section. 309 # 310 # @param self the object itself. 311 # @param key the dereferenced section name. 312 # @return the freshly created section. 313 def __missing__(self, key): 314 value = OptionDict(self, key) 315 self[key] = value 316 return value 317 318 ## 319 # Attempt to retrieve a section. 320 # 321 # @param self the object itself. 322 # @param chunk the section name. 323 def __getitem__(self, chunk: str) -> OptionDict: 324 return super().__getitem__(chunk.lower()) 325 326 ## 327 # Set a section. 328 # 329 # @param self the object itself. 330 # @param chunk the section name to set. 331 # @param value the value to set under that @a chunk. 332 def __setitem__(self, chunk: str, value: OptionDict) -> None: 333 super().__setitem__(chunk.lower(), value) 334 335 336 ## 337 # One loaded taler configuration, including base configuration 338 # files and included files. 339 class TalerConfig: 340 341 ## 342 # Init constructor.. 343 # 344 # @param self the object itself. 345 def __init__(self) -> None: 346 self.sections = SectionDict() # just plain dict 347 348 ## 349 # Load a configuration file, instantiating a config object. 350 # 351 # @param filename the filename where to load the configuration 352 # from. If None, it defaults "taler.conf". 353 # @param load_defaults if True, then defaults values are loaded 354 # (from canonical directories like "<prefix>/share/config.d/taler/") 355 # before the actual configuration file. This latter then 356 # can override some/all the defaults. 357 # @return the config object. 358 @staticmethod 359 def from_file(filename=None, load_defaults=True): 360 cfg = TalerConfig() 361 if filename is None: 362 xdg = os.environ.get("XDG_CONFIG_HOME") 363 if xdg: 364 filename = os.path.join(xdg, "taler.conf") 365 else: 366 filename = os.path.expanduser("~/.config/taler.conf") 367 logging.info("Loading default config: (%s)" % filename) 368 if load_defaults: 369 cfg.load_defaults() 370 cfg.load_file(os.path.expanduser(filename)) 371 return cfg 372 373 ## 374 # Get a string value from the config. 375 # 376 # @param self the config object itself. 377 # @param section the section to fetch the value from. 378 # @param option the value's option name. 379 # @param kwargs dict argument with instructions about 380 # the value retrieval logic. 381 # @return the wanted string (or a default / exception if 382 # a error occurs). 383 def value_string(self, section, option, **kwargs) -> str: 384 return self.sections[section][option].value_string( 385 kwargs.get("default"), kwargs.get("required"), kwargs.get("warn") 386 ) 387 388 ## 389 # Get a value from the config that should be a filename. 390 # The variable expansion for the path's components is internally managed. 391 # 392 # @param self the config object itself. 393 # @param section the section to fetch the value from. 394 # @param option the value's option name. 395 # @param kwargs dict argument with instructions about 396 # the value retrieval logic. 397 # @return the wanted filename (or a default / exception if 398 # a error occurs). 399 def value_filename(self, section, option, **kwargs) -> str: 400 return self.sections[section][option].value_filename( 401 kwargs.get("default"), kwargs.get("required"), kwargs.get("warn") 402 ) 403 404 ## 405 # Get a integer value from the config. 406 # 407 # @param self the config object itself. 408 # @param section the section to fetch the value from. 409 # @param option the value's option name. 410 # @param kwargs dict argument with instructions about 411 # the value retrieval logic. 412 # @return the wanted integer (or a default / exception if 413 # a error occurs). 414 def value_int(self, section, option, **kwargs) -> int: 415 return self.sections[section][option].value_int( 416 kwargs.get("default"), kwargs.get("required"), kwargs.get("warn") 417 ) 418 419 ## 420 # Load default values from canonical locations. 421 # 422 # @param self the object itself. 423 def load_defaults(self) -> None: 424 base_dir = os.environ.get("TALER_BASE_CONFIG") 425 if base_dir: 426 self.load_dir(base_dir) 427 return 428 prefix = os.environ.get("TALER_PREFIX") 429 if prefix: 430 tmp = os.path.split(os.path.normpath(prefix)) 431 if re.match("lib", tmp[1]): 432 prefix = tmp[0] 433 self.load_dir(os.path.join(prefix, "share/taler/config.d")) 434 return 435 LOGGER.warning("no base directory found") 436 437 ## 438 # Load configuration from environment variable 439 # TALER_CONFIG_FILE or from default location if the 440 # variable is not set. 441 # 442 # @param args currently unused. 443 # @param kwargs kwargs for subroutine @a from_file. 444 # @return freshly instantiated config object. 445 @staticmethod 446 def from_env(*args, **kwargs): 447 filename = os.environ.get("TALER_CONFIG_FILE") 448 return TalerConfig.from_file(filename, *args, **kwargs) 449 450 ## 451 # Load config values from _each_ file found in a directory. 452 # 453 # @param self the object itself. 454 # @param dirname the directory to crawl in the look for config files. 455 def load_dir(self, dirname) -> None: 456 try: 457 files = os.listdir(dirname) 458 except FileNotFoundError: 459 LOGGER.warning("can't read config directory '%s'", dirname) 460 return 461 for file in files: 462 if not file.endswith(".conf"): 463 continue 464 self.load_file(os.path.join(dirname, file)) 465 466 ## 467 # Load config values from a file. 468 # 469 # @param filename config file to take the values from. 470 def load_file(self, filename) -> None: 471 sections = self.sections 472 try: 473 with open(filename, "r") as file: 474 lineno = 0 475 current_section = None 476 for line in file: 477 lineno += 1 478 line = line.strip() 479 if line == "": 480 # empty line 481 continue 482 if line.startswith("#"): 483 # comment 484 continue 485 if line.startswith("@INLINE@"): 486 pair = line.split() 487 if 2 != len(pair): 488 LOGGER.error( 489 "invalid inlined config filename given ('%s')" % line 490 ) 491 continue 492 if pair[1].startswith("/"): 493 self.load_file(pair[1]) 494 else: 495 self.load_file( 496 os.path.join(os.path.dirname(filename), pair[1]) 497 ) 498 continue 499 if line.startswith("["): 500 if not line.endswith("]"): 501 LOGGER.error( 502 "invalid section header in line %s: %s", 503 lineno, 504 repr(line), 505 ) 506 section_name = line.strip("[]").strip().strip('"') 507 current_section = section_name 508 continue 509 if current_section is None: 510 LOGGER.error( 511 "option outside of section in line %s: %s", 512 lineno, 513 repr(line), 514 ) 515 continue 516 pair = line.split("=", 1) 517 if len(pair) != 2: 518 LOGGER.error( 519 "invalid option in line %s: %s", lineno, repr(line) 520 ) 521 key = pair[0].strip() 522 value = pair[1].strip() 523 if value.startswith('"'): 524 value = value[1:] 525 if not value.endswith('"'): 526 LOGGER.error( 527 "mismatched quotes in line %s: %s", lineno, repr(line) 528 ) 529 else: 530 value = value[:-1] 531 entry = Entry( 532 self.sections, 533 current_section, 534 key, 535 value=value, 536 filename=filename, 537 lineno=lineno, 538 ) 539 sections[current_section][key] = entry 540 except FileNotFoundError: 541 LOGGER.error("Configuration file (%s) not found", filename) 542 sys.exit(3) 543 544 ## 545 # Dump the textual representation of a config object. 546 # 547 # Format: 548 # 549 # [section] 550 # option = value # location (filename & line number) 551 # 552 # @param self the object itself, that will be dumped. 553 def dump(self) -> None: 554 for kv_section in list(self.sections.items()): 555 print("[%s]" % (kv_section[1].section_name,)) 556 for kv_option in list(kv_section[1].items()): 557 print( 558 "%s = %s # %s" 559 % (kv_option[1].option, kv_option[1].value, kv_option[1].location()) 560 ) 561 562 ## 563 # Return a whole section from this object. 564 # 565 # @param self the object itself. 566 # @param chunk name of the section to return. 567 # @return the section - note that if the section is 568 # not found, a empty one will created on the fly, 569 # then set under 'chunk', and returned. 570 def __getitem__(self, chunk: str) -> OptionDict: 571 if isinstance(chunk, str): 572 return self.sections[chunk] 573 raise TypeError("index must be string") 574 575 576 if __name__ == "__main__": 577 import argparse 578 579 PARSER = argparse.ArgumentParser() 580 PARSER.add_argument( 581 "--section", "-s", dest="section", default=None, metavar="SECTION" 582 ) 583 PARSER.add_argument("--option", "-o", dest="option", default=None, metavar="OPTION") 584 PARSER.add_argument("--config", "-c", dest="config", default=None, metavar="FILE") 585 PARSER.add_argument( 586 "--filename", "-f", dest="expand_filename", default=False, action="store_true" 587 ) 588 ARGS = PARSER.parse_args() 589 590 TC = TalerConfig.from_file(ARGS.config) 591 592 if ARGS.section is not None and ARGS.option is not None: 593 if ARGS.expand_filename: 594 X = TC.value_filename(ARGS.section, ARGS.option) 595 else: 596 X = TC.value_string(ARGS.section, ARGS.option) 597 if X is not None: 598 print(X) 599 else: 600 TC.dump()