diff options
author | ms <ms@taler.net> | 2021-09-26 23:44:43 +0200 |
---|---|---|
committer | ms <ms@taler.net> | 2021-09-26 23:44:43 +0200 |
commit | 78a499ecc87544b5ca95acf1d6addd05c7153cf5 (patch) | |
tree | 00650298c8bddfc5df766dd9edc02b9b1483021c | |
parent | e6397062b4e228b57a310bf877a9a5eaaa43e900 (diff) | |
download | deployment-78a499ecc87544b5ca95acf1d6addd05c7153cf5.tar.gz deployment-78a499ecc87544b5ca95acf1d6addd05c7153cf5.tar.bz2 deployment-78a499ecc87544b5ca95acf1d6addd05c7153cf5.zip |
taler-local: prepare key material
-rwxr-xr-x | bin/WIP/taler-local | 292 |
1 files changed, 208 insertions, 84 deletions
diff --git a/bin/WIP/taler-local b/bin/WIP/taler-local index 6863e90..af7d633 100755 --- a/bin/WIP/taler-local +++ b/bin/WIP/taler-local @@ -15,6 +15,7 @@ # You should have received a copy of the GNU General Public License # along with GNU Taler. If not, see <https://www.gnu.org/licenses/>. +import shutil import atexit import click import types @@ -24,12 +25,14 @@ import os.path import subprocess import time import random +import logging from os import listdir from os.path import isdir, join from pathlib import Path from dataclasses import dataclass from typing import List, Callable from shutil import copy +from multiprocessing import Process from string import ascii_letters, ascii_uppercase from sys import exit from urllib.parse import urljoin @@ -38,8 +41,12 @@ import requests from collections import OrderedDict import errno from pathlib import Path -from subprocess import check_call as Popen, DEVNULL +from subprocess import Popen, DEVNULL, PIPE from datetime import datetime +from requests_unixsocket import Session +from flask import Flask, request, Response +from werkzeug.datastructures import Headers +from werkzeug.exceptions import HTTPException @dataclass class Repo: @@ -320,6 +327,7 @@ def get_stale_repos(repos: List[Repo]) -> List[Repo]: if not s.exists(): timestamps[r.name] = time.time() stale.append(r) + continue ts = timestamps[r.name] = s.stat().st_mtime for dep in r.deps: if timestamps[dep] > ts: @@ -334,7 +342,9 @@ def get_stale_repos(repos: List[Repo]) -> List[Repo]: of the repositories to _exclude_ from compilation", default="") def build(without_repos) -> None: + """Build the deployment from source.""" + exclude = split_repos_list(without_repos) # Get the repositories names from the source directory repos_names = get_repos_names() @@ -385,47 +395,132 @@ def checkout_repos(repos: List[Repo]): def bootstrap(repos) -> None: """Clone all the specified repositories.""" - home = Path.home() reposList = split_repos_list(repos) checkout_repos(load_repos(reposList)) +class TalerReverseProxy(Flask): + def stop(self): + self.proc.terminate() + self.proc.join() + del os.environ["WERKZEUG_RUN_MAIN"] + + def start(self): + logging.basicConfig(filename="/tmp/reverse-proxy.log", filemode="a") + logger = logging.getLogger("werkzeug") + logger.setLevel(logging.DEBUG) + self.proc = Process( + target=self.run, + kwargs=dict(debug=False, port=8080, host="localhost") + ) + os.environ["WERKZEUG_RUN_MAIN"] = "true" + self.proc.start() + atexit.register(self.stop) + + def index(self): + return "I'm the Taler reverse proxy." + + def proxy(self, component, path=""): + logger = logging.getLogger("werkzeug") + logger.setLevel(logging.DEBUG) + s = Session() + uri = f"http+unix://%2Ftmp%2F{component}.sock/{path}" + raw_data = request.get_data() + if len(request.args) > 0: + uri += f"?{request.query_string.decode()}" + logger.debug("Proxying to: " + uri) + try: + if request.method == "GET": + method = s.get + if request.method == "POST": + method = s.post + if request.method == "PUT": + method = s.put + if request.method == "DELETE": + method = s.delete + resp = method(uri, headers=request.headers, data=raw_data) + except Exception as error: + logger.error(error) + return "Could not connect to upstream", 500 + logger.debug(f"Upstream responds: {resp.text}") + headers = Headers() + for k in resp.headers.keys(): + headers.set(k, resp.headers[k]) + return Response( + response=resp.text, + status=resp.status_code, + headers=headers + ) + + def get_app(self): + return self + + def __init__(self): + super().__init__("taler-proxy") + all_methods = [ + "GET", "POST", "HEAD", + "DELETE", "OPTIONS", "PUT" + ] + self.add_url_rule("/", view_func=self.index) + self.add_url_rule("/<component>", view_func=self.proxy, methods=all_methods) + self.add_url_rule("/<component>/", view_func=self.proxy, methods=all_methods) + self.add_url_rule("/<component>/<path:path>", view_func=self.proxy, methods=all_methods) + @cli.command() def prepare(): + + """Generate configuration, run-time blobs, instances, euFin accounts.""" + + def fail(reason=None): + if reason: + print("ERROR: " + reason) + print("See logs in /tmp/$component.log") + exit(1) + # Print No Newline. def print_nn(msg): - print(msg, env="") + print(msg, end="") # Runs a command synchronously. - def cmd(args, env=os.environ, custom_name=None): - handle = launch(args, env, custom_name) + def cmd( + args, env=os.environ, + custom_name=None, return_stdout=False + ): + handle = launch( + args, env=env, + custom_name=custom_name, + return_stdout=return_stdout + ) return_code = handle.wait() if return_code != 0: - print("Command: " + " ".join(args) + " failed, return code: " + str(return_code)) - print(f"See logs in /tmp/$command.log") - return False + print("\nCommand: " + " ".join(args) + " failed, return code: " + str(return_code)) + name = custom_name if custom_name else args[0] + print(f"See logs in /tmp/{name}.log") + exit(return_code) return handle - def kill(s): - s.terminate() - s.wait() + def kill(proc): + proc.terminate() + proc.wait() # Runs a command in the background. - def launch(cmd, env=os.environ, custom_name=None): + def launch(cmd, env=os.environ, custom_name=None, return_stdout=False): name = custom_name if custom_name else cmd[0] - log_file = open("/tmp/{name}.log", "a+") - handle = Popen( - cmd, - stdin=DEVNULL, - stdout=log_file, - stderr=log_file, - env=env - ) + log_file = open(f"/tmp/{name}.log", "a+") + try: + handle = Popen( + cmd, + stdin=DEVNULL, + stdout=log_file if not return_stdout else PIPE, + stderr=log_file, + env=env + ) + except Exception as error: + fail(f"Could not launch: {name}: {error}") atexit.register(lambda: kill(handle)) return handle def check_running(check_url): - print_nn(f"Checking {check_url} is served correctly...") for i in range(10): print_nn(".") try: @@ -434,11 +529,13 @@ def prepare(): # Raises if the request gets a non 200 OK. response.raise_for_status() except: - if i == 10: - print("ERROR: {name} is not running (correctly)") + if i == 9: + print("FAIL") return False - print(" OK") - return True + time.sleep(0.5) + continue + break + return True def get_nexus_cli_env( username, password, @@ -457,7 +554,12 @@ def prepare(): env["LIBEUFIN_SANDBOX_USERNAME"] = username env["LIBEUFIN_SANDBOX_PASSWORD"] = password return env - + + def get_sandbox_server_env(db_file): + env = os.environ.copy() + env["LIBEUFIN_SANDBOX_DB_CONNECTION"] = f"jdbc:sqlite:{db_file}" + return env + def prepare_nexus_account( ebics_url, ebics_host_id, @@ -606,11 +708,13 @@ def prepare(): currency, rev_proxy_url, wire_method, + exchange_master_pub, exchange_wire_address, merchant_wire_address, exchange_wire_gateway_username, exchange_wire_gateway_password, frontend_api_key, + taler_runtime_dir ): def coin( obj, @@ -637,10 +741,11 @@ def prepare(): obj.cfg_put(sec, "fee_deposit", currency + ":" + f_deposit) obj.cfg_put(sec, "rsa_keysize", rsa_keysize) - obj = ConfigFile("taler.conf") obj.cfg_put("paths", "TALER_DATA_HOME", "${HOME}/.taler-data") - obj.cfg_put("paths", "TALER_RUNTIME_DIR", "${HOME}/.taler-runtime") + if not os.path.isdir(taler_runtime_dir): + os.makedirs(taler_runtime_dir) + obj.cfg_put("paths", "TALER_RUNTIME_DIR", str(taler_runtime_dir)) obj.cfg_put("taler", "CURRENCY", currency) obj.cfg_put("taler", "CURRENCY_ROUND_UNIT", f"{currency}:0.01") @@ -689,6 +794,10 @@ def prepare(): "merchant-exchange-{}".format(currency), "exchange_base_url", rev_proxy_url + "/exchange/", ) + obj.cfg_put( + "merchant-exchange-{}".format(currency), + "master_key", exchange_master_pub, + ) obj.cfg_put("auditor", "serve", "unix") # FIXME: both below used? @@ -700,17 +809,18 @@ def prepare(): obj.cfg_put("taler-exchange-secmod-eddsa", "unixpath", "/tmp/exchange-secmod-eddsa.sock") obj.cfg_put("taler-exchange-secmod-rsa", "unixpath", "/tmp/exchange-secmod-rsa.sock") obj.cfg_put("taler-exchange-secmod-rsa", "sm_priv_key", - "/tmp/taler-data/taler-exchange-secmod-rsa/secmod-private-key" + "/tmp/.taler-data/taler-exchange-secmod-rsa/secmod-private-key" ) obj.cfg_put("exchange", "base_url", rev_proxy_url + "/exchange/") + obj.cfg_put("exchange", "master_public_key", exchange_master_pub) obj.cfg_put("exchange", "serve", "unix") obj.cfg_put("exchange", "unixpath", "/tmp/exchange.sock") obj.cfg_put("exchange", "terms_etag", "0") - obj.cfg_put("exchange", "terms_dir", "$HOME/local/share/taler-exchange/tos") + obj.cfg_put("exchange", "terms_dir", "$HOME/.local/share/taler-exchange/tos") obj.cfg_put("exchange", "privacy_etag", "0") - obj.cfg_put("exchange", "privacy_dir", "$HOME/local/share/taler-exchange/pp") + obj.cfg_put("exchange", "privacy_dir", "$HOME/.local/share/taler-exchange/pp") obj.cfg_put("exchangedb-postgres", "db_conn_str", "postgres:///taler") @@ -777,10 +887,14 @@ def prepare(): obj.cfg_put("authorization-question", "cost", f"{currency}:0") obj.cfg_put("authorization-question", "enabled", "yes") obj.cfg_write(outdir) - + CURRENCY = "EUR" WIRE_METHOD = "sepa" + + # Directories CFG_OUTDIR = Path.home() / ".config" + TALER_RUNTIME_DIR = Path.home() / ".taler-runtime" + TALER_DATA_DIR = Path.home() / ".taler-data" # IBANs IBAN_EXCHANGE = "EX00000000000000000000" @@ -814,17 +928,33 @@ def prepare(): EXCHANGE_BANK_CONNECTION = "exchange-ebics-connection" EXCHANGE_FACADE_NAME = "exchange-taler-facade" + print_nn("Remove stale data..") + if TALER_DATA_DIR.exists(): + shutil.rmtree(TALER_DATA_DIR) + if TALER_RUNTIME_DIR.exists(): + shutil.rmtree(TALER_RUNTIME_DIR) + print(" OK") + print_nn("Generate exchange's master key...") + res = cmd( + ["taler-exchange-offline", "setup"], + return_stdout=True + ) + EXCHANGE_MASTER_PUB = res.communicate()[0].decode("utf-8").rstrip() + print(" OK") + mc = config_main( "taler.conf", outdir=CFG_OUTDIR, currency=CURRENCY, rev_proxy_url=REV_PROXY, wire_method=WIRE_METHOD, + exchange_master_pub=EXCHANGE_MASTER_PUB, exchange_wire_address=IBAN_EXCHANGE, merchant_wire_address=IBAN_MERCHANT, exchange_wire_gateway_username=EXCHANGE_NEXUS_USERNAME, exchange_wire_gateway_password=EXCHANGE_NEXUS_PASSWORD, - frontend_api_key=FRONTENDS_API_TOKEN + frontend_api_key=FRONTENDS_API_TOKEN, + taler_runtime_dir=TALER_RUNTIME_DIR ) config_sync( "sync.conf", @@ -851,24 +981,32 @@ def prepare(): custom_name="remove-taler-data" ) print(" OK") - + + print_nn("Launching the reverse proxy...") + rev_proxy = TalerReverseProxy() + rev_proxy.start() + if not check_running(REV_PROXY + "/"): + fail("Reverse proxy did not start correctly.") + # Do check. + print(" OK") + print_nn("Launching the exchange RSA helper...") + exchange_rsa_handle = launch(["taler-exchange-secmod-rsa"]) + print(" OK") + print_nn("Launching the exchange EDDSA helper...") + exchange_eddsa_handle = launch(["taler-exchange-secmod-eddsa"]) + print(" OK") print_nn("Launching the exchange...") exchange_handle = launch(["taler-exchange-httpd"]) - if not check_running(REV_PROXY + "/exchange/keys"): - exit(1) + if not check_running(REV_PROXY + "/exchange/"): + fail("Exchange did not start correctly.") print(" OK") - print_nn("Launching the exchange RSA helper...") - exchange_rsa_handle = launch("exchange-rsa", ["taler-exchange-secmod-rsa"]) - print_nn("Launching the exchange RSA helper...") - exchange_eddsa_handle = launch("exchange-eddsa", ["taler-exchange-secmod-eddsa"]) - print_nn("exchange-offline: signing the exchange keys, hence testing the crypto helpers are correct...") + print_nn("exchange-offline: signing key material...") cmd(["taler-exchange-offline", "download", "sign", "upload"]) print(" OK") - PAYTO_URI=mc.sections["exchange-account-1"]["payto-uri"] - print_nn("exchange-offline: enabling {PAYTO_URI}...") + PAYTO_URI=mc.sections["exchange-account-1"]["payto_uri"] + print_nn(f"exchange-offline: enabling {PAYTO_URI}...") cmd(["taler-exchange-offline", "enable-account", PAYTO_URI, "upload"]) - print(" OK") - + print(" OK") # Set up wire fees for next 5 years NOW = datetime.now() YEAR = NOW.year @@ -878,7 +1016,7 @@ def prepare(): [ "taler-exchange-offline", "wire-fee", - year, + str(year), WIRE_METHOD, CURRENCY + ":0.01", CURRENCY + ":0.01", @@ -887,25 +1025,20 @@ def prepare(): custom_name="set-wire-fee" ) print(" OK") - print_nn("Getting exchange master public key via /keys..") - try: - response = requests.get(REV_PROXY + "/exchange/keys") - response.raise_for_status() - except: - EXCHANGE_MASTER_PUB = response.json().get("master_public_key") - print(" OK") print_nn("Stopping exchange HTTP daemon and crypto helpers...") kill(exchange_rsa_handle) kill(exchange_eddsa_handle) + kill(exchange_handle) print(" OK") print_nn("Add this exchange to the auditor...") cmd( [ "taler-auditor-exchange", "-m", EXCHANGE_MASTER_PUB, - "-u", REV_PROXY + "/exchange" + "-u", REV_PROXY + "/exchange/" ], ) + print(" OK") ## Step 4: Set up euFin print_nn("Resetting euFin databases...") try: @@ -924,17 +1057,22 @@ def prepare(): "--password", SANDBOX_ADMIN_PASSWORD ], custom_name="sandbox-superuser", + env=get_sandbox_server_env(SANDBOX_DB_FILE) ) print(" OK") print_nn("Launching Sandbox...") - handle_sandbox = launch("sandbox", [ - "libeufin-sandbox", "serve", - "--with-unix-socket", "/tmp/sandbox.sock" - ]) + handle_sandbox = launch( + [ + "libeufin-sandbox", "serve", + "--with-unix-socket", "/tmp/sandbox.sock", + ], + env=get_sandbox_server_env(SANDBOX_DB_FILE) + ) if not check_running(SANDBOX_URL): - exit(1) + fail("Sandbox did not start correctly") print(" OK") - + exit(22) + print_nn("Make Sandbox EBICS host...") cmd( [ @@ -1009,7 +1147,7 @@ def prepare(): "--with-unix-socket", "/tmp/nexus.sock" ]) if not check_running(NEXUS_URL): - exit(1) + fail("Nexus did not start correctly") print(" OK") prepare_nexus_account( @@ -1054,8 +1192,8 @@ def prepare(): ) response.raise_for_status() except Exception as error: - print(error) - exit(1) + fail(error) + FACADE_URL=response.json().get("facade")[0].get("baseUrl") print_nn("Terminating Nexus...") @@ -1065,17 +1203,6 @@ def prepare(): kill(handle_sandbox) print(" OK") - # Finish configuration now: - cmd( - [ - "taler-config", "-s" - f"merchant-exchange-{CURRENCY}", - "-o" "master_key", "-V", - EXCHANGE_MASTER_PUB - ], - custom_name="specify-exchange-pub-for-merchant", - ) - # Point the exchange to the facade. cmd( [ @@ -1086,7 +1213,6 @@ def prepare(): ], custom_name="specify-facade-url", ) - cmd( [ "taler-config", "-s" @@ -1151,7 +1277,7 @@ def prepare(): if resp.status_code < 200 or resp.status_code >= 300: print(f"Could not create (or patch) instance '{instance_id}'") print(f"Backend responds: {resp.status_code}/{resp.text}") - exit(1) + fail() print_nn("Start merchant (with TALER_MERCHANT_TOKEN into the env)...") auth_env = os.environ.copy() @@ -1161,15 +1287,13 @@ def prepare(): env=auth_env ) if not check_running("/merchant-backend"): - # check_running logs errors already. - exit(1) + fail("Merchant backend did not start correctly") print(" OK") ensure_instance( currency=CURRENCY, instance_id="default", backend_url = REV_PROXY + "/merchant-backend", - auth_token = TALER_MERCHANT_TOKEN, bank_hostname = REV_PROXY + "/sandbox", wire_method = "sepa", merchant_wire_address = "UNUSED_IBAN", # Won't pass Camt validation. @@ -1178,18 +1302,17 @@ def prepare(): print_nn("Stopping the merchant...") kill(merchant_handle) print(" OK") - print_nn("Restartin the merchant WITHOUT the auth-token in the env...") + print_nn("Restarting the merchant WITHOUT the auth-token in the env...") merchant_handle = launch(["taler-merchant-httpd"]) if not check_running("/merchant-backend"): # check_running logs errors already. - exit(1) + fail("Merchant backend did not start correctly") print(" OK") print_nn("Creating the 'donations' instance...") ensure_instance( currency=CURRENCY, instance_id="donations", backend_url = REV_PROXY + "/merchant-backend", - auth_token = TALER_MERCHANT_TOKEN, bank_hostname = REV_PROXY + "/sandbox", wire_method = "sepa", merchant_wire_address = "UNUSED_IBAN", # Won't pass Camt validation. @@ -1201,7 +1324,6 @@ def prepare(): currency=CURRENCY, instance_id="gnunet", backend_url = REV_PROXY + "/merchant-backend", - auth_token = TALER_MERCHANT_TOKEN, bank_hostname = REV_PROXY + "/sandbox", wire_method = "sepa", merchant_wire_address = IBAN_MERCHANT, @@ -1211,6 +1333,8 @@ def prepare(): print_nn("Stopping the merchant backend...") kill(merchant_handle) print(" OK") - + print_nn("Stopping the reverse proxy...") + rev_proxy.stop() + print(" OK") if __name__ == "__main__": cli() |