diff options
author | ms <ms@taler.net> | 2021-10-11 12:44:44 +0200 |
---|---|---|
committer | ms <ms@taler.net> | 2021-10-11 12:44:44 +0200 |
commit | 63b4d0293ac50325b61924d46bdb99c300d0c197 (patch) | |
tree | 3ec8fa684b1a32a18a8dc2fcabc37c3b57dd6157 | |
parent | d31e9557e0e1b43922764073e0b0cf8738eb2be5 (diff) | |
download | deployment-63b4d0293ac50325b61924d46bdb99c300d0c197.tar.gz deployment-63b4d0293ac50325b61924d46bdb99c300d0c197.tar.bz2 deployment-63b4d0293ac50325b61924d46bdb99c300d0c197.zip |
First complete version of taler-local's prepare step.
-rwxr-xr-x | bin/WIP/taler-local | 274 |
1 files changed, 172 insertions, 102 deletions
diff --git a/bin/WIP/taler-local b/bin/WIP/taler-local index 5eb89d5..f838fa9 100755 --- a/bin/WIP/taler-local +++ b/bin/WIP/taler-local @@ -449,7 +449,6 @@ class TalerReverseProxy(Flask): def stop(self): self.proc.terminate() self.proc.join() - del os.environ["WERKZEUG_RUN_MAIN"] def get_log_filename(self): return self.logger.handlers[0].baseFilename @@ -460,27 +459,36 @@ class TalerReverseProxy(Flask): logging.basicConfig(filename=self.log_dir / "reverse-proxy.log", filemode="a") self.logger = logging.getLogger("werkzeug") self.logger.setLevel(logging.DEBUG) + os.environ["WERKZEUG_RUN_MAIN"] = "true" self.proc = Process( target=self.run, kwargs=dict(debug=False, port=8080, host="localhost") ) - os.environ["WERKZEUG_RUN_MAIN"] = "true" + + # This proxy does set Content-Length, as it often + # reassembles chunked responses. Future versions + # will echo back chunks as they arrive. + # Response.automatically_set_content_length = False + self.proc.start() atexit.register(self.stop) def index(self): return "I'm the Taler reverse proxy." + + def iter_response(self, r): + for line in r.iter_lines(): + self.logger.debug("Responding chunk " + line.decode("utf-8")) + yield line def proxy(self, component, path=""): - logger = logging.getLogger("werkzeug") - logger.setLevel(logging.DEBUG) s = Session() sockets_dir_urlenc = quote(str(self.unix_sockets_dir), safe="") uri = f"http+unix://%2F{sockets_dir_urlenc}%2F{component}.sock/{path}" raw_data = request.get_data() if len(request.args) > 0: uri += f"?{request.query_string.decode()}" - logger.debug("Proxying to: " + uri) + self.logger.debug("Proxying to: " + uri) try: if request.method == "GET": method = s.get @@ -490,23 +498,38 @@ class TalerReverseProxy(Flask): method = s.put if request.method == "DELETE": method = s.delete - resp = method(uri, headers=request.headers, data=raw_data) + resp = method( + uri, + headers=request.headers, + data=raw_data, + ) except Exception as error: - logger.error(error) + self.logger.error(error) return "Could not connect to upstream", 500 - logger.debug(f"Upstream responds: {resp.text}") + self.logger.debug(f"Upstream responds: {resp.text}") + headers = Headers() for k in resp.headers.keys(): + # This version does not send chunked responses, so + # remove that header when it's found. Content-Length + # will be added before sending the response. + if k == "Transfer-Encoding" and resp.headers[k] == "chunked": + continue + # The content was already decompressed by this proxy, therefore + # the following header would confuse the client. + if k == "Content-Encoding" and (resp.headers[k] in ["deflate", "gzip"]): + continue headers.set(k, resp.headers[k]) + return Response( response=resp.text, status=resp.status_code, - headers=headers + headers=headers, ) def get_app(self): return self - + @cli.command() def prepare(): @@ -522,12 +545,13 @@ def prepare(): proc.wait() def get_nexus_cli_env( - username, password, + username, + password, nexus_url ): env = os.environ.copy() env["LIBEUFIN_NEXUS_USERNAME"] = username - env["LIBEUFIN_NEXUS_PASSWORD"] = password, + env["LIBEUFIN_NEXUS_PASSWORD"] = password env["LIBEUFIN_NEXUS_URL"] = nexus_url return env @@ -539,11 +563,27 @@ def prepare(): env["LIBEUFIN_SANDBOX_PASSWORD"] = password return env - def get_sandbox_server_env(db_file): + # Will be extended to include a SANDBOX_ADMIN_TOKEN + # that will obsolete the 'superuser' flag of ordinary + # user accounts. Likewise, the client side will be + # modified to use such token. + def get_sandbox_server_env(db_file, base_url): env = os.environ.copy() env["LIBEUFIN_SANDBOX_DB_CONNECTION"] = f"jdbc:sqlite:{db_file}" + env["LIBEUFIN_SANDBOX_BASE_URL"] = base_url return env + def get_nexus_server_env(db_file, base_url): + env = os.environ.copy() + env["LIBEUFIN_NEXUS_DB_CONNECTION"] = f"jdbc:sqlite:{db_file}" + env["LIBEUFIN_NEXUS_BASE_URL"] = base_url + return env + + def urljoin_nodrop(a, b): + a = a + "/" # urljoin will drop extra trailing slashes. + b = "/".join([x for x in b.split("/") if x != ""]) # remove leading slashes. + return urljoin(a, b) + def prepare_nexus_account( ebics_url, ebics_host_id, @@ -567,7 +607,6 @@ def prepare(): ], env ).run() - # connect Command( [ @@ -589,9 +628,9 @@ def prepare(): [ "libeufin-cli", "connections", "import-bank-account", - "--offered-account", + "--offered-account-id", bank_account_name_sandbox, - "--nexus-bank-account", + "--nexus-bank-account-id", bank_account_name_nexus, bank_connection_name ], @@ -672,27 +711,36 @@ def prepare(): # IBANs IBAN_EXCHANGE = "EX00000000000000000000" - IBAN_MERCHANT = "ME00000000000000000001" IBAN_CUSTOMER = "WA00000000000000000000" + IBAN_MERCHANT_DEFAULT = "ME00000000000000000000" + IBAN_MERCHANT_DEMOSHOP = "ME00000000000000000001" + + # Instances + INSTANCES = { + "demoshop": IBAN_MERCHANT_DEMOSHOP + } # Credentials / API keys SANDBOX_ADMIN_USERNAME = "admin" SANDBOX_ADMIN_PASSWORD = "secret" EXCHANGE_NEXUS_USERNAME = "exchange-nexus-user" EXCHANGE_NEXUS_PASSWORD = "exchange-nexus-password" - FRONTENDS_API_TOKEN = "secret:secret" - TALER_MERCHANT_TOKEN = "secret:secret" # same as above? - + FRONTENDS_API_TOKEN = "secret-token:secret" + TALER_MERCHANT_TOKEN = "secret-token:secret" + + # Network locations + REV_PROXY_NETLOC = "localhost:8080" + # URLs - REV_PROXY = "http://localhost:8080" - SANDBOX_URL = REV_PROXY + "/sandbox" - NEXUS_URL = REV_PROXY + "/nexus" + REV_PROXY_URL = f"http://{REV_PROXY_NETLOC}" + SANDBOX_URL = REV_PROXY_URL + "/sandbox" + NEXUS_URL = REV_PROXY_URL + "/nexus" # EBICS EBICS_HOST_ID = "ebicsDeployedHost" EXCHANGE_EBICS_USER_ID = "exchangeEbicsUserId" EXCHANGE_EBICS_PARTNER_ID = "exchangeEbicsPartnerId" - EBICS_URL = REV_PROXY + "/sandbox/ebicsweb" + EBICS_URL = REV_PROXY_URL + "/sandbox/ebicsweb" # euFin NEXUS_DB_FILE = "/tmp/nexus.sqlite" @@ -711,8 +759,8 @@ def prepare(): fail("Could not find a command to execute") self.name = custom_name if custom_name else cmd[0] self.cmd = cmd - self.env=env - self.capture_stdout=capture_stdout + self.env = env + self.capture_stdout = capture_stdout self.log_dir = log_dir @staticmethod @@ -755,21 +803,27 @@ def prepare(): def cleanup(self): if not self.log_file.closed: + self.log_file.flush() self.log_file.close() + def do(self): if not self.log_dir.is_dir(): os.makedirs(self.log_dir) - self.log_file = open(self.log_dir / (self.name + ".log"), "a+") + try: + log_filename = self.log_dir / f"{self.name}.log" + self.log_file = open(log_filename, "a+") + except Exception as error: + fail(f"Could not open log file: {log_filename}: {error}") try: self.handle = Popen( - self.cmd, + self.cmd, # list stdin=DEVNULL, stdout=self.log_file if not self.capture_stdout else PIPE, stderr=self.log_file, env=self.env ) except Exception as error: - fail(f"Could not launch: {self.name}: {error}") + fail(f"Could not execute: {' '.join(self.cmd)} (a.k.a. {self.name}): {error}") atexit.register(self.stop) class ConfigFile: @@ -822,6 +876,7 @@ def prepare(): # taler.conf is generated. Only after that, we'll # specify the master key where it is missing; namely # in the merchant backend and exchange HTTP daemon sections. + def config_main( filename, outdir, @@ -914,6 +969,10 @@ def prepare(): "merchant-exchange-{}".format(currency), "exchange_base_url", rev_proxy_url + "/exchange/", ) + obj.cfg_put( + "merchant-exchange-{}".format(currency), + "currency", currency + ) obj.cfg_put("auditor", "serve", "unix") # FIXME: both below used? obj.cfg_put("auditor", "base_url", rev_proxy_url + "/auditor") @@ -978,7 +1037,7 @@ def prepare(): obj.cfg_put("taler", "currency", currency) obj.cfg_put("sync", "serve", "unix") obj.cfg_put("sync", "unixpath", str(unix_sockets_dir / "sync.sock")) - obj.cfg_put("sync", "apikey", f"Bearer {api_key}") + obj.cfg_put("sync", "apikey", f"Bearer secret-token:{api_key}") obj.cfg_put("sync", "annual_fee", f"{currency}:0.1") obj.cfg_put("sync", "fulfillment_url", "taler://fulfillment-success/") obj.cfg_put("sync", "payment_backend_url", rev_proxy_url + "merchant-backend/instances/Taler/") @@ -1002,13 +1061,13 @@ def prepare(): "payment_backend_url", rev_proxy_url + "merchant-backend/instances/anastasis/" ) - obj.cfg_put("anastasis-merchant-backend", "api_key", f"Bearer {api_key}") + obj.cfg_put("anastasis-merchant-backend", "api_key", f"Bearer secret-token:{api_key}") obj.cfg_put("authorization-question", "cost", f"{currency}:0") obj.cfg_put("authorization-question", "enabled", "yes") obj.cfg_write(outdir) print_nn("Ensure no service is running...") - if Command.is_serving(REV_PROXY + "/"): + if Command.is_serving(REV_PROXY_URL + "/"): fail("Reverse proxy is unexpectedly running!") if UNIX_SOCKETS_DIR.is_dir(): for left_socket in os.listdir(UNIX_SOCKETS_DIR): @@ -1033,10 +1092,10 @@ def prepare(): outdir=CFG_OUTDIR, unix_sockets_dir=UNIX_SOCKETS_DIR, currency=CURRENCY, - rev_proxy_url=REV_PROXY, + rev_proxy_url=REV_PROXY_URL, wire_method=WIRE_METHOD, exchange_wire_address=IBAN_EXCHANGE, - merchant_wire_address=IBAN_MERCHANT, + merchant_wire_address=IBAN_MERCHANT_DEFAULT, exchange_wire_gateway_username=EXCHANGE_NEXUS_USERNAME, exchange_wire_gateway_password=EXCHANGE_NEXUS_PASSWORD, frontend_api_key=FRONTENDS_API_TOKEN, @@ -1067,7 +1126,7 @@ def prepare(): unix_sockets_dir=UNIX_SOCKETS_DIR, currency=CURRENCY, api_key=FRONTENDS_API_TOKEN, - rev_proxy_url=REV_PROXY + rev_proxy_url=REV_PROXY_URL ) print(" OK") print_nn("Generating anastasis.conf...") @@ -1076,7 +1135,7 @@ def prepare(): outdir=CFG_OUTDIR, unix_sockets_dir=UNIX_SOCKETS_DIR, currency=CURRENCY, - rev_proxy_url=REV_PROXY, + rev_proxy_url=REV_PROXY_URL, api_key=FRONTENDS_API_TOKEN ) print(" OK") @@ -1090,7 +1149,7 @@ def prepare(): print_nn("Launching the reverse proxy...") rev_proxy = TalerReverseProxy(LOG_DIR, UNIX_SOCKETS_DIR) rev_proxy.start() - if not Command.is_serving(REV_PROXY + "/"): + if not Command.is_serving(REV_PROXY_URL + "/"): fail(f"Reverse proxy did not start correctly. \ Logs: {rev_proxy.get_log_filename()}" ) @@ -1113,7 +1172,7 @@ Logs: {rev_proxy.get_log_filename()}" "taler-exchange-httpd", "-c", CFG_OUTDIR / "taler.conf" ]).launch() - if not Command.is_serving(REV_PROXY + "/exchange/"): + if not Command.is_serving(REV_PROXY_URL + "/exchange/"): fail(f"Exchange did not start correctly. Logs: {exchange_handle.get_log_filename()}") print(" OK") print_nn("exchange-offline: signing key material...") @@ -1161,7 +1220,7 @@ Logs: {rev_proxy.get_log_filename()}" "taler-auditor-exchange", "-c", CFG_OUTDIR / "taler.conf", "-m", EXCHANGE_MASTER_PUB, - "-u", REV_PROXY + "/exchange/" + "-u", REV_PROXY_URL + "/exchange/" ], ).run() print(" OK") @@ -1183,25 +1242,20 @@ Logs: {rev_proxy.get_log_filename()}" "--password", SANDBOX_ADMIN_PASSWORD ], custom_name="sandbox-superuser", - env=get_sandbox_server_env(SANDBOX_DB_FILE) + env=get_sandbox_server_env(SANDBOX_DB_FILE, SANDBOX_URL) ).run() print(" OK") - # FIXME: tested until here. To proceed, euFin needs to - # bind to Unix domain sockets. - exit(1) - print_nn("Launching Sandbox...") sandbox_handle = Command( [ "libeufin-sandbox", "serve", "--with-unix-socket", UNIX_SOCKETS_DIR / "sandbox.sock", ], - env=get_sandbox_server_env(SANDBOX_DB_FILE) + env=get_sandbox_server_env(SANDBOX_DB_FILE, SANDBOX_URL) ).launch() if not Command.is_serving(SANDBOX_URL): fail(f"Sandbox did not start correctly. Logs: {sandbox_handle.get_log_filename()}") print(" OK") - print_nn("Make Sandbox EBICS host...") Command( [ @@ -1217,7 +1271,8 @@ Logs: {rev_proxy.get_log_filename()}" custom_name="sandbox-create-ebicshost", ).run() print(" OK") - + + print_nn("Create Exchange account at Sandbox...") prepare_sandbox_account( currency=CURRENCY, sandbox_url=SANDBOX_URL, @@ -1225,27 +1280,36 @@ Logs: {rev_proxy.get_log_filename()}" ebics_partner_id=EXCHANGE_EBICS_PARTNER_ID, ebics_user_id=EXCHANGE_EBICS_USER_ID, person_name="Exchange Owner", - bank_account_name="sandbox-account-exchange", + bank_account_name=EXCHANGE_BANK_ACCOUNT_SANDBOX, bank_account_iban=IBAN_EXCHANGE, env=get_sandbox_cli_env( SANDBOX_ADMIN_USERNAME, SANDBOX_ADMIN_PASSWORD ) ) - prepare_sandbox_account( - currency=CURRENCY, - sandbox_url=SANDBOX_URL, - ebics_host_id=EBICS_HOST_ID, - ebics_partner_id="unusedMerchantEbicsPartnerId", - ebics_user_id="unusedMerchantEbicsUserId", - person_name="Shop Owner", - bank_account_name="sandbox-account-merchant", - bank_account_iban=IBAN_MERCHANT, - env=get_sandbox_cli_env( - SANDBOX_ADMIN_USERNAME, - SANDBOX_ADMIN_PASSWORD + print(" OK") + + # Give each instance a Sandbox account (note: 'default') + # won't have one, as it should typically only manage other + # instances. + for instance_id, iban in INSTANCES.items(): + print_nn(f"Create account of {instance_id} at Sandbox...") + prepare_sandbox_account( + currency=CURRENCY, + sandbox_url=SANDBOX_URL, + ebics_host_id=EBICS_HOST_ID, + ebics_partner_id="unusedMerchantEbicsPartnerId", + ebics_user_id=f"unused{instance_id}EbicsUserId", + person_name=f"Shop Owner of {instance_id}", + bank_account_name=f"sandbox-account-{instance_id}", + bank_account_iban=iban, + env=get_sandbox_cli_env( + SANDBOX_ADMIN_USERNAME, + SANDBOX_ADMIN_PASSWORD + ) ) - ) + print(" OK") + print_nn("Create Customer account at Sandbox...") prepare_sandbox_account( currency=CURRENCY, sandbox_url=SANDBOX_URL, @@ -1260,6 +1324,7 @@ Logs: {rev_proxy.get_log_filename()}" SANDBOX_ADMIN_PASSWORD ) ) + print(" OK") print_nn("Make Nexus superuser ...") Command( [ @@ -1267,18 +1332,27 @@ Logs: {rev_proxy.get_log_filename()}" EXCHANGE_NEXUS_USERNAME, "--password", EXCHANGE_NEXUS_PASSWORD ], + env=get_nexus_server_env( + NEXUS_DB_FILE, + NEXUS_URL + ), custom_name="nexus-superuser", ).run() print(" OK") + print_nn("Launching Nexus...") - nexus_handle = Command("nexus", [ - "libeufin-nexus", "serve", - "--with-unix-socket", UNIX_SOCKETS_DIR / "nexus.sock" - ]).launch() + nexus_handle = Command( + [ + "libeufin-nexus", "serve", + "--with-unix-socket", UNIX_SOCKETS_DIR / "nexus.sock" + ], + env=get_nexus_server_env(NEXUS_DB_FILE, NEXUS_URL) + ).launch() if not Command.is_serving(NEXUS_URL): fail(f"Nexus did not start correctly. Logs: {nexus_handle.get_log_filename()}") print(" OK") - + + print_nn("Create Exchange account at Nexus...") prepare_nexus_account( ebics_url=EBICS_URL, ebics_host_id=EBICS_HOST_ID, @@ -1293,6 +1367,8 @@ Logs: {rev_proxy.get_log_filename()}" NEXUS_URL ) ) + print(" OK") + print_nn("Create Taler facade ...") Command( [ @@ -1322,8 +1398,7 @@ Logs: {rev_proxy.get_log_filename()}" response.raise_for_status() except Exception as error: fail(error) - - FACADE_URL=response.json().get("facade")[0].get("baseUrl") + FACADE_URL=response.json().get("facades")[0].get("baseUrl") print_nn("Terminating Nexus...") nexus_handle.stop() @@ -1331,7 +1406,7 @@ Logs: {rev_proxy.get_log_filename()}" print_nn("Terminating Sandbox...") sandbox_handle.stop() print(" OK") - + # Point the exchange to the facade. Command( [ @@ -1363,7 +1438,7 @@ Logs: {rev_proxy.get_log_filename()}" ], custom_name="specify-password-for-facade", ).run() - + ## Step 6: Set up merchant print_nn("Reset and init merchant database...") @@ -1382,13 +1457,13 @@ Logs: {rev_proxy.get_log_filename()}" ): auth_header = {"Authorization": f"Bearer {auth_token}"} resp = requests.get( - urljoin(backend_url, f"management/instances/{instance_id}"), + urljoin_nodrop(backend_url, f"management/instances/{instance_id}"), headers = auth_header ) req = dict( id=instance_id, name=f"Name of '{instance_id}'", - payto_uris=["payto://{wire_method}/{bank_hostname}/{merchant_wire_address}"], + payto_uris=[f"payto://{wire_method}/{bank_hostname}/{merchant_wire_address}"], address=dict(), jurisdiction=dict(), default_max_wire_fee=f"{currency}:1", @@ -1400,13 +1475,15 @@ Logs: {rev_proxy.get_log_filename()}" ) http_method = requests.post endpoint = "management/instances" + # Instance exists, patching it. if resp.status_code == 200: print(f"Patching instance '{instance_id}'") http_method = requests.patch endpoint = f"management/instances/{instance_id}" + resp = http_method( - urljoin(backend_url, endpoint), + urljoin_nodrop(backend_url, endpoint), json=req, headers = auth_header ) @@ -1415,59 +1492,52 @@ Logs: {rev_proxy.get_log_filename()}" print(f"Backend responds: {resp.status_code}/{resp.text}") fail() - print_nn("Start merchant (with TALER_MERCHANT_TOKEN into the env)...") + print_nn(f"Start merchant (with TALER_MERCHANT_TOKEN into the env)...") auth_env = os.environ.copy() auth_env["TALER_MERCHANT_TOKEN"] = TALER_MERCHANT_TOKEN merchant_handle = Command( ["taler-merchant-httpd", "-c", CFG_OUTDIR / "taler.conf"], env=auth_env ).launch() - if not Command.is_serving(REV_PROXY + "/merchant-backend"): + if not Command.is_serving(REV_PROXY_URL + "/merchant-backend/config"): fail(f"Merchant backend did not start correctly. Logs: {merchant_handle.get_log_filename()}") print(" OK") ensure_instance( currency=CURRENCY, instance_id="default", - backend_url = REV_PROXY + "/merchant-backend", - bank_hostname = REV_PROXY + "/sandbox", + backend_url = REV_PROXY_URL + "/merchant-backend", + bank_hostname = REV_PROXY_NETLOC + "/sandbox", wire_method = "sepa", - merchant_wire_address = "UNUSED_IBAN", # Won't pass Camt validation. + merchant_wire_address = IBAN_MERCHANT_DEFAULT, auth_token=FRONTENDS_API_TOKEN ) print_nn("Stopping the merchant...") merchant_handle.stop() print(" OK") + print_nn("Restarting the merchant WITHOUT the auth-token in the env...") merchant_handle.launch() - if not Command.is_serving(REV_PROXY + "/merchant-backend"): + if not Command.is_serving(REV_PROXY_URL + "/merchant-backend/config"): # check_running logs errors already. fail(f"Merchant backend did not re start correctly. Logs: {merchant_handle.get_log_filename()}") print(" OK") - print_nn("Creating the 'donations' instance...") - ensure_instance( - currency=CURRENCY, - instance_id="donations", - backend_url = REV_PROXY + "/merchant-backend", - bank_hostname = REV_PROXY + "/sandbox", - wire_method = "sepa", - merchant_wire_address = "UNUSED_IBAN", # Won't pass Camt validation. - auth_token=FRONTENDS_API_TOKEN - ) - print(" OK") - print_nn("Creating the 'gnunet' instance...") - ensure_instance( - currency=CURRENCY, - instance_id="gnunet", - backend_url = REV_PROXY + "/merchant-backend", - bank_hostname = REV_PROXY + "/sandbox", - wire_method = "sepa", - merchant_wire_address = IBAN_MERCHANT, - auth_token=FRONTENDS_API_TOKEN - ) - print(" OK") + + for instance_id, iban in INSTANCES.items(): + print_nn(f"Creating the {instance_id} instance...") + ensure_instance( + currency=CURRENCY, + instance_id=instance_id, + backend_url = REV_PROXY_URL + "/merchant-backend", + bank_hostname = REV_PROXY_NETLOC + "/sandbox", + wire_method = "sepa", + merchant_wire_address = iban, + auth_token=FRONTENDS_API_TOKEN + ) + print(" OK") + print_nn("Stopping the merchant backend...") - kill(merchant_handle) + merchant_handle.stop() print(" OK") print_nn("Stopping the reverse proxy...") rev_proxy.stop() |