commit 45880ad633e9e7fc954233982f22eeed84b02556 parent b1d1a04590c18b57668b564fb91feb8af5a7d3c0 Author: Antoine A <> Date: Tue, 9 Jul 2024 15:14:55 +0200 common: centralize config logic, refactor repetitive code and clean code Diffstat:
19 files changed, 312 insertions(+), 407 deletions(-)
diff --git a/bank/src/main/kotlin/tech/libeufin/bank/Config.kt b/bank/src/main/kotlin/tech/libeufin/bank/Config.kt @@ -1,6 +1,6 @@ /* * This file is part of LibEuFin. - * Copyright (C) 2023 Taler Systems S.A. + * Copyright (C) 2023-2024 Taler Systems S.A. * LibEuFin is free software; you can redistribute it and/or modify * it under the terms of the GNU Affero General Public License as @@ -24,6 +24,7 @@ import org.slf4j.Logger import org.slf4j.LoggerFactory import tech.libeufin.common.* import tech.libeufin.common.db.DatabaseConfig +import tech.libeufin.bank.db.Database import java.nio.file.Path import java.time.Duration @@ -31,6 +32,7 @@ private val logger: Logger = LoggerFactory.getLogger("libeufin-bank") /** Configuration for libeufin-bank */ data class BankConfig( + private val cfg: TalerConfig, val name: String, val baseUrl: String?, val regionalCurrency: String, @@ -55,7 +57,18 @@ data class BankConfig( val gcAbortAfter: Duration, val gcCleanAfter: Duration, val gcDeleteAfter: Duration -) +) { + val dbCfg: DatabaseConfig by lazy { + val sect = cfg.section("libeufin-bankdb-postgres") + DatabaseConfig( + dbConnStr = sect.string("config").require(), + sqlDir = sect.path("sql_dir").require() + ) + } + val serverCfg: ServerConfig by lazy { + cfg.loadServerConfig("libeufin-bank") + } +} @Serializable data class ConversionRate ( @@ -71,17 +84,15 @@ data class ConversionRate ( val cashout_min_amount: TalerAmount, ) -fun talerConfig(configPath: Path?): TalerConfig = BANK_CONFIG_SOURCE.fromFile(configPath) +/** Load bank config at [configPath] */ +fun bankConfig(configPath: Path?): BankConfig = BANK_CONFIG_SOURCE.fromFile(configPath).loadBankConfig() -fun TalerConfig.loadDbConfig(): DatabaseConfig { - val section = section("libeufin-bankdb-postgres") - return DatabaseConfig( - dbConnStr = section.string("config").require(), - sqlDir = section.path("sql_dir").require() - ) +/** Run [lambda] with access to a database conn pool */ +suspend fun BankConfig.withDb(lambda: suspend (Database, BankConfig) -> Unit) { + Database(dbCfg, regionalCurrency, fiatCurrency).use { lambda(it, this) } } -fun TalerConfig.loadBankConfig(): BankConfig = section("libeufin-bank").run { +private fun TalerConfig.loadBankConfig(): BankConfig = section("libeufin-bank").run { val regionalCurrency = string("currency").require() var fiatCurrency: String? = null var fiatCurrencySpec: CurrencySpecification? = null @@ -116,6 +127,7 @@ fun TalerConfig.loadBankConfig(): BankConfig = section("libeufin-bank").run { val ZERO = TalerAmount.zero(regionalCurrency) val MAX = TalerAmount.max(regionalCurrency) BankConfig( + cfg = this@loadBankConfig, name = string("name").default("Taler Bank"), regionalCurrency = regionalCurrency, regionalCurrencySpec = currencySpecificationFor(regionalCurrency), @@ -166,4 +178,4 @@ private fun TalerConfigSection.loadCurrencySpecification(): CurrencySpecificatio num_fractional_trailing_zero_digits = number("fractional_trailing_zero_digits").require(), alt_unit_names = jsonMap("alt_unit_names").require() ) -} -\ No newline at end of file +} diff --git a/bank/src/main/kotlin/tech/libeufin/bank/Main.kt b/bank/src/main/kotlin/tech/libeufin/bank/Main.kt @@ -77,13 +77,12 @@ class BankDbInit : CliktCommand("Initialize the libeufin-bank database", name = ).flag() override fun run() = cliCmd(logger, common.log) { - val config = talerConfig(common.config) - val cfg = config.loadDbConfig() - val ctx = config.loadBankConfig() - pgDataSource(cfg.dbConnStr).dbInit(cfg, "libeufin-bank", reset) - Database(cfg, ctx.regionalCurrency, ctx.fiatCurrency).use { db -> + val cfg = bankConfig(common.config) + val dbCfg = cfg.dbCfg + pgDataSource(dbCfg.dbConnStr).dbInit(dbCfg, "libeufin-bank", reset) + cfg.withDb { db, cfg -> // Create admin account if missing - val res = createAdminAccount(db, ctx) // logs provided by the helper + val res = createAdminAccount(db, cfg) when (res) { AccountCreationResult.BonusBalanceInsufficient -> {} AccountCreationResult.LoginReuse -> {} @@ -100,36 +99,32 @@ class ServeBank : CliktCommand("Run libeufin-bank HTTP server", name = "serve") private val common by CommonOption() override fun run() = cliCmd(logger, common.log) { - val cfg = talerConfig(common.config) - val ctx = cfg.loadBankConfig() - val dbCfg = cfg.loadDbConfig() - val serverCfg = cfg.loadServerConfig("libeufin-bank") - Database(dbCfg, ctx.regionalCurrency, ctx.fiatCurrency).use { db -> - if (ctx.allowConversion) { + bankConfig(common.config).withDb { db, cfg -> + if (cfg.allowConversion) { logger.info("Ensure exchange account exists") - val info = db.account.bankInfo("exchange", ctx.payto) + val info = db.account.bankInfo("exchange", cfg.payto) if (info == null) { throw Exception("Exchange account missing: an exchange account named 'exchange' is required for conversion to be enabled") } else if (!info.isTalerExchange) { throw Exception("Account is not an exchange: an exchange account named 'exchange' is required for conversion to be enabled") } logger.info("Ensure conversion is enabled") - val sqlProcedures = Path("${dbCfg.sqlDir}/libeufin-conversion-setup.sql") + val sqlProcedures = Path("${cfg.dbCfg.sqlDir}/libeufin-conversion-setup.sql") if (!sqlProcedures.exists()) { throw Exception("Missing libeufin-conversion-setup.sql file") } db.conn { it.execSQLUpdate(sqlProcedures.readText()) } } else { logger.info("Ensure conversion is disabled") - val sqlProcedures = Path("${dbCfg.sqlDir}/libeufin-conversion-drop.sql") + val sqlProcedures = Path("${cfg.dbCfg.sqlDir}/libeufin-conversion-drop.sql") if (!sqlProcedures.exists()) { throw Exception("Missing libeufin-conversion-drop.sql file") } db.conn { it.execSQLUpdate(sqlProcedures.readText()) } // Remove conversion info from the database ? } - serve(serverCfg) { - corebankWebApp(db, ctx) + serve(cfg.serverCfg) { + corebankWebApp(db, cfg) } } } @@ -144,10 +139,7 @@ class ChangePw : CliktCommand("Change account password", name = "passwd") { ) override fun run() = cliCmd(logger, common.log) { - val cfg = talerConfig(common.config) - val ctx = cfg.loadBankConfig() - val dbCfg = cfg.loadDbConfig() - Database(dbCfg, ctx.regionalCurrency, ctx.fiatCurrency).use { db -> + bankConfig(common.config).withDb { db, _ -> val res = db.account.reconfigPassword(username, password, null, true) when (res) { AccountPatchAuthResult.UnknownAccount -> @@ -195,10 +187,7 @@ class EditAccount : CliktCommand( } override fun run() = cliCmd(logger, common.log) { - val cfg = talerConfig(common.config) - val ctx = cfg.loadBankConfig() - val dbCfg = cfg.loadDbConfig() - Database(dbCfg, ctx.regionalCurrency, ctx.fiatCurrency).use { db -> + bankConfig(common.config).withDb { db, cfg -> val req = AccountReconfiguration( name = name, is_taler_exchange = exchange, @@ -216,7 +205,7 @@ class EditAccount : CliktCommand( is Option.Some -> Option.Some(tmp.value) } ) - when (patchAccount(db, ctx, req, username, true, true)) { + when (patchAccount(db, cfg, req, username, true, true)) { AccountPatchResult.Success -> logger.info("Account '$username' edited") AccountPatchResult.UnknownAccount -> @@ -281,11 +270,7 @@ class CreateAccount : CliktCommand( override fun run() = cliCmd(logger, common.log) { // TODO support setting tan - val cfg = talerConfig(common.config) - val ctx = cfg.loadBankConfig() - val dbCfg = cfg.loadDbConfig() - - Database(dbCfg, ctx.regionalCurrency, ctx.fiatCurrency).use { db -> + bankConfig(common.config).withDb { db, cfg -> val req = json ?: options?.run { RegisterAccountRequest( username = username, @@ -304,7 +289,7 @@ class CreateAccount : CliktCommand( ) } req?.let { - when (val result = createAccount(db, ctx, req, true)) { + when (val result = createAccount(db, cfg, req, true)) { AccountCreationResult.BonusBalanceInsufficient -> throw Exception("Insufficient admin funds to grant bonus") AccountCreationResult.LoginReuse -> @@ -328,13 +313,9 @@ class GC : CliktCommand( private val common by CommonOption() override fun run() = cliCmd(logger, common.log) { - val cfg = talerConfig(common.config) - val ctx = cfg.loadBankConfig() - val dbCfg = cfg.loadDbConfig() - - Database(dbCfg, ctx.regionalCurrency, ctx.fiatCurrency).use { db -> + bankConfig(common.config).withDb { db, cfg -> logger.info("Run garbage collection") - db.gc.collect(Instant.now(), ctx.gcAbortAfter, ctx.gcCleanAfter, ctx.gcDeleteAfter) + db.gc.collect(Instant.now(), cfg.gcAbortAfter, cfg.gcCleanAfter, cfg.gcDeleteAfter) } } } diff --git a/bank/src/test/kotlin/helpers.kt b/bank/src/test/kotlin/helpers.kt @@ -60,19 +60,17 @@ fun setup( conf: String = "test.conf", lambda: suspend (Database, BankConfig) -> Unit ) = runBlocking { - val config = talerConfig(Path("conf/$conf")) - val dbCfg = config.loadDbConfig() - val ctx = config.loadBankConfig() - pgDataSource(dbCfg.dbConnStr).run { - dbInit(dbCfg, "libeufin-nexus", true) - dbInit(dbCfg, "libeufin-bank", true) + val cfg = bankConfig(Path("conf/$conf")) + pgDataSource(cfg.dbCfg.dbConnStr).run { + dbInit(cfg.dbCfg, "libeufin-nexus", true) + dbInit(cfg.dbCfg, "libeufin-bank", true) } - Database(dbCfg, ctx.regionalCurrency, ctx.fiatCurrency).use { - it.conn { conn -> - val sqlProcedures = Path("${dbCfg.sqlDir}/libeufin-conversion-setup.sql") + cfg.withDb { db, cfg -> + db.conn { conn -> + val sqlProcedures = Path("${cfg.dbCfg.sqlDir}/libeufin-conversion-setup.sql") conn.execSQLUpdate(sqlProcedures.readText()) } - lambda(it, ctx) + lambda(db, cfg) } } diff --git a/common/src/main/kotlin/Cli.kt b/common/src/main/kotlin/Cli.kt @@ -110,12 +110,14 @@ private class CliConfigGet(private val configSource: ConfigSource) : CliktComman val config = configSource.fromFile(common.config) val section = config.section(section) if (isPath) { - val res = section.path(option).orNull() - ?: throw Exception("option '$option' in section '$section' not found in config") + val res = requireNotNull(section.path(option).orNull()) { + "option '$option' in section '$section' not found in config" + } println(res) } else { - val res = section.string(option).orNull() - ?: throw Exception("option '$option' in section '$section' not found in config") + val res = requireNotNull(section.string(option).orNull()) { + "option '$option' in section '$section' not found in config" + } println(res) } } diff --git a/common/src/main/kotlin/TxMedatada.kt b/common/src/main/kotlin/TxMedatada.kt @@ -39,7 +39,6 @@ fun parseIncomingTxMetadata(subject: String): EddsaPublicKey { if (!matches.hasNext()) return null val match = matches.next() if (matches.hasNext()) { - //val count = matches.count() throw Exception("Found multiple reserve public key") } return EddsaPublicKey(match.value) diff --git a/common/src/main/kotlin/crypto/CryptoUtil.kt b/common/src/main/kotlin/crypto/CryptoUtil.kt @@ -51,18 +51,14 @@ object CryptoUtil { fun loadRSAPrivate(encodedPrivateKey: ByteArray): RSAPrivateCrtKey { val spec = PKCS8EncodedKeySpec(encodedPrivateKey) val priv = KeyFactory.getInstance("RSA").generatePrivate(spec) - if (priv !is RSAPrivateCrtKey) - throw Exception("wrong encoding") - return priv + return priv as RSAPrivateCrtKey } /** Load an RSA public key from its binary X509 encoding */ fun loadRSAPublic(encodedPublicKey: ByteArray): RSAPublicKey { val spec = X509EncodedKeySpec(encodedPublicKey) val pub = KeyFactory.getInstance("RSA").generatePublic(spec) - if (pub !is RSAPublicKey) - throw Exception("wrong encoding") - return pub + return pub as RSAPublicKey } /** Create an RSA public key from its components: [modulus] and [exponent] */ @@ -254,9 +250,7 @@ object CryptoUtil { /* Ready to decrypt */ val decryptedKeySpec: PKCS8EncodedKeySpec = data.getKeySpec(cipher) val priv = KeyFactory.getInstance("RSA").generatePrivate(decryptedKeySpec) - if (priv !is RSAPrivateCrtKey) - throw Exception("wrong encoding") - return priv + return priv as RSAPrivateCrtKey } fun hashStringSHA256(input: String): ByteArray = diff --git a/common/src/main/kotlin/db/DbPool.kt b/common/src/main/kotlin/db/DbPool.kt @@ -43,8 +43,8 @@ open class DbPool(cfg: DatabaseConfig, schema: String) : java.io.Closeable { val meta = con.metaData val majorVersion = meta.databaseMajorVersion val minorVersion = meta.databaseMinorVersion - if (majorVersion < MIN_VERSION) { - throw Exception("postgres version must be at least $MIN_VERSION.0 got $majorVersion.$minorVersion") + require(majorVersion >= MIN_VERSION) { + "postgres version must be at least $MIN_VERSION.0 got $majorVersion.$minorVersion" } checkMigrations(con, cfg, schema) } diff --git a/common/src/main/kotlin/db/config.kt b/common/src/main/kotlin/db/config.kt @@ -41,8 +41,8 @@ fun jdbcFromPg(pgConn: String): String { if (pgConn.startsWith("jdbc:")) { return pgConn } - if (!pgConn.startsWith("postgresql://") && !pgConn.startsWith("postgres://")) { - throw Exception("Not a Postgres connection string: $pgConn") + require(pgConn.startsWith("postgresql://") || pgConn.startsWith("postgres://")) { + "Not a Postgres connection string: $pgConn" } var maybeUnixSocket = false val uri = URI(pgConn) diff --git a/nexus/src/main/kotlin/tech/libeufin/nexus/Config.kt b/nexus/src/main/kotlin/tech/libeufin/nexus/Config.kt @@ -19,25 +19,46 @@ package tech.libeufin.nexus -import java.nio.file.Path import tech.libeufin.common.* +import tech.libeufin.common.db.* +import tech.libeufin.nexus.db.Database import tech.libeufin.nexus.ebics.Dialect +import java.nio.file.Path val NEXUS_CONFIG_SOURCE = ConfigSource("libeufin", "libeufin-nexus", "libeufin-nexus") class NexusFetchConfig(config: TalerConfig) { - val section = config.section("nexus-fetch") + private val section = config.section("nexus-fetch") val frequency = section.duration("frequency").require() + val frequencyRaw = section.string("frequency").require() val ignoreBefore = section.date("ignore_transactions_before").orNull() } +class NexusSubmitConfig(config: TalerConfig) { + private val section = config.section("nexus-submit") + val frequency = section.duration("frequency").require() + val frequencyRaw = section.string("frequency").require() +} + class ApiConfig(section: TalerConfigSection) { val authMethod = section.requireAuthMethod() } /** Configuration for libeufin-nexus */ -class NexusConfig(val config: TalerConfig) { - private val sect = config.section("nexus-ebics") +class NexusConfig internal constructor (private val cfg: TalerConfig) { + private val sect = cfg.section("nexus-ebics") + + val dbCfg: DatabaseConfig by lazy { + val sect = cfg.section("libeufin-nexusdb-postgres") + val configOption = sect.string("config") + DatabaseConfig( + dbConnStr = configOption.orNull() ?: cfg.section("nexus-postgres").string("config").orNull() ?: configOption.require(), + sqlDir = sect.path("sql_dir").require() + ) + } + val serverCfg: ServerConfig by lazy { + cfg.loadServerConfig("nexus-httpd") + } /** The bank's currency */ val currency = sect.string("currency").require() @@ -62,7 +83,9 @@ class NexusConfig(val config: TalerConfig) { /** Path where we store our private keys */ val clientPrivateKeysPath = sect.path("client_private_keys_file").require() - val fetch = NexusFetchConfig(config) + val fetch = NexusFetchConfig(cfg) + val submit = NexusSubmitConfig(cfg) + val dialect = sect.map("bank_dialect", "dialect", mapOf( "postfinance" to Dialect.postfinance, "gls" to Dialect.gls @@ -71,8 +94,8 @@ class NexusConfig(val config: TalerConfig) { "normal" to AccountType.normal, "exchange" to AccountType.exchange )).require() - val wireGatewayApiCfg = config.section("nexus-httpd-wire-gateway-api").apiConf() - val revenueApiCfg = config.section("nexus-httpd-revenue-api").apiConf() + val wireGatewayApiCfg = cfg.section("nexus-httpd-wire-gateway-api").apiConf() + val revenueApiCfg = cfg.section("nexus-httpd-revenue-api").apiConf() } fun NexusConfig.checkCurrency(amount: TalerAmount) { @@ -82,7 +105,7 @@ fun NexusConfig.checkCurrency(amount: TalerAmount) { ) } -fun TalerConfigSection.requireAuthMethod(): AuthMethod { +private fun TalerConfigSection.requireAuthMethod(): AuthMethod { return mapLambda("auth_method", "auth method", mapOf( "none" to { AuthMethod.None }, "bearer-token" to { @@ -92,7 +115,7 @@ fun TalerConfigSection.requireAuthMethod(): AuthMethod { )).require() } -fun TalerConfigSection.apiConf(): ApiConfig? { +private fun TalerConfigSection.apiConf(): ApiConfig? { val enabled = boolean("enabled").require() return if (enabled) { return ApiConfig(this) @@ -109,4 +132,15 @@ sealed interface AuthMethod { enum class AccountType { normal, exchange +} + +/** Load nexus config at [configPath] */ +fun nexusConfig(configPath: Path?): NexusConfig { + val config = NEXUS_CONFIG_SOURCE.fromFile(configPath) + return NexusConfig(config) +} + +/** Run [lambda] with access to a database conn pool */ +suspend fun NexusConfig.withDb(lambda: suspend (Database, NexusConfig) -> Unit) { + Database(dbCfg, currency).use { lambda(it, this) } } \ No newline at end of file diff --git a/nexus/src/main/kotlin/tech/libeufin/nexus/DbInit.kt b/nexus/src/main/kotlin/tech/libeufin/nexus/DbInit.kt @@ -40,7 +40,7 @@ class DbInit : CliktCommand("Initialize the libeufin-nexus database", name = "db ).flag() override fun run() = cliCmd(logger, common.log) { - val cfg = nexusConfig(common.config).dbConfig() + val cfg = nexusConfig(common.config).dbCfg pgDataSource(cfg.dbConnStr).dbInit(cfg, "libeufin-nexus", reset) } } \ No newline at end of file diff --git a/nexus/src/main/kotlin/tech/libeufin/nexus/EbicsFetch.kt b/nexus/src/main/kotlin/tech/libeufin/nexus/EbicsFetch.kt @@ -41,34 +41,6 @@ import kotlin.io.path.* import kotlin.time.toKotlinDuration /** - * Necessary data to perform a download. - */ -data class FetchContext( - /** - * Config handle. - */ - val cfg: NexusConfig, - /** - * HTTP client handle to reach the bank - */ - val httpClient: HttpClient, - /** - * EBICS subscriber private keys. - */ - val clientKeys: ClientPrivateKeysFile, - /** - * Bank public keys. - */ - val bankKeys: BankPublicKeysFile, - /** - * Start date of the returned documents. Only - * used in --transient mode. - */ - var pinnedStart: Instant?, - val fileLogger: FileLogger -) - -/** * Converts the 2-digits fraction value as given by the bank * (postfinance dialect), to the Taler 8-digit value (db representation). * @@ -278,11 +250,11 @@ private suspend fun ingestDocuments( * TODO update doc */ private suspend fun fetchDocuments( - db: Database, - ctx: FetchContext, - docs: List<EbicsDocument> + client: EbicsClient, + docs: List<EbicsDocument>, + pinnedStart: Instant?, ): Boolean { - val lastExecutionTime: Instant? = ctx.pinnedStart + val lastExecutionTime: Instant? = pinnedStart return docs.all { doc -> try { if (lastExecutionTime == null) { @@ -292,22 +264,17 @@ private suspend fun fetchDocuments( } // downloading the content val doc = doc.doc() - val order = ctx.cfg.dialect.downloadDoc(doc, false) - ebicsDownload( - ctx.httpClient, - ctx.cfg, - db, - ctx.clientKeys, - ctx.bankKeys, + val order = client.cfg.dialect.downloadDoc(doc, false) + client.download( order, lastExecutionTime, null ) { stream -> - val loggedStream = ctx.fileLogger.logFetch( + val loggedStream = client.fileLogger.logFetch( stream, doc == SupportedDocument.PAIN_002_LOGS ) - ingestDocuments(db, ctx.cfg, loggedStream, doc) + ingestDocuments(client.db, client.cfg, loggedStream, doc) } true } catch (e: Exception) { @@ -385,13 +352,13 @@ class EbicsFetch: CliktCommand("Fetches EBICS files") { override fun run() = cliCmd(logger, common.log) { nexusConfig(common.config).withDb { db, cfg -> val (clientKeys, bankKeys) = expectFullKeys(cfg) - val ctx = FetchContext( + val client = EbicsClient( cfg, httpClient(), + db, + FileLogger(ebicsLog), clientKeys, - bankKeys, - null, - FileLogger(ebicsLog) + bankKeys ) val docs = if (documents.isEmpty()) EbicsDocument.entries else documents.toList() if (transient) { @@ -401,18 +368,16 @@ class EbicsFetch: CliktCommand("Fetches EBICS files") { logger.debug("Pinning start date to: $pinnedStartVal") dateToInstant(pinnedStartVal) } else null - ctx.pinnedStart = pinnedStartArg - if (!fetchDocuments(db, ctx, docs)) { + if (!fetchDocuments(client, docs, pinnedStartArg)) { throw Exception("Failed to fetch documents") } } else { - val raw = cfg.config.section("nexus-fetch").string("frequency").require() - logger.debug("Running with a frequency of $raw") + logger.debug("Running with a frequency of ${cfg.fetch.frequencyRaw}") if (cfg.fetch.frequency == Duration.ZERO) { logger.warn("Long-polling not implemented, running therefore in transient mode") } do { - fetchDocuments(db, ctx, docs) + fetchDocuments(client, docs, null) delay(cfg.fetch.frequency.toKotlinDuration()) } while (cfg.fetch.frequency != Duration.ZERO) } diff --git a/nexus/src/main/kotlin/tech/libeufin/nexus/EbicsSetup.kt b/nexus/src/main/kotlin/tech/libeufin/nexus/EbicsSetup.kt @@ -182,6 +182,10 @@ class EbicsSetup: CliktCommand("Set up the EBICS subscriber") { private val generateRegistrationPdf by option( help = "Generates the PDF with the client public keys to send to the bank" ).flag(default = false) + private val ebicsLog by option( + "--debug-ebics", + help = "Log EBICS content at SAVEDIR", + ) /** * This function collects the main steps of setting up an EBICS access. */ @@ -234,7 +238,14 @@ class EbicsSetup: CliktCommand("Set up the EBICS subscriber") { logger.info("Doing administrative request HKD") try { cfg.withDb { db, cfg -> - ebicsDownload(client, cfg, db, clientKeys, bankKeys, EbicsOrder.V3("HKD"), null, null) { stream -> + EbicsClient( + cfg, + client, + db, + FileLogger(ebicsLog), + clientKeys, + bankKeys + ).download(EbicsOrder.V3("HKD"), null, null) { stream -> val hkd = EbicsAdministrative.parseHKD(stream) val account = hkd.account // TODO parse and check more information diff --git a/nexus/src/main/kotlin/tech/libeufin/nexus/EbicsSubmit.kt b/nexus/src/main/kotlin/tech/libeufin/nexus/EbicsSubmit.kt @@ -32,29 +32,6 @@ import java.util.* import kotlin.time.toKotlinDuration /** - * Groups useful parameters to submit pain.001 via EBICS. - */ -data class SubmissionContext( - /** - * HTTP connection handle. - */ - val httpClient: HttpClient, - /** - * Configuration handle. - */ - val cfg: NexusConfig, - /** - * Subscriber EBICS private keys. - */ - val clientPrivateKeysFile: ClientPrivateKeysFile, - /** - * Bank EBICS public keys. - */ - val bankPublicKeysFile: BankPublicKeysFile, - val fileLogger: FileLogger -) - -/** * Takes the initiated payment data as it was returned from the * database, sanity-checks it, gets the pain.001 from the helper * function and finally submits it via EBICS to the bank. @@ -64,7 +41,7 @@ data class SubmissionContext( * TODO update doc */ private suspend fun submitInitiatedPayment( - ctx: SubmissionContext, + client: EbicsClient, payment: InitiatedPayment ): String { val creditAccount = try { @@ -78,23 +55,18 @@ private suspend fun submitInitiatedPayment( throw e // TODO handle payto error } - val xml = createPain001( requestUid = payment.requestUid, initiationTimestamp = payment.initiationTime, amount = payment.amount, creditAccount = creditAccount, - debitAccount = ctx.cfg.account, + debitAccount = client.cfg.account, wireTransferSubject = payment.wireTransferSubject, - dialect = ctx.cfg.dialect + dialect = client.cfg.dialect ) - ctx.fileLogger.logSubmit(xml) - return doEbicsUpload( - ctx.httpClient, - ctx.cfg, - ctx.clientPrivateKeysFile, - ctx.bankPublicKeysFile, - ctx.cfg.dialect.directDebit(), + client.fileLogger.logSubmit(xml) + return client.upload( + client.cfg.dialect.directDebit(), xml ) } @@ -110,19 +82,16 @@ private suspend fun submitInitiatedPayment( * @param bankKeys bank public keys. * TODO update doc */ -private suspend fun submitBatch( - ctx: SubmissionContext, - db: Database, -) { - db.initiated.submittable(ctx.cfg.currency).forEach { +private suspend fun submitBatch(client: EbicsClient) { + client.db.initiated.submittable(client.cfg.currency).forEach { logger.debug("Submitting payment '${it.requestUid}'") - runCatching { submitInitiatedPayment(ctx, it) }.fold( + runCatching { submitInitiatedPayment(client, it) }.fold( onSuccess = { orderId -> - db.initiated.submissionSuccess(it.id, Instant.now(), orderId) + client.db.initiated.submissionSuccess(it.id, Instant.now(), orderId) logger.info("Payment '${it.requestUid}' submitted") }, onFailure = { e -> - db.initiated.submissionFailure(it.id, Instant.now(), e.message) + client.db.initiated.submissionFailure(it.id, Instant.now(), e.message) logger.error("Payment '${it.requestUid}' submission failure: ${e.fmt()}") throw e } @@ -151,29 +120,27 @@ class EbicsSubmit : CliktCommand("Submits any initiated payment found in the dat override fun run() = cliCmd(logger, common.log) { nexusConfig(common.config).withDb { db, cfg -> val (clientKeys, bankKeys) = expectFullKeys(cfg) - val ctx = SubmissionContext( - cfg = cfg, - bankPublicKeysFile = bankKeys, - clientPrivateKeysFile = clientKeys, - httpClient = HttpClient(), - fileLogger = FileLogger(ebicsLog) + val client = EbicsClient( + cfg, + httpClient(), + db, + FileLogger(ebicsLog), + clientKeys, + bankKeys ) val frequency: Duration = if (transient) { logger.info("Transient mode: submitting what found and returning.") Duration.ZERO } else { - val sect = cfg.config.section("nexus-submit") - val frequency = sect.duration("frequency").require() - val raw = sect.string("frequency").require() - logger.debug("Running with a frequency of $raw") - if (frequency == Duration.ZERO) { + logger.debug("Running with a frequency of ${cfg.submit.frequencyRaw}") + if (cfg.submit.frequency == Duration.ZERO) { logger.warn("Long-polling not implemented, running therefore in transient mode") } - frequency + cfg.submit.frequency } do { try { - submitBatch(ctx, db) + submitBatch(client) } catch (e: Exception) { throw Exception("Failed to submit payments", e) } diff --git a/nexus/src/main/kotlin/tech/libeufin/nexus/Main.kt b/nexus/src/main/kotlin/tech/libeufin/nexus/Main.kt @@ -47,7 +47,7 @@ import tech.libeufin.nexus.api.wireGatewayApi import tech.libeufin.nexus.db.Database import tech.libeufin.nexus.db.InitiatedPayment import tech.libeufin.nexus.ebics.EbicsOrder -import tech.libeufin.nexus.ebics.ebicsDownload +import tech.libeufin.nexus.ebics.EbicsClient import java.nio.file.Path import java.time.Instant import java.time.LocalDate @@ -153,9 +153,8 @@ class Serve : CliktCommand("Run libeufin-nexus HTTP server", name = "serve") { } } - val serverCfg = cfg.config.loadServerConfig("nexus-httpd") cfg.withDb { db, cfg -> - serve(serverCfg) { + serve(cfg.serverCfg) { nexusApi(db, cfg) } } @@ -178,11 +177,12 @@ class FakeIncoming: CliktCommand("Genere a fake incoming payment") { override fun run() = cliCmd(logger, common.log) { nexusConfig(common.config).withDb { db, cfg -> - val subject = payto.message ?: subject ?: throw Exception("Missing subject") - val amount = payto.amount ?: amount ?: throw Exception("Missing amount") + val subject = requireNotNull(payto.message ?: subject) { "Missing subject" } + val amount = requireNotNull(payto.amount ?: amount) { "Missing amount" } - if (amount.currency != cfg.currency) - throw Exception("Wrong currency: expected ${cfg.currency} got ${amount.currency}") + require(amount.currency == cfg.currency) { + "Wrong currency: expected ${cfg.currency} got ${amount.currency}" + } val bankId = run { val bytes = ByteArray(16).rand() @@ -259,27 +259,28 @@ class EbicsDownload: CliktCommand("Perform EBICS requests", name = "ebics-btd") logger.debug("Pinning start date to: $pinnedStartVal") dateToInstant(pinnedStartVal) } else null - val client = httpClient() - val fileLogger = FileLogger(ebicsLog) + val client = EbicsClient( + cfg, + httpClient(), + db, + FileLogger(ebicsLog), + clientKeys, + bankKeys + ) try { - ebicsDownload( - client, - cfg, - db, - clientKeys, - bankKeys, + client.download( EbicsOrder.V3(type, name, scope, messageName, messageVersion, container, option), pinnedStartArg, null ) { stream -> if (container == "ZIP") { - val stream = fileLogger.logFetch(stream, false) + val stream = client.fileLogger.logFetch(stream, false) stream.unzipEach { fileName, xmlContent -> println(fileName) println(xmlContent.readBytes().toString(Charsets.UTF_8)) } } else { - val stream = fileLogger.logFetch(stream, true) // TODO better name + val stream = client.fileLogger.logFetch(stream, true) // TODO better name println(stream.readBytes().toString(Charsets.UTF_8)) } if (dryRun) throw DryRun() diff --git a/nexus/src/main/kotlin/tech/libeufin/nexus/ebics/EbicsCommon.kt b/nexus/src/main/kotlin/tech/libeufin/nexus/ebics/EbicsCommon.kt @@ -97,14 +97,7 @@ sealed class EbicsError(msg: String, cause: Throwable? = null): Exception(msg, c class Protocol(msg: String, cause: Throwable? = null): EbicsError(msg, cause) } -/** - * POSTs the EBICS message to the bank. - * - * @param bankUrl where the bank serves EBICS requests. - * @param msg EBICS message as raw bytes. - * @return the raw bank response. - * TODO update doc - */ +/** POST an EBICS request [msg] to [bankUrl] returning a parsed XML response */ suspend fun HttpClient.postToBank(bankUrl: String, msg: ByteArray, phase: String): Document { val res = try { post(urlString = bankUrl) { @@ -127,6 +120,7 @@ suspend fun HttpClient.postToBank(bankUrl: String, msg: ByteArray, phase: String } } +/** POST an EBICS BTS request [xmlReq] using [client] returning a validated and parsed XML response */ suspend fun EbicsBTS.postBTS( client: HttpClient, xmlReq: ByteArray, @@ -148,112 +142,137 @@ suspend fun EbicsBTS.postBTS( return response } -/** - * Perform an EBICS download transaction. - * - * It conducts init -> transfer -> processing -> receipt phases. - * - * @param client HTTP client for POSTing to the bank. - * @param cfg configuration handle. - * @param clientKeys client EBICS private keys. - * @param bankKeys bank EBICS public keys. - * @param reqXml raw EBICS XML request of the init phase. - * @param processing processing lambda receiving EBICS files as a byte stream if the transaction was not empty. - * @return T if the transaction was successful. If the failure is at the EBICS - * level EbicsSideException is thrown else ités the exception of the processing lambda. - * TODO update doc - */ -suspend fun ebicsDownload( - client: HttpClient, - cfg: NexusConfig, - db: Database, - clientKeys: ClientPrivateKeysFile, - bankKeys: BankPublicKeysFile, - order: EbicsOrder, - startDate: Instant?, - endDate: Instant?, - processing: suspend (InputStream) -> Unit, +/** High level EBICS client */ +class EbicsClient( + val cfg: NexusConfig, + val client: HttpClient, + val db: Database, + val fileLogger: FileLogger, + val clientKeys: ClientPrivateKeysFile, + val bankKeys: BankPublicKeysFile ) { - val impl = EbicsBTS(cfg, bankKeys, clientKeys, order) + /** + * Performs an EBICS download transaction of [order] between [startDate] and [endDate]. + * Download content is passed to [processing] + * + * It conducts init -> transfer -> processing -> receipt phases. + * + * Cancellations and failures are handled. + */ + suspend fun download( + order: EbicsOrder, + startDate: Instant?, + endDate: Instant?, + processing: suspend (InputStream) -> Unit, + ) { + val impl = EbicsBTS(cfg, bankKeys, clientKeys, order) - // Close pending - while (true) { - val tId = db.ebics.first() - if (tId == null) break - val xml = impl.downloadReceipt(tId, false) - impl.postBTS(client, xml, "Closing pending") - db.ebics.remove(tId) - } - - // We need to run the logic in a non-cancelable context because we need to send - // a receipt for each open download transaction, otherwise we'll be stuck in an - // error loop until the pending transaction timeout. - val init = withContext(NonCancellable) { - // Init phase - val initReq = impl.downloadInitialization(startDate, endDate) - val initResp = impl.postBTS(client, initReq, "Download init phase") - if (initResp.bankCode == EbicsReturnCode.EBICS_NO_DOWNLOAD_DATA_AVAILABLE) { - logger.debug("Download content is empty") - return@withContext null + // Close pending + while (true) { + val tId = db.ebics.first() + if (tId == null) break + val xml = impl.downloadReceipt(tId, false) + impl.postBTS(client, xml, "Closing pending") + db.ebics.remove(tId) } - val initContent = initResp.okOrFail("Download init phase") - val tId = requireNotNull(initContent.transactionID) { - "Download init phase: missing transaction ID" + + // We need to run the logic in a non-cancelable context because we need to send + // a receipt for each open download transaction, otherwise we'll be stuck in an + // error loop until the pending transaction timeout. + val init = withContext(NonCancellable) { + // Init phase + val initReq = impl.downloadInitialization(startDate, endDate) + val initResp = impl.postBTS(client, initReq, "Download init phase") + if (initResp.bankCode == EbicsReturnCode.EBICS_NO_DOWNLOAD_DATA_AVAILABLE) { + logger.debug("Download content is empty") + return@withContext null + } + val initContent = initResp.okOrFail("Download init phase") + val tId = requireNotNull(initContent.transactionID) { + "Download init phase: missing transaction ID" + } + db.ebics.register(tId) + Pair(tId, initContent) + } + val (tId, initContent) = if (init == null) return else init + val howManySegments = requireNotNull(initContent.numSegments) { + "Download init phase: missing num segments" + } + val firstDataChunk = requireNotNull(initContent.payloadChunk) { + "Download init phase: missing OrderData" + } + val dataEncryptionInfo = requireNotNull(initContent.dataEncryptionInfo) { + "Download init phase: missing EncryptionInfo" } - db.ebics.register(tId) - Pair(tId, initContent) - } - val (tId, initContent) = if (init == null) return else init - val howManySegments = requireNotNull(initContent.numSegments) { - "Download init phase: missing num segments" - } - val firstDataChunk = requireNotNull(initContent.payloadChunk) { - "Download init phase: missing OrderData" - } - val dataEncryptionInfo = requireNotNull(initContent.dataEncryptionInfo) { - "Download init phase: missing EncryptionInfo" - } - logger.debug("Download init phase for transaction '$tId'") + logger.debug("Download init phase for transaction '$tId'") - /** Send download receipt */ - suspend fun receipt(success: Boolean) { - val xml = impl.downloadReceipt(tId, success) - impl.postBTS(client, xml, "Download receipt phase").okOrFail("Download receipt phase") - db.ebics.remove(tId) - } + // Transfer phase + val ebicsChunks = mutableListOf(firstDataChunk) + for (x in 2 .. howManySegments) { + val transReq = impl.downloadTransfer(x, howManySegments, tId) + val transResp = impl.postBTS(client, transReq, "Download transfer phase").okOrFail("Download transfer phase") + val chunk = requireNotNull(transResp.payloadChunk) { + "Download transfer phase: missing encrypted chunk" + } + ebicsChunks.add(chunk) + } - // Transfer phase - val ebicsChunks = mutableListOf(firstDataChunk) - for (x in 2 .. howManySegments) { - val transReq = impl.downloadTransfer(x, howManySegments, tId) - val transResp = impl.postBTS(client, transReq, "Download transfer phase").okOrFail("Download transfer phase") - val chunk = requireNotNull(transResp.payloadChunk) { - "Download transfer phase: missing encrypted chunk" + // Decompress encrypted chunks + val payloadStream = try { + decryptAndDecompressPayload( + clientKeys.encryption_private_key, + dataEncryptionInfo, + ebicsChunks + ) + } catch (e: Exception) { + throw EbicsError.Protocol("invalid chunks", e) } - ebicsChunks.add(chunk) - } - // Decompress encrypted chunks - val payloadStream = try { - decryptAndDecompressPayload( - clientKeys.encryption_private_key, - dataEncryptionInfo, - ebicsChunks - ) - } catch (e: Exception) { - throw EbicsError.Protocol("invalid chunks", e) - } + // Run business logic + val res = runCatching { + processing(payloadStream) + } - // Run business logic - val res = runCatching { - processing(payloadStream) + // First send a proper EBICS transaction receipt + val xml = impl.downloadReceipt(tId, res.isSuccess) + impl.postBTS(client, xml, "Download receipt phase").okOrFail("Download receipt phase") + runCatching { db.ebics.remove(tId) } + // Then throw business logic exception if any + res.getOrThrow() } - // First send a proper EBICS transaction receipt - receipt(res.isSuccess) - // Then throw business logic exception if any - res.getOrThrow() + /** + * Performs an EBICS upload transaction of [order] using [payload]. + * + * It conducts init -> upload phases. + * + * Returns upload orderID + */ + suspend fun upload( + order: EbicsOrder, + payload: ByteArray, + ): String { + val impl = EbicsBTS(cfg, bankKeys, clientKeys, order) + val preparedPayload = prepareUploadPayload(cfg, clientKeys, bankKeys, payload) + + // Init phase + val initXml = impl.uploadInitialization(preparedPayload) + val initResp = impl.postBTS(client, initXml, "Upload init phase").okOrFail("Upload init phase") + val tId = requireNotNull(initResp.transactionID) { + "Upload init phase: missing transaction ID" + } + val orderId = requireNotNull(initResp.orderID) { + "Upload init phase: missing order ID" + } + + // Transfer phase + for (i in 1..preparedPayload.segments.size) { + val transferXml = impl.uploadTransfer(tId, preparedPayload, i) + val transferResp = impl.postBTS(client, transferXml, "Upload transfer phase").okOrFail("Upload transfer phase") + } + return orderId + } } suspend fun HEV( @@ -266,16 +285,14 @@ suspend fun HEV( return EbicsAdministrative.parseHEV(xml).okOrFail("HEV") } -/** - * Signs and the encrypts the data to send via EBICS. - * - * @param cfg configuration handle. - * @param clientKeys client keys. - * @param bankKeys bank keys. - * @param payload business payload to send to the bank, typically ISO20022. - * @return [PreparedUploadData] - * TODO update doc - */ +class PreparedUploadData( + val transactionKey: ByteArray, + val userSignatureDataEncrypted: String, + val dataDigest: ByteArray, + val segments: List<String> +) + +/** Signs, encrypts and format data to send via EBICS */ fun prepareUploadPayload( cfg: NexusConfig, clientKeys: ClientPrivateKeysFile, @@ -318,63 +335,14 @@ fun prepareUploadPayload( ) } -/** - * Collects all the steps of an EBICS 3 upload transaction. - * NOTE: this function could conveniently be reused for an EBICS 2.x - * transaction, hence this function stays in this file. - * - * @param client HTTP client for POSTing to the bank. - * @param cfg configuration handle. - * @param clientKeys client EBICS private keys. - * @param bankKeys bank EBICS public keys. - * @param payload binary business paylaod. - * TODO update doc - */ -suspend fun doEbicsUpload( - client: HttpClient, - cfg: NexusConfig, - clientKeys: ClientPrivateKeysFile, - bankKeys: BankPublicKeysFile, - order: EbicsOrder, - payload: ByteArray, -): String { - val impl = EbicsBTS(cfg, bankKeys, clientKeys, order) - val preparedPayload = prepareUploadPayload(cfg, clientKeys, bankKeys, payload) - - // Init phase - val initXml = impl.uploadInitialization(preparedPayload) - val initResp = impl.postBTS(client, initXml, "Upload init phase").okOrFail("Upload init phase") - val tId = requireNotNull(initResp.transactionID) { - "Upload init phase: missing transaction ID" - } - val orderId = requireNotNull(initResp.orderID) { - "Upload init phase: missing order ID" - } - - // Transfer phase - for (i in 1..preparedPayload.segments.size) { - val transferXml = impl.uploadTransfer(tId, preparedPayload, i) - val transferResp = impl.postBTS(client, transferXml, "Upload transfer phase").okOrFail("Upload transfer phase") - } - return orderId -} - private val SECURE_RNG = SecureRandom() -/** - * @param size in bits - */ + +/** Generate a secure random nonce of [size] bytes */ fun getNonce(size: Int): ByteArray { return ByteArray(size / 8).rand(SECURE_RNG) } -class PreparedUploadData( - val transactionKey: ByteArray, - val userSignatureDataEncrypted: String, - val dataDigest: ByteArray, - val segments: List<String> -) - class DataEncryptionInfo( val transactionKey: ByteArray, val bankPubDigest: ByteArray diff --git a/nexus/src/main/kotlin/tech/libeufin/nexus/helpers.kt b/nexus/src/main/kotlin/tech/libeufin/nexus/helpers.kt @@ -19,34 +19,8 @@ package tech.libeufin.nexus -import tech.libeufin.nexus.db.Database -import tech.libeufin.common.* -import tech.libeufin.common.db.* import io.ktor.client.* import io.ktor.client.plugins.* -import java.nio.file.Path - -/** Load nexus config at [configFile] */ -fun nexusConfig(configFile: Path?): NexusConfig { - val config = NEXUS_CONFIG_SOURCE.fromFile(configFile) - return NexusConfig(config) -} - -/** Load nexus database config */ -fun NexusConfig.dbConfig(): DatabaseConfig { - val sect = config.section("libeufin-nexusdb-postgres") - val configOption = sect.string("config") - return DatabaseConfig( - dbConnStr = configOption.orNull() ?: config.section("nexus-postgres").string("config").orNull() ?: configOption.require(), - sqlDir = sect.path("sql_dir").require() - ) -} - -/** Run [lambda] with access to a database conn pool */ -suspend fun NexusConfig.withDb(lambda: suspend (Database, NexusConfig) -> Unit) { - val dbCfg = dbConfig() - Database(dbCfg, currency).use { lambda(it, this) } -} /** Create an HTTP client for EBICS requests */ fun httpClient(): HttpClient = HttpClient { diff --git a/nexus/src/main/kotlin/tech/libeufin/nexus/test/TxCheck.kt b/nexus/src/main/kotlin/tech/libeufin/nexus/test/TxCheck.kt @@ -33,9 +33,9 @@ data class TxCheckResult( /** * Test EBICS implementation's transactions semantic: - * - Can two fetch transactions run concurently ? - * - Can a fetch & submit transactions run concurently ? - * - Can two submit transactions run concurently ? + * - Can two fetch transactions run concurrently ? + * - Can a fetch & submit transactions run concurrently ? + * - Can two submit transactions run concurrently ? * - Is closing a submit transaction idempotent */ suspend fun txCheck( diff --git a/nexus/src/test/kotlin/helpers.kt b/nexus/src/test/kotlin/helpers.kt @@ -45,8 +45,7 @@ fun setup( conf: String = "test.conf", lambda: suspend (Database, NexusConfig) -> Unit ) = conf(conf) { cfg -> - val dbCfg = cfg.dbConfig() - pgDataSource(dbCfg.dbConnStr).dbInit(dbCfg, "libeufin-nexus", true) + pgDataSource(cfg.dbCfg.dbConnStr).dbInit(cfg.dbCfg, "libeufin-nexus", true) cfg.withDb(lambda) } diff --git a/testbench/src/main/kotlin/Main.kt b/testbench/src/main/kotlin/Main.kt @@ -127,8 +127,9 @@ class Cli : CliktCommand("Run integration tests on banks provider") { "CHF" to "payto://iban/CH4189144589712575493?receiver-name=John%20Smith", "EUR" to "payto://iban/DE54500105177452372744?receiver-name=John%20Smith" ) - val dummyPayto = dummyPaytos[currency] - ?: throw Exception("Missing dummy payto for $currency") + val dummyPayto = requireNotNull(dummyPaytos[currency]) { + "Missing dummy payto for $currency" + } val payto = benchCfg.payto[currency] ?: dummyPayto val recoverDoc = "report statement notification"