commit b1d1a04590c18b57668b564fb91feb8af5a7d3c0 parent 902cdf300456c929c0b1f5350f5e8147d94de538 Author: Antoine A <> Date: Mon, 8 Jul 2024 19:14:03 +0200 nexus: store pending transaction ids and clean code Diffstat:
20 files changed, 342 insertions(+), 268 deletions(-)
diff --git a/database-versioning/libeufin-nexus-0005.sql b/database-versioning/libeufin-nexus-0005.sql @@ -0,0 +1,26 @@ +-- +-- This file is part of TALER +-- Copyright (C) 2024 Taler Systems SA +-- +-- TALER is free software; you can redistribute it and/or modify it under the +-- terms of the GNU General Public License as published by the Free Software +-- Foundation; either version 3, or (at your option) any later version. +-- +-- TALER is distributed in the hope that it will be useful, but WITHOUT ANY +-- WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +-- A PARTICULAR PURPOSE. See the GNU General Public License for more details. +-- +-- You should have received a copy of the GNU General Public License along with +-- TALER; see the file COPYING. If not, see <http://www.gnu.org/licenses/> + +BEGIN; + +SELECT _v.register_patch('libeufin-nexus-0005', NULL, NULL); + +SET search_path TO libeufin_nexus; + +CREATE TABLE pending_ebics_transactions ( + tx_id TEXT NOT NULL UNIQUE PRIMARY KEY +); + +COMMIT; diff --git a/nexus/src/main/kotlin/tech/libeufin/nexus/DbInit.kt b/nexus/src/main/kotlin/tech/libeufin/nexus/DbInit.kt @@ -40,7 +40,7 @@ class DbInit : CliktCommand("Initialize the libeufin-nexus database", name = "db ).flag() override fun run() = cliCmd(logger, common.log) { - val cfg = loadConfig(common.config).dbConfig() + val cfg = nexusConfig(common.config).dbConfig() pgDataSource(cfg.dbConnStr).dbInit(cfg, "libeufin-nexus", reset) } } \ No newline at end of file diff --git a/nexus/src/main/kotlin/tech/libeufin/nexus/EbicsFetch.kt b/nexus/src/main/kotlin/tech/libeufin/nexus/EbicsFetch.kt @@ -296,6 +296,7 @@ private suspend fun fetchDocuments( ebicsDownload( ctx.httpClient, ctx.cfg, + db, ctx.clientKeys, ctx.bankKeys, order, @@ -382,19 +383,11 @@ class EbicsFetch: CliktCommand("Fetches EBICS files") { * mode when no flags are passed to the invocation. */ override fun run() = cliCmd(logger, common.log) { - val cfg = loadNexusConfig(common.config) - val dbCfg = cfg.config.dbConfig() - - Database(dbCfg, cfg.currency).use { db -> + nexusConfig(common.config).withDb { db, cfg -> val (clientKeys, bankKeys) = expectFullKeys(cfg) val ctx = FetchContext( cfg, - HttpClient { - install(HttpTimeout) { - // It can take a lot of time for the bank to generate documents - socketTimeoutMillis = 5 * 60 * 1000 - } - }, + httpClient(), clientKeys, bankKeys, null, diff --git a/nexus/src/main/kotlin/tech/libeufin/nexus/EbicsSetup.kt b/nexus/src/main/kotlin/tech/libeufin/nexus/EbicsSetup.kt @@ -1,6 +1,6 @@ /* * This file is part of LibEuFin. - * Copyright (C) 2023 Stanisci and Dold. + * Copyright (C) 2023-2024 Taler Systems S.A. * LibEuFin is free software; you can redistribute it and/or modify * it under the terms of the GNU Affero General Public License as @@ -23,7 +23,6 @@ import com.github.ajalt.clikt.core.CliktCommand import com.github.ajalt.clikt.parameters.groups.* import com.github.ajalt.clikt.parameters.options.* import io.ktor.client.* -import io.ktor.client.plugins.* import tech.libeufin.common.* import tech.libeufin.common.crypto.* import tech.libeufin.nexus.ebics.* @@ -151,17 +150,6 @@ suspend fun doKeysRequestAndUpdateState( } /** - * Mere collector of the steps to load and parse the config. - * - * @param configFile location of the configuration entry point. - * @return internal representation of the configuration. - */ -fun loadNexusConfig(configFile: Path?): NexusConfig { - val config = loadConfig(configFile) - return NexusConfig(config) -} - -/** * Mere collector of the PDF generation steps. Fails the * process if a problem occurs. * @@ -198,13 +186,8 @@ class EbicsSetup: CliktCommand("Set up the EBICS subscriber") { * This function collects the main steps of setting up an EBICS access. */ override fun run() = cliCmd(logger, common.log) { - val cfg = loadNexusConfig(common.config) - val client = HttpClient { - install(HttpTimeout) { - // It can take a lot of time for the bank to generate documents - socketTimeoutMillis = 5 * 60 * 1000 - } - } + val cfg = nexusConfig(common.config) + val client = httpClient() val clientKeys = loadOrGenerateClientKeys(cfg.clientPrivateKeysPath) var bankKeys = loadBankKeys(cfg.bankPublicKeysPath) @@ -250,19 +233,21 @@ class EbicsSetup: CliktCommand("Set up the EBICS subscriber") { // Check account information logger.info("Doing administrative request HKD") try { - ebicsDownload(client, cfg, clientKeys, bankKeys, EbicsOrder.V3("HKD"), null, null) { stream -> - val hkd = EbicsAdministrative.parseHKD(stream) - val account = hkd.account - // TODO parse and check more information - if (account.currency != null && account.currency != cfg.currency) - logger.error("Expected CURRENCY '${cfg.currency}' from config got '${account.currency}' from bank") - if (account.iban != null && account.iban != cfg.account.iban) - logger.error("Expected IBAN '${cfg.account.iban}' from config got '${account.iban}' from bank") - if (account.name != null && account.name != cfg.account.name) - logger.warn("Expected NAME '${cfg.account.name}' from config got '${account.name}' from bank") + cfg.withDb { db, cfg -> + ebicsDownload(client, cfg, db, clientKeys, bankKeys, EbicsOrder.V3("HKD"), null, null) { stream -> + val hkd = EbicsAdministrative.parseHKD(stream) + val account = hkd.account + // TODO parse and check more information + if (account.currency != null && account.currency != cfg.currency) + logger.error("Expected CURRENCY '${cfg.currency}' from config got '${account.currency}' from bank") + if (account.iban != null && account.iban != cfg.account.iban) + logger.error("Expected IBAN '${cfg.account.iban}' from config got '${account.iban}' from bank") + if (account.name != null && account.name != cfg.account.name) + logger.warn("Expected NAME '${cfg.account.name}' from config got '${account.name}' from bank") - for (order in hkd.orders) { - logger.debug("${order.type}${order.params}: ${order.description}") + for (order in hkd.orders) { + logger.debug("${order.type}${order.params}: ${order.description}") + } } } } catch (e: Exception) { diff --git a/nexus/src/main/kotlin/tech/libeufin/nexus/EbicsSubmit.kt b/nexus/src/main/kotlin/tech/libeufin/nexus/EbicsSubmit.kt @@ -1,6 +1,6 @@ /* * This file is part of LibEuFin. - * Copyright (C) 2023 Stanisci and Dold. + * Copyright (C) 2023-2024 Taler Systems S.A. * LibEuFin is free software; you can redistribute it and/or modify * it under the terms of the GNU Affero General Public License as @@ -149,17 +149,15 @@ class EbicsSubmit : CliktCommand("Submits any initiated payment found in the dat * FIXME: reduce code duplication with the fetch subcommand. */ override fun run() = cliCmd(logger, common.log) { - val cfg = loadNexusConfig(common.config) - val dbCfg = cfg.config.dbConfig() - val (clientKeys, bankKeys) = expectFullKeys(cfg) - val ctx = SubmissionContext( - cfg = cfg, - bankPublicKeysFile = bankKeys, - clientPrivateKeysFile = clientKeys, - httpClient = HttpClient(), - fileLogger = FileLogger(ebicsLog) - ) - Database(dbCfg, cfg.currency).use { db -> + nexusConfig(common.config).withDb { db, cfg -> + val (clientKeys, bankKeys) = expectFullKeys(cfg) + val ctx = SubmissionContext( + cfg = cfg, + bankPublicKeysFile = bankKeys, + clientPrivateKeysFile = clientKeys, + httpClient = HttpClient(), + fileLogger = FileLogger(ebicsLog) + ) val frequency: Duration = if (transient) { logger.info("Transient mode: submitting what found and returning.") Duration.ZERO diff --git a/nexus/src/main/kotlin/tech/libeufin/nexus/Main.kt b/nexus/src/main/kotlin/tech/libeufin/nexus/Main.kt @@ -78,26 +78,6 @@ fun Application.nexusApi(db: Database, cfg: NexusConfig) = talerApi(logger) { revenueApi(db, cfg) } -/** - * Abstracts the config loading - * - * @param configFile potentially NULL configuration file location. - * @return the configuration handle. - */ -fun loadConfig(configFile: Path?): TalerConfig = NEXUS_CONFIG_SOURCE.fromFile(configFile) - -/** - * Abstracts fetching the DB config values to set up Nexus. - */ -fun TalerConfig.dbConfig(): DatabaseConfig { - val sect = section("libeufin-nexusdb-postgres") - val configOption = sect.string("config") - return DatabaseConfig( - dbConnStr = configOption.orNull() ?: section("nexus-postgres").string("config").orNull() ?: configOption.require(), - sqlDir = sect.path("sql_dir").require() - ) -} - class InitiatePayment: CliktCommand("Initiate an outgoing payment") { private val common by CommonOption() private val amount by option( @@ -117,25 +97,20 @@ class InitiatePayment: CliktCommand("Initiate an outgoing payment") { ).convert { Payto.parse(it).expectIban() } override fun run() = cliCmd(logger, common.log) { - val cfg = loadConfig(common.config) - val dbCfg = cfg.dbConfig() - val currency = cfg.section("nexus-ebics").string("currency").require() - - val subject = payto.message ?: subject ?: throw Exception("Missing subject") - val amount = payto.amount ?: amount ?: throw Exception("Missing amount") - - if (payto.receiverName == null) - throw Exception("Missing receiver name in creditor payto") + nexusConfig(common.config).withDb { db, cfg -> + val subject = requireNotNull(payto.message ?: subject) { "Missing subject" } + val amount = requireNotNull(payto.amount ?: amount) { "Missing amount" } - if (amount.currency != currency) - throw Exception("Wrong currency: expected $currency got ${amount.currency}") + requireNotNull(payto.receiverName) { "Missing receiver name in creditor payto" } + require(amount.currency == cfg.currency) { + "Wrong currency: expected ${cfg.currency} got ${amount.currency}" + } - val requestUid = requestUid ?: run { - val bytes = ByteArray(16).rand() - Base32Crockford.encode(bytes) - } + val requestUid = requestUid ?: run { + val bytes = ByteArray(16).rand() + Base32Crockford.encode(bytes) + } - Database(dbCfg, currency).use { db -> db.initiated.create( InitiatedPayment( id = -1, @@ -155,7 +130,7 @@ class Serve : CliktCommand("Run libeufin-nexus HTTP server", name = "serve") { private val check by option().flag() override fun run() = cliCmd(logger, common.log) { - val cfg = loadNexusConfig(common.config) + val cfg = nexusConfig(common.config) if (check) { // Check if the server is to be started @@ -178,9 +153,8 @@ class Serve : CliktCommand("Run libeufin-nexus HTTP server", name = "serve") { } } - val dbCfg = cfg.config.dbConfig() val serverCfg = cfg.config.loadServerConfig("nexus-httpd") - Database(dbCfg, cfg.currency).use { db -> + cfg.withDb { db, cfg -> serve(serverCfg) { nexusApi(db, cfg) } @@ -203,21 +177,18 @@ class FakeIncoming: CliktCommand("Genere a fake incoming payment") { ).convert { Payto.parse(it).expectIban() } override fun run() = cliCmd(logger, common.log) { - val cfg = loadNexusConfig(common.config) - val dbCfg = cfg.config.dbConfig() + nexusConfig(common.config).withDb { db, cfg -> + val subject = payto.message ?: subject ?: throw Exception("Missing subject") + val amount = payto.amount ?: amount ?: throw Exception("Missing amount") - val subject = payto.message ?: subject ?: throw Exception("Missing subject") - val amount = payto.amount ?: amount ?: throw Exception("Missing amount") + if (amount.currency != cfg.currency) + throw Exception("Wrong currency: expected ${cfg.currency} got ${amount.currency}") - if (amount.currency != cfg.currency) - throw Exception("Wrong currency: expected ${cfg.currency} got ${amount.currency}") - - val bankId = run { - val bytes = ByteArray(16).rand() - Base32Crockford.encode(bytes) - } - - Database(dbCfg, amount.currency).use { db -> + val bankId = run { + val bytes = ByteArray(16).rand() + Base32Crockford.encode(bytes) + } + ingestIncomingPayment(db, IncomingPayment( amount = amount, @@ -236,16 +207,11 @@ class TxCheck: CliktCommand("Check transaction semantic") { private val common by CommonOption() override fun run() = cliCmd(logger, common.log) { - val cfg = loadNexusConfig(common.config) + val cfg = nexusConfig(common.config) val (clientKeys, bankKeys) = expectFullKeys(cfg) val doc = EbicsDocument.acknowledgement.doc() val order = cfg.dialect.downloadDoc(doc, false) - val client = HttpClient { - install(HttpTimeout) { - // It can take a lot of time for the bank to generate documents - socketTimeoutMillis = 5 * 60 * 1000 - } - } + val client = httpClient() val result = tech.libeufin.nexus.test.txCheck(client, cfg, clientKeys, bankKeys, order, cfg.dialect.directDebit()) println("$result") } @@ -286,45 +252,42 @@ class EbicsDownload: CliktCommand("Perform EBICS requests", name = "ebics-btd") class DryRun: Exception() override fun run() = cliCmd(logger, common.log) { - val cfg = loadNexusConfig(common.config) - val (clientKeys, bankKeys) = expectFullKeys(cfg) - val pinnedStartVal = pinnedStart - val pinnedStartArg = if (pinnedStartVal != null) { - logger.debug("Pinning start date to: $pinnedStartVal") - dateToInstant(pinnedStartVal) - } else null - val client = HttpClient { - install(HttpTimeout) { - // It can take a lot of time for the bank to generate documents - socketTimeoutMillis = 5 * 60 * 1000 - } - } - val fileLogger = FileLogger(ebicsLog) - try { - ebicsDownload( - client, - cfg, - clientKeys, - bankKeys, - EbicsOrder.V3(type, name, scope, messageName, messageVersion, container, option), - pinnedStartArg, - null - ) { stream -> - if (container == "ZIP") { - val stream = fileLogger.logFetch(stream, false) - stream.unzipEach { fileName, xmlContent -> - println(fileName) - println(xmlContent.readBytes().toString(Charsets.UTF_8)) + nexusConfig(common.config).withDb { db, cfg -> + val (clientKeys, bankKeys) = expectFullKeys(cfg) + val pinnedStartVal = pinnedStart + val pinnedStartArg = if (pinnedStartVal != null) { + logger.debug("Pinning start date to: $pinnedStartVal") + dateToInstant(pinnedStartVal) + } else null + val client = httpClient() + val fileLogger = FileLogger(ebicsLog) + try { + ebicsDownload( + client, + cfg, + db, + clientKeys, + bankKeys, + EbicsOrder.V3(type, name, scope, messageName, messageVersion, container, option), + pinnedStartArg, + null + ) { stream -> + if (container == "ZIP") { + val stream = fileLogger.logFetch(stream, false) + stream.unzipEach { fileName, xmlContent -> + println(fileName) + println(xmlContent.readBytes().toString(Charsets.UTF_8)) + } + } else { + val stream = fileLogger.logFetch(stream, true) // TODO better name + println(stream.readBytes().toString(Charsets.UTF_8)) } - } else { - val stream = fileLogger.logFetch(stream, true) // TODO better name - println(stream.readBytes().toString(Charsets.UTF_8)) + if (dryRun) throw DryRun() } - if (dryRun) throw DryRun() + } catch (e: DryRun) { + // We throw DryRun to not consume files while testing } - } catch (e: DryRun) { - // We throw DryRun to not consume files while testing - } + } } } @@ -336,11 +299,7 @@ class ListCmd: CliktCommand("List nexus transactions", name = "list") { ).enum<ListKind>() override fun run() = cliCmd(logger, common.log) { - val cfg = loadConfig(common.config) - val dbCfg = cfg.dbConfig() - val currency = cfg.section("nexus-ebics").string("currency").require() - - Database(dbCfg, currency).use { db -> + nexusConfig(common.config).withDb { db, cfg -> fun fmtPayto(payto: String?): String { if (payto == null) return "" try { diff --git a/nexus/src/main/kotlin/tech/libeufin/nexus/XMLUtil.kt b/nexus/src/main/kotlin/tech/libeufin/nexus/XMLUtil.kt @@ -99,13 +99,12 @@ object XMLUtil { */ fun signEbicsDocument( doc: Document, - signingPriv: PrivateKey, - schema: String + signingPriv: PrivateKey ) { val authSigNode = XPathFactory.newInstance().newXPath() - .evaluate("/*[1]/urn:org:ebics:$schema:AuthSignature", doc, XPathConstants.NODE) + .evaluate("/*[1]/*[local-name()='AuthSignature']", doc, XPathConstants.NODE) if (authSigNode !is Node) - throw java.lang.Exception("no AuthSignature") + throw java.lang.Exception("sign: no AuthSignature") val fac = XMLSignatureFactory.getInstance("DOM") val c14n = fac.newTransform(CanonicalizationMethod.INCLUSIVE, null as TransformParameterSpec?) val ref: Reference = @@ -135,14 +134,13 @@ object XMLUtil { fun verifyEbicsDocument( doc: Document, - signingPub: PublicKey, - schema: String + signingPub: PublicKey ): Boolean { val doc2: Document = doc.cloneNode(true) as Document val authSigNode = XPathFactory.newInstance().newXPath() - .evaluate("/*[1]/urn:org:ebics:$schema:AuthSignature", doc2, XPathConstants.NODE) + .evaluate("/*[1]/*[local-name()='AuthSignature']", doc2, XPathConstants.NODE) if (authSigNode !is Node) - throw java.lang.Exception("no AuthSignature") + throw java.lang.Exception("verify: no AuthSignature") val sigEl = doc2.createElementNS("http://www.w3.org/2000/09/xmldsig#", "ds:Signature") authSigNode.parentNode.insertBefore(sigEl, authSigNode) while (authSigNode.hasChildNodes()) { diff --git a/nexus/src/main/kotlin/tech/libeufin/nexus/db/Database.kt b/nexus/src/main/kotlin/tech/libeufin/nexus/db/Database.kt @@ -47,6 +47,7 @@ class Database(dbConfig: DatabaseConfig, val bankCurrency: String): DbPool(dbCon val payment = PaymentDAO(this) val initiated = InitiatedDAO(this) val exchange = ExchangeDAO(this) + val ebics = EbicsDAO(this) private val outgoingTxFlows: MutableSharedFlow<Long> = MutableSharedFlow() private val incomingTxFlows: MutableSharedFlow<Long> = MutableSharedFlow() diff --git a/nexus/src/main/kotlin/tech/libeufin/nexus/db/EbicsDAO.kt b/nexus/src/main/kotlin/tech/libeufin/nexus/db/EbicsDAO.kt @@ -0,0 +1,50 @@ +/* + * This file is part of LibEuFin. + * Copyright (C) 2024 Taler Systems S.A. + + * LibEuFin is free software; you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation; either version 3, or + * (at your option) any later version. + + * LibEuFin is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General + * Public License for more details. + + * You should have received a copy of the GNU Affero General Public + * License along with LibEuFin; see the file COPYING. If not, see + * <http://www.gnu.org/licenses/> + */ + +package tech.libeufin.nexus.db + +import tech.libeufin.common.db.* +import tech.libeufin.common.* +import java.time.Instant + +/** Data access logic for EBICS transaction */ +class EbicsDAO(private val db: Database) { + /** Register a pending transaction */ + suspend fun register(id: String) = db.serializable( + "INSERT INTO pending_ebics_transactions (tx_id) VALUES (?) ON CONFLICT DO NOTHING" + ) { + setString(1, id) + executeUpdate() + } + + /** Remove pending transaction */ + suspend fun remove(id: String) = db.serializable( + "DELETE FROM pending_ebics_transactions WHERE tx_id = ?" + ) { + setString(1, id) + executeUpdate() + } + + /** Get first pending transaction */ + suspend fun first(): String? = db.serializable( + "SELECT tx_id FROM pending_ebics_transactions LIMIT 1" + ) { + oneOrNull { it.getString(1) } + } +} +\ No newline at end of file diff --git a/nexus/src/main/kotlin/tech/libeufin/nexus/ebics/EbicsBTS.kt b/nexus/src/main/kotlin/tech/libeufin/nexus/ebics/EbicsBTS.kt @@ -244,8 +244,7 @@ class EbicsBTS( } XMLUtil.signEbicsDocument( doc, - clientKeys.authentication_private_key, - order.schema + clientKeys.authentication_private_key ) return XMLUtil.convertDomToBytes(doc) } diff --git a/nexus/src/main/kotlin/tech/libeufin/nexus/ebics/EbicsCommon.kt b/nexus/src/main/kotlin/tech/libeufin/nexus/ebics/EbicsCommon.kt @@ -44,6 +44,7 @@ import org.xml.sax.SAXException import tech.libeufin.common.* import tech.libeufin.common.crypto.* import tech.libeufin.nexus.* +import tech.libeufin.nexus.db.* import java.io.InputStream import java.io.SequenceInputStream import java.security.SecureRandom @@ -134,8 +135,7 @@ suspend fun EbicsBTS.postBTS( val doc = client.postToBank(cfg.hostBaseUrl, xmlReq, phase) if (!XMLUtil.verifyEbicsDocument( doc, - bankKeys.bank_authentication_public_key, - order.schema + bankKeys.bank_authentication_public_key )) { throw EbicsError.Protocol("$phase: bank signature did not verify") } @@ -166,96 +166,94 @@ suspend fun EbicsBTS.postBTS( suspend fun ebicsDownload( client: HttpClient, cfg: NexusConfig, + db: Database, clientKeys: ClientPrivateKeysFile, bankKeys: BankPublicKeysFile, order: EbicsOrder, startDate: Instant?, endDate: Instant?, processing: suspend (InputStream) -> Unit, -) = coroutineScope { +) { val impl = EbicsBTS(cfg, bankKeys, clientKeys, order) - val parentScope = this + + // Close pending + while (true) { + val tId = db.ebics.first() + if (tId == null) break + val xml = impl.downloadReceipt(tId, false) + impl.postBTS(client, xml, "Closing pending") + db.ebics.remove(tId) + } // We need to run the logic in a non-cancelable context because we need to send // a receipt for each open download transaction, otherwise we'll be stuck in an // error loop until the pending transaction timeout. - // TODO find a way to cancel the pending transaction ? - withContext(NonCancellable) { + val init = withContext(NonCancellable) { // Init phase val initReq = impl.downloadInitialization(startDate, endDate) val initResp = impl.postBTS(client, initReq, "Download init phase") if (initResp.bankCode == EbicsReturnCode.EBICS_NO_DOWNLOAD_DATA_AVAILABLE) { logger.debug("Download content is empty") - return@withContext + return@withContext null } val initContent = initResp.okOrFail("Download init phase") val tId = requireNotNull(initContent.transactionID) { "Download init phase: missing transaction ID" } - val howManySegments = requireNotNull(initContent.numSegments) { - "Download init phase: missing num segments" - } - val firstDataChunk = requireNotNull(initContent.payloadChunk) { - "Download init phase: missing OrderData" - } - val dataEncryptionInfo = requireNotNull(initContent.dataEncryptionInfo) { - "Download init phase: missing EncryptionInfo" - } - - logger.debug("Download init phase for transaction '$tId'") - - /** Send download receipt */ - suspend fun receipt(success: Boolean) { - val xml = impl.downloadReceipt(tId, success) - impl.postBTS(client, xml, "Download receipt phase").okOrFail("Download receipt phase") - } - /** Throw if parent scope have been canceled */ - suspend fun checkCancellation() { - if (!parentScope.isActive) { - // First send a proper EBICS transaction failure - receipt(false) - // Send throw cancellation exception - throw CancellationException() - } - } + db.ebics.register(tId) + Pair(tId, initContent) + } + val (tId, initContent) = if (init == null) return else init + val howManySegments = requireNotNull(initContent.numSegments) { + "Download init phase: missing num segments" + } + val firstDataChunk = requireNotNull(initContent.payloadChunk) { + "Download init phase: missing OrderData" + } + val dataEncryptionInfo = requireNotNull(initContent.dataEncryptionInfo) { + "Download init phase: missing EncryptionInfo" + } - // Transfer phase - val ebicsChunks = mutableListOf(firstDataChunk) - for (x in 2 .. howManySegments) { - checkCancellation() - val transReq = impl.downloadTransfer(x, howManySegments, tId) - val transResp = impl.postBTS(client, transReq, "Download transfer phase").okOrFail("Download transfer phase") - val chunk = requireNotNull(transResp.payloadChunk) { - "Download transfer phase: missing encrypted chunk" - } - ebicsChunks.add(chunk) - } + logger.debug("Download init phase for transaction '$tId'") - checkCancellation() + /** Send download receipt */ + suspend fun receipt(success: Boolean) { + val xml = impl.downloadReceipt(tId, success) + impl.postBTS(client, xml, "Download receipt phase").okOrFail("Download receipt phase") + db.ebics.remove(tId) + } - // Decompress encrypted chunks - val payloadStream = try { - decryptAndDecompressPayload( - clientKeys.encryption_private_key, - dataEncryptionInfo, - ebicsChunks - ) - } catch (e: Exception) { - throw EbicsError.Protocol("invalid chunks", e) + // Transfer phase + val ebicsChunks = mutableListOf(firstDataChunk) + for (x in 2 .. howManySegments) { + val transReq = impl.downloadTransfer(x, howManySegments, tId) + val transResp = impl.postBTS(client, transReq, "Download transfer phase").okOrFail("Download transfer phase") + val chunk = requireNotNull(transResp.payloadChunk) { + "Download transfer phase: missing encrypted chunk" } + ebicsChunks.add(chunk) + } - checkCancellation() - - // Run business logic - val res = runCatching { - processing(payloadStream) - } + // Decompress encrypted chunks + val payloadStream = try { + decryptAndDecompressPayload( + clientKeys.encryption_private_key, + dataEncryptionInfo, + ebicsChunks + ) + } catch (e: Exception) { + throw EbicsError.Protocol("invalid chunks", e) + } - // First send a proper EBICS transaction receipt - receipt(res.isSuccess) - // Then throw business logic exception if any - res.getOrThrow() + // Run business logic + val res = runCatching { + processing(payloadStream) } + + // First send a proper EBICS transaction receipt + receipt(res.isSuccess) + // Then throw business logic exception if any + res.getOrThrow() } suspend fun HEV( diff --git a/nexus/src/main/kotlin/tech/libeufin/nexus/ebics/EbicsKeyMng.kt b/nexus/src/main/kotlin/tech/libeufin/nexus/ebics/EbicsKeyMng.kt @@ -103,7 +103,7 @@ class EbicsKeyMng( if (data != null) el("DataTransfer/OrderData", data) } } - if (sign) XMLUtil.signEbicsDocument(doc, clientKeys.authentication_private_key, schema) + if (sign) XMLUtil.signEbicsDocument(doc, clientKeys.authentication_private_key) return XMLUtil.convertDomToBytes(doc) } diff --git a/nexus/src/main/kotlin/tech/libeufin/nexus/helpers.kt b/nexus/src/main/kotlin/tech/libeufin/nexus/helpers.kt @@ -0,0 +1,57 @@ +/* + * This file is part of LibEuFin. + * Copyright (C) 2024 Taler Systems S.A. + + * LibEuFin is free software; you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation; either version 3, or + * (at your option) any later version. + + * LibEuFin is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General + * Public License for more details. + + * You should have received a copy of the GNU Affero General Public + * License along with LibEuFin; see the file COPYING. If not, see + * <http://www.gnu.org/licenses/> + */ + +package tech.libeufin.nexus + +import tech.libeufin.nexus.db.Database +import tech.libeufin.common.* +import tech.libeufin.common.db.* +import io.ktor.client.* +import io.ktor.client.plugins.* +import java.nio.file.Path + +/** Load nexus config at [configFile] */ +fun nexusConfig(configFile: Path?): NexusConfig { + val config = NEXUS_CONFIG_SOURCE.fromFile(configFile) + return NexusConfig(config) +} + +/** Load nexus database config */ +fun NexusConfig.dbConfig(): DatabaseConfig { + val sect = config.section("libeufin-nexusdb-postgres") + val configOption = sect.string("config") + return DatabaseConfig( + dbConnStr = configOption.orNull() ?: config.section("nexus-postgres").string("config").orNull() ?: configOption.require(), + sqlDir = sect.path("sql_dir").require() + ) +} + +/** Run [lambda] with access to a database conn pool */ +suspend fun NexusConfig.withDb(lambda: suspend (Database, NexusConfig) -> Unit) { + val dbCfg = dbConfig() + Database(dbCfg, currency).use { lambda(it, this) } +} + +/** Create an HTTP client for EBICS requests */ +fun httpClient(): HttpClient = HttpClient { + install(HttpTimeout) { + // It can take a lot of time for the bank to generate documents + socketTimeoutMillis = 5 * 60 * 1000 + } +} +\ No newline at end of file diff --git a/nexus/src/test/kotlin/CliTest.kt b/nexus/src/test/kotlin/CliTest.kt @@ -1,6 +1,6 @@ /* * This file is part of LibEuFin. - * Copyright (C) 2023 Stanisci and Dold. + * Copyright (C) 2023-2024 Taler Systems S.A. * LibEuFin is free software; you can redistribute it and/or modify * it under the terms of the GNU Affero General Public License as @@ -50,10 +50,9 @@ class CliTest { val cmds = listOf("ebics-submit", "ebics-fetch") val allCmds = listOf("ebics-submit", "ebics-fetch", "ebics-setup") val conf = "conf/test.conf" - val cfg = loadConfig(Path(conf)) - val section = cfg.section("nexus-ebics") - val clientKeysPath = section.path("client_private_keys_file").require() - val bankKeysPath = section.path("bank_public_keys_file").require() + val cfg = nexusConfig(Path(conf)) + val clientKeysPath = cfg.clientPrivateKeysPath + val bankKeysPath = cfg.bankPublicKeysPath clientKeysPath.parent!!.createDirectories() clientKeysPath.parent!!.toFile().setWritable(true) bankKeysPath.parent!!.createDirectories() diff --git a/nexus/src/test/kotlin/DatabaseTest.kt b/nexus/src/test/kotlin/DatabaseTest.kt @@ -91,7 +91,7 @@ suspend fun Database.inTxExists(id: String): Boolean = conn { } class IncomingPaymentsTest { - // Tests creating and bouncing incoming payments in one DB transaction. + // Tests creating and bouncing incoming payments in one DB transaction @Test fun bounce() = setup { db, _ -> // creating and bouncing one incoming transaction. @@ -283,4 +283,23 @@ class PaymentInitiationsTest { db.initiated.submittable("KUDOS").map { it.requestUid } ) } +} + +class EbicsTxTest { + // Test pending transaction's id + @Test + fun pending() = setup { db, _ -> + val ids = setOf("first", "second", "third") + for (id in ids) { + db.ebics.register(id) + } + + repeat(ids.size) { + val id = db.ebics.first() + assert(ids.contains(id)) + db.ebics.remove(id!!) + } + + assertNull(db.ebics.first()) + } } \ No newline at end of file diff --git a/nexus/src/test/kotlin/XmlUtilTest.kt b/nexus/src/test/kotlin/XmlUtilTest.kt @@ -38,9 +38,9 @@ class XmlUtilTest { kpg.initialize(2048) val pair = kpg.genKeyPair() val otherPair = kpg.genKeyPair() - XMLUtil.signEbicsDocument(doc, pair.private, "H004") - kotlin.test.assertTrue(XMLUtil.verifyEbicsDocument(doc, pair.public, "H004")) - kotlin.test.assertFalse(XMLUtil.verifyEbicsDocument(doc, otherPair.public, "H004")) + XMLUtil.signEbicsDocument(doc, pair.private) + kotlin.test.assertTrue(XMLUtil.verifyEbicsDocument(doc, pair.public)) + kotlin.test.assertFalse(XMLUtil.verifyEbicsDocument(doc, otherPair.public)) } @Test @@ -55,8 +55,8 @@ class XmlUtilTest { val kpg = KeyPairGenerator.getInstance("RSA") kpg.initialize(2048) val pair = kpg.genKeyPair() - XMLUtil.signEbicsDocument(doc, pair.private, "H004") - kotlin.test.assertTrue(XMLUtil.verifyEbicsDocument(doc, pair.public, "H004")) + XMLUtil.signEbicsDocument(doc, pair.private) + kotlin.test.assertTrue(XMLUtil.verifyEbicsDocument(doc, pair.public)) } @Test @@ -67,6 +67,6 @@ class XmlUtilTest { val keyStream = classLoader.getResourceAsStream("signature1/public_key.txt") val keyBytes = keyStream.decodeBase64().readAllBytes() val key = CryptoUtil.loadRSAPublic(keyBytes) - assertTrue(XMLUtil.verifyEbicsDocument(doc, key, "H004")) + assertTrue(XMLUtil.verifyEbicsDocument(doc, key)) } } \ No newline at end of file diff --git a/nexus/src/test/kotlin/helpers.kt b/nexus/src/test/kotlin/helpers.kt @@ -37,22 +37,17 @@ fun conf( conf: String = "test.conf", lambda: suspend (NexusConfig) -> Unit ) = runBlocking { - val config = NEXUS_CONFIG_SOURCE.fromFile(Path("conf/$conf")) - val ctx = NexusConfig(config) - lambda(ctx) + val cfg = nexusConfig(Path("conf/$conf")) + lambda(cfg) } fun setup( conf: String = "test.conf", lambda: suspend (Database, NexusConfig) -> Unit -) = runBlocking { - val config = NEXUS_CONFIG_SOURCE.fromFile(Path("conf/$conf")) - val dbCfg = config.dbConfig() - val cfg = NexusConfig(config) +) = conf(conf) { cfg -> + val dbCfg = cfg.dbConfig() pgDataSource(dbCfg.dbConnStr).dbInit(dbCfg, "libeufin-nexus", true) - Database(dbCfg, cfg.currency).use { - lambda(it, cfg) - } + cfg.withDb(lambda) } fun serverSetup( diff --git a/testbench/src/main/kotlin/Main.kt b/testbench/src/main/kotlin/Main.kt @@ -98,11 +98,10 @@ class Cli : CliktCommand("Run integration tests on banks provider") { [libeufin-nexusdb-postgres] CONFIG = postgres:///libeufintestbench """) - val cfg = loadConfig(conf) - val section = cfg.section("nexus-ebics") + val cfg = nexusConfig(conf) // Check if platform is known - val kind = when (section.string("host_base_url").require()) { + val kind = when (cfg.hostBaseUrl) { "https://isotest.postfinance.ch/ebicsweb/ebicsweb" -> Kind("PostFinance IsoTest", "https://isotest.postfinance.ch/corporates/user/settings/ebics") "https://iso20022test.credit-suisse.com/ebicsweb/ebicsweb" -> @@ -120,9 +119,9 @@ class Cli : CliktCommand("Run integration tests on banks provider") { val log = "DEBUG" val flags = " -c $conf -L $log" val ebicsFlags = "$flags --transient --debug-ebics test/$platform" - val clientKeysPath = section.path("client_private_keys_file").require() - val bankKeysPath = section.path("bank_public_keys_file").require() - val currency = section.string("currency").require() + val clientKeysPath = cfg.clientPrivateKeysPath + val bankKeysPath = cfg.bankPublicKeysPath + val currency = cfg.currency val dummyPaytos = mapOf( "CHF" to "payto://iban/CH4189144589712575493?receiver-name=John%20Smith", diff --git a/testbench/src/test/kotlin/IntegrationTest.kt b/testbench/src/test/kotlin/IntegrationTest.kt @@ -69,11 +69,8 @@ fun server(lambda: () -> Unit) { fun setup(conf: String, lambda: suspend (NexusDb) -> Unit) { try { runBlocking { - val cfg = loadConfig(Path(conf)) - val dbCfg = cfg.dbConfig() - val currency = cfg.section("nexus-ebics").string("currency").require() - NexusDb(dbCfg, currency).use { - lambda(it) + nexusConfig(Path(conf)).withDb { db, _ -> + lambda(db) } } } finally { diff --git a/testbench/src/test/kotlin/Iso20022Test.kt b/testbench/src/test/kotlin/Iso20022Test.kt @@ -21,7 +21,7 @@ import org.junit.Test import tech.libeufin.nexus.parseCustomerAck import tech.libeufin.nexus.parseCustomerPaymentStatusReport import tech.libeufin.nexus.parseTx -import tech.libeufin.nexus.loadConfig +import tech.libeufin.nexus.nexusConfig import tech.libeufin.nexus.ebics.Dialect import java.nio.file.Files import kotlin.io.path.Path @@ -56,10 +56,9 @@ class Iso20022Test { if (!file.isDirectory()) continue val fetch = file.resolve("fetch") if (!fetch.exists()) continue - val cfg = loadConfig(platform.resolve("ebics.conf")) - val section = cfg.section("nexus-ebics") - val currency = section.string("currency").require() - val dialect = Dialect.valueOf(section.string("bank_dialect").require()) + val cfg = nexusConfig(platform.resolve("ebics.conf")) + val currency = cfg.currency + val dialect = cfg.dialect for (log in fetch.listDirectoryEntries()) { val content = Files.newInputStream(log) val name = log.toString()