libeufin

Integration and sandbox testing for FinTech APIs and data formats
Log | Files | Refs | Submodules | README | LICENSE

commit d5de1f4379c979cacb0b7e2c9830fd95332d8c20
parent 76c65b2d45e1f55df5fb68b868a644311c2d81e9
Author: Antoine A <>
Date:   Fri, 30 Aug 2024 12:55:08 +0200

nexus: support IGNORE_BOUNCES_BEFORE

Diffstat:
Mcontrib/nexus.conf | 4++++
Anexus/conf/skip.conf | 20++++++++++++++++++++
Mnexus/conf/test.conf | 3---
Mnexus/src/main/kotlin/tech/libeufin/nexus/Config.kt | 20+++++++++++++++++++-
Mnexus/src/main/kotlin/tech/libeufin/nexus/cli/EbicsFetch.kt | 66++++++++++++++++++++++++++++++++++++++++++------------------------
Mnexus/src/main/kotlin/tech/libeufin/nexus/cli/Testing.kt | 5++---
Mnexus/src/test/kotlin/DatabaseTest.kt | 99++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++-------------------
Mnexus/src/test/kotlin/bench.kt | 10++++++----
Mnexus/src/test/kotlin/helpers.kt | 46+++++++++++++++++++++++++++++-----------------
Mtestbench/src/test/kotlin/IntegrationTest.kt | 25++++++++++++-------------
10 files changed, 210 insertions(+), 88 deletions(-)

diff --git a/contrib/nexus.conf b/contrib/nexus.conf @@ -55,9 +55,13 @@ CONFIG = postgres:///libeufin [nexus-fetch] FREQUENCY = 30m + # Ignore all transactions prior to a certain date, useful when you want to use an existing account with old transactions that should not be bounced. # IGNORE_TRANSACTIONS_BEFORE = YYYY-MM-DD +# Ignore all malformed transactions prior to a certain date, useful when you want to import old transactions without bouncing the malformed ones a second time +# IGNORE_BOUNCES_BEFORE = YYYY-MM-DD + [nexus-submit] FREQUENCY = 30m diff --git a/nexus/conf/skip.conf b/nexus/conf/skip.conf @@ -0,0 +1,19 @@ +[nexus-ebics] +CURRENCY = CHF +BANK_DIALECT = postfinance +HOST_BASE_URL = https://isotest.postfinance.ch/ebicsweb/ebicsweb +BANK_PUBLIC_KEYS_FILE = test/tmp/bank-keys.json +CLIENT_PRIVATE_KEYS_FILE = test/tmp/client-keys.json +IBAN = CH7789144474425692816 +HOST_ID = PFEBICS +USER_ID = PFC00563 +PARTNER_ID = PFC00563 +BIC = BIC +NAME = myname + +[libeufin-nexusdb-postgres] +CONFIG = postgres:///libeufincheck + +[nexus-fetch] +IGNORE_TRANSACTIONS_BEFORE = 2024-04-04 +IGNORE_BOUNCES_BEFORE = 2024-06-12 +\ No newline at end of file diff --git a/nexus/conf/test.conf b/nexus/conf/test.conf @@ -14,9 +14,6 @@ NAME = myname [libeufin-nexusdb-postgres] CONFIG = postgres:///libeufincheck -[nexus-fetch] -IGNORE_TRANSACTIONS_BEFORE = 2024-04-04 - [nexus-httpd-wire-gateway-api] ENABLED = YES AUTH_METHOD = bearer-token diff --git a/nexus/src/main/kotlin/tech/libeufin/nexus/Config.kt b/nexus/src/main/kotlin/tech/libeufin/nexus/Config.kt @@ -24,14 +24,27 @@ import tech.libeufin.common.db.DatabaseConfig import tech.libeufin.nexus.db.Database import tech.libeufin.nexus.ebics.Dialect import java.nio.file.Path +import java.time.Instant val NEXUS_CONFIG_SOURCE = ConfigSource("libeufin", "libeufin-nexus", "libeufin-nexus") +data class NexusIngestConfig( + val accountType: AccountType, + val ignoreTransactionsBefore: Instant, + val ignoreBouncesBefore: Instant +) { + companion object { + fun default(accountType: AccountType) + = NexusIngestConfig(accountType, Instant.MIN, Instant.MIN) + } +} + class NexusFetchConfig(config: TalerConfig) { private val section = config.section("nexus-fetch") val frequency = section.duration("frequency").require() val frequencyRaw = section.string("frequency").require() - val ignoreBefore = section.date("ignore_transactions_before").orNull() + val ignoreTransactionsBefore = section.date("ignore_transactions_before").default(Instant.MIN) + val ignoreBouncesBefore = section.date("ignore_bounces_before").default(Instant.MIN) } class NexusSubmitConfig(config: TalerConfig) { @@ -75,6 +88,11 @@ class NexusEbicsConfig( val fetch = NexusFetchConfig(config) val submit = NexusSubmitConfig(config) + val ingest get() = NexusIngestConfig( + accountType, + fetch.ignoreTransactionsBefore, + fetch.ignoreBouncesBefore + ) } class ApiConfig(section: TalerConfigSection) { diff --git a/nexus/src/main/kotlin/tech/libeufin/nexus/cli/EbicsFetch.kt b/nexus/src/main/kotlin/tech/libeufin/nexus/cli/EbicsFetch.kt @@ -59,29 +59,38 @@ suspend fun ingestOutgoingPayment( /** * Ingest an incoming [payment] into [db] - * Stores the payment into valid talerable ones or bounces it, according to [accountType] . + * Stores the payment into valid talerable ones or bounces it */ suspend fun ingestIncomingPayment( db: Database, + cfg: NexusIngestConfig, payment: IncomingPayment, - accountType: AccountType ) { suspend fun bounce(msg: String) { if (payment.bankId == null) { logger.debug("{} ignored: missing bank ID", payment) return; } - when (accountType) { + when (cfg.accountType) { AccountType.exchange -> { - val result = db.payment.registerMalformedIncoming( - payment, - payment.amount, - Instant.now() - ) - if (result.new) { - logger.info("$payment bounced in '${result.bounceId}': $msg") + if (payment.executionTime < cfg.ignoreBouncesBefore) { + val res = db.payment.registerIncoming(payment) + if (res.new) { + logger.info("$payment ignored bounce: $msg") + } else { + logger.debug("{} already seen and ignored bounce: {}", payment, msg) + } } else { - logger.debug("{} already seen and bounced in '{}': {}", payment, result.bounceId, msg) + val result = db.payment.registerMalformedIncoming( + payment, + payment.amount, + Instant.now() + ) + if (result.new) { + logger.info("$payment bounced in '${result.bounceId}': $msg") + } else { + logger.debug("{} already seen and bounced in '{}': {}", payment, result.bounceId, msg) + } } } AccountType.normal -> { @@ -111,6 +120,26 @@ suspend fun ingestIncomingPayment( ) } +/** Ingest a [tx] notification into [db] */ +suspend fun ingestTransaction( + db: Database, + cfg: NexusIngestConfig, + tx: TxNotification, +) { + if (tx.executionTime < cfg.ignoreTransactionsBefore) { + logger.debug("IGNORE {}", tx) + } else { + when (tx) { + is IncomingPayment -> ingestIncomingPayment(db, cfg, tx) + is OutgoingPayment -> ingestOutgoingPayment(db, tx) + is TxNotification.Reversal -> { + logger.error("BOUNCE '${tx.msgId}': ${tx.reason}") + db.initiated.reversal(tx.msgId, "Payment bounced: ${tx.reason}") + } + } + } +} + /** Ingest an EBICS [payload] of [doc] into [db] */ private suspend fun ingestPayload( db: Database, @@ -123,19 +152,8 @@ private suspend fun ingestPayload( when (doc) { OrderDoc.report, OrderDoc.statement, OrderDoc.notification -> { try { - parseTx(xml, cfg.currency, cfg.dialect).forEach { - if (cfg.fetch.ignoreBefore != null && it.executionTime < cfg.fetch.ignoreBefore) { - logger.debug("IGNORE {}", it) - } else { - when (it) { - is IncomingPayment -> ingestIncomingPayment(db, it, cfg.accountType) - is OutgoingPayment -> ingestOutgoingPayment(db, it) - is TxNotification.Reversal -> { - logger.error("BOUNCE '${it.msgId}': ${it.reason}") - db.initiated.reversal(it.msgId, "Payment bounced: ${it.reason}") - } - } - } + parseTx(xml, cfg.currency, cfg.dialect).forEach { tx -> + ingestTransaction(db, cfg.ingest, tx) } } catch (e: Exception) { throw Exception("Ingesting notifications failed", e) diff --git a/nexus/src/main/kotlin/tech/libeufin/nexus/cli/Testing.kt b/nexus/src/main/kotlin/tech/libeufin/nexus/cli/Testing.kt @@ -91,15 +91,14 @@ class FakeIncoming: CliktCommand("Genere a fake incoming payment") { Base32Crockford.encode(bytes) } - ingestIncomingPayment(db, + ingestIncomingPayment(db, cfg.ebics.ingest, IncomingPayment( amount = amount, debitPaytoUri = payto.toString(), wireTransferSubject = subject, executionTime = Instant.now(), bankId = bankId - ), - cfg.accountType + ) ) } } diff --git a/nexus/src/test/kotlin/DatabaseTest.kt b/nexus/src/test/kotlin/DatabaseTest.kt @@ -18,13 +18,14 @@ */ import org.junit.Test -import tech.libeufin.common.ShortHashCode -import tech.libeufin.common.TalerAmount +import tech.libeufin.common.* import tech.libeufin.common.db.one import tech.libeufin.common.db.withStatement import tech.libeufin.nexus.AccountType +import tech.libeufin.nexus.NexusIngestConfig import tech.libeufin.nexus.cli.ingestIncomingPayment import tech.libeufin.nexus.cli.ingestOutgoingPayment +import tech.libeufin.nexus.cli.ingestTransaction import tech.libeufin.nexus.db.Database import tech.libeufin.nexus.db.InitiatedDAO.PaymentInitiationResult import java.time.Instant @@ -156,91 +157,143 @@ class IncomingPaymentsTest { // Test creating an incoming reserve taler transaction without and ID and reconcile it later again @Test fun reconcileMissingId() = setup { db, _ -> + val cfg = NexusIngestConfig.default(AccountType.exchange) val subject = "test with ${ShortHashCode.rand()} reserve pub" // Register with missing ID val incoming = genInPay(subject) val incomingMissingId = incoming.copy(bankId = null) - ingestIncomingPayment(db, incomingMissingId, AccountType.exchange) + ingestIncomingPayment(db, cfg, incomingMissingId) db.checkCount(1, 0, 1) assertFalse(db.inTxExists(incoming.bankId!!)) // Idempotent - ingestIncomingPayment(db, incomingMissingId, AccountType.exchange) + ingestIncomingPayment(db, cfg, incomingMissingId) db.checkCount(1, 0, 1) // Different metadata is bounced - ingestIncomingPayment(db, genInPay(subject, "KUDOS:9"), AccountType.exchange) - ingestIncomingPayment(db, genInPay("another $subject"), AccountType.exchange) + ingestIncomingPayment(db, cfg, genInPay(subject, "KUDOS:9")) + ingestIncomingPayment(db, cfg, genInPay("another $subject")) db.checkCount(3, 2, 1) // Different medata with missing id is ignored - ingestIncomingPayment(db, incomingMissingId.copy(amount = TalerAmount("KUDOS:9")), AccountType.exchange) - ingestIncomingPayment(db, incomingMissingId.copy(wireTransferSubject = "another $subject"), AccountType.exchange) + ingestIncomingPayment(db, cfg, incomingMissingId.copy(amount = TalerAmount("KUDOS:9"))) + ingestIncomingPayment(db, cfg, incomingMissingId.copy(wireTransferSubject = "another $subject")) db.checkCount(3, 2, 1) // Recover bank ID when metadata match - ingestIncomingPayment(db, incoming, AccountType.exchange) + ingestIncomingPayment(db, cfg, incoming) assertTrue(db.inTxExists(incoming.bankId!!)) // Idempotent - ingestIncomingPayment(db, incoming, AccountType.exchange) + ingestIncomingPayment(db, cfg, incoming) db.checkCount(3, 2, 1) // Missing ID is ignored - ingestIncomingPayment(db, incomingMissingId, AccountType.exchange) + ingestIncomingPayment(db, cfg, incomingMissingId) db.checkCount(3, 2, 1) // Other ID is bounced known that we know the id - ingestIncomingPayment(db, incomingMissingId.copy(bankId = "NEW"), AccountType.exchange) + ingestIncomingPayment(db, cfg, incomingMissingId.copy(bankId = "NEW")) db.checkCount(4, 3, 1) } // Test creating an incoming kyc taler transaction without and ID and reconcile it later again @Test fun reconcileMissingIdKyc() = setup { db, _ -> + val cfg = NexusIngestConfig.default(AccountType.exchange) val subject = "test with KYC:${ShortHashCode.rand()} account pub" // Register with missing ID val incoming = genInPay(subject) val incomingMissingId = incoming.copy(bankId = null) - ingestIncomingPayment(db, incomingMissingId, AccountType.exchange) + ingestIncomingPayment(db, cfg, incomingMissingId) db.checkCount(1, 0, 1) assertFalse(db.inTxExists(incoming.bankId!!)) // Idempotent - ingestIncomingPayment(db, incomingMissingId, AccountType.exchange) + ingestIncomingPayment(db, cfg, incomingMissingId) db.checkCount(1, 0, 1) // Different metadata is accepted - ingestIncomingPayment(db, genInPay(subject, "KUDOS:9"), AccountType.exchange) - ingestIncomingPayment(db, genInPay("another $subject"), AccountType.exchange) + ingestIncomingPayment(db, cfg, genInPay(subject, "KUDOS:9")) + ingestIncomingPayment(db, cfg, genInPay("another $subject")) db.checkCount(3, 0, 3) // Different medata with missing id are accepted - ingestIncomingPayment(db, incomingMissingId.copy(amount = TalerAmount("KUDOS:9.5")), AccountType.exchange) - ingestIncomingPayment(db, incomingMissingId.copy(wireTransferSubject = "again another $subject"), AccountType.exchange) + ingestIncomingPayment(db, cfg, incomingMissingId.copy(amount = TalerAmount("KUDOS:9.5"))) + ingestIncomingPayment(db, cfg, incomingMissingId.copy(wireTransferSubject = "again another $subject")) db.checkCount(5, 0, 5) // Recover bank ID when metadata match - ingestIncomingPayment(db, incoming, AccountType.exchange) + ingestIncomingPayment(db, cfg, incoming) assertTrue(db.inTxExists(incoming.bankId!!)) // Idempotent - ingestIncomingPayment(db, incoming, AccountType.exchange) + ingestIncomingPayment(db, cfg, incoming) db.checkCount(5, 0, 5) // Missing ID is ignored - ingestIncomingPayment(db, incomingMissingId, AccountType.exchange) + ingestIncomingPayment(db, cfg, incomingMissingId) db.checkCount(5, 0, 5) // Other ID is accepted - ingestIncomingPayment(db, incomingMissingId.copy(bankId = "NEW"), AccountType.exchange) + ingestIncomingPayment(db, cfg, incomingMissingId.copy(bankId = "NEW")) db.checkCount(6, 0, 6) } + + // Test skipping transaction based on config + @Test + fun skipping() = setup("skip.conf") { db, cfg -> + suspend fun checkCount(nbTxs: Int, nbBounce: Int) { + db.serializable( + """ + SELECT (SELECT count(*) FROM incoming_transactions) + (SELECT count(*) FROM outgoing_transactions) AS transactions, + (SELECT count(*) FROM bounced_transactions) AS bounce + """ + ) { + one { + assertEquals( + Pair(nbTxs, nbBounce), + Pair(it.getInt("transactions"), it.getInt("bounce")) + ) + } + } + } + + suspend fun ingest(executionTime: Instant) { + for (tx in sequenceOf( + genInPay("test at $executionTime", executionTime = executionTime), + genOutPay("test at $executionTime", executionTime = executionTime) + )) { + ingestTransaction(db, cfg.ebics.ingest, tx) + } + } + + assertEquals(cfg.ebics.fetch.ignoreTransactionsBefore, dateToInstant("2024-04-04")) + assertEquals(cfg.ebics.fetch.ignoreBouncesBefore, dateToInstant("2024-06-12")) + + // No transaction at the beginning + checkCount(0, 0) + + // Skipped transactions + ingest(cfg.ebics.fetch.ignoreTransactionsBefore.minusMillis(10)) + checkCount(0, 0) + + // Skipped bounces + ingest(cfg.ebics.fetch.ignoreTransactionsBefore) + ingest(cfg.ebics.fetch.ignoreTransactionsBefore.plusMillis(10)) + ingest(cfg.ebics.fetch.ignoreBouncesBefore.minusMillis(10)) + checkCount(6, 0) + + // Bounces + ingest(cfg.ebics.fetch.ignoreBouncesBefore) + ingest(cfg.ebics.fetch.ignoreBouncesBefore.plusMillis(10)) + checkCount(10, 2) + } } -class PaymentInitiationsTest { +class PaymentInitiationsTest { @Test fun status() = setup { db, _ -> assertIs<PaymentInitiationResult.Success>( diff --git a/nexus/src/test/kotlin/bench.kt b/nexus/src/test/kotlin/bench.kt @@ -89,6 +89,8 @@ class Bench { } println("Bench $ITER times with $AMOUNT rows") + val ingestCfg = NexusIngestConfig.default(AccountType.exchange) + bench(ITER) { serverSetup { db -> // Generate data db.conn { genData(it, AMOUNT) } @@ -111,13 +113,13 @@ class Bench { } measureAction("ingest_reserve_missing_id") { val incoming = genInPay("test with ${ShortHashCode.rand()} reserve pub") - ingestIncomingPayment(db, incoming.copy(bankId = null), AccountType.exchange) - ingestIncomingPayment(db, incoming, AccountType.exchange) + ingestIncomingPayment(db, ingestCfg, incoming.copy(bankId = null)) + ingestIncomingPayment(db, ingestCfg, incoming) } measureAction("ingest_kyc_missing_id") { val incoming = genInPay("test with KYC:${ShortHashCode.rand()} account pub") - ingestIncomingPayment(db, incoming.copy(bankId = null), AccountType.exchange) - ingestIncomingPayment(db, incoming, AccountType.exchange) + ingestIncomingPayment(db, ingestCfg, incoming.copy(bankId = null)) + ingestIncomingPayment(db, ingestCfg, incoming) } // Revenue API diff --git a/nexus/src/test/kotlin/helpers.kt b/nexus/src/test/kotlin/helpers.kt @@ -93,7 +93,11 @@ fun genInitPay( ) /** Generates an incoming payment, given its subject */ -fun genInPay(subject: String, amount: String = "KUDOS:44"): IncomingPayment { +fun genInPay( + subject: String, + amount: String = "KUDOS:44", + executionTime: Instant = Instant.now() +): IncomingPayment { val bankId = run { val bytes = ByteArray(16).rand() Base32Crockford.encode(bytes) @@ -102,13 +106,17 @@ fun genInPay(subject: String, amount: String = "KUDOS:44"): IncomingPayment { amount = TalerAmount(amount), debitPaytoUri = "payto://iban/not-used", wireTransferSubject = subject, - executionTime = Instant.now(), + executionTime = executionTime, bankId = bankId ) } /** Generates an outgoing payment, given its subject and messageId */ -fun genOutPay(subject: String, messageId: String? = null): OutgoingPayment { +fun genOutPay( + subject: String, + messageId: String? = null, + executionTime: Instant = Instant.now() +): OutgoingPayment { val id = messageId ?: run { val bytes = ByteArray(16).rand() Base32Crockford.encode(bytes) @@ -117,7 +125,7 @@ fun genOutPay(subject: String, messageId: String? = null): OutgoingPayment { amount = TalerAmount(44, 0, "KUDOS"), creditPaytoUri = "payto://iban/CH4189144589712575493?receiver-name=Test", wireTransferSubject = subject, - executionTime = Instant.now(), + executionTime = executionTime, messageId = id ) } @@ -166,30 +174,34 @@ suspend fun talerableOut(db: Database) { /** Ingest a talerable reserve incoming transaction */ suspend fun talerableIn(db: Database, nullId: Boolean = false, amount: String = "CHF:44") { val reserve_pub = ShortHashCode.rand() - ingestIncomingPayment(db, genInPay("test with $reserve_pub reserve pub", amount).run { - if (nullId) { - copy(bankId = null) - } else { - this + ingestIncomingPayment(db, NexusIngestConfig.default(AccountType.exchange), + genInPay("test with $reserve_pub reserve pub", amount).run { + if (nullId) { + copy(bankId = null) + } else { + this + } } - }, AccountType.exchange) + ) } /** Ingest a talerable KYC incoming transaction */ suspend fun talerableKycIn(db: Database, nullId: Boolean = false, amount: String = "CHF:44") { val account_pub = ShortHashCode.rand() - ingestIncomingPayment(db, genInPay("test with KYC:$account_pub account pub", amount).run { - if (nullId) { - copy(bankId = null) - } else { - this + ingestIncomingPayment(db, NexusIngestConfig.default(AccountType.exchange), + genInPay("test with KYC:$account_pub account pub", amount).run { + if (nullId) { + copy(bankId = null) + } else { + this + } } - }, AccountType.exchange) + ) } /** Ingest an incoming transaction */ suspend fun ingestIn(db: Database) { - ingestIncomingPayment(db, genInPay("ignored"), AccountType.normal) + ingestIncomingPayment(db, NexusIngestConfig.default(AccountType.normal), genInPay("ignored")) } /** Ingest an outgoing transaction */ diff --git a/testbench/src/test/kotlin/IntegrationTest.kt b/testbench/src/test/kotlin/IntegrationTest.kt @@ -35,8 +35,7 @@ import tech.libeufin.bank.cli.LibeufinBank import tech.libeufin.common.* import tech.libeufin.common.api.engine import tech.libeufin.common.db.one -import tech.libeufin.nexus.AccountType -import tech.libeufin.nexus.IncomingPayment +import tech.libeufin.nexus.* import tech.libeufin.nexus.cli.LibeufinNexus import tech.libeufin.nexus.cli.ingestIncomingPayment import tech.libeufin.nexus.nexusConfig @@ -150,6 +149,7 @@ class IntegrationTest { } setup("conf/integration.conf") { db -> + val cfg = NexusIngestConfig.default(AccountType.exchange) val userPayTo = IbanPayto.rand() val fiatPayTo = IbanPayto.rand() @@ -170,24 +170,23 @@ class IntegrationTest { ) assertException("ERROR: cashin failed: missing exchange account") { - ingestIncomingPayment(db, reservePayment, AccountType.exchange) + ingestIncomingPayment(db, cfg, reservePayment) } // But KYC works ingestIncomingPayment( - db, + db, cfg, reservePayment.copy( bankId = "kyc", wireTransferSubject = "Error test KYC:${EddsaPublicKey.rand()}" - ), - AccountType.exchange + ) ) // Create exchange account bankCmd.run("create-account $flags -u exchange -p password --name 'Mr Money' --exchange") assertException("ERROR: cashin currency conversion failed: missing conversion rates") { - ingestIncomingPayment(db, reservePayment, AccountType.exchange) + ingestIncomingPayment(db, cfg, reservePayment) } // Start server @@ -221,9 +220,9 @@ class IntegrationTest { // Too small amount db.checkCount(1, 0, 1) - ingestIncomingPayment(db, reservePayment.copy( + ingestIncomingPayment(db, cfg, reservePayment.copy( amount = TalerAmount("EUR:0.01"), - ), AccountType.exchange) + )) db.checkCount(2, 1, 1) client.get("http://0.0.0.0:8080/accounts/exchange/transactions") { basicAuth("exchange", "password") @@ -234,17 +233,17 @@ class IntegrationTest { wireTransferSubject = "Success $reservePub", bankId = "success" ) - ingestIncomingPayment(db, validPayment, AccountType.exchange) + ingestIncomingPayment(db, cfg, validPayment) db.checkCount(3, 1, 2) client.get("http://0.0.0.0:8080/accounts/exchange/transactions") { basicAuth("exchange", "password") }.assertOkJson<BankAccountTransactionsResponse>() // Check idempotency - ingestIncomingPayment(db, validPayment, AccountType.exchange) - ingestIncomingPayment(db, validPayment.copy( + ingestIncomingPayment(db, cfg, validPayment) + ingestIncomingPayment(db, cfg, validPayment.copy( wireTransferSubject="Success 2 $reservePub" - ), AccountType.exchange) + )) db.checkCount(3, 1, 2) } }