commit ce31d171834bcf493ab39f41db64ad1961b610c5
parent 5de415d9534784711a33b710f25dfb802399188c
Author: Antoine A <>
Date: Tue, 28 May 2024 19:09:03 +0900
nexus: support for incoming transactions without bank ID and fix instant transactions support from GLS
Diffstat:
10 files changed, 242 insertions(+), 96 deletions(-)
diff --git a/database-versioning/libeufin-nexus-0004.sql b/database-versioning/libeufin-nexus-0004.sql
@@ -0,0 +1,24 @@
+--
+-- This file is part of TALER
+-- Copyright (C) 2024 Taler Systems SA
+--
+-- TALER is free software; you can redistribute it and/or modify it under the
+-- terms of the GNU General Public License as published by the Free Software
+-- Foundation; either version 3, or (at your option) any later version.
+--
+-- TALER is distributed in the hope that it will be useful, but WITHOUT ANY
+-- WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+-- A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+--
+-- You should have received a copy of the GNU General Public License along with
+-- TALER; see the file COPYING. If not, see <http://www.gnu.org/licenses/>
+
+BEGIN;
+
+SELECT _v.register_patch('libeufin-nexus-0004', NULL, NULL);
+
+SET search_path TO libeufin_nexus;
+
+-- TODO fix this hack in a future update
+ALTER TABLE incoming_transactions ALTER COLUMN bank_id DROP NOT NULL;
+COMMIT;
diff --git a/database-versioning/libeufin-nexus-procedures.sql b/database-versioning/libeufin-nexus-procedures.sql
@@ -228,15 +228,28 @@ CREATE FUNCTION register_incoming_and_talerable(
,OUT out_tx_id INT8
)
LANGUAGE plpgsql AS $$
+DECLARE
+need_reconcile BOOLEAN;
BEGIN
--- Check conflict
-IF EXISTS (
- SELECT FROM talerable_incoming_transactions
- JOIN incoming_transactions USING(incoming_transaction_id)
- WHERE reserve_public_key = in_reserve_public_key
- AND bank_id != in_bank_id
-) THEN
- out_reserve_pub_reuse = TRUE;
+-- Check if exists
+SELECT incoming_transaction_id,
+ bank_id IS DISTINCT FROM in_bank_id,
+ bank_id IS NULL AND amount = in_amount
+ AND debit_payto_uri = in_debit_payto_uri
+ AND wire_transfer_subject = in_wire_transfer_subject
+ INTO out_tx_id, out_reserve_pub_reuse, need_reconcile
+ FROM talerable_incoming_transactions
+ JOIN incoming_transactions USING(incoming_transaction_id)
+ WHERE reserve_public_key = in_reserve_public_key;
+
+IF FOUND THEN
+ IF need_reconcile THEN
+ IF in_bank_id IS NOT NULL THEN
+ -- Update the bank_id now that we have it
+ UPDATE incoming_transactions SET bank_id = in_bank_id WHERE incoming_transaction_id = out_tx_id;
+ END IF;
+ out_reserve_pub_reuse=false;
+ END IF;
RETURN;
END IF;
diff --git a/nexus/src/main/kotlin/tech/libeufin/nexus/EbicsFetch.kt b/nexus/src/main/kotlin/tech/libeufin/nexus/EbicsFetch.kt
@@ -122,6 +122,10 @@ suspend fun ingestIncomingPayment(
accountType: AccountType
) {
suspend fun bounce(msg: String) {
+ if (payment.bankId == null) {
+ logger.debug("$payment ignored: missing bank ID")
+ return;
+ }
when (accountType) {
AccountType.exchange -> {
val result = db.payment.registerMalformedIncoming(
diff --git a/nexus/src/main/kotlin/tech/libeufin/nexus/Iso20022.kt b/nexus/src/main/kotlin/tech/libeufin/nexus/Iso20022.kt
@@ -271,8 +271,8 @@ sealed interface TxNotification {
/** ISO20022 incoming payment */
data class IncomingPayment(
- /** ISO20022 AccountServicerReference */
- val bankId: String,
+ /** ISO20022 UETR & TxID */
+ val bankId: String? = null, // Null when TxID is wrong with Atruvia's implementation of instant transactions
val amount: TalerAmount,
val wireTransferSubject: String,
override val executionTime: Instant,
@@ -288,9 +288,9 @@ data class OutgoingPayment(
/** ISO20022 MessageIdentification & EndToEndId */
val messageId: String,
val amount: TalerAmount,
- val wireTransferSubject: String? = null, // not showing in camt.054
+ val wireTransferSubject: String? = null, // Some implementation does not provide this for recovery
override val executionTime: Instant,
- val creditPaytoUri: String? = null, // not showing in camt.054
+ val creditPaytoUri: String? = null // Some implementation does not provide this for recovery
): TxNotification {
override fun toString(): String {
return "OUT ${executionTime.fmtDate()} $amount '$messageId' creditor=$creditPaytoUri subject=\"$wireTransferSubject\""
@@ -324,29 +324,53 @@ fun parseTx(
/*
In ISO 20022 specifications, most fields are optional and the same information
can be written several times in different places. For libeufin, we're only
- interested in a subset of the available values that can be found in both camt.053
- and camt.054. This function should not fail on legitimate files and should simply
- warn when available information are insufficient.
+ interested in a subset of the available values that can be found in both camt.052,
+ camt.053 and camt.054. This function should not fail on legitimate files and should
+ simply warn when available information are insufficient.
+
+ EBICS and ISO20022 do not provide a perfect transaction identifier. The best is the
+ UETR (unique end-to-end transaction reference), which is a universally unique
+ identifier (UUID). However, it is not supplied by all banks. TxId (TransactionIdentification)
+ is a unique identification as assigned by the first instructing agent. As its format
+ is ambiguous, its uniqueness is not guaranteed by the standard, and it is only
+ supposed to be unique for a “pre-agreed period”, whatever that means. These two
+ identifiers are optional in the standard, but have the advantage of being unique
+ and can be used to track a transaction between banks so we use them when available.
+
+ It is also possible to use AccountServicerReference, which is a unique reference
+ assigned by the account servicing institution. They can be present at several levels
+ (batch level, transaction level, etc.) and are often optional. They also have the
+ disadvantage of being known only by the account servicing institution. They should
+ therefore only be used as a last resort.
*/
- /** Assert that transaction status is BOOK */
- fun XmlDestructor.assertBooked(ref: String?) {
- one("Sts") {
+ /** Check if a transaction status is BOOK */
+ fun XmlDestructor.isBooked(): Boolean {
+ // We check at the Sts or Sts/Cd level for retrocompatibility
+ return one("Sts") {
val status = opt("Cd")?.text() ?: text()
- require(status == "BOOK") {
- "Found non booked entry $ref, stop parsing: expected BOOK got $status"
- }
+ status == "BOOK"
}
}
+ /** Parse the instruction execution date */
fun XmlDestructor.executionDate(): Instant {
// Value date if present else booking date
- return (opt("ValDt") ?: one("BookgDt")).one("Dt").date().atStartOfDay().toInstant(ZoneOffset.UTC)
+ val date = opt("ValDt") ?: one("BookgDt")
+ val parsed = date.opt("Dt") {
+ date().atStartOfDay()
+ } ?: date.one("DtTm") {
+ dateTime()
+ }
+ return parsed.toInstant(ZoneOffset.UTC)
}
+ /** Parse original transaction ID generated by libeufin-nexus */
fun XmlDestructor.nexusId(): String? =
+ // We check at the EndToEndId or MsgId level for retrocompatibility
opt("Refs") { opt("EndToEndId")?.textProvided() ?: opt("MsgId")?.text() }
+ /** Parse and format transaction return reasons */
fun XmlDestructor.returnReason(): String = one("RtrInf") {
val code = one("Rsn").one("Cd").enum<ExternalReturnReasonCode>()
val info = opt("AddtlInf")?.text()
@@ -383,13 +407,14 @@ fun parseTx(
XmlDestructor.fromStream(notifXml, "Document") { when (dialect) {
Dialect.gls -> {
+ /** Common parsing logic for camt.052 and camt.053 */
fun XmlDestructor.parseGlsInner() {
opt("Acct") {
// Sanity check on currency and IBAN ?
}
each("Ntry") {
+ if (!isBooked()) return@each
val entryRef = opt("AcctSvcrRef")?.text()
- assertBooked(entryRef)
val bookDate = executionDate()
val kind = one("CdtDbtInd").enum<Kind>()
val amount = amount(acceptedCurrency)
@@ -438,38 +463,41 @@ fun parseTx(
}
}
}
- opt("BkToCstmrAcctRpt")?.each("Rpt") { // Camt.052
+ opt("BkToCstmrStmt")?.each("Stmt") { // Camt.053
+ // All transactions appear here the day after they are booked
parseGlsInner()
}
- opt("BkToCstmrStmt")?.each("Stmt") { // Camt.053
+ opt("BkToCstmrAcctRpt")?.each("Rpt") { // Camt.052
+ // Transactions might appear here first before the end of the day
parseGlsInner()
}
opt("BkToCstmrDbtCdtNtfctn")?.each("Ntfctn") { // Camt.054
+ // Instant transactions appear here a few seconds after being booked
opt("Acct") {
// Sanity check on currency and IBAN ?
}
each("Ntry") {
+ if (!isBooked()) return@each
+ if (isReversalCode()) return@each
val entryRef = opt("AcctSvcrRef")?.text()
- assertBooked(entryRef)
val bookDate = executionDate()
val kind = one("CdtDbtInd").enum<Kind>()
val amount = amount(acceptedCurrency)
- if (!isReversalCode()) {
- one("NtryDtls").one("TxDtls") {
- val txRef = one("Refs").opt("AcctSvcrRef")?.text()
- val subject = opt("RmtInf")?.map("Ustrd") { text() }?.joinToString("")
- if (kind == Kind.CRDT) {
- val bankId = one("Refs").opt("TxId")?.text()
- val debtorPayto = opt("RltdPties") { payto("Dbtr") }
- txsInfo.add(TxInfo.Credit(
- ref = bankId ?: txRef ?: entryRef,
- bookDate = bookDate,
- bankId = bankId,
- amount = amount,
- subject = subject,
- debtorPayto = debtorPayto
- ))
- }
+ one("NtryDtls").one("TxDtls") {
+ val txRef = one("Refs").opt("AcctSvcrRef")?.text()
+ val subject = opt("RmtInf")?.map("Ustrd") { text() }?.joinToString("")
+ if (kind == Kind.CRDT) {
+ val bankId = one("Refs").opt("TxId")?.text()
+ val debtorPayto = opt("RltdPties") { payto("Dbtr") }
+ txsInfo.add(TxInfo.Credit(
+ ref = txRef ?: entryRef,
+ bookDate = bookDate,
+ // TODO use the bank ID again when Atruvia's implementation is fixed
+ bankId = null,
+ amount = amount,
+ subject = subject,
+ debtorPayto = debtorPayto
+ ))
}
}
}
@@ -477,70 +505,79 @@ fun parseTx(
}
Dialect.postfinance -> {
opt("BkToCstmrStmt")?.each("Stmt") { // Camt.053
+ /*
+ All transactions appear here on the day following their booking. Alas, some
+ necessary metadata is missing, which is only present in camt.054. However,
+ this file contains the structured return reasons that are missing from the
+ camt.054 files. That's why we only use this file for this purpose.
+ */
opt("Acct") {
// Sanity check on currency and IBAN ?
}
each("Ntry") {
+ if (!isBooked()) return@each
+ // Non reversal transaction are handled in camt.054
+ if (!isReversalCode()) return@each
+
val entryRef = opt("AcctSvcrRef")?.text()
- assertBooked(entryRef)
val bookDate = executionDate()
- if (isReversalCode()) {
- one("NtryDtls").one("TxDtls") {
- val kind = one("CdtDbtInd").enum<Kind>()
- if (kind == Kind.CRDT) {
- val txRef = opt("Refs")?.opt("AcctSvcrRef")?.text()
- val nexusId = nexusId()
- val reason = returnReason()
- txsInfo.add(TxInfo.CreditReversal(
- ref = nexusId ?: txRef ?: entryRef,
- bookDate = bookDate,
- nexusId = nexusId,
- reason = reason
- ))
- }
+ one("NtryDtls").one("TxDtls") {
+ val kind = one("CdtDbtInd").enum<Kind>()
+ if (kind == Kind.CRDT) {
+ val txRef = opt("Refs")?.opt("AcctSvcrRef")?.text()
+ val nexusId = nexusId()
+ val reason = returnReason()
+ txsInfo.add(TxInfo.CreditReversal(
+ ref = nexusId ?: txRef ?: entryRef,
+ bookDate = bookDate,
+ nexusId = nexusId,
+ reason = reason
+ ))
}
}
}
}
opt("BkToCstmrDbtCdtNtfctn")?.each("Ntfctn") { // Camt.054
+ // Instant transactions appear here a moment after being booked
opt("Acct") {
// Sanity check on currency and IBAN ?
}
each("Ntry") {
+ if (!isBooked()) return@each
+ // Reversal are handled from camt.053
+ if (isReversalCode()) return@each
+
val entryRef = opt("AcctSvcrRef")?.text()
- assertBooked(entryRef)
val bookDate = executionDate()
- if (!isReversalCode()) {
- one("NtryDtls").each("TxDtls") {
- val kind = one("CdtDbtInd").enum<Kind>()
- val amount = amount(acceptedCurrency)
- val txRef = one("Refs").opt("AcctSvcrRef")?.text()
- val subject = opt("RmtInf")?.map("Ustrd") { text() }?.joinToString("")
- when (kind) {
- Kind.CRDT -> {
- val bankId = one("Refs").opt("UETR")?.text()
- val debtorPayto = opt("RltdPties") { payto("Dbtr") }
- txsInfo.add(TxInfo.Credit(
- ref = bankId ?: txRef ?: entryRef,
- bookDate = bookDate,
- bankId = bankId,
- amount = amount,
- subject = subject,
- debtorPayto = debtorPayto
- ))
- }
- Kind.DBIT -> {
- val nexusId = nexusId()
- val creditorPayto = opt("RltdPties") { payto("Cdtr") }
- txsInfo.add(TxInfo.Debit(
- ref = nexusId ?: txRef ?: entryRef,
- bookDate = bookDate,
- nexusId = nexusId,
- amount = amount,
- subject = subject,
- creditorPayto = creditorPayto
- ))
- }
+ one("NtryDtls").each("TxDtls") {
+ val kind = one("CdtDbtInd").enum<Kind>()
+ val amount = amount(acceptedCurrency)
+ val txRef = one("Refs").opt("AcctSvcrRef")?.text()
+ val subject = opt("RmtInf")?.map("Ustrd") { text() }?.joinToString("")
+ when (kind) {
+ Kind.CRDT -> {
+ val bankId = one("Refs").opt("UETR")?.text()
+ val debtorPayto = opt("RltdPties") { payto("Dbtr") }
+ txsInfo.add(TxInfo.Credit(
+ ref = bankId ?: txRef ?: entryRef,
+ bookDate = bookDate,
+ bankId = bankId,
+ amount = amount,
+ subject = subject,
+ debtorPayto = debtorPayto
+ ))
+ }
+ Kind.DBIT -> {
+ val nexusId = nexusId()
+ val creditorPayto = opt("RltdPties") { payto("Cdtr") }
+ txsInfo.add(TxInfo.Debit(
+ ref = nexusId ?: txRef ?: entryRef,
+ bookDate = bookDate,
+ nexusId = nexusId,
+ amount = amount,
+ subject = subject,
+ creditorPayto = creditorPayto
+ ))
}
}
}
@@ -604,8 +641,8 @@ private fun parseTxLogic(info: TxInfo): TxNotification {
)
}
is TxInfo.Credit -> {
- if (info.bankId == null)
- throw TxErr("missing bank ID for Credit ${info.ref}")
+ /*if (info.bankId == null) TODO use the bank ID again when Atruvia's implementation is fixed
+ throw TxErr("missing bank ID for Credit ${info.ref}")*/
if (info.subject == null)
throw TxErr("missing subject for Credit ${info.ref}")
if (info.debtorPayto == null)
diff --git a/nexus/src/main/kotlin/tech/libeufin/nexus/Main.kt b/nexus/src/main/kotlin/tech/libeufin/nexus/Main.kt
@@ -346,7 +346,7 @@ class ListCmd: CliktCommand("List nexus transactions", name = "list") {
txs.map {
listOf(
"${it.date} ${it.amount}",
- it.id,
+ it.id.toString(),
it.reservePub?.toString() ?: "",
fmtPayto(it.debtor),
it.subject
diff --git a/nexus/src/main/kotlin/tech/libeufin/nexus/db/PaymentDAO.kt b/nexus/src/main/kotlin/tech/libeufin/nexus/db/PaymentDAO.kt
@@ -283,7 +283,7 @@ data class IncomingTxMetadata(
val amount: DecimalNumber,
val subject: String,
val debtor: String,
- val id: String,
+ val id: String?,
val reservePub: EddsaPublicKey?
)
diff --git a/nexus/src/test/kotlin/CliTest.kt b/nexus/src/test/kotlin/CliTest.kt
@@ -151,5 +151,8 @@ class CliTest {
talerableOut(db)
talerableIn(db)
check()
+ // Check with null id
+ talerableIn(db, true)
+ check()
}
}
\ No newline at end of file
diff --git a/nexus/src/test/kotlin/DatabaseTest.kt b/nexus/src/test/kotlin/DatabaseTest.kt
@@ -19,7 +19,9 @@
import org.junit.Test
import tech.libeufin.common.*
+import tech.libeufin.common.db.*
import tech.libeufin.nexus.db.InitiatedDAO.PaymentInitiationResult
+import tech.libeufin.nexus.db.*
import tech.libeufin.nexus.*
import java.time.Instant
import kotlin.test.assertEquals
@@ -71,6 +73,23 @@ class OutgoingPaymentsTest {
}
}
+suspend fun Database.checkCount(nbIncoming: Int, nbBounce: Int, nbTalerable: Int) {
+ conn {
+ val cIncoming = it.prepareStatement("SELECT count(*) FROM incoming_transactions").one { it.getInt(1) }
+ val cBounce = it.prepareStatement("SELECT count(*) FROM bounced_transactions").one { it.getInt(1) }
+ val cTalerable = it.prepareStatement("SELECT count(*) FROM talerable_incoming_transactions").one { it.getInt(1) }
+ assertEquals(Triple(nbIncoming, nbBounce, nbTalerable), Triple(cIncoming, cBounce, cTalerable))
+ }
+}
+
+suspend fun Database.inTxExists(id: String): Boolean = conn {
+ it.prepareStatement("SELECT EXISTS(SELECT FROM incoming_transactions WHERE bank_id = ?)").apply {
+ setString(1, id)
+ }.one {
+ it.getBoolean(1)
+ }
+}
+
class IncomingPaymentsTest {
// Tests creating and bouncing incoming payments in one DB transaction.
@Test
@@ -125,6 +144,46 @@ class IncomingPaymentsTest {
)
}
}
+
+ // Test creating an incoming taler transaction without and ID and reconcile it later again
+ @Test
+ fun reconcileMissingId() = setup { db, _ ->
+ // Register with missing ID
+ val reserve_pub = ShortHashCode.rand()
+ val incoming = genInPay("history test with $reserve_pub reserve pub")
+ val incomingMissingId = incoming.copy(bankId = null)
+ ingestIncomingPayment(db, incomingMissingId, AccountType.exchange)
+ db.checkCount(1, 0, 1)
+ assertFalse(db.inTxExists(incoming.bankId!!))
+
+ // Idempotent
+ ingestIncomingPayment(db, incomingMissingId, AccountType.exchange)
+ db.checkCount(1, 0, 1)
+
+ // Different metadata is bounced
+ ingestIncomingPayment(db, genInPay("another $reserve_pub reserve pub"), AccountType.exchange)
+ db.checkCount(2, 1, 1)
+
+ // Different medata with missing id is ignored
+ ingestIncomingPayment(db, incomingMissingId.copy(amount = TalerAmount("KUDOS:9")), AccountType.exchange)
+ db.checkCount(2, 1, 1)
+
+ // Recover bank ID when metadata match
+ ingestIncomingPayment(db, incoming, AccountType.exchange)
+ assertTrue(db.inTxExists(incoming.bankId!!))
+
+ // Idempotent
+ ingestIncomingPayment(db, incoming, AccountType.exchange)
+ db.checkCount(2, 1, 1)
+
+ // Missing ID is ignored
+ ingestIncomingPayment(db, incomingMissingId, AccountType.exchange)
+ db.checkCount(2, 1, 1)
+
+ // Other ID is bounced known that we know the id
+ ingestIncomingPayment(db, incomingMissingId.copy(bankId = "NEW"), AccountType.exchange)
+ db.checkCount(3, 2, 1)
+ }
}
class PaymentInitiationsTest {
diff --git a/nexus/src/test/kotlin/Iso20022Test.kt b/nexus/src/test/kotlin/Iso20022Test.kt
@@ -171,7 +171,7 @@ class Iso20022Test {
assertEquals(
listOf(
IncomingPayment(
- bankId = "IS11PGENODEFF2DA8899900378806",
+ bankId = null, //"IS11PGENODEFF2DA8899900378806",
amount = TalerAmount("EUR:2.5"),
wireTransferSubject = "Test ICT",
executionTime = instant("2024-05-05"),
diff --git a/nexus/src/test/kotlin/helpers.kt b/nexus/src/test/kotlin/helpers.kt
@@ -147,9 +147,15 @@ suspend fun talerableOut(db: Database) {
}
/** Ingest a talerable incoming transaction */
-suspend fun talerableIn(db: Database) {
+suspend fun talerableIn(db: Database, nullId: Boolean = false) {
val reserve_pub = ShortHashCode.rand()
- ingestIncomingPayment(db, genInPay("history test with $reserve_pub reserve pub"), AccountType.exchange)
+ ingestIncomingPayment(db, genInPay("history test with $reserve_pub reserve pub").run {
+ if (nullId) {
+ copy(bankId = null)
+ } else {
+ this
+ }
+ }, AccountType.exchange)
}
/** Ingest an incoming transaction */
