diff options
author | Antoine A <> | 2024-04-18 12:43:19 +0900 |
---|---|---|
committer | Antoine A <> | 2024-04-18 12:43:19 +0900 |
commit | 9d8599f68e063c47eba1a9b1348f79f6357023b2 (patch) | |
tree | 4ee8780e3d2013b3879d23b66c2199aef8013711 | |
parent | a8c67aad250c62f0db3baebe5ea8bb67571382a8 (diff) | |
download | libeufin-9d8599f68e063c47eba1a9b1348f79f6357023b2.tar.gz libeufin-9d8599f68e063c47eba1a9b1348f79f6357023b2.tar.bz2 libeufin-9d8599f68e063c47eba1a9b1348f79f6357023b2.zip |
Merge camt.053 and camt.054 logic and handle missing information with warnings instead of failures
6 files changed, 299 insertions, 208 deletions
diff --git a/nexus/src/main/kotlin/tech/libeufin/nexus/EbicsFetch.kt b/nexus/src/main/kotlin/tech/libeufin/nexus/EbicsFetch.kt index 03c127c0..978c7d2d 100644 --- a/nexus/src/main/kotlin/tech/libeufin/nexus/EbicsFetch.kt +++ b/nexus/src/main/kotlin/tech/libeufin/nexus/EbicsFetch.kt @@ -156,9 +156,9 @@ private suspend fun ingestDocument( whichDocument: SupportedDocument ) { when (whichDocument) { - SupportedDocument.CAMT_054 -> { + SupportedDocument.CAMT_053, SupportedDocument.CAMT_054 -> { try { - parseTxNotif(xml, cfg.currency).forEach { + parseTx(xml, cfg.currency).forEach { if (cfg.fetch.ignoreBefore != null && it.executionTime < cfg.fetch.ignoreBefore) { logger.debug("IGNORE $it") } else { @@ -212,26 +212,6 @@ private suspend fun ingestDocument( db.initiated.bankMessage(status.msgId, msg) } } - SupportedDocument.CAMT_053 -> { - try { - parseTxStatement(xml, cfg.currency).forEach { - if (cfg.fetch.ignoreBefore != null && it.executionTime < cfg.fetch.ignoreBefore) { - logger.debug("IGNORE $it") - } else { - when (it) { - is IncomingPayment -> ingestIncomingPayment(db, it) - is OutgoingPayment -> ingestOutgoingPayment(db, it) - is TxNotification.Reversal -> { - logger.error("BOUNCE '${it.msgId}': ${it.reason}") - db.initiated.reversal(it.msgId, "Payment bounced: ${it.reason}") - } - } - } - } - } catch (e: Exception) { - throw Exception("Ingesting statements failed", e) - } - } SupportedDocument.CAMT_052 -> { // TODO parsing // TODO ingesting @@ -246,10 +226,10 @@ private suspend fun ingestDocuments( whichDocument: SupportedDocument ) { when (whichDocument) { - SupportedDocument.CAMT_054, SupportedDocument.PAIN_002, + SupportedDocument.CAMT_052, SupportedDocument.CAMT_053, - SupportedDocument.CAMT_052 -> { + SupportedDocument.CAMT_054 -> { try { content.unzipEach { fileName, xmlContent -> logger.trace("parse $fileName") diff --git a/nexus/src/main/kotlin/tech/libeufin/nexus/Iso20022.kt b/nexus/src/main/kotlin/tech/libeufin/nexus/Iso20022.kt index 3ff3e76f..124f90e4 100644 --- a/nexus/src/main/kotlin/tech/libeufin/nexus/Iso20022.kt +++ b/nexus/src/main/kotlin/tech/libeufin/nexus/Iso20022.kt @@ -303,203 +303,184 @@ data class OutgoingPayment( } } -/** Parse camt.054 XML file */ -fun parseTxNotif( +private fun XmlDestructor.payto(prefix: String): String? { + val iban = opt("${prefix}Acct")?.one("Id")?.one("IBAN")?.text() + return if (iban != null) { + val payto = StringBuilder("payto://iban/$iban") + val name = opt(prefix)?.opt("Pty")?.one("Nm")?.text() + if (name != null) { + val urlEncName = URLEncoder.encode(name, "utf-8") + payto.append("?receiver-name=$urlEncName") + } + return payto.toString() + } else { + null + } +} + +/** Parse camt.054 or camt.053 file */ +fun parseTx( notifXml: InputStream, acceptedCurrency: String ): List<TxNotification> { - fun notificationForEachTx( - directionLambda: XmlDestructor.(Instant, Boolean, String?) -> Unit - ) { - XmlDestructor.fromStream(notifXml, "Document") { - opt("BkToCstmrDbtCdtNtfctn")?.each("Ntfctn") { - each("Ntry") { - val reversal = opt("RvslInd")?.bool() ?: false - val info = opt("AddtlNtryInf")?.text() - one("Sts") { - if (text() != "BOOK") { - one("Cd") { - if (text() != "BOOK") - throw Exception("Found non booked transaction, " + - "stop parsing. Status was: ${text()}" - ) - } - } - } - val bookDate: Instant = one("BookgDt").one("Dt").date().atStartOfDay().toInstant(ZoneOffset.UTC) - one("NtryDtls").each("TxDtls") { - directionLambda(this, bookDate, reversal, info) - } + fun XmlDestructor.parseNotif(): List<RawTx> { + one("Sts") { + if (text() != "BOOK") { + one("Cd") { + if (text() != "BOOK") + throw Exception("Found non booked transaction, " + + "stop parsing. Status was: ${text()}" + ) } } } + val reversal = opt("RvslInd")?.bool() ?: false + val info = opt("AddtlNtryInf")?.text() + val bookDate: Instant = one("BookgDt").one("Dt").date().atStartOfDay().toInstant(ZoneOffset.UTC) + val ref = opt("AcctSvcrRef")?.text() + return one("NtryDtls").map("TxDtls") { + val kind = one("CdtDbtInd").text() + val amount: TalerAmount = one("Amt") { + val currency = attr("Ccy") + /** FIXME: test by sending non-CHF to PoFi and see which currency gets here. */ + if (currency != acceptedCurrency) throw Exception("Currency $currency not supported") + TalerAmount("$currency:${text()}") + } + var msgId = opt("Refs")?.opt("MsgId")?.text() + val subject = opt("RmtInf")?.map("Ustrd") { text() }?.joinToString("") + var debtorPayto = opt("RltdPties") { payto("Dbtr") } + RawTx( + kind, + bookDate, + amount, + reversal, + info, + ref, + msgId, + subject, + debtorPayto + ) + } } - - val notifications = mutableListOf<TxNotification>() - notificationForEachTx { bookDate, reversal, info -> + fun XmlDestructor.parseStatement(): RawTx { + one("Sts") { + if (text() != "BOOK") { + one("Cd") { + if (text() != "BOOK") + throw Exception("Found non booked transaction, " + + "stop parsing. Status was: ${text()}" + ) + } + } + } + val reversal = opt("RvslInd")?.bool() ?: false + val info = opt("AddtlNtryInf")?.text() + val bookDate: Instant = one("BookgDt").one("Dt").date().atStartOfDay().toInstant(ZoneOffset.UTC) val kind = one("CdtDbtInd").text() val amount: TalerAmount = one("Amt") { val currency = attr("Ccy") - /** - * FIXME: test by sending non-CHF to PoFi and see which currency gets here. - */ + /** FIXME: test by sending non-CHF to PoFi and see which currency gets here. */ if (currency != acceptedCurrency) throw Exception("Currency $currency not supported") TalerAmount("$currency:${text()}") } - if (reversal) { - require("CRDT" == kind) - val msgId = one("Refs").opt("MsgId")?.text() - if (msgId == null) { - logger.debug("Unsupported reversal without message id") - } else { - notifications.add(TxNotification.Reversal( - msgId = msgId, - reason = info, - executionTime = bookDate - )) - } - return@notificationForEachTx + val ref = opt("AcctSvcrRef")?.text() + return one("NtryDtls").one("TxDtls") { + var msgId = opt("Refs")?.opt("MsgId")?.text() + val subject = opt("RmtInf")?.map("Ustrd") { text() }?.joinToString("") + var debtorPayto = opt("RltdPties") { payto("Dbtr") } + RawTx( + kind, + bookDate, + amount, + reversal, + info, + ref, + msgId, + subject, + debtorPayto + ) } - when (kind) { - "CRDT" -> { - val bankId: String = one("Refs").one("AcctSvcrRef").text() - // Obtaining payment subject. - val subject = opt("RmtInf")?.map("Ustrd") { text() }?.joinToString("") - if (subject == null) { - logger.debug("Skip notification '$bankId', missing subject") - return@notificationForEachTx - } - - // Obtaining the payer's details - val debtorPayto = StringBuilder("payto://iban/") - one("RltdPties") { - one("DbtrAcct").one("Id").one("IBAN") { - debtorPayto.append(text()) - } - // warn: it might need the postal address too.. - one("Dbtr").opt("Pty")?.one("Nm") { - val urlEncName = URLEncoder.encode(text(), "utf-8") - debtorPayto.append("?receiver-name=$urlEncName") - } + } + val raws = mutableListOf<RawTx>() + XmlDestructor.fromStream(notifXml, "Document") { + opt("BkToCstmrDbtCdtNtfctn") { + each("Ntfctn") { + each("Ntry") { + raws.addAll(parseNotif()) } - notifications.add(IncomingPayment( - amount = amount, - bankId = bankId, - debitPaytoUri = debtorPayto.toString(), - executionTime = bookDate, - wireTransferSubject = subject.toString() - )) - } - "DBIT" -> { - val messageId = one("Refs").one("MsgId").text() - notifications.add(OutgoingPayment( - amount = amount, - messageId = messageId, - executionTime = bookDate - )) } - else -> throw Exception("Unknown transaction notification kind '$kind'") - } - } - return notifications -} - -/** Parse camt.053 XML file */ -fun parseTxStatement( - notifXml: InputStream, - acceptedCurrency: String -): List<TxNotification> { - fun notificationForEachTx( - directionLambda: XmlDestructor.(Instant, Boolean, String?) -> Unit - ) { - XmlDestructor.fromStream(notifXml, "Document") { - one("BkToCstmrStmt").each("Stmt") { + } ?: opt("BkToCstmrStmt") { + each("Stmt") { one("Acct") { - // Sanity check on currency and IBAN + // Sanity check on currency and IBAN ? } each("Ntry") { - val reversal = opt("RvslInd")?.bool() ?: false - val info = opt("AddtlNtryInf")?.text() - one("Sts") { - if (text() != "BOOK") { - throw Exception("Found non booked transaction, " + - "stop parsing. Status was: ${text()}" - ) - } - } - val bookDate: Instant = one("BookgDt").one("Dt").date().atStartOfDay().toInstant(ZoneOffset.UTC) - directionLambda(this, bookDate, reversal, info) + raws.add(parseStatement()) } } - } + } ?: throw Exception("Missing BkToCstmrDbtCdtNtfctn or BkToCstmrStmt") + } + return raws.mapNotNull { it -> + try { + parseTxLogic(it) + } catch (e: TxErr) { + // TODO: add more info in doc or in log message? + logger.warn("skip incomplete tx: ${e.msg}") + null + } } +} - val notifications = mutableListOf<TxNotification>() - notificationForEachTx { bookDate, reversal, info -> - val kind = one("CdtDbtInd").text() - val amount: TalerAmount = one("Amt") { - val currency = attr("Ccy") - /** - * FIXME: test by sending non-CHF to PoFi and see which currency gets here. - */ - if (currency != acceptedCurrency) throw Exception("Currency $currency not supported") - TalerAmount("$currency:${text()}") +private data class RawTx( + val kind: String, + val bookDate: Instant, + val amount: TalerAmount, + val reversal: Boolean, + val info: String?, + val ref: String?, + val msgId: String?, + val subject: String?, + val debtorPayto: String? +) + +private class TxErr(val msg: String): Exception(msg) + +private fun parseTxLogic(raw: RawTx): TxNotification { + if (raw.reversal) { + require("CRDT" == raw.kind) // TODO handle DBIT reversal + if (raw.msgId == null) + throw TxErr("missing msg ID for Credit reversal ${raw.ref}") + return TxNotification.Reversal( + msgId = raw.msgId, + reason = raw.info, + executionTime = raw.bookDate + ) + } + return when (raw.kind) { + "CRDT" -> { + if (raw.ref == null) + throw TxErr("missing subject for Credit ${raw.ref}") + if (raw.subject == null) + throw TxErr("missing subject for Credit ${raw.ref}") + if (raw.debtorPayto == null) + throw TxErr("missing debtor info for Credit ${raw.ref}") + IncomingPayment( + amount = raw.amount, + bankId = raw.ref, + debitPaytoUri = raw.debtorPayto, + executionTime = raw.bookDate, + wireTransferSubject = raw.subject + ) } - if (reversal) { - throw Exception("Reversal !!") - require("CRDT" == kind) - val msgId = one("Refs").opt("MsgId")?.text() - if (msgId == null) { - logger.debug("Unsupported reversal without message id") - } else { - notifications.add(TxNotification.Reversal( - msgId = msgId, - reason = info, - executionTime = bookDate - )) - } - return@notificationForEachTx + "DBIT" -> { + if (raw.msgId == null) + throw TxErr("missing msg ID for Debit ${raw.ref}") + OutgoingPayment( + amount = raw.amount, + messageId = raw.msgId, + executionTime = raw.bookDate + ) } - when (kind) { - "CRDT" -> { - val bankId: String = one("AcctSvcrRef").text() - one("NtryDtls").one("TxDtls") { - // Obtaining payment subject. - val subject = opt("RmtInf")?.map("Ustrd") { text() }?.joinToString("") - if (subject == null) { - logger.debug("Skip notification '$bankId', missing subject") - //return@notificationForEachTx - } - // Obtaining the payer's details - val debtorPayto = StringBuilder("payto://iban/") - one("RltdPties") { - one("DbtrAcct").one("Id").one("IBAN") { - debtorPayto.append(text()) - } - one("Dbtr").one("Nm") { - val urlEncName = URLEncoder.encode(text(), "utf-8") - debtorPayto.append("?receiver-name=$urlEncName") - } - } - notifications.add(IncomingPayment( - amount = amount, - bankId = bankId, - debitPaytoUri = debtorPayto.toString(), - executionTime = bookDate, - wireTransferSubject = subject.toString() - )) - } - } - "DBIT" -> { - /*val messageId = one("Refs").one("MsgId").text() - notifications.add(OutgoingPayment( - amount = amount, - messageId = messageId, - executionTime = bookDate - ))*/ - } - else -> throw Exception("Unknown transaction notification kind '$kind'") - } + else -> throw Exception("Unknown transaction notification kind '${raw.kind}'") } - return notifications }
\ No newline at end of file diff --git a/nexus/src/main/kotlin/tech/libeufin/nexus/XmlCombinators.kt b/nexus/src/main/kotlin/tech/libeufin/nexus/XmlCombinators.kt index 26eb080d..a836e6ad 100644 --- a/nexus/src/main/kotlin/tech/libeufin/nexus/XmlCombinators.kt +++ b/nexus/src/main/kotlin/tech/libeufin/nexus/XmlCombinators.kt @@ -158,7 +158,7 @@ class XmlDestructor internal constructor(private val el: Element) { } val el = children.next() if (children.hasNext()) { - throw DestructionError("expected unique '${el.tagName}.$path', got ${children.asSequence() + 1}") + throw DestructionError("expected unique '${el.tagName}.$path', got ${children.asSequence().count() + 1}") } return XmlDestructor(el) } diff --git a/testbench/sample/200519_camt054-ESR-ASR_P_CH2909000000250094239_1110092704_0_2019042500372179_v2019.xml b/testbench/sample/200519_camt054-ESR-ASR_P_CH2909000000250094239_1110092704_0_2019042500372179_v2019.xml new file mode 100644 index 00000000..1967851a --- /dev/null +++ b/testbench/sample/200519_camt054-ESR-ASR_P_CH2909000000250094239_1110092704_0_2019042500372179_v2019.xml @@ -0,0 +1,131 @@ +<?xml version="1.0" encoding="UTF-8"?> +<Document xmlns="urn:iso:std:iso:20022:tech:xsd:camt.054.001.04" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="urn:iso:std:iso:20022:tech:xsd:camt.054.001.04 file:///C:/Users/alihodzica/Desktop/camt.054/camt.054.001.04.xsd"> + <BkToCstmrDbtCdtNtfctn> + <GrpHdr> + <MsgId>20190424375204228750928</MsgId> + <CreDtTm>2019-04-25T00:20:05</CreDtTm> + <MsgPgntn> + <PgNb>1</PgNb> + <LastPgInd>true</LastPgInd> + </MsgPgntn> + <AddtlInf>SPS/2.0/PROD</AddtlInf> + </GrpHdr> + <Ntfctn> + <Id>20190424375204228750931</Id> + <CreDtTm>2019-04-25T00:20:05</CreDtTm> + <FrToDt> + <FrDtTm>2019-04-24T00:00:00</FrDtTm> + <ToDtTm>2019-04-24T23:59:59</ToDtTm> + </FrToDt> + <Acct> + <Id> + <IBAN>CH2909000000250094239</IBAN> + </Id> + <Ccy>CHF</Ccy> + <Ownr> + <Nm>Robert Schneider SA</Nm> + </Ownr> + </Acct> + <Ntry> + <NtryRef>020010001</NtryRef> + <Amt Ccy="CHF">147.00</Amt> + <CdtDbtInd>DBIT</CdtDbtInd> + <RvslInd>false</RvslInd> + <Sts> + <Cd>BOOK</Cd> + </Sts> + <BookgDt> + <Dt>2019-04-24</Dt> + </BookgDt> + <ValDt> + <Dt>2019-04-25</Dt> + </ValDt> + <AcctSvcrRef>100820002V496ZRA</AcctSvcrRef> + <BkTxCd> + <Domn> + <Cd>PMNT</Cd> + <Fmly> + <Cd>CNTR</Cd> + <SubFmlyCd>CWDL</SubFmlyCd> + </Fmly> + </Domn> + </BkTxCd> + <Chrgs> + <TtlChrgsAndTaxAmt Ccy="CHF">4.40</TtlChrgsAndTaxAmt> + <Rcrd> + <Amt Ccy="CHF">4.40</Amt> + <CdtDbtInd>DBIT</CdtDbtInd> + <ChrgInclInd>false</ChrgInclInd> + <Tp> + <Prtry> + <Id>6</Id> + </Prtry> + </Tp> + </Rcrd> + </Chrgs> + <NtryDtls> + <Btch> + <NbOfTxs>1</NbOfTxs> + </Btch> + <TxDtls> + <Refs> + <AcctSvcrRef>180410CH02UZ2PC1</AcctSvcrRef> + <Prtry> + <Tp>06</Tp> + <Ref>20190423848301000100105</Ref> + </Prtry> + </Refs> + <Amt Ccy="CHF">147.00</Amt> + <CdtDbtInd>DBIT</CdtDbtInd> + <BkTxCd> + <Domn> + <Cd>PMNT</Cd> + <Fmly> + <Cd>CNTR</Cd> + <SubFmlyCd>CWDL</SubFmlyCd> + </Fmly> + </Domn> + </BkTxCd> + <Chrgs> + <TtlChrgsAndTaxAmt Ccy="CHF">4.40</TtlChrgsAndTaxAmt> + <Rcrd> + <Amt Ccy="CHF">4.40</Amt> + <CdtDbtInd>DBIT</CdtDbtInd> + <ChrgInclInd>false</ChrgInclInd> + <Tp> + <Prtry> + <Id>6</Id> + </Prtry> + </Tp> + </Rcrd> + </Chrgs> + <RltdPties> + <Cdtr> + <Pty> + <Nm>Maria Bernasconi</Nm> + </Pty> + </Cdtr> + <CdtrAcct> + <Id> + <IBAN>CH5109000000250092291</IBAN> + </Id> + </CdtrAcct> + </RltdPties> + <RmtInf> + <Strd> + <CdtrRefInf> + <Ref>100041698214115449371805278</Ref> + </CdtrRefInf> + <AddtlRmtInf>?REJECT?0</AddtlRmtInf> + </Strd> + </RmtInf> + <RltdDts> + <AccptncDtTm>2019-04-23T20:00:00</AccptncDtTm> + </RltdDts> + </TxDtls> + </NtryDtls> + <AddtlNtryInf>SAMMELLASTSCHRIFT ASR VERARBEITUNG VOM 24.04.2019 KUNDENNUMMER 02-1000-1 PAKET ID: 180410CH00000AL0</AddtlNtryInf> + </Ntry> + </Ntfctn> + </BkToCstmrDbtCdtNtfctn> +</Document> diff --git a/testbench/src/main/kotlin/Main.kt b/testbench/src/main/kotlin/Main.kt index d7d0a321..ed7fcd02 100644 --- a/testbench/src/main/kotlin/Main.kt +++ b/testbench/src/main/kotlin/Main.kt @@ -147,6 +147,7 @@ class Cli : CliktCommand("Run integration tests on banks provider") { put("ack", "Fetch CustomerAcknowledgement", "ebics-fetch $ebicsFlags acknowledgement") put("status", "Fetch CustomerPaymentStatusReport", "ebics-fetch $ebicsFlags status") put("notification", "Fetch BankToCustomerDebitCreditNotification", "ebics-fetch $ebicsFlags notification") + put("statement", "Fetch BankToCustomerStatement", "ebics-fetch $ebicsFlags statement") put("submit", "Submit pending transactions", "ebics-submit $ebicsFlags") put("setup", "Setup", "ebics-setup $flags") put("reset-keys", suspend { diff --git a/testbench/src/test/kotlin/Iso20022Test.kt b/testbench/src/test/kotlin/Iso20022Test.kt index 04a7ea82..55a0f6d7 100644 --- a/testbench/src/test/kotlin/Iso20022Test.kt +++ b/testbench/src/test/kotlin/Iso20022Test.kt @@ -20,8 +20,8 @@ import org.junit.Test import tech.libeufin.nexus.parseCustomerAck import tech.libeufin.nexus.parseCustomerPaymentStatusReport -import tech.libeufin.nexus.parseTxNotif -import tech.libeufin.nexus.parseTxStatement +import tech.libeufin.nexus.parseTx +import tech.libeufin.nexus.loadConfig import java.nio.file.Files import kotlin.io.path.Path import kotlin.io.path.exists @@ -37,12 +37,10 @@ class Iso20022Test { println(name) if (name.contains("HAC")) { parseCustomerAck(content) - } else if (name.contains("pain.002")) { + } else if (name.contains("pain.002") || name.contains("pain002") ) { parseCustomerPaymentStatusReport(content) - } else if (name.contains("C53")) { - parseTxStatement(content, "EUR") } else { - parseTxNotif(content, "CHF") + parseTx(content, "CHF") } } } @@ -55,6 +53,8 @@ class Iso20022Test { for (file in platform.listDirectoryEntries()) { val fetch = file.resolve("fetch") if (file.isDirectory() && fetch.exists()) { + val cfg = loadConfig(platform.resolve("ebics.conf")) + val currency = cfg.requireString("nexus-ebics", "currency") for (log in fetch.listDirectoryEntries()) { val content = Files.newInputStream(log) val name = log.toString() @@ -63,10 +63,8 @@ class Iso20022Test { parseCustomerAck(content) } else if (name.contains("pain.002")) { parseCustomerPaymentStatusReport(content) - } else if (name.contains("C53")) { - parseTxStatement(content, "EUR") - } else { - parseTxNotif(content, "CHF") + } else if (!name.contains("camt.052") && !name.contains("_C52_") && !name.contains("_Z01_")) { + parseTx(content, currency) } } } |