commit 340506abc60e1c3e30767b1a14e54bd6cf620f71
parent 34283e93011ff3add8d67dea3299efa8b34794f5
Author: Florian Dold <florian.dold@gmail.com>
Date: Thu, 2 Jul 2020 15:19:51 +0530
more ISO parsing/schema work
Diffstat:
9 files changed, 128 insertions(+), 103 deletions(-)
diff --git a/.idea/dictionaries/dold.xml b/.idea/dictionaries/dold.xml
@@ -2,11 +2,15 @@
<dictionary name="dold">
<words>
<w>affero</w>
+ <w>camt</w>
<w>combinators</w>
+ <w>crdt</w>
<w>cronspec</w>
+ <w>dbit</w>
<w>ebics</w>
<w>libeufin</w>
<w>payto</w>
+ <w>pdng</w>
<w>sqlite</w>
</words>
</dictionary>
diff --git a/.idea/misc.xml b/.idea/misc.xml
@@ -1,5 +1,10 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
+ <component name="EntryPointsManager">
+ <list size="1">
+ <item index="0" class="java.lang.String" itemvalue="com.fasterxml.jackson.annotation.JsonValue" />
+ </list>
+ </component>
<component name="ExternalStorageConfigurationManager" enabled="true" />
<component name="ProjectRootManager" version="2" languageLevel="JDK_11" default="true" project-jdk-name="11" project-jdk-type="JavaSDK" />
</project>
\ No newline at end of file
diff --git a/nexus/src/main/kotlin/tech/libeufin/nexus/Iso20022.kt b/nexus/src/main/kotlin/tech/libeufin/nexus/Iso20022.kt
@@ -27,6 +27,7 @@ import com.fasterxml.jackson.annotation.JsonSubTypes
import com.fasterxml.jackson.annotation.JsonTypeInfo
import com.fasterxml.jackson.annotation.JsonValue
import org.w3c.dom.Document
+import tech.libeufin.nexus.server.CurrencyAmount
import tech.libeufin.util.*
import java.time.Instant
import java.time.ZoneId
@@ -54,24 +55,30 @@ enum class TransactionStatus {
INFO,
}
-enum class CashManagementResponseType(@get:JsonValue val jsonname: String) {
+enum class CashManagementResponseType(@get:JsonValue val jsonName: String) {
Report("report"), Statement("statement"), Notification("notification")
}
-/**
- * Schemes to identify a transaction within an account.
- * An identifier from such a scheme will be used to reconcile transactions
- * from multiple sources.
- * (LibEuFin-specific, not defined by ISO 20022)
- */
-enum class TransactionIdentifierSchemes {
+data class CamtReport(
+ val account: AccountIdentification,
+ val entries: List<CamtBankAccountEntry>
+)
+
+data class CamtParseResult(
+ val reports: List<CamtReport>,
+ val messageId: String,
/**
- * Reconcile based on the account servicer reference.
+ * Message type in form of the ISO 20022 message name.
*/
- AcctSvcrRef
-}
+ val messageType: CashManagementResponseType,
+ val creationDateTime: String
+)
+
+@JsonInclude(JsonInclude.Include.NON_NULL)
+data class TransactionInfo(
+ val batchPaymentInformationId: String?,
+ val batchMessageId: String?,
-data class TransactionDetails(
/**
* Related parties as JSON.
*/
@@ -100,22 +107,15 @@ data class AccountIdentificationGeneric(
val proprietary: String?
) : AccountIdentification()
-data class BankTransaction(
- val account: AccountIdentification,
- /**
- * Identifier for the transaction that should be unique within an account.
- * Prefix by the identifier scheme name followed by a colon.
- */
- val transactionIdentifier: String,
- /**
- * Scheme used for the account identifier.
- */
- val currency: String,
- val amount: String,
+
+data class CamtBankAccountEntry(
+ val entryAmount: CurrencyAmount,
+
/**
* Booked, pending, etc.
*/
val status: TransactionStatus,
+
/**
* Is this transaction debiting or crediting the account
* it is reported for?
@@ -127,13 +127,13 @@ data class BankTransaction(
*/
val bankTransactionCode: BankTransactionCode,
/**
- * Is this a batch booking?
+ * Transaction details, if this entry contains a single transaction.
*/
- val isBatch: Boolean,
- val details: List<TransactionDetails>,
+ val transactionInfos: List<TransactionInfo>,
val valueDate: DateOrDateTime?,
val bookingDate: DateOrDateTime?,
- val accountServicerReference: String?
+ val accountServicerRef: String?,
+ val entryRef: String?
)
@JsonTypeInfo(
@@ -175,15 +175,11 @@ class DateTime(
@JsonInclude(JsonInclude.Include.NON_NULL)
data class BankTransactionCode(
- /**
- * Standardized bank transaction code, as "$domain/$family/$subfamily"
- */
- val iso: String?,
-
- /**
- * Proprietary code, as "$issuer/$code".
- */
- val proprietary: String?
+ val domain: String?,
+ val family: String?,
+ val subfamily: String?,
+ val proprietaryCode: String?,
+ val proprietaryIssuer: String?
)
data class AmountAndCurrencyExchangeDetails(
@@ -441,10 +437,12 @@ private fun XmlElementDestructor.extractAmountAndCurrencyExchangeDetails(): Amou
)
}
-private fun XmlElementDestructor.extractTransactionDetails(): List<TransactionDetails> {
+private fun XmlElementDestructor.extractTransactionInfos(): List<TransactionInfo> {
return requireUniqueChildNamed("NtryDtls") {
mapEachChildNamed("TxDtls") {
- TransactionDetails(
+ TransactionInfo(
+ batchMessageId = null,
+ batchPaymentInformationId = null,
relatedParties = extractPartiesAndAgents(),
amountDetails = maybeUniqueChildNamed("AmtDtls") {
AmountDetails(
@@ -467,9 +465,9 @@ private fun XmlElementDestructor.extractTransactionDetails(): List<TransactionDe
}
}
-private fun XmlElementDestructor.extractInnerTransactions(): List<BankTransaction> {
+private fun XmlElementDestructor.extractInnerTransactions(): CamtReport {
val account = requireUniqueChildNamed("Acct") { extractAccount() }
- return mapEachChildNamed("Ntry") {
+ val entries = mapEachChildNamed("Ntry") {
val amount = requireUniqueChildNamed("Amt") { it.textContent }
val currency = requireUniqueChildNamed("Amt") { it.getAttribute("Ccy") }
val status = requireUniqueChildNamed("Sts") { it.textContent }.let {
@@ -480,54 +478,44 @@ private fun XmlElementDestructor.extractInnerTransactions(): List<BankTransactio
}
val btc = requireUniqueChildNamed("BkTxCd") {
BankTransactionCode(
- proprietary = maybeUniqueChildNamed("Prtry") {
- val cd = requireUniqueChildNamed("Cd") { it.textContent }
- val issr = requireUniqueChildNamed("Issr") { it.textContent }
- "$issr/$cd"
+ domain = maybeUniqueChildNamed("Domn") { maybeUniqueChildNamed("Cd") { it.textContent} },
+ family = maybeUniqueChildNamed("Domn") {
+ maybeUniqueChildNamed("Fmly") {
+ maybeUniqueChildNamed("Cd") { it.textContent }
+ }
},
- iso = maybeUniqueChildNamed("Domn") {
- val cd = requireUniqueChildNamed("Cd") { it.textContent }
- val r = requireUniqueChildNamed("Fmly") {
- object {
- val fmlyCd = requireUniqueChildNamed("Cd") { it.textContent }
- val subFmlyCd = requireUniqueChildNamed("SubFmlyCd") { it.textContent }
- }
+ subfamily = maybeUniqueChildNamed("Domn") {
+ maybeUniqueChildNamed("Fmly") {
+ maybeUniqueChildNamed("SubFmlyCd") { it.textContent }
}
- "$cd/${r.fmlyCd}/${r.subFmlyCd}"
+ },
+ proprietaryCode = maybeUniqueChildNamed("Prtry") {
+ maybeUniqueChildNamed("Cd") { it.textContent }
+ },
+ proprietaryIssuer = maybeUniqueChildNamed("Prtry") {
+ maybeUniqueChildNamed("Issr") { it.textContent }
}
)
}
val acctSvcrRef = maybeUniqueChildNamed("AcctSvcrRef") { it.textContent }
+ val entryRef = maybeUniqueChildNamed("NtryRef") { it.textContent }
// For now, only support account servicer reference as id
- val txId = "AcctSvcrRef:" + (acctSvcrRef ?: throw Exception("currently, AcctSvcrRef is mandatory in LibEuFin"))
- val details = extractTransactionDetails()
- BankTransaction(
- account = account,
- amount = amount,
- currency = currency,
+ val transactionInfos = extractTransactionInfos()
+ CamtBankAccountEntry(
+ entryAmount = CurrencyAmount(currency, amount),
status = status,
creditDebitIndicator = creditDebitIndicator,
bankTransactionCode = btc,
- details = details,
- isBatch = details.size > 1,
+ transactionInfos = transactionInfos,
bookingDate = maybeUniqueChildNamed("BookgDt") { extractDateOrDateTime() },
valueDate = maybeUniqueChildNamed("ValDt") { extractDateOrDateTime() },
- accountServicerReference = acctSvcrRef,
- transactionIdentifier = txId
+ accountServicerRef = acctSvcrRef,
+ entryRef = entryRef
)
}
+ return CamtReport(account, entries);
}
-data class CamtParseResult(
- val transactions: List<BankTransaction>,
- val messageId: String,
- /**
- * Message type in form of the ISO 20022 message name.
- */
- val messageType: CashManagementResponseType,
- val creationDateTime: String
-)
-
/**
* Extract a list of transactions from an ISO20022 camt.052 / camt.053 message.
*/
@@ -535,7 +523,7 @@ fun parseCamtMessage(doc: Document): CamtParseResult {
return destructXml(doc) {
requireRootElement("Document") {
// Either bank to customer statement or report
- val transactions = requireOnlyChild {
+ val reports = requireOnlyChild {
when (it.localName) {
"BkToCstmrAcctRpt" -> {
mapEachChildNamed("Rpt") {
@@ -551,7 +539,7 @@ fun parseCamtMessage(doc: Document): CamtParseResult {
throw CamtParsingError("expected statement or report")
}
}
- }.flatten()
+ }
val messageId = requireOnlyChild {
requireUniqueChildNamed("GrpHdr") {
requireUniqueChildNamed("MsgId") { it.textContent }
@@ -571,7 +559,7 @@ fun parseCamtMessage(doc: Document): CamtParseResult {
}
}
}
- CamtParseResult(transactions, messageId, messageType, creationDateTime)
+ CamtParseResult(reports, messageId, messageType, creationDateTime)
}
}
}
diff --git a/nexus/src/main/kotlin/tech/libeufin/nexus/Taler.kt b/nexus/src/main/kotlin/tech/libeufin/nexus/Taler.kt
@@ -348,7 +348,7 @@ private suspend fun talerAddIncoming(call: ApplicationCall, httpClient: HttpClie
}
-private fun ingestIncoming(payment: NexusBankTransactionEntity, txDtls: TransactionDetails) {
+private fun ingestIncoming(payment: NexusBankTransactionEntity, txDtls: TransactionInfo) {
val subject = txDtls.unstructuredRemittanceInformation
val debtorName = txDtls.relatedParties.debtor?.name
if (debtorName == null) {
@@ -414,13 +414,13 @@ fun ingestTalerTransactions() {
(NexusBankTransactionsTable.id.greater(lastId))
}.orderBy(Pair(NexusBankTransactionsTable.id, SortOrder.ASC)).forEach {
// Incoming payment.
- val tx = jacksonObjectMapper().readValue(it.transactionJson, BankTransaction::class.java)
- if (tx.isBatch) {
+ val tx = jacksonObjectMapper().readValue(it.transactionJson, CamtBankAccountEntry::class.java)
+ if (tx.transactionInfos.size != 1) {
// We don't support batch transactions at the moment!
logger.warn("batch transactions not supported")
} else {
when (tx.creditDebitIndicator) {
- CreditDebitIndicator.CRDT -> ingestIncoming(it, txDtls = tx.details[0])
+ CreditDebitIndicator.CRDT -> ingestIncoming(it, txDtls = tx.transactionInfos[0])
}
}
lastId = it.id.value
diff --git a/nexus/src/main/kotlin/tech/libeufin/nexus/bankaccount/BankAccount.kt b/nexus/src/main/kotlin/tech/libeufin/nexus/bankaccount/BankAccount.kt
@@ -21,21 +21,17 @@ package tech.libeufin.nexus.bankaccount
import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper
import io.ktor.application.ApplicationCall
-import io.ktor.application.call
import io.ktor.client.HttpClient
import io.ktor.http.HttpStatusCode
import org.jetbrains.exposed.sql.*
import org.jetbrains.exposed.sql.transactions.transaction
import org.w3c.dom.Document
import tech.libeufin.nexus.*
-import tech.libeufin.nexus.OfferedBankAccountsTable.iban
-import tech.libeufin.nexus.OfferedBankAccountsTable.imported
import tech.libeufin.nexus.ebics.fetchEbicsBySpec
import tech.libeufin.nexus.ebics.submitEbicsPaymentInitiation
import tech.libeufin.nexus.server.FetchSpecJson
import tech.libeufin.nexus.server.Pain001Data
import tech.libeufin.nexus.server.requireBankConnection
-import tech.libeufin.nexus.server.requireBankConnectionInternal
import tech.libeufin.util.XMLUtil
import java.time.Instant
import java.time.ZonedDateTime
@@ -139,10 +135,10 @@ fun processCamtMessage(
}
}
- val transactions = res.transactions
- logger.info("found ${transactions.size} transactions")
- txloop@ for (tx in transactions) {
- val acctSvcrRef = tx.accountServicerReference
+ val entries = res.reports.map { it.entries }.flatten()
+ logger.info("found ${entries.size} transactions")
+ txloop@ for (tx in entries) {
+ val acctSvcrRef = tx.accountServicerRef
if (acctSvcrRef == null) {
// FIXME(dold): Report this!
logger.error("missing account servicer reference in transaction")
@@ -158,15 +154,15 @@ fun processCamtMessage(
val rawEntity = NexusBankTransactionEntity.new {
bankAccount = acct
accountTransactionId = "AcctSvcrRef:$acctSvcrRef"
- amount = tx.amount
- currency = tx.currency
+ amount = tx.entryAmount.amount
+ currency = tx.entryAmount.currency
transactionJson = jacksonObjectMapper().writerWithDefaultPrettyPrinter().writeValueAsString(tx)
creditDebitIndicator = tx.creditDebitIndicator.name
status = tx.status
}
rawEntity.flush()
if (tx.creditDebitIndicator == CreditDebitIndicator.DBIT) {
- val t0 = tx.details.getOrNull(0)
+ val t0 = tx.transactionInfos.getOrNull(0)
val msgId = t0?.references?.messageIdentification
val pmtInfId = t0?.references?.paymentInformationIdentification
if (t0 != null && msgId != null && pmtInfId != null) {
diff --git a/nexus/src/main/kotlin/tech/libeufin/nexus/server/JSON.kt b/nexus/src/main/kotlin/tech/libeufin/nexus/server/JSON.kt
@@ -24,7 +24,9 @@ import com.fasterxml.jackson.annotation.JsonTypeInfo
import com.fasterxml.jackson.annotation.JsonTypeName
import com.fasterxml.jackson.annotation.JsonValue
import com.fasterxml.jackson.databind.JsonNode
-import tech.libeufin.nexus.BankTransaction
+import tech.libeufin.nexus.CamtBankAccountEntry
+import tech.libeufin.nexus.CreditDebitIndicator
+import tech.libeufin.nexus.TransactionStatus
import tech.libeufin.util.*
import java.time.Instant
import java.time.ZoneId
@@ -224,7 +226,7 @@ data class PaymentStatus(
)
data class Transactions(
- val transactions: MutableList<BankTransaction> = mutableListOf()
+ val transactions: MutableList<CamtBankAccountEntry> = mutableListOf()
)
data class BankProtocolsResponse(
@@ -338,4 +340,24 @@ data class CreateAccountTaskRequest(
data class ImportBankAccount(
val offeredAccountId: String,
val nexusBankAccountId: String
-)
-\ No newline at end of file
+)
+
+data class CurrencyAmount(
+ val currency: String,
+ val amount: String
+)
+
+
+/**
+ * Account entry item as returned by the /bank-accounts/{acctId}/transactions API.
+ */
+data class AccountEntryItemJson(
+ val nexusEntryId: String,
+ val nexusStatusSequenceId: Int,
+
+ val entryId: String?,
+ val accountServicerRef: String?,
+ val creditDebitIndicator: CreditDebitIndicator,
+ val entryAmount: CurrencyAmount,
+ val status: TransactionStatus
+)
diff --git a/nexus/src/main/kotlin/tech/libeufin/nexus/server/NexusServer.kt b/nexus/src/main/kotlin/tech/libeufin/nexus/server/NexusServer.kt
@@ -48,7 +48,6 @@ import io.ktor.server.netty.Netty
import io.ktor.utils.io.ByteReadChannel
import io.ktor.utils.io.jvm.javaio.toByteReadChannel
import io.ktor.utils.io.jvm.javaio.toInputStream
-import org.jetbrains.exposed.dao.id.EntityID
import org.jetbrains.exposed.sql.and
import org.jetbrains.exposed.sql.select
import org.jetbrains.exposed.sql.transactions.transaction
@@ -574,7 +573,7 @@ fun serverMain(dbName: String, host: String) {
transaction {
authenticateRequest(call.request).id.value
NexusBankTransactionEntity.all().map {
- val tx = jacksonObjectMapper().readValue(it.transactionJson, BankTransaction::class.java)
+ val tx = jacksonObjectMapper().readValue(it.transactionJson, CamtBankAccountEntry::class.java)
ret.transactions.add(tx)
}
}
diff --git a/nexus/src/test/kotlin/Iso20022Test.kt b/nexus/src/test/kotlin/Iso20022Test.kt
@@ -22,11 +22,24 @@ class Iso20022Test {
assertEquals(r.messageId, "27632364572")
assertEquals(r.creationDateTime, "2016-05-11T19:30:47.0+01:00")
assertEquals(r.messageType, CashManagementResponseType.Statement)
- for (tx in r.transactions) {
- // Make sure that roundtripping works
- val txStr = jacksonObjectMapper().writerWithDefaultPrettyPrinter().writeValueAsString(tx)
+ assertEquals(r.reports.size, 1)
+ assertEquals(r.reports[0].entries[0].entryAmount.amount, "100.00")
+ assertEquals(r.reports[0].entries[0].entryAmount.currency, "EUR")
+ assertEquals(r.reports[0].entries[0].status, TransactionStatus.BOOK)
+ assertEquals(r.reports[0].entries[0].entryRef, null)
+ assertEquals(r.reports[0].entries[0].accountServicerRef, "Bankreferenz")
+ assertEquals(r.reports[0].entries[0].bankTransactionCode.domain, "PMNT")
+ assertEquals(r.reports[0].entries[0].bankTransactionCode.family, "RCDT")
+ assertEquals(r.reports[0].entries[0].bankTransactionCode.subfamily, "ESCT")
+ assertEquals(r.reports[0].entries[0].bankTransactionCode.proprietaryCode, "166")
+ assertEquals(r.reports[0].entries[0].bankTransactionCode.proprietaryIssuer, "DK")
+ assertEquals(r.reports[0].entries[0].transactionInfos.size, 1)
+
+ // Make sure that round-tripping of entry CamtBankAccountEntry JSON works
+ for (entry in r.reports.flatMap { it.entries }) {
+ val txStr = jacksonObjectMapper().writerWithDefaultPrettyPrinter().writeValueAsString(entry)
println(txStr)
- val tx2 = jacksonObjectMapper().readValue(txStr, BankTransaction::class.java)
+ val tx2 = jacksonObjectMapper().readValue(txStr, CamtBankAccountEntry::class.java)
val tx2Str = jacksonObjectMapper().writerWithDefaultPrettyPrinter().writeValueAsString(tx2)
assertEquals(jacksonObjectMapper().readTree(txStr), jacksonObjectMapper().readTree(tx2Str))
}
diff --git a/sandbox/src/main/kotlin/tech/libeufin/sandbox/DB.kt b/sandbox/src/main/kotlin/tech/libeufin/sandbox/DB.kt
@@ -315,4 +315,4 @@ fun dbCreateTables(dbName: String) {
BankAccountsTable
)
}
-}
-\ No newline at end of file
+}
