commit 5060b676aa729e2e2e57adbdf1a025554728fecf
parent 6bac5cf1c5d642d6e0a0a9afbc6c90cb0291c78b
Author: Antoine A <>
Date: Tue, 9 Jan 2024 16:30:47 +0000
Semi automated test for netzbon ebics-fetch
Diffstat:
9 files changed, 206 insertions(+), 126 deletions(-)
diff --git a/.gitignore b/.gitignore
@@ -26,6 +26,7 @@ __pycache__
*.log
.DS_Store
*.mk
+*.xsd
util/src/main/resources/version.txt
debian/usr/share/libeufin/demobank-ui/index.js
debian/usr/share/libeufin/demobank-ui/*.html
diff --git a/Makefile b/Makefile
@@ -102,6 +102,10 @@ check: install-nobuild-bank-files
test: install-nobuild-bank-files
./gradlew test --tests $(test) -i
+.PHONY: integration
+integration:
+ ./gradlew :integration:run --console=plain --args="$(test)"
+
.PHONY: doc
doc:
./gradlew dokkaHtmlMultiModule
diff --git a/bank/src/main/resources/logback.xml b/bank/src/main/resources/logback.xml
@@ -12,6 +12,9 @@
<logger name="tech.libeufin.util" level="ALL" additivity="false">
<appender-ref ref="STDERR" />
</logger>
+ <logger name="tech.libeufin.nexus" level="ALL" additivity="false">
+ <appender-ref ref="STDERR" />
+ </logger>
<logger name="io.netty" level="INFO" />
<logger name="ktor" level="TRACE" />
diff --git a/integration/conf/netzbon.conf b/integration/conf/netzbon.conf
@@ -0,0 +1,25 @@
+[nexus-ebics]
+CURRENCY = CHF
+
+# Bank
+HOST_BASE_URL = https://ebics.postfinance.ch/ebics/ebics.aspx
+BANK_DIALECT = postfinance
+
+# EBICS IDs
+HOST_ID = PFEBICS
+USER_ID = 5183101
+PARTNER_ID = 51831
+
+
+BANK_PUBLIC_KEYS_FILE = test/netzbon/bank-keys.json
+CLIENT_PRIVATE_KEYS_FILE = test/netzbon/client-keys.json
+
+IBAN = CH4009000000160948810
+BIC = POFICHBEXXX
+NAME = Genossenschaft Netz Soziale Oekonomie
+
+[nexus-fetch]
+STATEMENT_LOG_DIRECTORY = test/netzbon/fetch
+
+[nexus-postgres]
+CONFIG = postgres:///libeufincheck
diff --git a/integration/conf/postfinance.conf b/integration/conf/postfinance.conf
@@ -1,14 +1,24 @@
[nexus-ebics]
currency = CHF
-BANK_DIALECT = postfinance
+
+# Bank
HOST_BASE_URL = https://isotest.postfinance.ch/ebicsweb/ebicsweb
+BANK_DIALECT = postfinance
+
+# EBICS IDs
+HOST_ID = PFEBICS
+USER_ID = PFC00563
+PARTNER_ID = PFC00563
+
+# Key files
BANK_PUBLIC_KEYS_FILE = test/postfinance/bank-keys.json
CLIENT_PRIVATE_KEYS_FILE = test/postfinance/client-keys.json
+
#IBAN = CH2989144971918294289
IBAN = CH7789144474425692816
-HOST_ID = PFEBICS
-USER_ID = PFC00563
-PARTNER_ID = PFC00563
+
+[nexus-fetch]
+STATEMENT_LOG_DIRECTORY = test/postfinance/fetch
[nexus-postgres]
CONFIG = postgres:///libeufincheck
diff --git a/integration/src/main/kotlin/Main.kt b/integration/src/main/kotlin/Main.kt
@@ -25,6 +25,8 @@ import tech.libeufin.nexus.*
import tech.libeufin.bank.*
import tech.libeufin.util.*
import com.github.ajalt.clikt.core.*
+import com.github.ajalt.clikt.parameters.arguments.*
+import com.github.ajalt.clikt.parameters.types.*
import com.github.ajalt.clikt.testing.*
import io.ktor.client.*
import io.ktor.client.engine.cio.*
@@ -64,84 +66,104 @@ fun CliktCommandTestResult.assertErr(msg: String? = null) {
assertEquals(1, statusCode, msg)
}
-class PostFinanceCli : CliktCommand("Run tests on postfinance", name="postfinance") {
+enum class Kind {
+ postfinance,
+ netzbon
+}
+
+class Cli : CliktCommand("Run integration tests on banks provider") {
+ val kind: Kind by argument().enum<Kind>()
override fun run() {
+ val name = kind.name
+ step("Test init $name")
+
runBlocking {
- Path("test/postfinance").createDirectories()
- val conf = "conf/postfinance.conf"
+ Path("test/$name").createDirectories()
+ val conf = "conf/$name.conf"
val cfg = loadConfig(conf)
+
val clientKeysPath = Path(cfg.requireString("nexus-ebics", "client_private_keys_file"))
val bankKeysPath = Path(cfg.requireString("nexus-ebics", "bank_public_keys_file"))
var hasClientKeys = clientKeysPath.exists()
var hasBankKeys = bankKeysPath.exists()
- if (hasClientKeys || hasBankKeys) {
- if (ask("Reset keys ? y/n>") == "y") {
- if (hasClientKeys) clientKeysPath.deleteIfExists()
- if (hasBankKeys) bankKeysPath.deleteIfExists()
- hasClientKeys = false
- hasBankKeys = false
- }
- }
-
- if (!hasClientKeys) {
- step("Test INI order")
- ask("Got to https://testplattform.postfinance.ch/corporates/user/settings/ebics and click on 'Reset EBICS user'.\nPress Enter when done>")
- nexusCmd.test("ebics-setup -c $conf")
- .assertErr("ebics-setup should failed the first time")
- }
-
- if (!hasBankKeys) {
- step("Test HIA order")
- ask("Got to https://testplattform.postfinance.ch/corporates/user/settings/ebics and click on 'Activate EBICS user'.\nPress Enter when done>")
- nexusCmd.test("ebics-setup --auto-accept-keys -c $conf")
- .assertOk("ebics-setup should succeed the second time")
- }
-
- if (ask("Submit transactions ? y/n>") == "y") {
- val payto = "payto://iban/CH2989144971918294289?receiver-name=Test"
-
- step("Test submit one transaction")
- val nexusDb = NexusDb("postgresql:///libeufincheck")
- nexusCmd.test("dbinit -r -c $conf").assertOk()
- nexusDb.initiatedPaymentCreate(InitiatedPayment(
- amount = NexusAmount(42L, 0, "CFH"),
- creditPaytoUri = payto,
- wireTransferSubject = "single transaction test",
- initiationTime = Instant.now(),
- requestUid = Base32Crockford.encode(randBytes(16))
- ))
- nexusCmd.test("ebics-submit --transient -c $conf").assertOk()
-
- step("Test submit many transaction")
- repeat(4) {
+ nexusCmd.test("dbinit -r -c $conf").assertOk()
+ val nexusDb = NexusDb("postgresql:///libeufincheck")
+
+ when (kind) {
+ Kind.postfinance -> {
+ if (hasClientKeys || hasBankKeys) {
+ if (ask("Reset keys ? y/n>") == "y") {
+ if (hasClientKeys) clientKeysPath.deleteIfExists()
+ if (hasBankKeys) bankKeysPath.deleteIfExists()
+ hasClientKeys = false
+ hasBankKeys = false
+ }
+ }
+
+ if (!hasClientKeys) {
+ step("Test INI order")
+ ask("Got to https://testplattform.postfinance.ch/corporates/user/settings/ebics and click on 'Reset EBICS user'.\nPress Enter when done>")
+ nexusCmd.test("ebics-setup -c $conf")
+ .assertErr("ebics-setup should failed the first time")
+ }
+
+ if (!hasBankKeys) {
+ step("Test HIA order")
+ ask("Got to https://testplattform.postfinance.ch/corporates/user/settings/ebics and click on 'Activate EBICS user'.\nPress Enter when done>")
+ nexusCmd.test("ebics-setup --auto-accept-keys -c $conf")
+ .assertOk("ebics-setup should succeed the second time")
+ }
+
+ if (ask("Submit transactions ? y/n>") == "y") {
+ val payto = "payto://iban/CH2989144971918294289?receiver-name=Test"
+
+ step("Test submit one transaction")
nexusDb.initiatedPaymentCreate(InitiatedPayment(
- amount = NexusAmount(100L + it, 0, "CFH"),
- creditPaytoUri = payto,
- wireTransferSubject = "multi transaction test $it",
- initiationTime = Instant.now(),
- requestUid = Base32Crockford.encode(randBytes(16))
- ))
+ amount = NexusAmount(42L, 0, "CFH"),
+ creditPaytoUri = payto,
+ wireTransferSubject = "single transaction test",
+ initiationTime = Instant.now(),
+ requestUid = Base32Crockford.encode(randBytes(16))
+ ))
+ nexusCmd.test("ebics-submit --transient -c $conf").assertOk()
+
+ step("Test submit many transaction")
+ repeat(4) {
+ nexusDb.initiatedPaymentCreate(InitiatedPayment(
+ amount = NexusAmount(100L + it, 0, "CFH"),
+ creditPaytoUri = payto,
+ wireTransferSubject = "multi transaction test $it",
+ initiationTime = Instant.now(),
+ requestUid = Base32Crockford.encode(randBytes(16))
+ ))
+ }
+ nexusCmd.test("ebics-submit --transient -c $conf").assertOk()
+ }
+
+ step("Test fetch transactions")
+ nexusCmd.test("ebics-fetch --transient -c $conf --pinned-start 2022-01-01").assertOk()
+ }
+ Kind.netzbon -> {
+ if (!hasClientKeys)
+ throw Exception("Clients keys are required to run netzbon tests")
+
+ if (!hasBankKeys) {
+ step("Test HIA order")
+ nexusCmd.test("ebics-setup --auto-accept-keys -c $conf").assertOk("ebics-setup should succeed the second time")
+ }
+
+ step("Test fetch transactions")
+ nexusCmd.test("ebics-fetch --transient -c $conf --pinned-start 2022-01-01").assertOk()
}
- nexusCmd.test("ebics-submit --transient -c $conf").assertOk()
}
-
- step("Test fetch transactions")
- nexusCmd.test("ebics-fetch --transient -c $conf --pinned-start 2022-01-01").assertOk()
-
- step("Test succeed")
}
+
+ step("Test succeed")
}
}
-class Cli : CliktCommand() {
- init {
- subcommands(PostFinanceCli())
- }
- override fun run() = Unit
-}
-
fun main(args: Array<String>) {
Cli().main(args)
}
diff --git a/nexus/src/main/kotlin/tech/libeufin/nexus/Database.kt b/nexus/src/main/kotlin/tech/libeufin/nexus/Database.kt
@@ -9,6 +9,14 @@ import tech.libeufin.util.*
import java.sql.PreparedStatement
import java.sql.SQLException
import java.time.Instant
+import java.util.Date
+import java.text.SimpleDateFormat
+
+fun Instant.fmtDate(): String {
+ val formatter = SimpleDateFormat("yyyy-MM-dd")
+ return formatter.format(Date.from(this))
+}
+
// Remove this once TalerAmount from the bank
// module gets moved to the 'util' module (#7987).
@@ -16,7 +24,16 @@ data class TalerAmount(
val value: Long,
val fraction: Int, // has at most 8 digits.
val currency: String
-)
+) {
+ override fun toString(): String {
+ if (fraction == 0) {
+ return "$currency:$value"
+ } else {
+ return "$currency:$value.${fraction.toString().padStart(8, '0')}"
+ .dropLastWhile { it == '0' } // Trim useless fractional trailing 0
+ }
+ }
+}
// INCOMING PAYMENTS STRUCTS
@@ -29,7 +46,12 @@ data class IncomingPayment(
val debitPaytoUri: String,
val executionTime: Instant,
val bankTransferId: String
-)
+) {
+ override fun toString(): String {
+ return ">> ${executionTime.fmtDate()} $amount $bankTransferId debitor=$debitPaytoUri subject=$wireTransferSubject"
+ }
+}
+
// INITIATED PAYMENTS STRUCTS
@@ -104,7 +126,11 @@ data class OutgoingPayment(
val bankTransferId: String,
val creditPaytoUri: String? = null, // not showing in camt.054
val wireTransferSubject: String? = null // not showing in camt.054
-)
+) {
+ override fun toString(): String {
+ return "<< ${executionTime.fmtDate()} $amount $bankTransferId creditor=$creditPaytoUri subject=$wireTransferSubject"
+ }
+}
/**
* Witnesses the outcome of inserting an outgoing
diff --git a/nexus/src/main/kotlin/tech/libeufin/nexus/EbicsFetch.kt b/nexus/src/main/kotlin/tech/libeufin/nexus/EbicsFetch.kt
@@ -18,7 +18,7 @@ import java.time.LocalDate
import java.time.ZoneId
import java.util.UUID
import kotlin.concurrent.fixedRateTimer
-import kotlin.io.path.createDirectories
+import kotlin.io.path.*
/**
* Necessary data to perform a download.
@@ -140,25 +140,20 @@ fun maybeLogFile(
val asUtcDate = LocalDate.ofInstant(now, ZoneId.of("UTC"))
val subDir = "${asUtcDate.year}-${asUtcDate.monthValue}-${asUtcDate.dayOfMonth}"
// Creating the combined dir.
- val dirs = Path.of(maybeLogDir, subDir)
+ val dirs = Path(maybeLogDir, subDir)
dirs.createDirectories()
- fun maybeWrite(f: File, xml: String) {
- if (f.exists()) {
- throw Exception("Log file exists already at: ${f.path}")
- }
- f.writeText(xml)
- }
if (nonZip) {
- val f = File(dirs.toString(), "${now.toDbMicros()}_HAC_response.pain.002.xml")
- maybeWrite(f, content.toString(Charsets.UTF_8))
- return
- }
- // Write each ZIP entry in the combined dir.
- content.unzipForEach { fileName, xmlContent ->
- val f = File(dirs.toString(), "${now.toDbMicros()}_$fileName")
- // Rare: cannot download the same file twice in the same microsecond.
- maybeWrite(f, xmlContent)
+ val f = Path(dirs.toString(), "${now.toDbMicros()}_HAC_response.pain.002.xml")
+ f.writeBytes(content)
+ } else {
+ // Write each ZIP entry in the combined dir.
+ content.unzipForEach { fileName, xmlContent ->
+ val f = Path(dirs.toString(), "${now.toDbMicros()}_$fileName")
+ // Rare: cannot download the same file twice in the same microsecond.
+ f.writeText(xmlContent)
+ }
}
+
}
/**
@@ -371,50 +366,36 @@ fun firstLessThanSecond(
* @param db database connection.
* @param content the ZIP file that contains the EBICS
* notification as camt.054 records.
- * @return true if the ingestion succeeded, false otherwise.
- * False should fail the process, since it means that
- * the notification could not be parsed.
*/
private fun ingestNotification(
db: Database,
ctx: FetchContext,
content: ByteArray
-): Boolean {
+) {
val incomingPayments = mutableListOf<IncomingPayment>()
val outgoingPayments = mutableListOf<OutgoingPayment>()
- val filenamePrefixForIncoming = "camt.054_P_${ctx.cfg.myIbanAccount.iban}"
- val filenamePrefixForOutgoing = "camt.054-Debit_P_${ctx.cfg.myIbanAccount.iban}"
+
try {
content.unzipForEach { fileName, xmlContent ->
if (!fileName.contains("camt.054", ignoreCase = true))
throw Exception("Asked for notification but did NOT get a camt.054")
-
+ logger.debug("parse $fileName")
parseTxNotif(xmlContent, ctx.cfg.currency, incomingPayments, outgoingPayments)
}
} catch (e: IOException) {
- logger.error("Could not open any ZIP archive")
- return false
- } catch (e: Exception) {
- logger.error(e.message)
- return false
+ throw Exception("Could not open any ZIP archive", e)
}
- try {
- runBlocking {
- incomingPayments.forEach {
- logger.debug("incoming tx: $it")
- ingestIncomingPayment(db, it)
- }
- outgoingPayments.forEach {
- logger.debug("outgoing tx: $it")
- ingestOutgoingPayment(db, it)
- }
+ runBlocking {
+ incomingPayments.forEach {
+ logger.debug("$it")
+ ingestIncomingPayment(db, it)
+ }
+ outgoingPayments.forEach {
+ logger.debug("$it")
+ ingestOutgoingPayment(db, it)
}
- } catch (e: Exception) {
- logger.error(e.message)
- return false
}
- return true
}
/**
@@ -469,8 +450,10 @@ private suspend fun fetchDocuments(
logger.warn("Not ingesting ${ctx.whichDocument}. Only camt.054 notifications supported.")
return
}
- if (!ingestNotification(db, ctx, maybeContent)) {
- throw Exception("Ingesting notifications failed")
+ try {
+ ingestNotification(db, ctx, maybeContent)
+ } catch (e: Exception) {
+ throw Exception("Ingesting notifications failed", e)
}
}
@@ -545,7 +528,6 @@ class EbicsFetch: CliktCommand("Fetches bank records. Defaults to camt.054 noti
SupportedDocument.CAMT_054 -> {
val incomingTxs = mutableListOf<IncomingPayment>()
val outgoingTxs = mutableListOf<OutgoingPayment>()
-
parseTxNotif(maybeStdin, cfg.currency, incomingTxs, outgoingTxs)
println(incomingTxs)
println(outgoingTxs)
diff --git a/nexus/src/main/kotlin/tech/libeufin/nexus/Iso20022.kt b/nexus/src/main/kotlin/tech/libeufin/nexus/Iso20022.kt
@@ -189,13 +189,18 @@ fun parseTxNotif(
focusElement.textContent
}
}
- // Obtaining payment subject.
- val subject = StringBuilder()
- requireUniqueChildNamed("RmtInf") {
- this.mapEachChildNamed("Ustrd") {
- val piece = this.focusElement.textContent
+ // Obtaining payment subject.
+ val subject = maybeUniqueChildNamed("RmtInf") {
+ val subject = StringBuilder()
+ mapEachChildNamed("Ustrd") {
+ val piece = focusElement.textContent
subject.append(piece)
}
+ subject
+ }
+ if (subject == null) {
+ logger.debug("Skip notification $uidFromBank, missing subject")
+ return@notificationForEachTx
}
// Obtaining the payer's details
@@ -210,7 +215,7 @@ fun parseTxNotif(
}
// warn: it might need the postal address too..
requireUniqueChildNamed("Dbtr") {
- requireUniqueChildNamed("Pty") {
+ maybeUniqueChildNamed("Pty") {
requireUniqueChildNamed("Nm") {
val urlEncName = URLEncoder.encode(focusElement.textContent, "utf-8")
debtorPayto.append("?receiver-name=$urlEncName")
@@ -275,11 +280,13 @@ private fun notificationForEachTx(
mapEachChildNamed("Ntfctn") {
mapEachChildNamed("Ntry") {
requireUniqueChildNamed("Sts") {
- requireUniqueChildNamed("Cd") {
- if (focusElement.textContent != "BOOK")
- throw Exception("Found non booked transaction, " +
- "stop parsing. Status was: ${focusElement.textContent}"
- )
+ if (focusElement.textContent != "BOOK") {
+ requireUniqueChildNamed("Cd") {
+ if (focusElement.textContent != "BOOK")
+ throw Exception("Found non booked transaction, " +
+ "stop parsing. Status was: ${focusElement.textContent}"
+ )
+ }
}
}
val bookDate: Instant = requireUniqueChildNamed("BookgDt") {
