From d72290e523da8aa1dda3f072e3ecf232700f4cfe Mon Sep 17 00:00:00 2001 From: MS Date: Fri, 28 Jul 2023 16:57:34 +0200 Subject: Testing the EBICS time-framed history request. --- nexus/src/main/kotlin/tech/libeufin/nexus/Main.kt | 4 +- .../main/kotlin/tech/libeufin/nexus/Scheduling.kt | 3 +- .../tech/libeufin/nexus/ebics/EbicsClient.kt | 5 +- .../kotlin/tech/libeufin/nexus/ebics/EbicsNexus.kt | 5 +- .../tech/libeufin/nexus/iso20022/Iso20022.kt | 1 + .../main/kotlin/tech/libeufin/nexus/server/JSON.kt | 2 +- .../tech/libeufin/nexus/server/NexusServer.kt | 27 ++++++--- nexus/src/test/kotlin/ConversionServiceTest.kt | 11 ---- nexus/src/test/kotlin/MakeEnv.kt | 17 +++++- nexus/src/test/kotlin/NexusApiTest.kt | 64 +++++++++++++++++++++- nexus/src/test/kotlin/SandboxAccessApiTest.kt | 3 + .../src/main/kotlin/tech/libeufin/sandbox/Main.kt | 7 ++- .../kotlin/tech/libeufin/sandbox/bankAccount.kt | 4 +- util/src/main/kotlin/Ebics.kt | 9 ++- util/src/main/kotlin/time.kt | 10 ++++ 15 files changed, 135 insertions(+), 37 deletions(-) diff --git a/nexus/src/main/kotlin/tech/libeufin/nexus/Main.kt b/nexus/src/main/kotlin/tech/libeufin/nexus/Main.kt index 0886e177..5b453530 100644 --- a/nexus/src/main/kotlin/tech/libeufin/nexus/Main.kt +++ b/nexus/src/main/kotlin/tech/libeufin/nexus/Main.kt @@ -30,6 +30,7 @@ import com.github.ajalt.clikt.parameters.types.int import execThrowableOrTerminate import com.github.ajalt.clikt.core.* import com.github.ajalt.clikt.parameters.options.* +import io.ktor.server.application.* import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.launch @@ -38,7 +39,6 @@ import tech.libeufin.nexus.iso20022.NexusPaymentInitiationData import tech.libeufin.nexus.iso20022.createPain001document import tech.libeufin.nexus.iso20022.parseCamtMessage import tech.libeufin.nexus.server.EbicsDialects -import tech.libeufin.nexus.server.client import tech.libeufin.nexus.server.nexusApp import tech.libeufin.util.* import java.io.File @@ -77,7 +77,7 @@ class Serve : CliktCommand("Run nexus HTTP server") { override fun run() { setLogLevel(logLevel) execThrowableOrTerminate { dbCreateTables(getDbConnFromEnv(NEXUS_DB_ENV_VAR_NAME)) } - CoroutineScope(Dispatchers.IO).launch(fallback) { whileTrueOperationScheduler(client) } + CoroutineScope(Dispatchers.IO).launch(fallback) { whileTrueOperationScheduler() } if (withUnixSocket != null) { startServer( withUnixSocket!!, diff --git a/nexus/src/main/kotlin/tech/libeufin/nexus/Scheduling.kt b/nexus/src/main/kotlin/tech/libeufin/nexus/Scheduling.kt index 8b413b44..2e67eb3b 100644 --- a/nexus/src/main/kotlin/tech/libeufin/nexus/Scheduling.kt +++ b/nexus/src/main/kotlin/tech/libeufin/nexus/Scheduling.kt @@ -30,6 +30,7 @@ import org.jetbrains.exposed.sql.transactions.transaction import tech.libeufin.nexus.bankaccount.fetchBankAccountTransactions import tech.libeufin.nexus.bankaccount.submitAllPaymentInitiations import tech.libeufin.nexus.server.FetchSpecJson +import tech.libeufin.nexus.server.client import java.lang.IllegalArgumentException import java.time.Duration import java.time.Instant @@ -162,7 +163,7 @@ suspend fun javaTimerOperationScheduler(httpClient: HttpClient) { } */ -suspend fun whileTrueOperationScheduler(httpClient: HttpClient) { +suspend fun whileTrueOperationScheduler(httpClient: HttpClient = client) { while (true) { operationScheduler(httpClient) // Wait the shortest period that the cron spec would allow. diff --git a/nexus/src/main/kotlin/tech/libeufin/nexus/ebics/EbicsClient.kt b/nexus/src/main/kotlin/tech/libeufin/nexus/ebics/EbicsClient.kt index e2706d8b..e2fe7264 100644 --- a/nexus/src/main/kotlin/tech/libeufin/nexus/ebics/EbicsClient.kt +++ b/nexus/src/main/kotlin/tech/libeufin/nexus/ebics/EbicsClient.kt @@ -46,18 +46,17 @@ private suspend inline fun HttpClient.postToBank(url: String, body: String): Str setBody(body) } } catch (e: ClientRequestException) { - logger.error(e.message) + logger.error("Exception during request to $url: ${e.message}") val returnStatus = if (e.response.status.value == HttpStatusCode.RequestTimeout.value) HttpStatusCode.GatewayTimeout else HttpStatusCode.BadGateway - throw NexusError( returnStatus, e.message ) } catch (e: Exception) { - logger.error("Exception during request ${e.message}") + logger.error("Exception during request to $url: ${e.message}") throw NexusError( HttpStatusCode.BadGateway, e.message ?: "Could not reach the bank" diff --git a/nexus/src/main/kotlin/tech/libeufin/nexus/ebics/EbicsNexus.kt b/nexus/src/main/kotlin/tech/libeufin/nexus/ebics/EbicsNexus.kt index fbb128f8..0f007d77 100644 --- a/nexus/src/main/kotlin/tech/libeufin/nexus/ebics/EbicsNexus.kt +++ b/nexus/src/main/kotlin/tech/libeufin/nexus/ebics/EbicsNexus.kt @@ -681,8 +681,9 @@ class EbicsBankConnectionProtocol: BankConnectionProtocol { is FetchSpecTimeRangeJson -> { // the parse() method defaults to the YYYY-MM-DD format. // If parsing fails, the global catcher intervenes. - val start: LocalDate = LocalDate.parse(fetchSpec.start) - val end: LocalDate = LocalDate.parse(fetchSpec.end) + + val start: LocalDate = parseDashedDate(fetchSpec.start) + val end: LocalDate = parseDashedDate(fetchSpec.end) val p = EbicsStandardOrderParams( EbicsDateRange( start = start.atStartOfDay().atZone(ZoneId.systemDefault()), diff --git a/nexus/src/main/kotlin/tech/libeufin/nexus/iso20022/Iso20022.kt b/nexus/src/main/kotlin/tech/libeufin/nexus/iso20022/Iso20022.kt index d5229d94..128b7a1b 100644 --- a/nexus/src/main/kotlin/tech/libeufin/nexus/iso20022/Iso20022.kt +++ b/nexus/src/main/kotlin/tech/libeufin/nexus/iso20022/Iso20022.kt @@ -59,6 +59,7 @@ import java.time.LocalDateTime import java.time.ZoneId import java.time.ZonedDateTime import java.time.format.DateTimeFormatter +import tech.libeufin.nexus.logger enum class CashManagementResponseType(@get:JsonValue val jsonName: String) { diff --git a/nexus/src/main/kotlin/tech/libeufin/nexus/server/JSON.kt b/nexus/src/main/kotlin/tech/libeufin/nexus/server/JSON.kt index a0dc6b0c..39955778 100644 --- a/nexus/src/main/kotlin/tech/libeufin/nexus/server/JSON.kt +++ b/nexus/src/main/kotlin/tech/libeufin/nexus/server/JSON.kt @@ -227,7 +227,7 @@ class FetchSpecTimeRangeJson( start: String, end: String, bankConnection: String? -) : FetchSpecJson(level, bankConnection) +) : FetchSpecJson(level, bankConnection, start, end) @JsonTypeName("previous-days") class FetchSpecPreviousDaysJson(level: FetchLevel, bankConnection: String?, val number: Int) : diff --git a/nexus/src/main/kotlin/tech/libeufin/nexus/server/NexusServer.kt b/nexus/src/main/kotlin/tech/libeufin/nexus/server/NexusServer.kt index 7f0fa09f..a9d31e63 100644 --- a/nexus/src/main/kotlin/tech/libeufin/nexus/server/NexusServer.kt +++ b/nexus/src/main/kotlin/tech/libeufin/nexus/server/NexusServer.kt @@ -47,6 +47,7 @@ import tech.libeufin.nexus.ebics.* import tech.libeufin.nexus.iso20022.ingestCamtMessageIntoAccount import tech.libeufin.util.* import java.net.URLEncoder +import tech.libeufin.nexus.logger // Return facade state depending on the type. fun getFacadeState(type: String, facade: FacadeEntity): JsonNode { @@ -780,16 +781,26 @@ val nexusApp: Application.() -> Unit = { * fetches, it is ALSO possible that although one error is reported, * SOME transactions made it to the database! */ - if (ingestionResult.errors != null) - /** - * Nexus could not handle the error (regardless of it being generated - * here or gotten from the bank). The response body should inform the - * client about what failed. - */ - statusCode = HttpStatusCode.InternalServerError + if (ingestionResult.errors != null) { + /** + * Nexus could not handle the error (regardless of it being generated + * here or gotten from the bank). The response body should inform the + * client about what failed. + */ + statusCode = HttpStatusCode.InternalServerError + } + call.respond( status = statusCode, - ingestionResult + object { + val newTransactions = ingestionResult.newTransactions + val downloadedTransactions = ingestionResult.downloadedTransactions + val errors = mutableListOf().apply { + ingestionResult.errors?.forEach { + this.add(it.message ?: "Error message not found.") + } + } + } ) return@post } diff --git a/nexus/src/test/kotlin/ConversionServiceTest.kt b/nexus/src/test/kotlin/ConversionServiceTest.kt index bec0d4e9..f38dece0 100644 --- a/nexus/src/test/kotlin/ConversionServiceTest.kt +++ b/nexus/src/test/kotlin/ConversionServiceTest.kt @@ -238,17 +238,6 @@ class ConversionServiceTest { } } - // Abstracts the mock handler installation. - private fun getMockedClient(handler: MockRequestHandleScope.(HttpRequestData) -> HttpResponseData): HttpClient { - return HttpClient(MockEngine) { - followRedirects = false - engine { - addHandler { - request -> handler(request) - } - } - } - } /** * Checks that the cash-out monitor reacts after * a CRDT transaction arrives at the designated account. diff --git a/nexus/src/test/kotlin/MakeEnv.kt b/nexus/src/test/kotlin/MakeEnv.kt index eb84e628..32c6e607 100644 --- a/nexus/src/test/kotlin/MakeEnv.kt +++ b/nexus/src/test/kotlin/MakeEnv.kt @@ -1,4 +1,7 @@ import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper +import io.ktor.client.* +import io.ktor.client.engine.mock.* +import io.ktor.client.request.* import org.jetbrains.exposed.sql.Database import org.jetbrains.exposed.sql.statements.api.ExposedBlob import org.jetbrains.exposed.sql.transactions.TransactionManager @@ -754,4 +757,16 @@ val poFiCamt054_2019_incoming: String = """ -""".trimIndent() \ No newline at end of file +""".trimIndent() + +// Abstracts the mock handler installation. +fun getMockedClient(handler: MockRequestHandleScope.(HttpRequestData) -> HttpResponseData): HttpClient { + return HttpClient(MockEngine) { + followRedirects = false + engine { + addHandler { + request -> handler(request) + } + } + } +} \ No newline at end of file diff --git a/nexus/src/test/kotlin/NexusApiTest.kt b/nexus/src/test/kotlin/NexusApiTest.kt index ab6dcb34..899f6dad 100644 --- a/nexus/src/test/kotlin/NexusApiTest.kt +++ b/nexus/src/test/kotlin/NexusApiTest.kt @@ -1,9 +1,12 @@ import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper +import io.ktor.client.engine.mock.* import io.ktor.client.plugins.* import io.ktor.client.request.* import io.ktor.client.statement.* import io.ktor.http.* +import io.ktor.server.application.* +import io.ktor.server.config.* import io.ktor.server.testing.* import io.netty.handler.codec.http.HttpResponseStatus import kotlinx.coroutines.async @@ -13,7 +16,14 @@ import kotlinx.coroutines.runBlocking import org.jetbrains.exposed.sql.transactions.transaction import org.junit.Test import tech.libeufin.nexus.PaymentInitiationEntity -import tech.libeufin.nexus.server.nexusApp +import tech.libeufin.nexus.bankaccount.ingestBankMessagesIntoAccount +import tech.libeufin.nexus.getConnectionPlugin +import tech.libeufin.nexus.iso20022.ingestCamtMessageIntoAccount +import tech.libeufin.nexus.server.* +import tech.libeufin.sandbox.BankAccountTransactionEntity +import tech.libeufin.sandbox.BankAccountTransactionsTable +import tech.libeufin.sandbox.sandboxApp +import tech.libeufin.sandbox.wireTransfer /** * This class tests the API offered by Nexus, @@ -143,7 +153,6 @@ class NexusApiTest { contentType(ContentType.Application.Json) expectSuccess = true basicAuth("foo", "foo") - // NOTE: current API doesn't allow to omit the 'params' field. setBody("""{ "name": "send-payments", "cronspec": "* * *", @@ -205,7 +214,58 @@ class NexusApiTest { expectSuccess = false } assert(maybeConflict.status.value == HttpStatusCode.Conflict.value) + } + } + } + @Test + fun timeRangeFetch() { + withTestDatabase { + prepSandboxDb() + prepNexusDb() + val ref = wireTransfer( + "admin", + "foo", + subject = "past payment", + amount = "TESTKUDOS:30" + ) + transaction { + BankAccountTransactionEntity.find { + BankAccountTransactionsTable.accountServicerReference eq ref + }.first().date = 1577833200000L // Jan, 1st, 2020 + } + testApplication { + application(sandboxApp) + val conn = getConnectionPlugin("ebics") + // Asking a time range where the one payment is expected to exist + conn.fetchTransactions( + fetchSpec = FetchSpecTimeRangeJson( + FetchLevel.REPORT, + start = "2019-12-31", + end = "2020-01-02", + bankConnection = null + ), + accountId = "foo", + bankConnectionId = "foo", + client = client + ) + val res = ingestBankMessagesIntoAccount("foo", "foo") + assert(res.newTransactions == 1) + // Asking a time range where the one payment is NOT expected to exist + conn.fetchTransactions( + fetchSpec = FetchSpecTimeRangeJson( + FetchLevel.REPORT, + start = "2019-10-31", + end = "2020-11-30", + bankConnection = null + ), + accountId = "foo", + bankConnectionId = "foo", + client = client + ) + val resNoData = ingestBankMessagesIntoAccount("foo", "foo") + assert(resNoData.downloadedTransactions == 0) + assert(resNoData.newTransactions == 0) } } } diff --git a/nexus/src/test/kotlin/SandboxAccessApiTest.kt b/nexus/src/test/kotlin/SandboxAccessApiTest.kt index b2833890..ac64d327 100644 --- a/nexus/src/test/kotlin/SandboxAccessApiTest.kt +++ b/nexus/src/test/kotlin/SandboxAccessApiTest.kt @@ -15,6 +15,7 @@ import org.junit.Ignore import org.junit.Test import tech.libeufin.nexus.bankaccount.getBankAccount import tech.libeufin.sandbox.* +import tech.libeufin.util.getDatabaseName import java.util.* import kotlin.concurrent.schedule @@ -250,7 +251,9 @@ class SandboxAccessApiTest { // Check the withdrawal amount in the unique transaction. val t = client.get("/demobanks/default/access-api/accounts/foo/transactions") { basicAuth("foo", "foo") + expectSuccess = true } + println(t.bodyAsText()) val amount = mapper.readTree(t.readBytes()).get("transactions").get(0).get("amount").asText() assert(amount == "500000000") } diff --git a/sandbox/src/main/kotlin/tech/libeufin/sandbox/Main.kt b/sandbox/src/main/kotlin/tech/libeufin/sandbox/Main.kt index bfd521cc..0dc7b742 100644 --- a/sandbox/src/main/kotlin/tech/libeufin/sandbox/Main.kt +++ b/sandbox/src/main/kotlin/tech/libeufin/sandbox/Main.kt @@ -518,6 +518,7 @@ private suspend fun getWithdrawal(call: ApplicationCall) { private suspend fun confirmWithdrawal(call: ApplicationCall) { val withdrawalId = call.expectUriComponent("withdrawal_id") + logger.debug("Maybe confirming withdrawal: $withdrawalId") transaction { val wo = getWithdrawalOperation(withdrawalId) if (wo.aborted) throw SandboxError( @@ -541,6 +542,7 @@ private suspend fun confirmWithdrawal(call: ApplicationCall) { "Cannot withdraw without an exchange." ) ) + logger.debug("Withdrawal ${wo.wopid} confirmed? ${wo.confirmationDone}") if (!wo.confirmationDone) { wireTransfer( debitAccount = wo.walletBankAccount, @@ -1557,15 +1559,18 @@ val sandboxApp: Application.() -> Unit = { var ret: List = transaction { extractTxHistory(historyParams) } + logger.debug("Is payment data empty? ${ret.isEmpty()}") // Data was found already, UNLISTEN and respond. if (listenHandle != null && ret.isNotEmpty()) { + logger.debug("No need to wait DB events, payment data found.") listenHandle.postgresUnlisten() call.respond(object {val transactions = ret}) return@get } // No data was found, sleep until the timeout or getting woken up. // Third condition only silences the compiler. - if (listenHandle != null && ret.isEmpty() && longPollMs != null) { + if (listenHandle != null && longPollMs != null) { + logger.debug("Waiting DB event for new payment data.") val notificationArrived = listenHandle.waitOnIODispatchers(longPollMs) // Only if the awaited event fired, query again the DB. if (notificationArrived) diff --git a/sandbox/src/main/kotlin/tech/libeufin/sandbox/bankAccount.kt b/sandbox/src/main/kotlin/tech/libeufin/sandbox/bankAccount.kt index dc361a52..9a449f84 100644 --- a/sandbox/src/main/kotlin/tech/libeufin/sandbox/bankAccount.kt +++ b/sandbox/src/main/kotlin/tech/libeufin/sandbox/bankAccount.kt @@ -228,11 +228,9 @@ fun wireTransfer( } // Adjusting the balances (acceptable debit conditions checked before). - debitAccount.refresh() - creditAccount.refresh() // Debit: val newDebitBalance = (BigDecimal(debitAccount.balance) - amountAsNumber).roundToTwoDigits() - debitAccount.balance = newDebitBalance.toPlainString() // FIXME: that's ignored! + debitAccount.balance = newDebitBalance.toPlainString() // Credit: val newCreditBalance = (BigDecimal(creditAccount.balance) + amountAsNumber).roundToTwoDigits() creditAccount.balance = newCreditBalance.toPlainString() diff --git a/util/src/main/kotlin/Ebics.kt b/util/src/main/kotlin/Ebics.kt index e951016a..036e1c54 100644 --- a/util/src/main/kotlin/Ebics.kt +++ b/util/src/main/kotlin/Ebics.kt @@ -37,6 +37,7 @@ import java.math.BigInteger import java.security.SecureRandom import java.security.interfaces.RSAPrivateCrtKey import java.security.interfaces.RSAPublicKey +import java.time.ZoneId import java.time.ZonedDateTime import java.util.* import java.util.zip.DeflaterInputStream @@ -382,7 +383,9 @@ fun createEbicsRequestForDownloadInitialization( subscriberDetails.partnerId, subscriberDetails.hostId, nonce, - DatatypeFactory.newInstance().newXMLGregorianCalendar(GregorianCalendar()), + DatatypeFactory.newInstance().newXMLGregorianCalendar(GregorianCalendar( + TimeZone.getTimeZone(ZoneId.systemDefault()) + )), subscriberDetails.bankEncPub ?: throw EbicsProtocolError( HttpStatusCode.BadRequest, "Invalid subscriber state 'bankEncPub' missing, please send HPB first" @@ -415,7 +418,9 @@ fun createEbicsRequestForDownloadInitialization( subscriberDetails.partnerId, subscriberDetails.hostId, nonce, - DatatypeFactory.newInstance().newXMLGregorianCalendar(GregorianCalendar()), + DatatypeFactory.newInstance().newXMLGregorianCalendar(GregorianCalendar( + TimeZone.getTimeZone(ZoneId.systemDefault()) + )), subscriberDetails.bankEncPub ?: throw EbicsProtocolError( HttpStatusCode.BadRequest, "Invalid subscriber state 'bankEncPub' missing, please send HPB first" diff --git a/util/src/main/kotlin/time.kt b/util/src/main/kotlin/time.kt index bbdb3143..867d1950 100644 --- a/util/src/main/kotlin/time.kt +++ b/util/src/main/kotlin/time.kt @@ -52,4 +52,14 @@ fun importDateFromMillis(millis: Long): ZonedDateTime { fun LocalDateTime.millis(): Long { val instant = Instant.from(this.atZone(ZoneOffset.UTC)) return instant.toEpochMilli() +} + +fun parseDashedDate(maybeDashedDate: String?): LocalDate { + if (maybeDashedDate == null) + throw badRequest("dashed date found as null") + return try { + LocalDate.parse(maybeDashedDate) + } catch (e: Exception) { + throw badRequest("bad dashed date: $maybeDashedDate. ${e.message}") + } } \ No newline at end of file -- cgit v1.2.3