Testing the EBICS time-framed history request.

15 files changed, 135 insertions, 37 deletions

diff --git a/nexus/src/main/kotlin/tech/libeufin/nexus/Main.kt b/nexus/src/main/kotlin/tech/libeufin/nexus/Main.kt
index 0886e177..5b453530 100644
--- a/nexus/src/main/kotlin/tech/libeufin/nexus/Main.kt
+++ b/nexus/src/main/kotlin/tech/libeufin/nexus/Main.kt
@@ -30,6 +30,7 @@ import com.github.ajalt.clikt.parameters.types.int
 import execThrowableOrTerminate
 import com.github.ajalt.clikt.core.*
 import com.github.ajalt.clikt.parameters.options.*
+import io.ktor.server.application.*
 import kotlinx.coroutines.CoroutineScope
 import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.launch
@@ -38,7 +39,6 @@ import tech.libeufin.nexus.iso20022.NexusPaymentInitiationData
 import tech.libeufin.nexus.iso20022.createPain001document
 import tech.libeufin.nexus.iso20022.parseCamtMessage
 import tech.libeufin.nexus.server.EbicsDialects
-import tech.libeufin.nexus.server.client
 import tech.libeufin.nexus.server.nexusApp
 import tech.libeufin.util.*
 import java.io.File
@@ -77,7 +77,7 @@ class Serve : CliktCommand("Run nexus HTTP server") {
     override fun run() {
         setLogLevel(logLevel)
         execThrowableOrTerminate { dbCreateTables(getDbConnFromEnv(NEXUS_DB_ENV_VAR_NAME)) }
-        CoroutineScope(Dispatchers.IO).launch(fallback) { whileTrueOperationScheduler(client) }
+        CoroutineScope(Dispatchers.IO).launch(fallback) { whileTrueOperationScheduler() }
         if (withUnixSocket != null) {
             startServer(
                 withUnixSocket!!,
diff --git a/nexus/src/main/kotlin/tech/libeufin/nexus/Scheduling.kt b/nexus/src/main/kotlin/tech/libeufin/nexus/Scheduling.kt
index 8b413b44..2e67eb3b 100644
--- a/nexus/src/main/kotlin/tech/libeufin/nexus/Scheduling.kt
+++ b/nexus/src/main/kotlin/tech/libeufin/nexus/Scheduling.kt
@@ -30,6 +30,7 @@ import org.jetbrains.exposed.sql.transactions.transaction
 import tech.libeufin.nexus.bankaccount.fetchBankAccountTransactions
 import tech.libeufin.nexus.bankaccount.submitAllPaymentInitiations
 import tech.libeufin.nexus.server.FetchSpecJson
+import tech.libeufin.nexus.server.client
 import java.lang.IllegalArgumentException
 import java.time.Duration
 import java.time.Instant
@@ -162,7 +163,7 @@ suspend fun javaTimerOperationScheduler(httpClient: HttpClient) {
 }
 */
 
-suspend fun whileTrueOperationScheduler(httpClient: HttpClient) {
+suspend fun whileTrueOperationScheduler(httpClient: HttpClient = client) {
     while (true) {
         operationScheduler(httpClient)
         // Wait the shortest period that the cron spec would allow.
diff --git a/nexus/src/main/kotlin/tech/libeufin/nexus/ebics/EbicsClient.kt b/nexus/src/main/kotlin/tech/libeufin/nexus/ebics/EbicsClient.kt
index e2706d8b..e2fe7264 100644
--- a/nexus/src/main/kotlin/tech/libeufin/nexus/ebics/EbicsClient.kt
+++ b/nexus/src/main/kotlin/tech/libeufin/nexus/ebics/EbicsClient.kt
@@ -46,18 +46,17 @@ private suspend inline fun HttpClient.postToBank(url: String, body: String): Str
                 setBody(body)
         }
     } catch (e: ClientRequestException) {
-        logger.error(e.message)
+        logger.error("Exception during request to $url: ${e.message}")
         val returnStatus = if (e.response.status.value == HttpStatusCode.RequestTimeout.value)
             HttpStatusCode.GatewayTimeout
         else HttpStatusCode.BadGateway
-        
         throw NexusError(
             returnStatus,
             e.message
         )
     }
     catch (e: Exception) {
-        logger.error("Exception during request ${e.message}")
+        logger.error("Exception during request to $url: ${e.message}")
         throw NexusError(
             HttpStatusCode.BadGateway,
             e.message ?: "Could not reach the bank"
diff --git a/nexus/src/main/kotlin/tech/libeufin/nexus/ebics/EbicsNexus.kt b/nexus/src/main/kotlin/tech/libeufin/nexus/ebics/EbicsNexus.kt
index fbb128f8..0f007d77 100644
--- a/nexus/src/main/kotlin/tech/libeufin/nexus/ebics/EbicsNexus.kt
+++ b/nexus/src/main/kotlin/tech/libeufin/nexus/ebics/EbicsNexus.kt
@@ -681,8 +681,9 @@ class EbicsBankConnectionProtocol: BankConnectionProtocol {
             is FetchSpecTimeRangeJson -> {
                 // the parse() method defaults to the YYYY-MM-DD format.
                 // If parsing fails, the global catcher intervenes.
-                val start: LocalDate = LocalDate.parse(fetchSpec.start)
-                val end: LocalDate = LocalDate.parse(fetchSpec.end)
+
+                val start: LocalDate = parseDashedDate(fetchSpec.start)
+                val end: LocalDate = parseDashedDate(fetchSpec.end)
                 val p = EbicsStandardOrderParams(
                     EbicsDateRange(
                         start = start.atStartOfDay().atZone(ZoneId.systemDefault()),
diff --git a/nexus/src/main/kotlin/tech/libeufin/nexus/iso20022/Iso20022.kt b/nexus/src/main/kotlin/tech/libeufin/nexus/iso20022/Iso20022.kt
index d5229d94..128b7a1b 100644
--- a/nexus/src/main/kotlin/tech/libeufin/nexus/iso20022/Iso20022.kt
+++ b/nexus/src/main/kotlin/tech/libeufin/nexus/iso20022/Iso20022.kt
@@ -59,6 +59,7 @@ import java.time.LocalDateTime
 import java.time.ZoneId
 import java.time.ZonedDateTime
 import java.time.format.DateTimeFormatter
+import tech.libeufin.nexus.logger
 
 
 enum class CashManagementResponseType(@get:JsonValue val jsonName: String) {
diff --git a/nexus/src/main/kotlin/tech/libeufin/nexus/server/JSON.kt b/nexus/src/main/kotlin/tech/libeufin/nexus/server/JSON.kt
index a0dc6b0c..39955778 100644
--- a/nexus/src/main/kotlin/tech/libeufin/nexus/server/JSON.kt
+++ b/nexus/src/main/kotlin/tech/libeufin/nexus/server/JSON.kt
@@ -227,7 +227,7 @@ class FetchSpecTimeRangeJson(
     start: String,
     end: String,
     bankConnection: String?
-) : FetchSpecJson(level, bankConnection)
+) : FetchSpecJson(level, bankConnection, start, end)
 
 @JsonTypeName("previous-days")
 class FetchSpecPreviousDaysJson(level: FetchLevel, bankConnection: String?, val number: Int) :
diff --git a/nexus/src/main/kotlin/tech/libeufin/nexus/server/NexusServer.kt b/nexus/src/main/kotlin/tech/libeufin/nexus/server/NexusServer.kt
index 7f0fa09f..a9d31e63 100644
--- a/nexus/src/main/kotlin/tech/libeufin/nexus/server/NexusServer.kt
+++ b/nexus/src/main/kotlin/tech/libeufin/nexus/server/NexusServer.kt
@@ -47,6 +47,7 @@ import tech.libeufin.nexus.ebics.*
 import tech.libeufin.nexus.iso20022.ingestCamtMessageIntoAccount
 import tech.libeufin.util.*
 import java.net.URLEncoder
+import tech.libeufin.nexus.logger
 
 // Return facade state depending on the type.
 fun getFacadeState(type: String, facade: FacadeEntity): JsonNode {
@@ -780,16 +781,26 @@ val nexusApp: Application.() -> Unit = {
              * fetches, it is ALSO possible that although one error is reported,
              * SOME transactions made it to the database!
              */
-            if (ingestionResult.errors != null)
-            /**
-             * Nexus could not handle the error (regardless of it being generated
-             * here or gotten from the bank).  The response body should inform the
-             * client about what failed.
-             */
-            statusCode = HttpStatusCode.InternalServerError
+            if (ingestionResult.errors != null) {
+                /**
+                 * Nexus could not handle the error (regardless of it being generated
+                 * here or gotten from the bank).  The response body should inform the
+                 * client about what failed.
+                 */
+                statusCode = HttpStatusCode.InternalServerError
+            }
+
             call.respond(
                 status = statusCode,
-                ingestionResult
+                object {
+                    val newTransactions = ingestionResult.newTransactions
+                    val downloadedTransactions = ingestionResult.downloadedTransactions
+                    val errors = mutableListOf<String>().apply {
+                        ingestionResult.errors?.forEach {
+                            this.add(it.message ?: "Error message not found.")
+                        }
+                    }
+                }
             )
             return@post
         }
diff --git a/nexus/src/test/kotlin/ConversionServiceTest.kt b/nexus/src/test/kotlin/ConversionServiceTest.kt
index bec0d4e9..f38dece0 100644
--- a/nexus/src/test/kotlin/ConversionServiceTest.kt
+++ b/nexus/src/test/kotlin/ConversionServiceTest.kt
@@ -238,17 +238,6 @@ class ConversionServiceTest {
         }
     }
 
-    // Abstracts the mock handler installation.
-    private fun getMockedClient(handler: MockRequestHandleScope.(HttpRequestData) -> HttpResponseData): HttpClient {
-        return HttpClient(MockEngine) {
-            followRedirects = false
-            engine {
-                addHandler {
-                        request -> handler(request)
-                }
-            }
-        }
-    }
     /**
      * Checks that the cash-out monitor reacts after
      * a CRDT transaction arrives at the designated account.
diff --git a/nexus/src/test/kotlin/MakeEnv.kt b/nexus/src/test/kotlin/MakeEnv.kt
index eb84e628..32c6e607 100644
--- a/nexus/src/test/kotlin/MakeEnv.kt
+++ b/nexus/src/test/kotlin/MakeEnv.kt
@@ -1,4 +1,7 @@
 import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper
+import io.ktor.client.*
+import io.ktor.client.engine.mock.*
+import io.ktor.client.request.*
 import org.jetbrains.exposed.sql.Database
 import org.jetbrains.exposed.sql.statements.api.ExposedBlob
 import org.jetbrains.exposed.sql.transactions.TransactionManager
@@ -754,4 +757,16 @@ val poFiCamt054_2019_incoming: String = """
 		</Ntfctn>
 	</BkToCstmrDbtCdtNtfctn>
 </Document>
-""".trimIndent()
\ No newline at end of file
+""".trimIndent()
+
+// Abstracts the mock handler installation.
+fun getMockedClient(handler: MockRequestHandleScope.(HttpRequestData) -> HttpResponseData): HttpClient {
+    return HttpClient(MockEngine) {
+        followRedirects = false
+        engine {
+            addHandler {
+                    request -> handler(request)
+            }
+        }
+    }
+}
\ No newline at end of file
diff --git a/nexus/src/test/kotlin/NexusApiTest.kt b/nexus/src/test/kotlin/NexusApiTest.kt
index ab6dcb34..899f6dad 100644
--- a/nexus/src/test/kotlin/NexusApiTest.kt
+++ b/nexus/src/test/kotlin/NexusApiTest.kt
@@ -1,9 +1,12 @@
 import com.fasterxml.jackson.databind.ObjectMapper
 import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper
+import io.ktor.client.engine.mock.*
 import io.ktor.client.plugins.*
 import io.ktor.client.request.*
 import io.ktor.client.statement.*
 import io.ktor.http.*
+import io.ktor.server.application.*
+import io.ktor.server.config.*
 import io.ktor.server.testing.*
 import io.netty.handler.codec.http.HttpResponseStatus
 import kotlinx.coroutines.async
@@ -13,7 +16,14 @@ import kotlinx.coroutines.runBlocking
 import org.jetbrains.exposed.sql.transactions.transaction
 import org.junit.Test
 import tech.libeufin.nexus.PaymentInitiationEntity
-import tech.libeufin.nexus.server.nexusApp
+import tech.libeufin.nexus.bankaccount.ingestBankMessagesIntoAccount
+import tech.libeufin.nexus.getConnectionPlugin
+import tech.libeufin.nexus.iso20022.ingestCamtMessageIntoAccount
+import tech.libeufin.nexus.server.*
+import tech.libeufin.sandbox.BankAccountTransactionEntity
+import tech.libeufin.sandbox.BankAccountTransactionsTable
+import tech.libeufin.sandbox.sandboxApp
+import tech.libeufin.sandbox.wireTransfer
 
 /**
  * This class tests the API offered by Nexus,
@@ -143,7 +153,6 @@ class NexusApiTest {
                     contentType(ContentType.Application.Json)
                     expectSuccess = true
                     basicAuth("foo", "foo")
-                    // NOTE: current API doesn't allow to omit the 'params' field.
                     setBody("""{
                         "name": "send-payments",
                         "cronspec": "* * *",
@@ -205,7 +214,58 @@ class NexusApiTest {
                     expectSuccess = false
                 }
                 assert(maybeConflict.status.value == HttpStatusCode.Conflict.value)
+            }
+        }
+    }
+    @Test
+    fun timeRangeFetch() {
+        withTestDatabase {
+            prepSandboxDb()
+            prepNexusDb()
+            val ref = wireTransfer(
+                "admin",
+                "foo",
+                subject = "past payment",
+                amount = "TESTKUDOS:30"
+                )
+            transaction {
+                BankAccountTransactionEntity.find {
+                    BankAccountTransactionsTable.accountServicerReference eq ref
+                }.first().date = 1577833200000L // Jan, 1st, 2020
+            }
+            testApplication {
+                application(sandboxApp)
+                val conn = getConnectionPlugin("ebics")
 
+                // Asking a time range where the one payment is expected to exist
+                conn.fetchTransactions(
+                    fetchSpec = FetchSpecTimeRangeJson(
+                        FetchLevel.REPORT,
+                        start = "2019-12-31",
+                        end = "2020-01-02",
+                        bankConnection = null
+                    ),
+                    accountId = "foo",
+                    bankConnectionId = "foo",
+                    client = client
+                )
+                val res = ingestBankMessagesIntoAccount("foo", "foo")
+                assert(res.newTransactions == 1)
+                // Asking a time range where the one payment is NOT expected to exist
+                conn.fetchTransactions(
+                    fetchSpec = FetchSpecTimeRangeJson(
+                        FetchLevel.REPORT,
+                        start = "2019-10-31",
+                        end = "2020-11-30",
+                        bankConnection = null
+                    ),
+                    accountId = "foo",
+                    bankConnectionId = "foo",
+                    client = client
+                )
+                val resNoData = ingestBankMessagesIntoAccount("foo", "foo")
+                assert(resNoData.downloadedTransactions == 0)
+                assert(resNoData.newTransactions == 0)
             }
         }
     }
diff --git a/nexus/src/test/kotlin/SandboxAccessApiTest.kt b/nexus/src/test/kotlin/SandboxAccessApiTest.kt
index b2833890..ac64d327 100644
--- a/nexus/src/test/kotlin/SandboxAccessApiTest.kt
+++ b/nexus/src/test/kotlin/SandboxAccessApiTest.kt
@@ -15,6 +15,7 @@ import org.junit.Ignore
 import org.junit.Test
 import tech.libeufin.nexus.bankaccount.getBankAccount
 import tech.libeufin.sandbox.*
+import tech.libeufin.util.getDatabaseName
 import java.util.*
 import kotlin.concurrent.schedule
 
@@ -250,7 +251,9 @@ class SandboxAccessApiTest {
                 // Check the withdrawal amount in the unique transaction.
                 val t = client.get("/demobanks/default/access-api/accounts/foo/transactions") {
                     basicAuth("foo", "foo")
+                    expectSuccess = true
                 }
+                println(t.bodyAsText())
                 val amount = mapper.readTree(t.readBytes()).get("transactions").get(0).get("amount").asText()
                 assert(amount == "500000000")
             }
diff --git a/sandbox/src/main/kotlin/tech/libeufin/sandbox/Main.kt b/sandbox/src/main/kotlin/tech/libeufin/sandbox/Main.kt
index bfd521cc..0dc7b742 100644
--- a/sandbox/src/main/kotlin/tech/libeufin/sandbox/Main.kt
+++ b/sandbox/src/main/kotlin/tech/libeufin/sandbox/Main.kt
@@ -518,6 +518,7 @@ private suspend fun getWithdrawal(call: ApplicationCall) {
 
 private suspend fun confirmWithdrawal(call: ApplicationCall) {
     val withdrawalId = call.expectUriComponent("withdrawal_id")
+    logger.debug("Maybe confirming withdrawal: $withdrawalId")
     transaction {
         val wo = getWithdrawalOperation(withdrawalId)
         if (wo.aborted) throw SandboxError(
@@ -541,6 +542,7 @@ private suspend fun confirmWithdrawal(call: ApplicationCall) {
                 "Cannot withdraw without an exchange."
             )
         )
+        logger.debug("Withdrawal ${wo.wopid} confirmed? ${wo.confirmationDone}")
         if (!wo.confirmationDone) {
             wireTransfer(
                 debitAccount = wo.walletBankAccount,
@@ -1557,15 +1559,18 @@ val sandboxApp: Application.() -> Unit = {
                     var ret: List<XLibeufinBankTransaction> = transaction {
                         extractTxHistory(historyParams)
                     }
+                    logger.debug("Is payment data empty? ${ret.isEmpty()}")
                     // Data was found already, UNLISTEN and respond.
                     if (listenHandle != null && ret.isNotEmpty()) {
+                        logger.debug("No need to wait DB events, payment data found.")
                         listenHandle.postgresUnlisten()
                         call.respond(object {val transactions = ret})
                         return@get
                     }
                     // No data was found, sleep until the timeout or getting woken up.
                     // Third condition only silences the compiler.
-                    if (listenHandle != null && ret.isEmpty() && longPollMs != null) {
+                    if (listenHandle != null && longPollMs != null) {
+                        logger.debug("Waiting DB event for new payment data.")
                         val notificationArrived = listenHandle.waitOnIODispatchers(longPollMs)
                         // Only if the awaited event fired, query again the DB.
                         if (notificationArrived)
diff --git a/sandbox/src/main/kotlin/tech/libeufin/sandbox/bankAccount.kt b/sandbox/src/main/kotlin/tech/libeufin/sandbox/bankAccount.kt
index dc361a52..9a449f84 100644
--- a/sandbox/src/main/kotlin/tech/libeufin/sandbox/bankAccount.kt
+++ b/sandbox/src/main/kotlin/tech/libeufin/sandbox/bankAccount.kt
@@ -228,11 +228,9 @@ fun wireTransfer(
         }
 
         // Adjusting the balances (acceptable debit conditions checked before).
-        debitAccount.refresh()
-        creditAccount.refresh()
         // Debit:
         val newDebitBalance = (BigDecimal(debitAccount.balance) - amountAsNumber).roundToTwoDigits()
-        debitAccount.balance = newDebitBalance.toPlainString() // FIXME: that's ignored!
+        debitAccount.balance = newDebitBalance.toPlainString()
         // Credit:
         val newCreditBalance = (BigDecimal(creditAccount.balance) + amountAsNumber).roundToTwoDigits()
         creditAccount.balance = newCreditBalance.toPlainString()
diff --git a/util/src/main/kotlin/Ebics.kt b/util/src/main/kotlin/Ebics.kt
index e951016a..036e1c54 100644
--- a/util/src/main/kotlin/Ebics.kt
+++ b/util/src/main/kotlin/Ebics.kt
@@ -37,6 +37,7 @@ import java.math.BigInteger
 import java.security.SecureRandom
 import java.security.interfaces.RSAPrivateCrtKey
 import java.security.interfaces.RSAPublicKey
+import java.time.ZoneId
 import java.time.ZonedDateTime
 import java.util.*
 import java.util.zip.DeflaterInputStream
@@ -382,7 +383,9 @@ fun createEbicsRequestForDownloadInitialization(
         subscriberDetails.partnerId,
         subscriberDetails.hostId,
         nonce,
-        DatatypeFactory.newInstance().newXMLGregorianCalendar(GregorianCalendar()),
+        DatatypeFactory.newInstance().newXMLGregorianCalendar(GregorianCalendar(
+            TimeZone.getTimeZone(ZoneId.systemDefault())
+        )),
         subscriberDetails.bankEncPub ?: throw EbicsProtocolError(
             HttpStatusCode.BadRequest,
             "Invalid subscriber state 'bankEncPub' missing, please send HPB first"
@@ -415,7 +418,9 @@ fun createEbicsRequestForDownloadInitialization(
         subscriberDetails.partnerId,
         subscriberDetails.hostId,
         nonce,
-        DatatypeFactory.newInstance().newXMLGregorianCalendar(GregorianCalendar()),
+        DatatypeFactory.newInstance().newXMLGregorianCalendar(GregorianCalendar(
+            TimeZone.getTimeZone(ZoneId.systemDefault())
+        )),
         subscriberDetails.bankEncPub ?: throw EbicsProtocolError(
             HttpStatusCode.BadRequest,
             "Invalid subscriber state 'bankEncPub' missing, please send HPB first"
diff --git a/util/src/main/kotlin/time.kt b/util/src/main/kotlin/time.kt
index bbdb3143..867d1950 100644
--- a/util/src/main/kotlin/time.kt
+++ b/util/src/main/kotlin/time.kt
@@ -52,4 +52,14 @@ fun importDateFromMillis(millis: Long): ZonedDateTime {
 fun LocalDateTime.millis(): Long {
     val instant = Instant.from(this.atZone(ZoneOffset.UTC))
     return instant.toEpochMilli()
+}
+
+fun parseDashedDate(maybeDashedDate: String?): LocalDate {
+    if (maybeDashedDate == null)
+        throw badRequest("dashed date found as null")
+    return try {
+        LocalDate.parse(maybeDashedDate)
+    } catch (e: Exception) {
+        throw badRequest("bad dashed date: $maybeDashedDate.  ${e.message}")
+    }
 }
\ No newline at end of file