commit b2b49493967b71b58910274f8d3fe87427891c7a
parent d985b3a4a8980792489fc975d8aeef7682ec1ca9
Author: MS <ms@taler.net>
Date: Thu, 9 Nov 2023 17:08:14 +0100
nexus fetch
drafting the main logic to get the last incoming transaction
timestamp from the database -> ask EBICS notifications based
on it -> (optionally) store the plain camt.054 to disk.
Diffstat:
6 files changed, 210 insertions(+), 75 deletions(-)
diff --git a/nexus/src/main/kotlin/tech/libeufin/nexus/Database.kt b/nexus/src/main/kotlin/tech/libeufin/nexus/Database.kt
@@ -308,6 +308,25 @@ class Database(dbConfig: String): java.io.Closeable {
}
/**
+ * Get the last execution time of an incoming transaction. This
+ * serves as the start date for new requests to the bank.
+ *
+ * @return [Instant] or null if no results were found
+ */
+ suspend fun incomingPaymentLastExecTime(): Instant? = runConn { conn ->
+ val stmt = conn.prepareStatement(
+ "SELECT MAX(execution_time) as latest_execution_time FROM incoming_transactions"
+ )
+ stmt.executeQuery().use {
+ if (!it.next()) return@runConn null
+ val timestamp = it.getLong("latest_execution_time").microsToJavaInstant()
+ if (timestamp == null)
+ throw Exception("Could not convert latest_execution_time to Instant")
+ return@runConn timestamp
+ }
+ }
+
+ /**
* Creates a new incoming payment record in the database.
*
* @param paymentData information related to the incoming payment.
diff --git a/nexus/src/main/kotlin/tech/libeufin/nexus/EbicsFetch.kt b/nexus/src/main/kotlin/tech/libeufin/nexus/EbicsFetch.kt
@@ -4,12 +4,22 @@ import com.github.ajalt.clikt.core.CliktCommand
import com.github.ajalt.clikt.parameters.options.flag
import com.github.ajalt.clikt.parameters.options.option
import io.ktor.client.*
+import kotlinx.coroutines.runBlocking
import org.apache.commons.compress.archivers.zip.ZipFile
import org.apache.commons.compress.utils.SeekableInMemoryByteChannel
+import tech.libeufin.nexus.ebics.EbicsSideError
+import tech.libeufin.nexus.ebics.EbicsSideException
+import tech.libeufin.nexus.ebics.createEbics3DownloadInitialization
+import tech.libeufin.nexus.ebics.doEbicsDownload
import tech.libeufin.util.ebics_h005.Ebics3Request
import tech.libeufin.util.getXmlDate
+import tech.libeufin.util.toDbMicros
+import java.nio.file.Path
import java.time.Instant
+import java.time.LocalDate
+import java.time.ZoneId
import kotlin.concurrent.fixedRateTimer
+import kotlin.io.path.createDirectories
import kotlin.system.exitProcess
/**
@@ -194,8 +204,100 @@ fun prepReportRequest(
}
/**
- * Fetches the banking records via EBICS, calling the CAMT
- * parsing logic and finally updating the database accordingly.
+ * Downloads content via EBICS, according to the order params passed
+ * by the caller.
+ *
+ * @param cfg configuration handle.
+ * @param bankKeys bank public keys.
+ * @param clientKeys EBICS subscriber private keys.
+ * @param httpClient handle to the HTTP layer.
+ * @param req contains the instructions for the download, namely
+ * which document is going to be downloaded from the bank.
+ * @return the [ByteArray] payload. On an empty response, the array
+ * length is zero. It returns null, if the bank assigned an
+ * error to the EBICS transaction.
+ */
+suspend fun downloadRecords(
+ cfg: EbicsSetupConfig,
+ bankKeys: BankPublicKeysFile,
+ clientKeys: ClientPrivateKeysFile,
+ httpClient: HttpClient,
+ req: Ebics3Request.OrderDetails.BTOrderParams
+): ByteArray? {
+ val initXml = createEbics3DownloadInitialization(
+ cfg,
+ bankKeys,
+ clientKeys,
+ orderParams = req
+ )
+ try {
+ return doEbicsDownload(
+ httpClient,
+ cfg,
+ clientKeys,
+ bankKeys,
+ initXml,
+ isEbics3 = true,
+ tolerateEmptyResult = true
+ )
+ } catch (e: EbicsSideException) {
+ logger.error(e.message)
+ /**
+ * Failing regardless of the error being at the client or at the
+ * bank side. A client with an unreliable bank is not useful, hence
+ * failing here.
+ */
+ exitProcess(1)
+ }
+}
+
+/**
+ * Extracts the archive entries and logs them to the location
+ * optionally specified in the configuration. It does nothing,
+ * if the configuration lacks the log directory.
+ *
+ * @param cfg config handle.
+ * @param content ZIP bytes from the server.
+ */
+fun maybeLogFile(cfg: EbicsSetupConfig, content: ByteArray) {
+ val maybeLogDir = cfg.config.lookupString(
+ "[neuxs-fetch]",
+ "STATEMENT_LOG_DIRECTORY"
+ ) ?: return
+ try { Path.of(maybeLogDir).createDirectories() }
+ catch (e: Exception) {
+ logger.error("Could not create log directory of path: $maybeLogDir")
+ exitProcess(1)
+ }
+ val now = Instant.now()
+ val asUtcDate = LocalDate.ofInstant(now, ZoneId.of("UTC"))
+ content.unzipForEach { fileName, xmlContent ->
+ val f = Path.of(
+ "${asUtcDate.year}-${asUtcDate.monthValue}-${asUtcDate.dayOfMonth}",
+ "${now.toDbMicros()}_$fileName"
+ ).toFile()
+ val completePath = Path.of(maybeLogDir, f.path)
+ // Rare: cannot download the same file twice in the same microsecond.
+ if (f.exists()) {
+ logger.error("Log file exists already at: $completePath")
+ exitProcess(1)
+ }
+ completePath.toFile().writeText(xmlContent)
+ }
+}
+
+/**
+ * Fetches the banking records via EBICS notifications requests.
+ *
+ * It first checks the last execution_time (db column) among the
+ * incoming transactions. If that's not found, it asks the bank
+ * about 'unseen notifications' (= does not specify any date range
+ * in the request). If that's found, it crafts a notification
+ * request with such execution_time as the start date and now as
+ * the end date.
+ *
+ * What this function does NOT do (now): linking documents between
+ * different camt.05x formats and/or pain.002 acknowledgements.
*
* @param cfg config handle.
* @param db database connection
@@ -203,14 +305,28 @@ fun prepReportRequest(
* @param clientKeys EBICS subscriber private keys.
* @param bankKeys bank public keys.
*/
-fun fetchHistory(
+suspend fun fetchHistory(
cfg: EbicsSetupConfig,
db: Database,
httpClient: HttpClient,
clientKeys: ClientPrivateKeysFile,
bankKeys: BankPublicKeysFile
) {
- throw NotImplementedError()
+ // maybe get last execution_date.
+ val lastExecutionTime = db.incomingPaymentLastExecTime()
+ // Asking unseen records.
+ val req = if (lastExecutionTime == null) prepNotificationRequest(isAppendix = false)
+ else prepNotificationRequest(lastExecutionTime, isAppendix = false)
+ val maybeContent = downloadRecords(
+ cfg,
+ bankKeys,
+ clientKeys,
+ httpClient,
+ req
+ ) ?: exitProcess(1) // client is wrong, failing.
+
+ if (maybeContent.isEmpty()) return
+ maybeLogFile(cfg, maybeContent)
}
class EbicsFetch: CliktCommand("Fetches bank records") {
@@ -252,7 +368,7 @@ class EbicsFetch: CliktCommand("Fetches bank records") {
val httpClient = HttpClient()
if (transient) {
logger.info("Transient mode: fetching once and returning.")
- fetchHistory(cfg, db, httpClient, clientKeys, bankKeys)
+ runBlocking { fetchHistory(cfg, db, httpClient, clientKeys, bankKeys) }
return
}
val frequency: NexusFrequency = doOrFail {
@@ -263,14 +379,14 @@ class EbicsFetch: CliktCommand("Fetches bank records") {
logger.debug("Running with a frequency of ${frequency.fromConfig}")
if (frequency.inSeconds == 0) {
logger.warn("Long-polling not implemented, running therefore in transient mode")
- fetchHistory(cfg, db, httpClient, clientKeys, bankKeys)
+ runBlocking { fetchHistory(cfg, db, httpClient, clientKeys, bankKeys) }
return
}
fixedRateTimer(
name = "ebics submit period",
period = (frequency.inSeconds * 1000).toLong(),
action = {
- fetchHistory(cfg, db, httpClient, clientKeys, bankKeys)
+ runBlocking { fetchHistory(cfg, db, httpClient, clientKeys, bankKeys) }
}
)
}
diff --git a/nexus/src/main/kotlin/tech/libeufin/nexus/EbicsSubmit.kt b/nexus/src/main/kotlin/tech/libeufin/nexus/EbicsSubmit.kt
@@ -24,24 +24,19 @@ import com.github.ajalt.clikt.parameters.options.flag
import com.github.ajalt.clikt.parameters.options.option
import io.ktor.client.*
import kotlinx.coroutines.runBlocking
-import tech.libeufin.nexus.ebics.EbicsEarlyErrorCode
-import tech.libeufin.nexus.ebics.EbicsEarlyException
+import tech.libeufin.nexus.ebics.EbicsSideError
+import tech.libeufin.nexus.ebics.EbicsSideException
import tech.libeufin.nexus.ebics.EbicsUploadException
import tech.libeufin.nexus.ebics.submitPain001
import tech.libeufin.util.parsePayto
import tech.libeufin.util.toDbMicros
-import java.io.File
import java.nio.file.Path
-import java.text.DateFormat
import java.time.Instant
import java.time.LocalDate
import java.time.ZoneId
import java.util.*
-import javax.xml.crypto.Data
import kotlin.concurrent.fixedRateTimer
import kotlin.io.path.createDirectories
-import kotlin.io.path.createParentDirectories
-import kotlin.math.log
import kotlin.system.exitProcess
/**
@@ -112,12 +107,12 @@ private suspend fun submitInitiatedPayment(
bankPublicKeysFile,
httpClient
)
- } catch (early: EbicsEarlyException) {
- val errorStage = when (early.earlyEc) {
- EbicsEarlyErrorCode.HTTP_POST_FAILED ->
+ } catch (early: EbicsSideException) {
+ val errorStage = when (early.sideEc) {
+ EbicsSideError.HTTP_POST_FAILED ->
NexusSubmissionStage.http // transient error
/**
- * Any other [EbicsEarlyErrorCode] should be treated as permanent,
+ * Any other [EbicsSideError] should be treated as permanent,
* as they involve invalid signatures or an unexpected response
* format. For this reason, they get the "ebics" stage assigned
* below, that will cause the payment as permanently failed and
diff --git a/nexus/src/main/kotlin/tech/libeufin/nexus/ebics/Ebics3.kt b/nexus/src/main/kotlin/tech/libeufin/nexus/ebics/Ebics3.kt
@@ -193,7 +193,7 @@ fun createEbics3RequestForUploadTransferPhase(
/**
* Collects all the steps to prepare the submission of a pain.001
* document to the bank, and finally send it. Indirectly throws
- * [EbicsEarlyException] or [EbicsUploadException]. The first means
+ * [EbicsSideException] or [EbicsUploadException]. The first means
* that the bank sent an invalid response or signature, the second
* that a proper EBICS or business error took place. The caller must
* catch those exceptions and decide the retry policy.
diff --git a/nexus/src/main/kotlin/tech/libeufin/nexus/ebics/EbicsCommon.kt b/nexus/src/main/kotlin/tech/libeufin/nexus/ebics/EbicsCommon.kt
@@ -214,7 +214,7 @@ fun generateKeysPdf(
* @param tolerateBankReturnCode Business return code that may be accepted instead of
* EBICS_OK. Typically, EBICS_NO_DOWNLOAD_DATA_AVAILABLE is tolerated
* when asking for new incoming payments.
- * @return [EbicsResponseContent] or throws [EbicsEarlyException]
+ * @return [EbicsResponseContent] or throws [EbicsSideException]
*/
suspend fun postEbics(
client: HttpClient,
@@ -224,9 +224,9 @@ suspend fun postEbics(
isEbics3: Boolean
): EbicsResponseContent {
val respXml = client.postToBank(cfg.hostBaseUrl, xmlReq)
- ?: throw EbicsEarlyException(
+ ?: throw EbicsSideException(
"POSTing to ${cfg.hostBaseUrl} failed",
- earlyEc = EbicsEarlyErrorCode.HTTP_POST_FAILED
+ sideEc = EbicsSideError.HTTP_POST_FAILED
)
return parseAndValidateEbicsResponse(
bankKeys,
@@ -257,11 +257,12 @@ private fun areCodesOk(ebicsResponseContent: EbicsResponseContent) =
* @param isEbics3 true for EBICS 3, false otherwise.
* @param tolerateEmptyResult true if the EC EBICS_NO_DOWNLOAD_DATA_AVAILABLE
* should be tolerated as the bank-technical error, false otherwise.
- * @return the bank response as an uncompressed [ByteArray], or null if one error took place.
- * If the request tolerates an empty download content, then the empty
- * array is returned. If the request does not tolerate an empty response
- * any non-EBICS_OK error as the EBICS- or bank-technical EC constitutes
- * an error.
+ * @return the bank response as an uncompressed [ByteArray], or null if one
+ * error took place. Definition of error: any EBICS- or bank-technical
+ * EC pairs where at least one is not EBICS_OK, or if tolerateEmptyResult
+ * is true, the bank-technical EC EBICS_NO_DOWNLOAD_DATA_AVAILABLE is allowed
+ * other than EBICS_OK. If the request tolerates an empty download content,
+ * then the empty array is returned. The function may throw [EbicsAdditionalErrors].
*/
suspend fun doEbicsDownload(
client: HttpClient,
@@ -287,11 +288,11 @@ suspend fun doEbicsDownload(
return null
}
val tId = initResp.transactionID
- if (tId == null) {
- tech.libeufin.nexus.logger.error("Transaction ID not found in the init response, cannot do transfer phase, failing.")
- return null
- }
- logger.debug("EBICS download transaction got ID: $tId")
+ ?: throw EbicsSideException(
+ "EBICS download init phase did not return a transaction ID, cannot do the transfer phase.",
+ sideEc = EbicsSideError.EBICS_UPLOAD_TRANSACTION_ID_MISSING
+ )
+ logger.debug("EBICS download transaction passed the init phase, got ID: $tId")
val howManySegments = initResp.numSegments
if (howManySegments == null) {
tech.libeufin.nexus.logger.error("Init response lacks the quantity of segments, failing.")
@@ -300,13 +301,15 @@ suspend fun doEbicsDownload(
val ebicsChunks = mutableListOf<String>()
// Getting the chunk(s)
val firstDataChunk = initResp.orderDataEncChunk
- if (firstDataChunk == null) {
- tech.libeufin.nexus.logger.error("Could not get the first data chunk, although the EBICS_OK return code, failing.")
- return null
- }
+ ?: throw EbicsSideException(
+ "OrderData element not found, despite non empty payload, failing.",
+ sideEc = EbicsSideError.ORDER_DATA_ELEMENT_NOT_FOUND
+ )
val dataEncryptionInfo = initResp.dataEncryptionInfo ?: run {
- tech.libeufin.nexus.logger.error("EncryptionInfo element not found, despite non empty payload, failing.")
- return null
+ throw EbicsSideException(
+ "EncryptionInfo element not found, despite non empty payload, failing.",
+ sideEc = EbicsSideError.ENCRYPTION_INFO_ELEMENT_NOT_FOUND
+ )
}
ebicsChunks.add(firstDataChunk)
// proceed with the transfer phase.
@@ -317,9 +320,11 @@ suspend fun doEbicsDownload(
else createEbics25DownloadTransferPhase(cfg, clientKeys, x, howManySegments, tId)
val transResp = postEbics(client, cfg, bankKeys, transReq, isEbics3)
- if (!areCodesOk(transResp)) { // FIXME: consider tolerating EBICS_NO_DOWNLOAD_DATA_AVAILABLE.
- tech.libeufin.nexus.logger.error("EBICS transfer segment #$x failed.")
- return null
+ if (!areCodesOk(transResp)) {
+ throw EbicsSideException(
+ "EBICS transfer segment #$x failed.",
+ sideEc = EbicsSideError.TRANSFER_SEGMENT_FAILED
+ )
}
val chunk = transResp.orderDataEncChunk
if (chunk == null) {
@@ -334,38 +339,35 @@ suspend fun doEbicsDownload(
dataEncryptionInfo,
ebicsChunks
)
- // payload reconstructed, ack to the bank.
- val ackXml = if (isEbics3)
+ // payload reconstructed, receipt to the bank.
+ val receiptXml = if (isEbics3)
createEbics3DownloadReceiptPhase(cfg, clientKeys, tId)
else createEbics25DownloadReceiptPhase(cfg, clientKeys, tId)
- try {
- postEbics(
- client,
- cfg,
- bankKeys,
- ackXml,
- isEbics3
- )
- } catch (e: EbicsEarlyException) {
- logger.error("Download receipt phase failed: " + e.message)
- return null
- }
- // receipt phase OK, can now return the payload as an XML string.
- return try {
- payloadBytes
- } catch (e: Exception) {
- logger.error("Could not get the XML string out of payload bytes.")
- null
- }
+ // Sending the receipt to the bank.
+ postEbics(
+ client,
+ cfg,
+ bankKeys,
+ receiptXml,
+ isEbics3
+ )
+ // Receipt didn't throw, can now return the payload.
+ return payloadBytes
}
-enum class EbicsEarlyErrorCode {
+/**
+ * These errors affect an EBICS transaction regardless
+ * of the standard error codes.
+ */
+enum class EbicsSideError {
BANK_SIGNATURE_DIDNT_VERIFY,
BANK_RESPONSE_IS_INVALID,
+ ENCRYPTION_INFO_ELEMENT_NOT_FOUND,
+ ORDER_DATA_ELEMENT_NOT_FOUND,
+ TRANSFER_SEGMENT_FAILED,
/**
- * That's the bank fault, as this value should be there even
- * if there was an error.
+ * This might indicate that the EBICS transaction had errors.
*/
EBICS_UPLOAD_TRANSACTION_ID_MISSING,
/**
@@ -381,9 +383,9 @@ enum class EbicsEarlyErrorCode {
* and successfully verify its signature. They bring therefore NO
* business meaning and may be retried.
*/
-class EbicsEarlyException(
+class EbicsSideException(
msg: String,
- val earlyEc: EbicsEarlyErrorCode
+ val sideEc: EbicsSideError
) : Exception(msg)
/**
@@ -393,7 +395,7 @@ class EbicsEarlyException(
* @param bankKeys provides the bank auth pub, to verify the signature.
* @param responseStr raw XML response from the bank
* @param withEbics3 true if the communication is EBICS 3, false otherwise.
- * @return [EbicsResponseContent] or throw [EbicsEarlyException]
+ * @return [EbicsResponseContent] or throw [EbicsSideException]
*/
fun parseAndValidateEbicsResponse(
bankKeys: BankPublicKeysFile,
@@ -403,9 +405,9 @@ fun parseAndValidateEbicsResponse(
val responseDocument = try {
XMLUtil.parseStringIntoDom(responseStr)
} catch (e: Exception) {
- throw EbicsEarlyException(
+ throw EbicsSideException(
"Bank response apparently invalid",
- earlyEc = EbicsEarlyErrorCode.BANK_RESPONSE_IS_INVALID
+ sideEc = EbicsSideError.BANK_RESPONSE_IS_INVALID
)
}
if (!XMLUtil.verifyEbicsDocument(
@@ -413,9 +415,9 @@ fun parseAndValidateEbicsResponse(
bankKeys.bank_authentication_public_key,
withEbics3
)) {
- throw EbicsEarlyException(
+ throw EbicsSideException(
"Bank signature did not verify",
- earlyEc = EbicsEarlyErrorCode.BANK_SIGNATURE_DIDNT_VERIFY
+ sideEc = EbicsSideError.BANK_SIGNATURE_DIDNT_VERIFY
)
}
if (withEbics3)
@@ -555,9 +557,9 @@ suspend fun doEbicsUpload(
)
// Init phase OK, proceeding with the transfer phase.
val tId = initResp.transactionID
- ?: throw EbicsEarlyException(
+ ?: throw EbicsSideException(
"EBICS upload init phase did not return a transaction ID, cannot do the transfer phase.",
- earlyEc = EbicsEarlyErrorCode.EBICS_UPLOAD_TRANSACTION_ID_MISSING
+ sideEc = EbicsSideError.EBICS_UPLOAD_TRANSACTION_ID_MISSING
)
val transferXml = createEbics3RequestForUploadTransferPhase(
cfg,
diff --git a/nexus/src/test/kotlin/PostFinance.kt b/nexus/src/test/kotlin/PostFinance.kt
@@ -1,5 +1,6 @@
import io.ktor.client.*
import kotlinx.coroutines.runBlocking
+import org.junit.Ignore
import org.junit.Test
import tech.libeufin.nexus.*
import tech.libeufin.nexus.ebics.*
@@ -21,6 +22,7 @@ private fun prep(): EbicsSetupConfig {
return EbicsSetupConfig(handle)
}
+@Ignore
class Iso20022 {
private val yesterday: Instant = Instant.now().minus(1, ChronoUnit.DAYS)
@@ -40,7 +42,7 @@ class Iso20022 {
*/
@Test
fun getStatement() {
- val inflatedBytes = download(prepStatementRequest(yesterday))
+ val inflatedBytes = download(prepStatementRequest())
inflatedBytes?.unzipForEach { name, content ->
println(name)
println(content)
@@ -153,6 +155,7 @@ class Iso20022 {
}
}
+@Ignore
class PostFinance {
// Tests sending client keys to the PostFinance test platform.
@Test
