commit 812122ba1de2b7eb4bb52c11df05db3c9bf01675
parent cdf522a9c77db2547cab2fc26919f48a1bb606e7
Author: MS <ms@taler.net>
Date: Tue, 13 Dec 2022 22:58:32 +0100
Disk-space policy: fix database query.
Revert use of index-based search, because the
deletion of bank messages is now optional. That
avoids to process stored messages more than once.
Diffstat:
4 files changed, 35 insertions(+), 14 deletions(-)
diff --git a/nexus/src/main/kotlin/tech/libeufin/nexus/DB.kt b/nexus/src/main/kotlin/tech/libeufin/nexus/DB.kt
@@ -285,6 +285,8 @@ object NexusBankAccountsTable : LongIdTable() {
val lastStatementCreationTimestamp = long("lastStatementCreationTimestamp").nullable()
val lastReportCreationTimestamp = long("lastReportCreationTimestamp").nullable()
val lastNotificationCreationTimestamp = long("lastNotificationCreationTimestamp").nullable()
+ // Highest bank message ID that this bank account is aware of.
+ val highestSeenBankMessageSerialId = long("highestSeenBankMessageSerialId")
val pain001Counter = long("pain001counter").default(1)
}
@@ -294,11 +296,13 @@ class NexusBankAccountEntity(id: EntityID<Long>) : LongEntity(id) {
return find { NexusBankAccountsTable.bankAccountName eq name }.firstOrNull()
}
}
+
var bankAccountName by NexusBankAccountsTable.bankAccountName
var accountHolder by NexusBankAccountsTable.accountHolder
var iban by NexusBankAccountsTable.iban
var bankCode by NexusBankAccountsTable.bankCode
var defaultBankConnection by NexusBankConnectionEntity optionalReferencedOn NexusBankAccountsTable.defaultBankConnection
+ var highestSeenBankMessageSerialId by NexusBankAccountsTable.highestSeenBankMessageSerialId
var pain001Counter by NexusBankAccountsTable.pain001Counter
var lastStatementCreationTimestamp by NexusBankAccountsTable.lastStatementCreationTimestamp
var lastReportCreationTimestamp by NexusBankAccountsTable.lastReportCreationTimestamp
diff --git a/nexus/src/main/kotlin/tech/libeufin/nexus/bankaccount/BankAccount.kt b/nexus/src/main/kotlin/tech/libeufin/nexus/bankaccount/BankAccount.kt
@@ -39,14 +39,14 @@ import java.time.Instant
import java.time.ZonedDateTime
import java.time.format.DateTimeFormatter
-// Defaults to false. Gets true if defined as "yes".
private val keepBankMessages: String? = System.getenv("LIBEUFIN_NEXUS_KEEP_BANK_MESSAGES")
-
fun requireBankAccount(call: ApplicationCall, parameterKey: String): NexusBankAccountEntity {
val name = call.parameters[parameterKey]
- if (name == null) {
- throw NexusError(HttpStatusCode.InternalServerError, "no parameter for bank account")
- }
+ if (name == null)
+ throw NexusError(
+ HttpStatusCode.InternalServerError,
+ "no parameter for bank account"
+ )
val account = transaction { NexusBankAccountEntity.findByName(name) }
if (account == null) {
throw NexusError(HttpStatusCode.NotFound, "bank connection '$name' not found")
@@ -291,7 +291,10 @@ fun processCamtMessage(
* Create new transactions for an account based on bank messages it
* did not see before.
*/
-fun ingestBankMessagesIntoAccount(bankConnectionId: String, bankAccountId: String): CamtTransactionsCount {
+fun ingestBankMessagesIntoAccount(
+ bankConnectionId: String,
+ bankAccountId: String
+): CamtTransactionsCount {
var totalNew = 0
var downloadedTransactions = 0
transaction {
@@ -304,25 +307,35 @@ fun ingestBankMessagesIntoAccount(bankConnectionId: String, bankAccountId: Strin
if (acct == null) {
throw NexusError(HttpStatusCode.InternalServerError, "account not found")
}
+ var lastId = acct.highestSeenBankMessageSerialId
NexusBankMessageEntity.find {
- (NexusBankMessagesTable.bankConnection eq conn.id) and not(
- NexusBankMessagesTable.errors
- )
+ (NexusBankMessagesTable.bankConnection eq conn.id) and
+ (NexusBankMessagesTable.id greater acct.highestSeenBankMessageSerialId) and
+ // Wrong messages got already skipped by the
+ // index check above. Below is a extra check.
+ not(NexusBankMessagesTable.errors)
}.orderBy(Pair(NexusBankMessagesTable.id, SortOrder.ASC)).forEach {
val doc = XMLUtil.parseStringIntoDom(it.message.bytes.toString(Charsets.UTF_8))
val processingResult = processCamtMessage(bankAccountId, doc, it.code)
if (processingResult.newTransactions == -1) {
it.errors = true
+ lastId = it.id.value
return@forEach
}
- /**
- * The XML document from the bank parsed correctly,
- * remove now from the database.
- */
- if (keepBankMessages == null || keepBankMessages != "yes") it.delete()
totalNew += processingResult.newTransactions
downloadedTransactions += processingResult.downloadedTransactions
+ /**
+ * Disk-space conservative check: only store if "yes" was
+ * explicitly set into the environment variable. Any other
+ * value or non given falls back to deletion.
+ */
+ if (keepBankMessages == null || keepBankMessages != "yes") {
+ it.delete()
+ return@forEach
+ }
+ lastId = it.id.value
}
+ acct.highestSeenBankMessageSerialId = lastId
}
// return totalNew
return CamtTransactionsCount(
@@ -449,6 +462,7 @@ fun importBankAccount(call: ApplicationCall, offeredBankAccountId: String, nexus
iban = offeredAccount[OfferedBankAccountsTable.iban]
bankCode = offeredAccount[OfferedBankAccountsTable.bankCode]
defaultBankConnection = conn
+ highestSeenBankMessageSerialId = 0
accountHolder = offeredAccount[OfferedBankAccountsTable.accountHolder]
}
logger.info("Account ${newImportedAccount.id} gets imported")
diff --git a/nexus/src/main/kotlin/tech/libeufin/nexus/ebics/EbicsNexus.kt b/nexus/src/main/kotlin/tech/libeufin/nexus/ebics/EbicsNexus.kt
@@ -305,6 +305,7 @@ fun Route.ebicsBankConnectionRoutes(client: HttpClient) {
reason = "bank gave no BIC"
)
defaultBankConnection = conn
+ highestSeenBankMessageSerialId = 0
}
}
}
diff --git a/nexus/src/test/kotlin/MakeEnv.kt b/nexus/src/test/kotlin/MakeEnv.kt
@@ -89,6 +89,7 @@ fun prepNexusDb() {
iban = FOO_USER_IBAN
bankCode = "SANDBOXX"
defaultBankConnection = c
+ highestSeenBankMessageSerialId = 0
accountHolder = "foo"
}
val b = NexusBankAccountEntity.new {
@@ -96,6 +97,7 @@ fun prepNexusDb() {
iban = BAR_USER_IBAN
bankCode = "SANDBOXX"
defaultBankConnection = c
+ highestSeenBankMessageSerialId = 0
accountHolder = "bar"
}
}
