libeufin

commit cdf522a9c77db2547cab2fc26919f48a1bb606e7
parent b8041b128e787dfcdfb4b8edeadb342aedf5c515
Author: MS <ms@taler.net>
Date:   Tue, 13 Dec 2022 20:12:53 +0100

Disk space policy at Nexus.

Make the conservative storing of bank messages
optional, defaulting to non storing them.

Diffstat:
M
nexus/src/main/kotlin/tech/libeufin/nexus/DB.kt
 | 
5
-----
M
nexus/src/main/kotlin/tech/libeufin/nexus/bankaccount/BankAccount.kt
 | 
17
+++++++++++------
M
nexus/src/main/kotlin/tech/libeufin/nexus/ebics/EbicsNexus.kt
 | 
1
-
M
nexus/src/test/kotlin/MakeEnv.kt
 | 
2
--

4 files changed, 11 insertions(+), 14 deletions(-)
diff --git a/nexus/src/main/kotlin/tech/libeufin/nexus/DB.kt b/nexus/src/main/kotlin/tech/libeufin/nexus/DB.kt
@@ -285,9 +285,6 @@ object NexusBankAccountsTable : LongIdTable() {
     val lastStatementCreationTimestamp = long("lastStatementCreationTimestamp").nullable()
     val lastReportCreationTimestamp = long("lastReportCreationTimestamp").nullable()
     val lastNotificationCreationTimestamp = long("lastNotificationCreationTimestamp").nullable()
-
-    // Highest bank message ID that this bank account is aware of.
-    val highestSeenBankMessageSerialId = long("highestSeenBankMessageSerialId")
     val pain001Counter = long("pain001counter").default(1)
 }
 
@@ -297,13 +294,11 @@ class NexusBankAccountEntity(id: EntityID<Long>) : LongEntity(id) {
             return find { NexusBankAccountsTable.bankAccountName eq name }.firstOrNull()
         }
     }
-
     var bankAccountName by NexusBankAccountsTable.bankAccountName
     var accountHolder by NexusBankAccountsTable.accountHolder
     var iban by NexusBankAccountsTable.iban
     var bankCode by NexusBankAccountsTable.bankCode
     var defaultBankConnection by NexusBankConnectionEntity optionalReferencedOn NexusBankAccountsTable.defaultBankConnection
-    var highestSeenBankMessageSerialId by NexusBankAccountsTable.highestSeenBankMessageSerialId
     var pain001Counter by NexusBankAccountsTable.pain001Counter
     var lastStatementCreationTimestamp by NexusBankAccountsTable.lastStatementCreationTimestamp
     var lastReportCreationTimestamp by NexusBankAccountsTable.lastReportCreationTimestamp
diff --git a/nexus/src/main/kotlin/tech/libeufin/nexus/bankaccount/BankAccount.kt b/nexus/src/main/kotlin/tech/libeufin/nexus/bankaccount/BankAccount.kt
@@ -39,6 +39,9 @@ import java.time.Instant
 import java.time.ZonedDateTime
 import java.time.format.DateTimeFormatter
 
+// Defaults to false.  Gets true if defined as "yes".
+private val keepBankMessages: String? = System.getenv("LIBEUFIN_NEXUS_KEEP_BANK_MESSAGES")
+
 fun requireBankAccount(call: ApplicationCall, parameterKey: String): NexusBankAccountEntity {
     val name = call.parameters[parameterKey]
     if (name == null) {
@@ -301,10 +304,10 @@ fun ingestBankMessagesIntoAccount(bankConnectionId: String, bankAccountId: Strin
         if (acct == null) {
             throw NexusError(HttpStatusCode.InternalServerError, "account not found")
         }
-        var lastId = acct.highestSeenBankMessageSerialId
         NexusBankMessageEntity.find {
-            (NexusBankMessagesTable.bankConnection eq conn.id) and
-                    (NexusBankMessagesTable.id greater acct.highestSeenBankMessageSerialId)
+            (NexusBankMessagesTable.bankConnection eq conn.id) and not(
+                NexusBankMessagesTable.errors
+            )
         }.orderBy(Pair(NexusBankMessagesTable.id, SortOrder.ASC)).forEach {
             val doc = XMLUtil.parseStringIntoDom(it.message.bytes.toString(Charsets.UTF_8))
             val processingResult = processCamtMessage(bankAccountId, doc, it.code)
@@ -312,11 +315,14 @@ fun ingestBankMessagesIntoAccount(bankConnectionId: String, bankAccountId: Strin
                 it.errors = true
                 return@forEach
             }
-            lastId = it.id.value
+            /**
+             * The XML document from the bank parsed correctly,
+             * remove now from the database.
+             */
+            if (keepBankMessages == null || keepBankMessages != "yes") it.delete()
             totalNew += processingResult.newTransactions
             downloadedTransactions += processingResult.downloadedTransactions
         }
-        acct.highestSeenBankMessageSerialId = lastId
     }
     // return totalNew
     return CamtTransactionsCount(
@@ -443,7 +449,6 @@ fun importBankAccount(call: ApplicationCall, offeredBankAccountId: String, nexus
                         iban = offeredAccount[OfferedBankAccountsTable.iban]
                         bankCode = offeredAccount[OfferedBankAccountsTable.bankCode]
                         defaultBankConnection = conn
-                        highestSeenBankMessageSerialId = 0
                         accountHolder = offeredAccount[OfferedBankAccountsTable.accountHolder]
                     }
                     logger.info("Account ${newImportedAccount.id} gets imported")
diff --git a/nexus/src/main/kotlin/tech/libeufin/nexus/ebics/EbicsNexus.kt b/nexus/src/main/kotlin/tech/libeufin/nexus/ebics/EbicsNexus.kt
@@ -305,7 +305,6 @@ fun Route.ebicsBankConnectionRoutes(client: HttpClient) {
                                     reason = "bank gave no BIC"
                                 )
                             defaultBankConnection = conn
-                            highestSeenBankMessageSerialId = 0
                         }
                     }
                 }
diff --git a/nexus/src/test/kotlin/MakeEnv.kt b/nexus/src/test/kotlin/MakeEnv.kt
@@ -89,7 +89,6 @@ fun prepNexusDb() {
             iban = FOO_USER_IBAN
             bankCode = "SANDBOXX"
             defaultBankConnection = c
-            highestSeenBankMessageSerialId = 0
             accountHolder = "foo"
         }
         val b = NexusBankAccountEntity.new {
@@ -97,7 +96,6 @@ fun prepNexusDb() {
             iban = BAR_USER_IBAN
             bankCode = "SANDBOXX"
             defaultBankConnection = c
-            highestSeenBankMessageSerialId = 0
             accountHolder = "bar"
         }
     }