From a047fb5c1076c0afe2c04c8beb2edd6026f595b3 Mon Sep 17 00:00:00 2001 From: Antoine A <> Date: Wed, 20 Mar 2024 18:03:20 +0100 Subject: Add GC command --- bank/src/main/kotlin/tech/libeufin/bank/Config.kt | 11 ++++++-- bank/src/main/kotlin/tech/libeufin/bank/Main.kt | 21 +++++++++++++++- .../src/main/kotlin/tech/libeufin/bank/db/GcDAO.kt | 18 ++++++-------- bank/src/test/kotlin/GcTest.kt | 12 ++++----- common/src/main/kotlin/TalerConfig.kt | 29 ++++++++++++++++++++++ contrib/bank.conf | 11 +++++++- testbench/src/test/kotlin/IntegrationTest.kt | 2 ++ 7 files changed, 82 insertions(+), 22 deletions(-) diff --git a/bank/src/main/kotlin/tech/libeufin/bank/Config.kt b/bank/src/main/kotlin/tech/libeufin/bank/Config.kt index 9c252b8f..54f2c44b 100644 --- a/bank/src/main/kotlin/tech/libeufin/bank/Config.kt +++ b/bank/src/main/kotlin/tech/libeufin/bank/Config.kt @@ -24,6 +24,7 @@ import org.slf4j.Logger import org.slf4j.LoggerFactory import tech.libeufin.common.* import java.nio.file.Path +import java.time.Duration private val logger: Logger = LoggerFactory.getLogger("libeufin-bank") @@ -47,7 +48,10 @@ data class BankConfig( val spaPath: Path?, val tanChannels: Map>>, val payto: BankPaytoCtx, - val wireMethod: WireMethod + val wireMethod: WireMethod, + val gcAbortAfter: Duration, + val gcCleanAfter: Duration, + val gcDeleteAfter: Duration ) @Serializable @@ -141,7 +145,10 @@ fun TalerConfig.loadBankConfig(): BankConfig { fiatCurrencySpec = fiatCurrencySpec, tanChannels = tanChannels, payto = payto, - wireMethod = method + wireMethod = method, + gcAbortAfter = requireDuration("libeufin-bank", "gc_abort_after"), + gcCleanAfter = requireDuration("libeufin-bank", "gc_clean_after"), + gcDeleteAfter = requireDuration("libeufin-bank", "gc_delete_after"), ) } diff --git a/bank/src/main/kotlin/tech/libeufin/bank/Main.kt b/bank/src/main/kotlin/tech/libeufin/bank/Main.kt index ecdc5810..0663e49e 100644 --- a/bank/src/main/kotlin/tech/libeufin/bank/Main.kt +++ b/bank/src/main/kotlin/tech/libeufin/bank/Main.kt @@ -58,6 +58,7 @@ import java.net.InetAddress import java.sql.SQLException import java.util.zip.DataFormatException import java.util.zip.Inflater +import java.time.Instant import kotlin.io.path.Path import kotlin.io.path.exists import kotlin.io.path.readText @@ -505,10 +506,28 @@ class CreateAccount : CliktCommand( } } +class GC : CliktCommand( + "Run garbage collection: abort expired operations and clean expired data", + name = "gc" +) { + private val common by CommonOption() + + override fun run() = cliCmd(logger, common.log) { + val cfg = talerConfig(common.config) + val ctx = cfg.loadBankConfig() + val dbCfg = cfg.loadDbConfig() + + Database(dbCfg.dbConnStr, ctx.regionalCurrency, ctx.fiatCurrency).use { db -> + logger.info("Run garbage collection") + db.gc.collect(Instant.now(), ctx.gcAbortAfter, ctx.gcCleanAfter, ctx.gcDeleteAfter) + } + } +} + class LibeufinBankCommand : CliktCommand() { init { versionOption(getVersion()) - subcommands(ServeBank(), BankDbInit(), CreateAccount(), EditAccount(), ChangePw(), CliConfigCmd(BANK_CONFIG_SOURCE)) + subcommands(ServeBank(), BankDbInit(), CreateAccount(), EditAccount(), ChangePw(), GC(), CliConfigCmd(BANK_CONFIG_SOURCE)) } override fun run() = Unit diff --git a/bank/src/main/kotlin/tech/libeufin/bank/db/GcDAO.kt b/bank/src/main/kotlin/tech/libeufin/bank/db/GcDAO.kt index 9f5e9431..7e1c7a08 100644 --- a/bank/src/main/kotlin/tech/libeufin/bank/db/GcDAO.kt +++ b/bank/src/main/kotlin/tech/libeufin/bank/db/GcDAO.kt @@ -23,24 +23,20 @@ import tech.libeufin.bank.* import tech.libeufin.common.* import tech.libeufin.common.crypto.* import java.time.Instant -import java.time.ZoneOffset -import java.time.LocalDateTime -import java.time.temporal.TemporalAmount -import java.time.chrono.ChronoLocalDateTime +import java.time.Duration /** Data access logic for garbage collection */ class GcDAO(private val db: Database) { /** Run garbage collection */ suspend fun collect( now: Instant, - abortAfter: TemporalAmount, - cleanAfter: TemporalAmount, - deleteAfter: TemporalAmount + abortAfter: Duration, + cleanAfter: Duration, + deleteAfter: Duration ) = db.conn { conn -> - val dateTime = LocalDateTime.ofInstant(now, ZoneOffset.UTC) - val abortAfterMicro = dateTime.minus(abortAfter).toInstant(ZoneOffset.UTC).micros() - val cleanAfterMicro = dateTime.minus(cleanAfter).toInstant(ZoneOffset.UTC).micros() - val deleteAfterMicro = dateTime.minus(deleteAfter).toInstant(ZoneOffset.UTC).micros() + val abortAfterMicro = now.minus(abortAfter).micros() + val cleanAfterMicro = now.minus(cleanAfter).micros() + val deleteAfterMicro = now.minus(deleteAfter).micros() // Abort pending operations conn.prepareStatement( diff --git a/bank/src/test/kotlin/GcTest.kt b/bank/src/test/kotlin/GcTest.kt index 550178a4..92c4bd41 100644 --- a/bank/src/test/kotlin/GcTest.kt +++ b/bank/src/test/kotlin/GcTest.kt @@ -51,14 +51,12 @@ class GcTest { // Time calculation val abortAfter = Duration.ofMinutes(15) - val cleanAfter = Period.ofDays(14) - val deleteAfter = Period.ofYears(10) + val cleanAfter = Duration.ofDays(14) + val deleteAfter = Duration.ofDays(350) val now = Instant.now() - val dateTime = LocalDateTime.ofInstant(now, ZoneOffset.UTC) - val abort = dateTime.minus(abortAfter).toInstant(ZoneOffset.UTC) - val clean = dateTime.minus(cleanAfter).toInstant(ZoneOffset.UTC) - val delete = dateTime.minus(deleteAfter).toInstant(ZoneOffset.UTC) - + val abort = now.minus(abortAfter) + val clean = now.minus(cleanAfter) + val delete = now.minus(deleteAfter) // Create test accounts val payto = IbanPayto.rand() diff --git a/common/src/main/kotlin/TalerConfig.kt b/common/src/main/kotlin/TalerConfig.kt index ed11ee94..edcc2faa 100644 --- a/common/src/main/kotlin/TalerConfig.kt +++ b/common/src/main/kotlin/TalerConfig.kt @@ -24,6 +24,8 @@ import org.slf4j.LoggerFactory import java.nio.file.AccessDeniedException import java.nio.file.NoSuchFileException import java.nio.file.Path +import java.time.temporal.ChronoUnit +import java.time.Duration import kotlin.io.path.* private val logger: Logger = LoggerFactory.getLogger("libeufin-config") @@ -465,4 +467,31 @@ class TalerConfig internal constructor( fun requirePath(section: String, option: String): Path = lookupPath(section, option) ?: throw TalerConfigError.missing("path", section, option) + + fun lookupDuration(section: String, option: String): Duration? { + val entry = lookupString(section, option) ?: return null + return TIME_AMOUNT_PATTERN.findAll(entry).map { match -> + val (rawAmount, unit) = match.destructured + val amount = rawAmount.toLongOrNull() ?: throw TalerConfigError.invalid("temporal", section, option, "'$rawAmount' not a valid temporal amount") + val value = when (unit) { + "us" -> 1 + "ms" -> 1000 + "s", "second", "seconds", "\"" -> 1000 * 1000L + "m", "min", "minute", "minutes", "'" -> 60 * 1000 * 1000L + "h", "hour", "hours" -> 60 * 60 * 1000 * 1000L + "d", "day", "days" -> 24 * 60 * 60 * 1000L * 1000L + "week", "weeks" -> 7 * 24 * 60 * 60 * 1000L * 1000L + "year", "years", "a" -> 31536000000000L + else -> throw TalerConfigError.invalid("temporal", section, option, "'$unit' not a valid temporal unit") + } + Duration.of(amount * value, ChronoUnit.MICROS) + }.fold(Duration.ZERO) { a, b -> a.plus(b) } + } + + fun requireDuration(section: String, option: String): Duration = + lookupDuration(section, option) ?: throw TalerConfigError.missing("temporal", section, option) + + companion object { + private val TIME_AMOUNT_PATTERN = Regex("([0-9]+) ?([a-z'\"]+)") + } } diff --git a/contrib/bank.conf b/contrib/bank.conf index 29239e1e..372143e6 100644 --- a/contrib/bank.conf +++ b/contrib/bank.conf @@ -70,7 +70,16 @@ BIND_TO = 0.0.0.0 SPA = $DATADIR/spa/ # Exchange that is suggested to wallets when withdrawing. -#SUGGESTED_WITHDRAWAL_EXCHANGE = https://exchange.demo.taler.net/ +# SUGGESTED_WITHDRAWAL_EXCHANGE = https://exchange.demo.taler.net/ + +# Time after which pending operations are aborted +GC_ABORT_AFTER = 15m + +# Time after which aborted operations and expired items are deleted +GC_CLEAN_AFTER = 14d + +# Time after which all bank transactions, operations and deleted accounts are deleted +GC_DELETE_AFTER = 10year [libeufin-bankdb-postgres] # Where are the SQL files to setup our tables? diff --git a/testbench/src/test/kotlin/IntegrationTest.kt b/testbench/src/test/kotlin/IntegrationTest.kt index b6ede6c0..441ec143 100644 --- a/testbench/src/test/kotlin/IntegrationTest.kt +++ b/testbench/src/test/kotlin/IntegrationTest.kt @@ -108,6 +108,8 @@ class IntegrationTest { // Check bank is running client.get("http://0.0.0.0:8080/public-accounts").assertNoContent() } + + bankCmd.run("gc $flags") } @Test -- cgit v1.2.3