commit 11bf2426f46eaf4d970656b40815337827f14277 parent 5b19ae252484010728591ffe504b1da8301b9403 Author: Antoine A <> Date: Tue, 30 Jul 2024 23:51:45 +0200 Merge branch 'master' into dev/antoinea/kyc Diffstat:
42 files changed, 606 insertions(+), 349 deletions(-)
diff --git a/RELEASE.md b/RELEASE.md @@ -0,0 +1,20 @@ +# Release Process + +## Checklist + +- [ ] bump version in build.gradle +- [ ] add entry to debian/changelog +- [ ] check CI (contrib/ci, buildbot.taler.net) +- [ ] tag with dev tag, test in staging environment +- [ ] tag with release tag +- [ ] upload to GNU mirrors +- [ ] upload Debian packages to deb.taler.net + +## Versioning + +Releases use `$major.$minor.$patch` semantic versions. The corresponding git +tag is `v$major.minor.$patch`. + +Versions that are tested in staging environments typically use +`v$major.$minor.$patch-dev.$n` tags. + diff --git a/bank/conf/test.conf b/bank/conf/test.conf @@ -12,6 +12,7 @@ allow_conversion = YES FIAT_CURRENCY = EUR tan_sms = libeufin-tan-file.sh tan_email = libeufin-tan-file.sh +PWD_HASH_CONFIG = { "cost": 4 } [libeufin-bankdb-postgres] CONFIG = postgresql:///libeufincheck diff --git a/bank/conf/test_bonus.conf b/bank/conf/test_bonus.conf @@ -5,6 +5,7 @@ IBAN_PAYTO_BIC = SANDBOXX REGISTRATION_BONUS = KUDOS:100 ALLOW_REGISTRATION = yes ALLOW_ACCOUNT_DELETION = yes +PWD_HASH_CONFIG = { "cost": 4 } [libeufin-bankdb-postgres] CONFIG = postgresql:///libeufincheck \ No newline at end of file diff --git a/bank/conf/test_no_conversion.conf b/bank/conf/test_no_conversion.conf @@ -4,6 +4,7 @@ WIRE_TYPE = iban IBAN_PAYTO_BIC = SANDBOXX ALLOW_REGISTRATION = yes ALLOW_ACCOUNT_DELETION = yes +PWD_HASH_CONFIG = { "cost": 4 } [libeufin-bankdb-postgres] CONFIG = postgresql:///libeufincheck \ No newline at end of file diff --git a/bank/conf/test_restrict.conf b/bank/conf/test_restrict.conf @@ -5,6 +5,7 @@ IBAN_PAYTO_BIC = SANDBOXX DEFAULT_DEBT_LIMIT = KUDOS:100 allow_conversion = YES FIAT_CURRENCY = EUR +PWD_HASH_CONFIG = { "cost": 4 } [libeufin-bankdb-postgres] CONFIG = postgresql:///libeufincheck \ No newline at end of file diff --git a/bank/conf/test_tan_err.conf b/bank/conf/test_tan_err.conf @@ -10,6 +10,7 @@ ALLOW_REGISTRATION = yes ALLOW_ACCOUNT_DELETION = yes ALLOW_EDIT_CASHOUT_PAYTO_URI = yes tan_sms = libeufin-tan-fail.sh +PWD_HASH_CONFIG = { "cost": 4 } [libeufin-bankdb-postgres] CONFIG = postgresql:///libeufincheck diff --git a/bank/conf/test_with_fees.conf b/bank/conf/test_with_fees.conf @@ -15,6 +15,7 @@ tan_email = libeufin-tan-file.sh wire_transfer_fees = KUDOS:0.1 min_wire_transfer_amount = KUDOS:0.01 max_wire_transfer_amount = KUDOS:100 +PWD_HASH_CONFIG = { "cost": 4 } [libeufin-bankdb-postgres] CONFIG = postgresql:///libeufincheck diff --git a/bank/conf/test_x_taler_bank.conf b/bank/conf/test_x_taler_bank.conf @@ -8,6 +8,7 @@ ALLOW_REGISTRATION = yes ALLOW_ACCOUNT_DELETION = yes ALLOW_EDIT_NAME = yes ALLOW_EDIT_CASHOUT_PAYTO_URI = yes +PWD_HASH_CONFIG = { "cost": 4 } [libeufin-bankdb-postgres] CONFIG = postgresql:///libeufincheck diff --git a/bank/src/main/kotlin/tech/libeufin/bank/Config.kt b/bank/src/main/kotlin/tech/libeufin/bank/Config.kt @@ -21,6 +21,7 @@ package tech.libeufin.bank import kotlinx.serialization.Serializable import tech.libeufin.bank.db.Database import tech.libeufin.common.* +import tech.libeufin.common.crypto.PwCrypto import tech.libeufin.common.db.DatabaseConfig import java.nio.file.Path import java.time.Duration @@ -49,6 +50,7 @@ data class BankConfig( val tanChannels: Map<TanChannel, Pair<Path, Map<String, String>>>, val payto: BankPaytoCtx, val wireMethod: WireMethod, + val pwCrypto: PwCrypto, val gcAbortAfter: Duration, val gcCleanAfter: Duration, val gcDeleteAfter: Duration @@ -119,6 +121,10 @@ private fun TalerConfig.loadBankConfig(): BankConfig = section("libeufin-bank"). ) } + val pwCrypto = map("pwd_hash_algorithm", "password hash algorithm", mapOf( + "bcrypt" to json<PwCrypto.Bcrypt>("pwd_hash_config", "bcrypt JSON config").require() + )).require() + val ZERO = TalerAmount.zero(regionalCurrency) val MAX = TalerAmount.max(regionalCurrency) BankConfig( @@ -144,13 +150,14 @@ private fun TalerConfig.loadBankConfig(): BankConfig = section("libeufin-bank"). tanChannels = tanChannels, payto = payto, wireMethod = method, + pwCrypto = pwCrypto, gcAbortAfter = duration("gc_abort_after").require(), gcCleanAfter = duration("gc_clean_after").require(), gcDeleteAfter = duration("gc_delete_after").require(), ) } -fun TalerConfig.currencySpecificationFor(currency: String): CurrencySpecification +private fun TalerConfig.currencySpecificationFor(currency: String): CurrencySpecification = sections.find { val section = section(it) it.startsWith("CURRENCY-") && section.boolean("enabled").require() && section.string("code").require() == currency diff --git a/bank/src/main/kotlin/tech/libeufin/bank/api/ConversionApi.kt b/bank/src/main/kotlin/tech/libeufin/bank/api/ConversionApi.kt @@ -101,7 +101,7 @@ fun Routing.conversionApi(db: Database, ctx: BankConfig) = conditional(ctx.allow } } } - authAdmin(db, TokenScope.readwrite) { + authAdmin(db, ctx.pwCrypto, TokenScope.readwrite) { post("/conversion-info/conversion-rate") { val req = call.receive<ConversionRate>() for (regionalAmount in sequenceOf(req.cashin_fee, req.cashin_tiny_amount, req.cashout_min_amount)) { diff --git a/bank/src/main/kotlin/tech/libeufin/bank/api/CoreBankApi.kt b/bank/src/main/kotlin/tech/libeufin/bank/api/CoreBankApi.kt @@ -47,45 +47,45 @@ import java.util.* private val logger: Logger = LoggerFactory.getLogger("libeufin-bank-api") -fun Routing.coreBankApi(db: Database, ctx: BankConfig) { +fun Routing.coreBankApi(db: Database, cfg: BankConfig) { get("/config") { call.respond( Config( - bank_name = ctx.name, - base_url = ctx.baseUrl, - currency = ctx.regionalCurrency, - currency_specification = ctx.regionalCurrencySpec, - allow_conversion = ctx.allowConversion, - allow_registrations = ctx.allowRegistration, - allow_deletions = ctx.allowAccountDeletion, - default_debit_threshold = ctx.defaultDebtLimit, - supported_tan_channels = ctx.tanChannels.keys, - allow_edit_name = ctx.allowEditName, - allow_edit_cashout_payto_uri = ctx.allowEditCashout, - wire_type = ctx.wireMethod, - wire_transfer_fees = ctx.wireTransferFees, - min_wire_transfer_amount = ctx.minAmount, - max_wire_transfer_amount = ctx.maxAmount + bank_name = cfg.name, + base_url = cfg.baseUrl, + currency = cfg.regionalCurrency, + currency_specification = cfg.regionalCurrencySpec, + allow_conversion = cfg.allowConversion, + allow_registrations = cfg.allowRegistration, + allow_deletions = cfg.allowAccountDeletion, + default_debit_threshold = cfg.defaultDebtLimit, + supported_tan_channels = cfg.tanChannels.keys, + allow_edit_name = cfg.allowEditName, + allow_edit_cashout_payto_uri = cfg.allowEditCashout, + wire_type = cfg.wireMethod, + wire_transfer_fees = cfg.wireTransferFees, + min_wire_transfer_amount = cfg.minAmount, + max_wire_transfer_amount = cfg.maxAmount ) ) } - authAdmin(db, TokenScope.readonly) { + authAdmin(db, cfg.pwCrypto, TokenScope.readonly) { get("/monitor") { val params = MonitorParams.extract(call.request.queryParameters) call.respond(db.monitor(params)) } } - coreBankTokenApi(db) - coreBankAccountsApi(db, ctx) - coreBankTransactionsApi(db, ctx) - coreBankWithdrawalApi(db, ctx) - coreBankCashoutApi(db, ctx) - coreBankTanApi(db, ctx) + coreBankTokenApi(db, cfg) + coreBankAccountsApi(db, cfg) + coreBankTransactionsApi(db, cfg) + coreBankWithdrawalApi(db, cfg) + coreBankCashoutApi(db, cfg) + coreBankTanApi(db, cfg) } -private fun Routing.coreBankTokenApi(db: Database) { +private fun Routing.coreBankTokenApi(db: Database, cfg: BankConfig) { val TOKEN_DEFAULT_DURATION: Duration = Duration.ofDays(1L) - auth(db, TokenScope.refreshable) { + auth(db, cfg.pwCrypto, TokenScope.refreshable) { post("/accounts/{USERNAME}/token") { val existingToken = call.authToken val req = call.receive<TokenRequest>() @@ -111,7 +111,7 @@ private fun Routing.coreBankTokenApi(db: Database) { Instant.MAX } else { try { - logger.debug("Creating token with days duration: ${tokenDuration.toDays()}") + logger.debug { "Creating token with days duration: ${tokenDuration.toDays()}" } creationTime.plus(tokenDuration) } catch (e: Exception) { throw badRequest("Bad token duration: ${e.message}") @@ -136,7 +136,7 @@ private fun Routing.coreBankTokenApi(db: Database) { ) } } - auth(db, TokenScope.readonly) { + auth(db, cfg.pwCrypto, TokenScope.readonly) { delete("/accounts/{USERNAME}/token") { val token = call.authToken ?: throw badRequest("Basic auth not supported here.") db.token.delete(token) @@ -223,7 +223,8 @@ suspend fun createAccount( tanChannel = req.tan_channel, checkPaytoIdempotent = req.payto_uri != null, ctx = cfg.payto, - minCashout = req.min_cashout + minCashout = req.min_cashout, + pwCrypto = cfg.pwCrypto ) when (cfg.wireMethod) { @@ -297,7 +298,7 @@ suspend fun patchAccount( } private fun Routing.coreBankAccountsApi(db: Database, ctx: BankConfig) { - authAdmin(db, TokenScope.readwrite, !ctx.allowRegistration) { + authAdmin(db, ctx.pwCrypto, TokenScope.readwrite, !ctx.allowRegistration) { post("/accounts") { val req = call.receive<RegisterAccountRequest>() when (val result = createAccount(db, ctx, req, isAdmin)) { @@ -319,6 +320,7 @@ private fun Routing.coreBankAccountsApi(db: Database, ctx: BankConfig) { } auth( db, + ctx.pwCrypto, TokenScope.readwrite, allowAdmin = true, requireAdmin = !ctx.allowAccountDeletion @@ -349,7 +351,7 @@ private fun Routing.coreBankAccountsApi(db: Database, ctx: BankConfig) { } } } - auth(db, TokenScope.readwrite, allowAdmin = true) { + auth(db, ctx.pwCrypto, TokenScope.readwrite, allowAdmin = true) { patch("/accounts/{USERNAME}") { val (req, challenge) = call.receiveChallenge<AccountReconfiguration>(db, Operation.account_reconfig) val res = patchAccount(db, ctx, req, username, isAdmin, challenge != null, challenge?.channel, challenge?.info) @@ -390,7 +392,7 @@ private fun Routing.coreBankAccountsApi(db: Database, ctx: BankConfig) { TalerErrorCode.BANK_NON_ADMIN_PATCH_MISSING_OLD_PASSWORD ) } - when (db.account.reconfigPassword(username, req.new_password, req.old_password, isAdmin || challenge != null)) { + when (db.account.reconfigPassword(username, req.new_password, req.old_password, isAdmin || challenge != null, ctx.pwCrypto)) { AccountPatchAuthResult.Success -> call.respond(HttpStatusCode.NoContent) AccountPatchAuthResult.TanRequired -> call.respondChallenge(db, Operation.account_auth_reconfig, req) AccountPatchAuthResult.UnknownAccount -> throw unknownAccount(username) @@ -410,7 +412,7 @@ private fun Routing.coreBankAccountsApi(db: Database, ctx: BankConfig) { call.respond(PublicAccountsResponse(publicAccounts)) } } - authAdmin(db, TokenScope.readonly) { + authAdmin(db, ctx.pwCrypto, TokenScope.readonly) { get("/accounts") { val params = AccountParams.extract(call.request.queryParameters) val accounts = db.account.pageAdmin(params, ctx.payto) @@ -421,7 +423,7 @@ private fun Routing.coreBankAccountsApi(db: Database, ctx: BankConfig) { } } } - auth(db, TokenScope.readonly, allowAdmin = true) { + auth(db, ctx.pwCrypto, TokenScope.readonly, allowAdmin = true) { get("/accounts/{USERNAME}") { val account = db.account.get(username, ctx.payto) ?: throw unknownAccount(username) call.respond(account) @@ -430,7 +432,7 @@ private fun Routing.coreBankAccountsApi(db: Database, ctx: BankConfig) { } private fun Routing.coreBankTransactionsApi(db: Database, ctx: BankConfig) { - auth(db, TokenScope.readonly, allowAdmin = true) { + auth(db, ctx.pwCrypto, TokenScope.readonly, allowAdmin = true) { get("/accounts/{USERNAME}/transactions") { val params = HistoryParams.extract(call.request.queryParameters) val bankAccount = call.bankInfo(db, ctx.payto) @@ -452,7 +454,7 @@ private fun Routing.coreBankTransactionsApi(db: Database, ctx: BankConfig) { call.respond(tx) } } - auth(db, TokenScope.readwrite) { + auth(db, ctx.pwCrypto, TokenScope.readwrite) { post("/accounts/{USERNAME}/transactions") { val (req, challenge) = call.receiveChallenge<TransactionCreateRequest>(db, Operation.bank_transaction) @@ -506,7 +508,7 @@ private fun Routing.coreBankTransactionsApi(db: Database, ctx: BankConfig) { } private fun Routing.coreBankWithdrawalApi(db: Database, ctx: BankConfig) { - auth(db, TokenScope.readwrite) { + auth(db, ctx.pwCrypto, TokenScope.readwrite) { post("/accounts/{USERNAME}/withdrawals") { val req = call.receive<BankAccountCreateWithdrawalRequest>() req.amount?.run(ctx::checkRegionalCurrency) @@ -614,7 +616,7 @@ private fun Routing.coreBankWithdrawalApi(db: Database, ctx: BankConfig) { } private fun Routing.coreBankCashoutApi(db: Database, ctx: BankConfig) = conditional(ctx.allowConversion) { - auth(db, TokenScope.readwrite) { + auth(db, ctx.pwCrypto, TokenScope.readwrite) { post("/accounts/{USERNAME}/cashouts") { val (req, challenge) = call.receiveChallenge<CashoutRequest>(db, Operation.cashout) @@ -663,7 +665,7 @@ private fun Routing.coreBankCashoutApi(db: Database, ctx: BankConfig) = conditio } } } - auth(db, TokenScope.readonly, allowAdmin = true) { + auth(db, ctx.pwCrypto, TokenScope.readonly, allowAdmin = true) { get("/accounts/{USERNAME}/cashouts/{CASHOUT_ID}") { val id = call.longPath("CASHOUT_ID") val cashout = db.cashout.get(id, username) ?: throw notFound( @@ -682,7 +684,7 @@ private fun Routing.coreBankCashoutApi(db: Database, ctx: BankConfig) = conditio } } } - authAdmin(db, TokenScope.readonly) { + authAdmin(db, ctx.pwCrypto, TokenScope.readonly) { get("/cashouts") { val params = PageParams.extract(call.request.queryParameters) val cashouts = db.cashout.pageAll(params) @@ -696,7 +698,7 @@ private fun Routing.coreBankCashoutApi(db: Database, ctx: BankConfig) = conditio } private fun Routing.coreBankTanApi(db: Database, ctx: BankConfig) { - auth(db, TokenScope.readwrite) { + auth(db, ctx.pwCrypto, TokenScope.readwrite) { post("/accounts/{USERNAME}/challenge/{CHALLENGE_ID}") { val id = call.longPath("CHALLENGE_ID") val res = db.tan.send( diff --git a/bank/src/main/kotlin/tech/libeufin/bank/api/RevenueApi.kt b/bank/src/main/kotlin/tech/libeufin/bank/api/RevenueApi.kt @@ -32,7 +32,7 @@ import tech.libeufin.common.RevenueConfig import tech.libeufin.common.RevenueIncomingHistory fun Routing.revenueApi(db: Database, ctx: BankConfig) { - auth(db, TokenScope.revenue) { + auth(db, ctx.pwCrypto, TokenScope.revenue) { get("/accounts/{USERNAME}/taler-revenue/config") { call.respond(RevenueConfig( currency = ctx.regionalCurrency diff --git a/bank/src/main/kotlin/tech/libeufin/bank/api/WireGatewayApi.kt b/bank/src/main/kotlin/tech/libeufin/bank/api/WireGatewayApi.kt @@ -40,7 +40,7 @@ import java.time.Instant fun Routing.wireGatewayApi(db: Database, ctx: BankConfig) { - auth(db, TokenScope.readwrite) { + auth(db, ctx.pwCrypto, TokenScope.readwrite) { get("/accounts/{USERNAME}/taler-wire-gateway/config") { call.respond(WireGatewayConfig( currency = ctx.regionalCurrency @@ -82,7 +82,7 @@ fun Routing.wireGatewayApi(db: Database, ctx: BankConfig) { } } } - auth(db, TokenScope.readonly) { + auth(db, ctx.pwCrypto, TokenScope.readonly) { suspend fun <T> PipelineContext<Unit, ApplicationCall>.historyEndpoint( reduce: (List<T>, String) -> Any, dbLambda: suspend ExchangeDAO.(HistoryParams, Long, BankPaytoCtx) -> List<T> @@ -111,7 +111,7 @@ fun Routing.wireGatewayApi(db: Database, ctx: BankConfig) { historyEndpoint(::OutgoingHistory, ExchangeDAO::outgoingHistory) } } - authAdmin(db, TokenScope.readwrite) { + authAdmin(db, ctx.pwCrypto, TokenScope.readwrite) { suspend fun ApplicationCall.addIncoming( amount: TalerAmount, debitAccount: Payto, diff --git a/bank/src/main/kotlin/tech/libeufin/bank/auth/auth.kt b/bank/src/main/kotlin/tech/libeufin/bank/auth/auth.kt @@ -30,6 +30,7 @@ import tech.libeufin.bank.TokenScope import tech.libeufin.bank.db.AccountDAO.CheckPasswordResult import tech.libeufin.bank.db.Database import tech.libeufin.common.* +import tech.libeufin.common.crypto.PwCrypto import tech.libeufin.common.api.intercept import java.time.Instant @@ -61,17 +62,17 @@ val ApplicationCall.authToken: ByteArray? get() = attributes.getOrNull(AUTH_TOKE * * You can check is the currently authenticated user is admin using [isAdmin]. **/ -fun Route.authAdmin(db: Database, scope: TokenScope, enforce: Boolean = true, callback: Route.() -> Unit): Route = +fun Route.authAdmin(db: Database, pwCrypto: PwCrypto, scope: TokenScope, enforce: Boolean = true, callback: Route.() -> Unit): Route = intercept(callback) { if (enforce) { - val login = context.authenticateBankRequest(db, scope) + val login = context.authenticateBankRequest(db, pwCrypto, scope) if (login != "admin") { throw unauthorized("Only administrator allowed") } context.attributes.put(AUTH_IS_ADMIN, true) } else { val login = try { - context.authenticateBankRequest(db, scope) + context.authenticateBankRequest(db, pwCrypto, scope) } catch (e: Exception) { null } @@ -88,9 +89,9 @@ fun Route.authAdmin(db: Database, scope: TokenScope, enforce: Boolean = true, ca * * You can check is the currently authenticated user is admin using [isAdmin]. **/ -fun Route.auth(db: Database, scope: TokenScope, allowAdmin: Boolean = false, requireAdmin: Boolean = false, callback: Route.() -> Unit): Route = +fun Route.auth(db: Database, pwCrypto: PwCrypto ,scope: TokenScope, allowAdmin: Boolean = false, requireAdmin: Boolean = false, callback: Route.() -> Unit): Route = intercept(callback) { - val authLogin = context.authenticateBankRequest(db, scope) + val authLogin = context.authenticateBankRequest(db, pwCrypto, scope) if (requireAdmin && authLogin != "admin") { throw unauthorized("Only administrator allowed") } else { @@ -109,7 +110,7 @@ fun Route.auth(db: Database, scope: TokenScope, allowAdmin: Boolean = false, req * * Returns the authenticated customer login. */ -private suspend fun ApplicationCall.authenticateBankRequest(db: Database, requiredScope: TokenScope): String { +private suspend fun ApplicationCall.authenticateBankRequest(db: Database, pwCrypto: PwCrypto, requiredScope: TokenScope): String { val header = request.headers[HttpHeaders.Authorization] // Basic auth challenge @@ -127,7 +128,7 @@ private suspend fun ApplicationCall.authenticateBankRequest(db: Database, requir TalerErrorCode.GENERIC_HTTP_HEADERS_MALFORMED ) return when (scheme) { - "Basic" -> doBasicAuth(db, content) + "Basic" -> doBasicAuth(db, content, pwCrypto) "Bearer" -> doTokenAuth(db, content, requiredScope) else -> throw unauthorized("Authorization method wrong or not supported") } @@ -138,13 +139,13 @@ private suspend fun ApplicationCall.authenticateBankRequest(db: Database, requir * * Returns the authenticated customer login */ -private suspend fun doBasicAuth(db: Database, encoded: String): String { +private suspend fun doBasicAuth(db: Database, encoded: String, pwCrypto: PwCrypto): String { val decoded = String(encoded.decodeBase64(), Charsets.UTF_8) val (login, plainPassword) = decoded.splitOnce(":") ?: throw badRequest( "Malformed Basic auth credentials found in the Authorization header", TalerErrorCode.GENERIC_HTTP_HEADERS_MALFORMED ) - return when (db.account.checkPassword(login, plainPassword)) { + return when (db.account.checkPassword(login, plainPassword, pwCrypto)) { CheckPasswordResult.UnknownAccount -> throw unauthorized("Unknown account") CheckPasswordResult.PasswordMismatch -> throw unauthorized("Bad password") CheckPasswordResult.Success -> login diff --git a/bank/src/main/kotlin/tech/libeufin/bank/cli/BenchPwh.kt b/bank/src/main/kotlin/tech/libeufin/bank/cli/BenchPwh.kt @@ -0,0 +1,64 @@ +/* + * This file is part of LibEuFin. + * Copyright (C) 2024 Taler Systems S.A. + + * LibEuFin is free software; you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation; either version 3, or + * (at your option) any later version. + + * LibEuFin is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General + * Public License for more details. + + * You should have received a copy of the GNU Affero General Public + * License along with LibEuFin; see the file COPYING. If not, see + * <http://www.gnu.org/licenses/> + */ + +package tech.libeufin.bank.cli + +import com.github.ajalt.clikt.core.CliktCommand +import com.github.ajalt.clikt.parameters.arguments.argument +import com.github.ajalt.clikt.parameters.groups.provideDelegate +import tech.libeufin.bank.bankConfig +import tech.libeufin.bank.db.AccountDAO.AccountPatchAuthResult +import tech.libeufin.bank.logger +import tech.libeufin.bank.withDb +import tech.libeufin.common.CommonOption +import tech.libeufin.common.cliCmd +import tech.libeufin.common.crypto.PwCrypto + +class BenchPwh : CliktCommand("Benchmark password hashin algorithm and configuration", name = "bench-pwh") { + private val common by CommonOption() + + override fun run() = cliCmd(logger, common.log) { + val pwCrypto = bankConfig(common.config).pwCrypto + + when (pwCrypto) { + is PwCrypto.Bcrypt -> println("Benching bcrypt with cost=${pwCrypto.cost} for 10s") + } + + val start = System.currentTimeMillis() + val stop = start + 10000 // 10s + var count = 0 + + while (true) { + val now = System.currentTimeMillis() + if (now < stop) { + val password = (0..10).map { + "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!@#".random() + }.joinToString("") + pwCrypto.hashpw(password) + count ++ + } else { + val elapsed = (now-start).toDouble() + val perSec = count.toDouble() / (elapsed / 1000.0) + val iterTime = elapsed / count.toDouble() + println("hash password in ${String.format("%.0f", iterTime)}ms ${String.format("%.2f", perSec)} H/s") + break + } + } + } +} +\ No newline at end of file diff --git a/bank/src/main/kotlin/tech/libeufin/bank/cli/ChangePw.kt b/bank/src/main/kotlin/tech/libeufin/bank/cli/ChangePw.kt @@ -38,8 +38,8 @@ class ChangePw : CliktCommand("Change account password", name = "passwd") { ) override fun run() = cliCmd(logger, common.log) { - bankConfig(common.config).withDb { db, _ -> - val res = db.account.reconfigPassword(username, password, null, true) + bankConfig(common.config).withDb { db, cfg -> + val res = db.account.reconfigPassword(username, password, null, true, cfg.pwCrypto) when (res) { AccountPatchAuthResult.UnknownAccount -> throw Exception("Password change for '$username' account failed: unknown account") diff --git a/bank/src/main/kotlin/tech/libeufin/bank/cli/LibeufinBank.kt b/bank/src/main/kotlin/tech/libeufin/bank/cli/LibeufinBank.kt @@ -29,7 +29,7 @@ import tech.libeufin.common.getVersion class LibeufinBank : CliktCommand() { init { versionOption(getVersion()) - subcommands(Serve(), DbInit(), CreateAccount(), EditAccount(), ChangePw(), GC(), CliConfigCmd(BANK_CONFIG_SOURCE)) + subcommands(Serve(), DbInit(), CreateAccount(), EditAccount(), ChangePw(), BenchPwh(), GC(), CliConfigCmd(BANK_CONFIG_SOURCE)) } override fun run() = Unit diff --git a/bank/src/main/kotlin/tech/libeufin/bank/db/AccountDAO.kt b/bank/src/main/kotlin/tech/libeufin/bank/db/AccountDAO.kt @@ -52,7 +52,8 @@ class AccountDAO(private val db: Database) { tanChannel: TanChannel?, // Whether to check [internalPaytoUri] for idempotency checkPaytoIdempotent: Boolean, - ctx: BankPaytoCtx + ctx: BankPaytoCtx, + pwCrypto: PwCrypto ): AccountCreationResult = db.serializableTransaction { conn -> val timestamp = Instant.now().micros() val idempotent = conn.withStatement(""" @@ -83,7 +84,7 @@ class AccountDAO(private val db: Database) { setString(10, login) oneOrNull { Pair( - PwCrypto.checkpw(password, it.getString(1)).match && it.getBoolean(2), + pwCrypto.checkpw(password, it.getString(1)).match && it.getBoolean(2), it.getBankPayto("internal_payto_uri", "name", ctx) ) } @@ -125,7 +126,7 @@ class AccountDAO(private val db: Database) { """ ) { setString(1, login) - setString(2, PwCrypto.hashpw(password)) + setString(2, pwCrypto.hashpw(password)) setString(3, name) setString(4, email) setString(5, phone) @@ -411,7 +412,8 @@ class AccountDAO(private val db: Database) { login: String, newPw: String, oldPw: String?, - is2fa: Boolean + is2fa: Boolean, + pwCrypto: PwCrypto ): AccountPatchAuthResult = db.serializableTransaction { conn -> val (customerId, currentPwh, tanRequired) = conn.withStatement(""" SELECT customer_id, password_hash, (NOT ? AND tan_channel IS NOT NULL) @@ -425,10 +427,10 @@ class AccountDAO(private val db: Database) { } if (tanRequired) { AccountPatchAuthResult.TanRequired - } else if (oldPw != null && !PwCrypto.checkpw(oldPw, currentPwh).match) { + } else if (oldPw != null && !pwCrypto.checkpw(oldPw, currentPwh).match) { AccountPatchAuthResult.OldPasswordMismatch } else { - val newPwh = PwCrypto.hashpw(newPw) + val newPwh = pwCrypto.hashpw(newPw) conn.withStatement("UPDATE customers SET password_hash=? WHERE customer_id=?") { setString(1, newPwh) setLong(2, customerId) @@ -447,7 +449,7 @@ class AccountDAO(private val db: Database) { } /** Check password of account [login] against [pw], rehashing it if outdated */ - suspend fun checkPassword(login: String, pw: String): CheckPasswordResult { + suspend fun checkPassword(login: String, pw: String, pwCrypto: PwCrypto): CheckPasswordResult { // Get user current password hash val info = db.serializable( "SELECT customer_id, password_hash FROM customers WHERE login=? AND deleted_at IS NULL" @@ -461,12 +463,12 @@ class AccountDAO(private val db: Database) { val (customerId, currentPwh) = info // Check password - val check = PwCrypto.checkpw(pw, currentPwh) + val check = pwCrypto.checkpw(pw, currentPwh) if (!check.match) return CheckPasswordResult.PasswordMismatch // Reshah if outdated if (check.outdated) { - val newPwh = PwCrypto.hashpw(pw) + val newPwh = pwCrypto.hashpw(pw) db.serializable( "UPDATE customers SET password_hash=? where customer_id=? AND password_hash=?" ) { diff --git a/bank/src/main/kotlin/tech/libeufin/bank/helpers.kt b/bank/src/main/kotlin/tech/libeufin/bank/helpers.kt @@ -128,7 +128,8 @@ suspend fun createAdminAccount(db: Database, cfg: BankConfig, pw: String? = null cashoutPayto = null, tanChannel = null, minCashout = null, - ctx = cfg.payto + ctx = cfg.payto, + pwCrypto = cfg.pwCrypto ) } diff --git a/bank/src/test/kotlin/bench.kt b/bank/src/test/kotlin/bench.kt @@ -50,7 +50,8 @@ class Bench { // In general half of the data is for generated account and half is for customer val mid = amount / 2 - val password = PwCrypto.hashpw("password") + val password = PwCrypto.Bcrypt(cost = 4).hashpw("password") + val token32 = ByteArray(32) val token64 = ByteArray(64) diff --git a/bank/src/test/kotlin/helpers.kt b/bank/src/test/kotlin/helpers.kt @@ -95,7 +95,8 @@ fun bankSetup( cashoutPayto = null, tanChannel = null, minCashout = null, - ctx = cfg.payto + ctx = cfg.payto, + pwCrypto = cfg.pwCrypto )) assertIs<AccountCreationResult.Success>(db.account.create( login = "exchange", @@ -112,7 +113,8 @@ fun bankSetup( cashoutPayto = null, tanChannel = null, minCashout = null, - ctx = cfg.payto + ctx = cfg.payto, + pwCrypto = cfg.pwCrypto )) assertIs<AccountCreationResult.Success>(db.account.create( login = "customer", @@ -129,7 +131,8 @@ fun bankSetup( cashoutPayto = null, tanChannel = null, minCashout = null, - ctx = cfg.payto + ctx = cfg.payto, + pwCrypto = cfg.pwCrypto )) // Create admin account assertIs<AccountCreationResult.Success>(createAdminAccount(db, cfg, "admin-password")) diff --git a/build.gradle b/build.gradle @@ -9,7 +9,7 @@ plugins { } group = "tech.libeufin" -version = "0.11.3" +version = "0.12.0" if (!JavaVersion.current().isCompatibleWith(JavaVersion.VERSION_17)){ throw new GradleException( diff --git a/common/src/main/kotlin/Cli.kt b/common/src/main/kotlin/Cli.kt @@ -39,21 +39,6 @@ import org.slf4j.event.Level private val logger: Logger = LoggerFactory.getLogger("libeufin-config") -fun Throwable.fmt(): String = buildString { - append(message ?: this@fmt::class.simpleName) - var cause = cause - while (cause != null) { - append(": ") - append(cause.message ?: cause::class.simpleName) - cause = cause.cause - } -} - -fun Throwable.fmtLog(logger: Logger) { - logger.error(this.fmt()) - logger.trace("", this) -} - fun cliCmd(logger: Logger, level: Level, lambda: suspend () -> Unit) { // Set root log level val root = LoggerFactory.getLogger(Logger.ROOT_LOGGER_NAME) as ch.qos.logback.classic.Logger diff --git a/common/src/main/kotlin/TalerConfig.kt b/common/src/main/kotlin/TalerConfig.kt @@ -51,7 +51,7 @@ class TalerConfigError private constructor (m: String, cause: Throwable? = null) } /** Configuration error when converting option value */ -private class ValueError(val msg: String): Exception(msg) +class ValueError(val msg: String): Exception(msg) /** Information about how the configuration is loaded */ data class ConfigSource( @@ -447,7 +447,7 @@ class TalerConfigSection internal constructor( val section: String ) { /** Setup an accessor/converted for a [type] at [option] using [transform] */ - private fun <T> option(option: String, type: String, transform: TalerConfigSection.(String) -> T): TalerConfigOption<T> { + fun <T> option(option: String, type: String, transform: TalerConfigSection.(String) -> T): TalerConfigOption<T> { val canonOption = option.uppercase() var raw = entries?.get(canonOption) if (raw == "") raw = null @@ -511,15 +511,18 @@ class TalerConfigSection internal constructor( } } - /** Access [option] as Map<String, String> */ - fun jsonMap(option: String) = option(option, "json key/value map") { + /** Access [option] as JSON object [T] */ + inline fun <reified T> json(option: String, type: String) = option(option, type) { try { - Json.decodeFromString<Map<String, String>>(it) + Json.decodeFromString<T>(it) } catch (e: Exception) { throw ValueError("'$it' is malformed") } } + /** Access [option] as Map<String, String> */ + fun jsonMap(option: String) = json<Map<String, String>>(option, "json key/value map") + /** Access [option] as TalerAmount */ fun amount(option: String, currency: String) = option(option, "amount") { val amount = try { diff --git a/common/src/main/kotlin/crypto/CryptoUtil.kt b/common/src/main/kotlin/crypto/CryptoUtil.kt @@ -26,6 +26,7 @@ import org.bouncycastle.asn1.x509.KeyUsage import org.bouncycastle.cert.jcajce.JcaX509CertificateConverter import org.bouncycastle.cert.jcajce.JcaX509v3CertificateBuilder import org.bouncycastle.crypto.generators.Argon2BytesGenerator +import org.bouncycastle.crypto.generators.BCrypt import org.bouncycastle.crypto.params.Argon2Parameters import org.bouncycastle.jce.provider.BouncyCastleProvider import org.bouncycastle.operator.jcajce.JcaContentSignerBuilder @@ -261,6 +262,11 @@ object CryptoUtil { fun hashStringSHA256(input: String): ByteArray = MessageDigest.getInstance("SHA-256").digest(input.toByteArray(Charsets.UTF_8)) + + fun bcrypt(password: String, salt: ByteArray, cost: Int): ByteArray { + val pwBytes = BCrypt.passwordToByteArray(password.toCharArray()) + return BCrypt.generate(pwBytes, salt, cost) + } fun hashArgon2id(password: String, salt: ByteArray): ByteArray { // OSWAP recommended config https://cheatsheetseries.owasp.org/cheatsheets/Password_Storage_Cheat_Sheet.html#argon2id diff --git a/common/src/main/kotlin/crypto/PwCrypto.kt b/common/src/main/kotlin/crypto/PwCrypto.kt @@ -19,6 +19,8 @@ package tech.libeufin.common.crypto +import kotlinx.serialization.Serializable +import tech.libeufin.common.decodeBase64 import tech.libeufin.common.encodeBase64 import tech.libeufin.common.rand import java.security.SecureRandom @@ -29,24 +31,32 @@ data class PasswordHashCheck( ) /** Cryptographic operations for secure password storage and verification */ -object PwCrypto { - private val SECURE_RNG = SecureRandom() +sealed interface PwCrypto { + @Serializable + data class Bcrypt(val cost: Int = 8): PwCrypto - /** Hash [pw] using the strongest supported hashing method */ + /** Hash [pw] using [cfg] hashing method */ fun hashpw(pw: String): String { - val salt = ByteArray(16).rand(SECURE_RNG) - /* TODO Argon2id - val pwh = CryptoUtil.hashArgon2id(pw, salt) - return "argon2id\$${salt.encodeBase64()}\$${pwh.encodeBase64()}" - */ - val saltEncoded = salt.encodeBase64() - val pwh = CryptoUtil.hashStringSHA256("$saltEncoded|$pw").encodeBase64() - return "sha256-salted\$$saltEncoded\$$pwh" + when (this) { + is Bcrypt -> { + val salt = ByteArray(16).rand(SECURE_RNG) + val pwh = CryptoUtil.bcrypt(pw, salt, cost) + return "bcrypt\$$cost\$${salt.encodeBase64()}\$${pwh.encodeBase64()}" + } + /* TODO Argon2id + "argon2id" -> { + require(components.size == 3) { "bad password hash format" } + val salt = components[1].decodeBase64() + val hash = components[2] + val pwh = CryptoUtil.hashArgon2id(pw, salt).encodeBase64() + PasswordHashCheck(pwh == hash, false) + } */ + } } /** Check whether [pw] match hashed [storedPwHash] and if it should be rehashed */ fun checkpw(pw: String, storedPwHash: String): PasswordHashCheck { - val components = storedPwHash.split('$', limit = 4) + val components = storedPwHash.split('$', limit = 5) return when (val algo = components[0]) { "sha256" -> { require(components.size == 2) { "bad password hash format" } @@ -59,17 +69,20 @@ object PwCrypto { val salt = components[1] val hash = components[2] val pwh = CryptoUtil.hashStringSHA256("$salt|$pw").encodeBase64() - PasswordHashCheck(pwh == hash, false) + PasswordHashCheck(pwh == hash, true) + } + "bcrypt" -> { + require(components.size == 4) { "bad password hash format" } + val cost = components[1].toInt() + val salt = components[2].decodeBase64() + val hash = components[3] + val pwh = CryptoUtil.bcrypt(pw, salt, cost).encodeBase64() + PasswordHashCheck(pwh == hash, !(this is Bcrypt && this.cost == cost)) } - /* TODO Argon2id - "argon2id" -> { - require(components.size == 3) { "bad password hash format" } - val salt = components[1].decodeBase64() - val hash = components[2] - val pwh = CryptoUtil.hashArgon2id(pw, salt).encodeBase64() - PasswordHashCheck(pwh == hash, false) - } */ else -> throw Exception("unsupported hash algo: '$algo'") } } -} + companion object { + private val SECURE_RNG = SecureRandom() + } +} +\ No newline at end of file diff --git a/common/src/main/kotlin/helpers.kt b/common/src/main/kotlin/helpers.kt @@ -33,6 +33,7 @@ import java.util.zip.DeflaterInputStream import java.util.zip.InflaterInputStream import java.util.zip.ZipInputStream import kotlin.random.Random +import org.slf4j.Logger /* ----- String ----- */ @@ -119,4 +120,27 @@ fun InputStream.deflate(): DeflaterInputStream /** Inflate an input stream */ fun InputStream.inflate(): InflaterInputStream - = InflaterInputStream(this) -\ No newline at end of file + = InflaterInputStream(this) + +/* ----- Throwable ----- */ + +fun Throwable.fmt(): String = buildString { + append(message ?: this@fmt::class.simpleName) + var cause = cause + while (cause != null) { + append(": ") + append(cause.message ?: cause::class.simpleName) + cause = cause.cause + } +} + +fun Throwable.fmtLog(logger: Logger) { + logger.error(this.fmt()) + logger.trace("", this) +} + +/* ----- Logger ----- */ + +inline fun Logger.debug(lambda: () -> String) { + if (isDebugEnabled()) debug(lambda()) +} +\ No newline at end of file diff --git a/common/src/test/kotlin/CryptoUtilTest.kt b/common/src/test/kotlin/CryptoUtilTest.kt @@ -110,16 +110,21 @@ class CryptoUtilTest { @Test fun passwordHashing() { val password = "myinsecurepw" - + val pwCrypto = PwCrypto.Bcrypt(cost = 4) // Check roundtrip - val hash = PwCrypto.hashpw(password) - assertEquals(PwCrypto.checkpw(password, hash), PasswordHashCheck(true, false)) - assertEquals(PwCrypto.checkpw("other", hash), PasswordHashCheck(false, false)) + val hash = pwCrypto.hashpw(password) + assertEquals(pwCrypto.checkpw(password, hash), PasswordHashCheck(true, false)) + assertEquals(pwCrypto.checkpw("other", hash), PasswordHashCheck(false, false)) - // Check outdated + // Check outdated algorithm val pwh = CryptoUtil.hashStringSHA256(password).encodeBase64() val outdatedHash = "sha256\$$pwh" - assertEquals(PwCrypto.checkpw(password, outdatedHash), PasswordHashCheck(true, true)) - assertEquals(PwCrypto.checkpw("other", outdatedHash), PasswordHashCheck(false, true)) + assertEquals(pwCrypto.checkpw(password, outdatedHash), PasswordHashCheck(true, true)) + assertEquals(pwCrypto.checkpw("other", outdatedHash), PasswordHashCheck(false, true)) + + // Check outdated options + val betterCrypto = pwCrypto.copy(cost = 5) + assertEquals(betterCrypto.checkpw(password, hash), PasswordHashCheck(true, true)) + assertEquals(betterCrypto.checkpw("other", hash), PasswordHashCheck(false, true)) } } diff --git a/contrib/bank.conf b/contrib/bank.conf @@ -57,11 +57,11 @@ WIRE_TYPE = # Path to TAN challenge transmission script via email. If not specified, this TAN channel will not be supported. # TAN_EMAIL = libeufin-tan-email.sh -# Environment variables for the sms TAN script. -# TAN_SMS_ENV = AUTH_TOKEN=secret-token +# Environment variables for the sms TAN script as a single-line JSON object +# TAN_SMS_ENV = { "AUTH_TOKEN": "secret-token" } -# Environment variables for the email TAN script. -# TAN_EMAIL_ENV = AUTH_TOKEN=secret-token +# Environment variables for the email TAN script as a single-line JSON object +# TAN_EMAIL_ENV = { "AUTH_TOKEN": "secret-token" } # How "libeufin-bank serve" serves its API, this can either be tcp or unix SERVE = tcp @@ -84,6 +84,13 @@ SPA = $DATADIR/spa/ # Exchange that is suggested to wallets when withdrawing. # SUGGESTED_WITHDRAWAL_EXCHANGE = https://exchange.demo.taler.net/ +# Password hash algorithm, this can only be bcrypt +PWD_HASH_ALGORITHM = bcrypt + +# Password hash algorithm configuration as a single-line JSON object +# When PWD_HASH_ALGORITHM = bcrypt you can configure cost +PWD_HASH_CONFIG = { "cost": 8 } + # Time after which pending operations are aborted during garbage collection GC_ABORT_AFTER = 15m diff --git a/debian/changelog b/debian/changelog @@ -1,3 +1,9 @@ +libeufin (0.12.0) unstable; urgency=low + + * Release version 0.12.0 + + -- Florian Dold <dold@taler.net> Wed, 24 Jul 2024 06:53:07 +0200 + libeufin (0.11.3) unstable; urgency=low * Update to latest bank SPA. diff --git a/nexus/src/main/kotlin/tech/libeufin/nexus/EbicsLogger.kt b/nexus/src/main/kotlin/tech/libeufin/nexus/EbicsLogger.kt @@ -59,25 +59,7 @@ class EbicsLogger(private val dir: Path?) { /** Create a new [order] EBICS transaction logger */ fun tx(order: EbicsOrder): TxLogger { if (dir == null) return TxLogger(null) - val name = buildString { - when (order) { - is EbicsOrder.V2_5 -> { - append(order.type) - append('-') - append(order.attribute) - } - is EbicsOrder.V3 -> { - append(order.type) - for (part in sequenceOf(order.name, order.messageName, order.option)) { - if (part != null) { - append('-') - append(part) - } - } - } - } - } - return tx(name) + return tx(order.description()) } companion object { diff --git a/nexus/src/main/kotlin/tech/libeufin/nexus/XmlCombinators.kt b/nexus/src/main/kotlin/tech/libeufin/nexus/XmlCombinators.kt @@ -151,25 +151,25 @@ class XmlDestructor internal constructor(private val el: Element) { }.toList() } - fun one(path: String): XmlDestructor { - val children = el.childrenByTag(path).iterator() + fun one(tag: String): XmlDestructor { + val children = el.childrenByTag(tag).iterator() if (!children.hasNext()) { - throw DestructionError("expected unique '${el.tagName}.$path', got none") + throw DestructionError("expected unique '${el.tagName}.$tag', got none") } val el = children.next() if (children.hasNext()) { - throw DestructionError("expected unique '${el.tagName}.$path', got ${children.asSequence().count() + 1}") + throw DestructionError("expected unique '${el.tagName}.$tag', got ${children.asSequence().count() + 1}") } return XmlDestructor(el) } - fun opt(path: String): XmlDestructor? { - val children = el.childrenByTag(path).iterator() + fun opt(tag: String): XmlDestructor? { + val children = el.childrenByTag(tag).iterator() if (!children.hasNext()) { return null } val el = children.next() if (children.hasNext()) { - throw DestructionError("expected optional '${el.tagName}.$path', got ${children.asSequence().count() + 1}") + throw DestructionError("expected optional '${el.tagName}.$tag', got ${children.asSequence().count() + 1}") } return XmlDestructor(el) } diff --git a/nexus/src/main/kotlin/tech/libeufin/nexus/cli/EbicsFetch.kt b/nexus/src/main/kotlin/tech/libeufin/nexus/cli/EbicsFetch.kt @@ -26,13 +26,12 @@ import com.github.ajalt.clikt.parameters.arguments.unique import com.github.ajalt.clikt.parameters.groups.provideDelegate import com.github.ajalt.clikt.parameters.options.option import com.github.ajalt.clikt.parameters.types.enum -import kotlinx.coroutines.delay +import kotlinx.coroutines.* import tech.libeufin.common.* import tech.libeufin.nexus.* import tech.libeufin.nexus.db.Database import tech.libeufin.nexus.db.PaymentDAO.IncomingRegistrationResult -import tech.libeufin.nexus.ebics.EbicsClient -import tech.libeufin.nexus.ebics.SupportedDocument +import tech.libeufin.nexus.ebics.* import java.io.IOException import java.io.InputStream import java.time.Duration @@ -69,7 +68,7 @@ suspend fun ingestIncomingPayment( ) { suspend fun bounce(msg: String) { if (payment.bankId == null) { - logger.debug("$payment ignored: missing bank ID") + logger.debug("{} ignored: missing bank ID", payment) return; } when (accountType) { @@ -112,17 +111,17 @@ suspend fun ingestIncomingPayment( ) } -/** Ingest an EBICS [payload] of [document] into [db] */ +/** Ingest an EBICS [payload] of [doc] into [db] */ private suspend fun ingestPayload( db: Database, cfg: NexusEbicsConfig, payload: InputStream, - document: SupportedDocument + doc: OrderDoc ) { /** Ingest a single EBICS [xml] [document] into [db] */ suspend fun ingest(xml: InputStream) { - when (document) { - SupportedDocument.CAMT_052, SupportedDocument.CAMT_053, SupportedDocument.CAMT_054 -> { + when (doc) { + OrderDoc.report, OrderDoc.statement, OrderDoc.notification -> { try { parseTx(xml, cfg.currency, cfg.dialect).forEach { if (cfg.fetch.ignoreBefore != null && it.executionTime < cfg.fetch.ignoreBefore) { @@ -142,7 +141,7 @@ private suspend fun ingestPayload( throw Exception("Ingesting notifications failed", e) } } - SupportedDocument.PAIN_002_LOGS -> { + OrderDoc.acknowledgement -> { val acks = parseCustomerAck(xml) for (ack in acks) { when (ack.actionType) { @@ -167,7 +166,7 @@ private suspend fun ingestPayload( } } } - SupportedDocument.PAIN_002 -> { + OrderDoc.status -> { val status = parseCustomerPaymentStatusReport(xml) val msg = status.msg() logger.debug("{}", status) @@ -182,11 +181,11 @@ private suspend fun ingestPayload( } // Unzip payload if necessary - when (document) { - SupportedDocument.PAIN_002, - SupportedDocument.CAMT_052, - SupportedDocument.CAMT_053, - SupportedDocument.CAMT_054 -> { + when (doc) { + OrderDoc.status, + OrderDoc.report, + OrderDoc.statement, + OrderDoc.notification -> { try { payload.unzipEach { fileName, xml -> logger.trace("parse $fileName") @@ -196,12 +195,12 @@ private suspend fun ingestPayload( throw Exception("Could not open any ZIP archive", e) } } - SupportedDocument.PAIN_002_LOGS -> ingest(payload) + OrderDoc.acknowledgement -> ingest(payload) } } /** - * Fetch and ingest banking records of type [docs] using EBICS [client] starting from [pinnedStart] + * Fetch and ingest banking records from [orders] using EBICS [client] starting from [pinnedStart] * * If [pinnedStart] is null fetch new records. * @@ -209,80 +208,46 @@ private suspend fun ingestPayload( */ private suspend fun fetchEbicsDocuments( client: EbicsClient, - docs: List<EbicsDocument>, + orders: List<EbicsOrder>, pinnedStart: Instant?, ): Boolean { val lastExecutionTime: Instant? = pinnedStart - return docs.all { doc -> - try { - if (lastExecutionTime == null) { - logger.info("Fetching new '${doc.fullDescription()}'") - } else { - logger.info("Fetching '${doc.fullDescription()}' from timestamp: $lastExecutionTime") - } - // downloading the content - val doc = doc.doc() - val order = client.cfg.dialect.downloadDoc(doc, false) - client.download( - order, - lastExecutionTime, - null - ) { payload -> - ingestPayload(client.db, client.cfg, payload, doc) - } + return orders.all { order -> + val doc = order.doc() + if (doc == null) { + logger.debug("Skip unsupported order {}", order) true - } catch (e: Exception) { - e.fmtLog(logger) - false + } else { + try { + if (lastExecutionTime == null) { + logger.info("Fetching new '${doc.fullDescription()}'") + } else { + logger.info("Fetching '${doc.fullDescription()}' from timestamp: $lastExecutionTime") + } + // downloading the content + client.download( + order, + lastExecutionTime, + null + ) { payload -> + ingestPayload(client.db, client.cfg, payload, doc) + } + true + } catch (e: Exception) { + e.fmtLog(logger) + false + } } } } -enum class EbicsDocument { - /// EBICS acknowledgement - CustomerAcknowledgement HAC pain.002 - acknowledgement, - /// Payment status - CustomerPaymentStatusReport pain.002 - status, - /// Account intraday reports - BankToCustomerAccountReport camt.052 - report, - /// Account statements - BankToCustomerStatement camt.053 - statement, - /// Debit & credit notifications - BankToCustomerDebitCreditNotification camt.054 - notification, - ; - - fun shortDescription(): String = when (this) { - acknowledgement -> "EBICS acknowledgement" - status -> "Payment status" - report -> "Account intraday reports" - statement -> "Account statements" - notification -> "Debit & credit notifications" - } - - fun fullDescription(): String = when (this) { - acknowledgement -> "EBICS acknowledgement - CustomerAcknowledgement HAC pain.002" - status -> "Payment status - CustomerPaymentStatusReport pain.002" - report -> "Account intraday reports - BankToCustomerAccountReport camt.052" - statement -> "Account statements - BankToCustomerStatement camt.053" - notification -> "Debit & credit notifications - BankToCustomerDebitCreditNotification camt.054" - } - - fun doc(): SupportedDocument = when (this) { - acknowledgement -> SupportedDocument.PAIN_002_LOGS - status -> SupportedDocument.PAIN_002 - report -> SupportedDocument.CAMT_052 - statement -> SupportedDocument.CAMT_053 - notification -> SupportedDocument.CAMT_054 - } -} - class EbicsFetch: CliktCommand("Downloads and parse EBICS files from the bank and ingest them into the database") { private val common by CommonOption() private val transient by transientOption() - private val documents: Set<EbicsDocument> by argument( + private val documents: Set<OrderDoc> by argument( help = "Which documents should be fetched? If none are specified, all supported documents will be fetched", - helpTags = EbicsDocument.entries.associate { Pair(it.name, it.shortDescription()) }, - ).enum<EbicsDocument>().multiple().unique() + helpTags = OrderDoc.entries.associate { Pair(it.name, it.shortDescription()) }, + ).enum<OrderDoc>().multiple().unique() private val pinnedStart by option( help = "Only supported in --transient mode, this option lets specify the earliest timestamp of the downloaded documents", metavar = "YYYY-MM-DD" @@ -301,26 +266,42 @@ class EbicsFetch: CliktCommand("Downloads and parse EBICS files from the bank an clientKeys, bankKeys ) - val docs = if (documents.isEmpty()) EbicsDocument.entries else documents.toList() + val docs = if (documents.isEmpty()) OrderDoc.entries else documents.toList() + val orders = docs.map { cfg.dialect.downloadDoc(it, false) } if (transient) { logger.info("Transient mode: fetching once and returning.") val pinnedStartVal = pinnedStart val pinnedStartArg = if (pinnedStartVal != null) { - logger.debug("Pinning start date to: $pinnedStartVal") + logger.debug("Pinning start date to: {}", pinnedStartVal) dateToInstant(pinnedStartVal) } else null - if (!fetchEbicsDocuments(client, docs, pinnedStartArg)) { + if (!fetchEbicsDocuments(client, orders, pinnedStartArg)) { throw Exception("Failed to fetch documents") } } else { - logger.debug("Running with a frequency of ${cfg.fetch.frequencyRaw}") - if (cfg.fetch.frequency == Duration.ZERO) { - logger.warn("Long-polling not implemented, running therefore in transient mode") + val wssNotification = listenForNotification(client) + logger.info("Running with a frequency of ${cfg.fetch.frequencyRaw}") + var nextFullRun = 0L + while (true) { + val now = System.currentTimeMillis() + if (nextFullRun < now) { + fetchEbicsDocuments(client, orders, null) + nextFullRun = now + cfg.fetch.frequency.toMillis() + } + val delay = nextFullRun - now + if (wssNotification == null) { + logger.info("Running at frequency") + delay(delay) + } else { + val notifications = withTimeoutOrNull(delay) { + wssNotification.receive() + } + if (notifications != null) { + logger.info("Running at real-time notifications reception") + fetchEbicsDocuments(client, notifications, null) + } + } } - do { - fetchEbicsDocuments(client, docs, null) - delay(cfg.fetch.frequency.toKotlinDuration()) - } while (cfg.fetch.frequency != Duration.ZERO) } } } diff --git a/nexus/src/main/kotlin/tech/libeufin/nexus/cli/EbicsSubmit.kt b/nexus/src/main/kotlin/tech/libeufin/nexus/cli/EbicsSubmit.kt @@ -105,25 +105,22 @@ class EbicsSubmit : CliktCommand("Submits pending initiated payments found in th clientKeys, bankKeys ) - val frequency: Duration = if (transient) { + + if (transient) { logger.info("Transient mode: submitting what found and returning.") - Duration.ZERO + submitBatch(client) } else { logger.debug("Running with a frequency of ${cfg.submit.frequencyRaw}") - if (cfg.submit.frequency == Duration.ZERO) { - logger.warn("Long-polling not implemented, running therefore in transient mode") + while (true) { + try { + submitBatch(client) + } catch (e: Exception) { + throw Exception("Failed to submit payments", e) + } + // TODO take submitBatch taken time in the delay + delay(cfg.submit.frequency.toKotlinDuration()) } - cfg.submit.frequency } - do { - try { - submitBatch(client) - } catch (e: Exception) { - throw Exception("Failed to submit payments", e) - } - // TODO take submitBatch taken time in the delay - delay(frequency.toKotlinDuration()) - } while (frequency != Duration.ZERO) } } } \ No newline at end of file diff --git a/nexus/src/main/kotlin/tech/libeufin/nexus/cli/Testing.kt b/nexus/src/main/kotlin/tech/libeufin/nexus/cli/Testing.kt @@ -32,10 +32,7 @@ import com.github.ajalt.clikt.parameters.types.enum import kotlinx.coroutines.delay import tech.libeufin.common.* import tech.libeufin.nexus.* -import tech.libeufin.nexus.ebics.EbicsClient -import tech.libeufin.nexus.ebics.EbicsOrder -import tech.libeufin.nexus.ebics.connect -import tech.libeufin.nexus.ebics.wssParams +import tech.libeufin.nexus.ebics.* import java.time.Instant class Wss: CliktCommand("Listen to EBICS instant notification over websocket") { @@ -43,7 +40,6 @@ class Wss: CliktCommand("Listen to EBICS instant notification over websocket") { private val ebicsLog by ebicsLogOption() override fun run() = cliCmd(logger, common.log) { - val backoff = ExpoBackoffDecorr() nexusConfig(common.config).withDb { db, nexusCgf -> val cfg = nexusCgf.ebics val (clientKeys, bankKeys) = expectFullKeys(cfg) @@ -56,17 +52,11 @@ class Wss: CliktCommand("Listen to EBICS instant notification over websocket") { clientKeys, bankKeys ) - while (true) { - try { - logger.info("Fetch WSS params") - val params = client.wssParams() - logger.debug("{}", params) - logger.info("Start listening") - params.connect(httpClient) { - backoff.reset() - } - } catch (e: Exception) { - delay(backoff.next()) + val wssNotifications = listenForNotification(client) + if (wssNotifications != null) { + while (true) { + val notifications = wssNotifications.receive() + logger.debug("{}", wssNotifications) } } } @@ -122,8 +112,7 @@ class TxCheck: CliktCommand("Check transaction semantic") { val nexusCgf = nexusConfig(common.config) val cfg = nexusCgf.ebics val (clientKeys, bankKeys) = expectFullKeys(cfg) - val doc = EbicsDocument.acknowledgement.doc() - val order = cfg.dialect.downloadDoc(doc, false) + val order = cfg.dialect.downloadDoc(OrderDoc.acknowledgement, false) val client = httpClient() val result = tech.libeufin.nexus.test.txCheck(client, cfg, clientKeys, bankKeys, order, cfg.dialect.directDebit()) println("$result") diff --git a/nexus/src/main/kotlin/tech/libeufin/nexus/ebics/EbicsBTS.kt b/nexus/src/main/kotlin/tech/libeufin/nexus/ebics/EbicsBTS.kt @@ -268,7 +268,7 @@ class EbicsBTS( private fun XmlBuilder.service(order: EbicsOrder.V3) { el("Service") { - el("ServiceName", order.name!!) + el("ServiceName", order.service!!) if (order.scope != null) { el("Scope", order.scope) } @@ -281,9 +281,9 @@ class EbicsBTS( } } el("MsgName") { - if (order.messageVersion != null) - attr("version", order.messageVersion) - text(order.messageName!!) + if (order.version != null) + attr("version", order.version) + text(order.message!!) } } } diff --git a/nexus/src/main/kotlin/tech/libeufin/nexus/ebics/EbicsCommon.kt b/nexus/src/main/kotlin/tech/libeufin/nexus/ebics/EbicsCommon.kt @@ -29,10 +29,7 @@ import kotlinx.coroutines.withContext import org.w3c.dom.Document import org.xml.sax.SAXException import tech.libeufin.common.crypto.CryptoUtil -import tech.libeufin.common.deflate -import tech.libeufin.common.encodeBase64 -import tech.libeufin.common.inflate -import tech.libeufin.common.rand +import tech.libeufin.common.* import tech.libeufin.nexus.* import tech.libeufin.nexus.db.Database import java.io.InputStream @@ -53,10 +50,14 @@ enum class SupportedDocument { /** EBICS related errors */ sealed class EbicsError(msg: String, cause: Throwable? = null): Exception(msg, cause) { - /** Http and network errors */ - class Transport(msg: String, cause: Throwable? = null): EbicsError(msg, cause) + /** Network errors */ + class Network(msg: String, cause: Throwable): EbicsError(msg, cause) + /** Http errors */ + class HTTP(msg: String, val status: HttpStatusCode): EbicsError(msg) /** EBICS protocol & XML format error */ class Protocol(msg: String, cause: Throwable? = null): EbicsError(msg, cause) + /** EBICS protocol & XML format error */ + class Code(msg: String, val technicalCode: EbicsReturnCode, val bankCode: EbicsReturnCode): EbicsError(msg) } /** POST an EBICS request [msg] to [bankUrl] returning a parsed XML response */ @@ -72,11 +73,11 @@ suspend fun HttpClient.postToBank( setBody(msg) } } catch (e: Exception) { - throw EbicsError.Transport("$phase: failed to contact bank", e) + throw EbicsError.Network("$phase: failed to contact bank", e) } if (res.status != HttpStatusCode.OK) { - throw EbicsError.Transport("$phase: bank HTTP error: ${res.status}") + throw EbicsError.HTTP("$phase: bank HTTP error: ${res.status}", res.status) } try { val bodyStream = res.bodyAsChannel().toInputStream(); @@ -85,7 +86,7 @@ suspend fun HttpClient.postToBank( } catch (e: SAXException) { throw EbicsError.Protocol("$phase: invalid XML bank response", e) } catch (e: Exception) { - throw EbicsError.Transport("$phase: failed read bank response", e) + throw EbicsError.Network("$phase: failed read bank response", e) } } @@ -108,7 +109,19 @@ suspend fun EbicsBTS.postBTS( } catch (e: Exception) { throw EbicsError.Protocol("$phase: invalid ebics response", e) } - logger.debug("{} return codes: {} & {}", phase, response.technicalCode, response.bankCode) + logger.debug { + buildString { + append(phase) + response.content.transactionID?.let { + append(" for ") + append(it) + } + append(": ") + append(response.technicalCode) + append(" & ") + append(response.bankCode) + } + } return response } @@ -135,6 +148,7 @@ class EbicsClient( endDate: Instant?, processing: suspend (InputStream) -> Unit, ) { + logger.debug { "Download order ${order.description()}" } val txLog = ebicsLogger.tx(order) val impl = EbicsBTS(cfg, bankKeys, clientKeys, order) @@ -146,46 +160,43 @@ class EbicsClient( impl.postBTS(client, xml, "Closing pending") db.ebics.remove(tId) } - + // We need to run the logic in a non-cancelable context because we need to send // a receipt for each open download transaction, otherwise we'll be stuck in an // error loop until the pending transaction timeout. val init = withContext(NonCancellable) { // Init phase val initReq = impl.downloadInitialization(startDate, endDate) - val initResp = impl.postBTS(client, initReq, "Download init phase", txLog.step("init")) + val initResp = impl.postBTS(client, initReq, "Download init", txLog.step("init")) if (initResp.bankCode == EbicsReturnCode.EBICS_NO_DOWNLOAD_DATA_AVAILABLE) { - logger.debug("Download content is empty") - return@withContext null + return@withContext null } - val initContent = initResp.okOrFail("Download init phase") + val initContent = initResp.okOrFail("Download init") val tId = requireNotNull(initContent.transactionID) { - "Download init phase: missing transaction ID" + "Download init: missing transaction ID" } db.ebics.register(tId) Pair(tId, initContent) } val (tId, initContent) = if (init == null) return else init val howManySegments = requireNotNull(initContent.numSegments) { - "Download init phase: missing num segments" + "Download init: missing num segments" } val firstSegment = requireNotNull(initContent.segment) { - "Download init phase: missing OrderData" + "Download init: missing OrderData" } val dataEncryptionInfo = requireNotNull(initContent.dataEncryptionInfo) { - "Download init phase: missing EncryptionInfo" + "Download init: missing EncryptionInfo" } - logger.debug("Download init phase for transaction '$tId'") - // Transfer phase val segments = mutableListOf(firstSegment) for (x in 2 .. howManySegments) { val transReq = impl.downloadTransfer(x, howManySegments, tId) - val transResp = impl.postBTS(client, transReq, "Download transfer phase", txLog.step("transfer$x")) - .okOrFail("Download transfer phase") + val transResp = impl.postBTS(client, transReq, "Download transfer", txLog.step("transfer$x")) + .okOrFail("Download transfer") val segment = requireNotNull(transResp.segment) { - "Download transfer phase: missing encrypted segment" + "Download transfer: missing encrypted segment" } segments.add(segment) } @@ -214,8 +225,8 @@ class EbicsClient( // First send a proper EBICS transaction receipt val xml = impl.downloadReceipt(tId, res.isSuccess) - impl.postBTS(client, xml, "Download receipt phase", txLog.step("receipt")) - .okOrFail("Download receipt phase") + impl.postBTS(client, xml, "Download receipt", txLog.step("receipt")) + .okOrFail("Download receipt") runCatching { db.ebics.remove(tId) } // Then throw business logic exception if any res.getOrThrow() @@ -232,19 +243,20 @@ class EbicsClient( order: EbicsOrder, payload: ByteArray, ): String { + logger.debug { "Upload order ${order.description()}" } val txLog = ebicsLogger.tx(order) val impl = EbicsBTS(cfg, bankKeys, clientKeys, order) val preparedPayload = prepareUploadPayload(cfg, clientKeys, bankKeys, payload) // Init phase val initXml = impl.uploadInitialization(preparedPayload) - val initResp = impl.postBTS(client, initXml, "Upload init phase", txLog.step("init")) - .okOrFail("Upload init phase") + val initResp = impl.postBTS(client, initXml, "Upload init", txLog.step("init")) + .okOrFail("Upload init") val tId = requireNotNull(initResp.transactionID) { - "Upload init phase: missing transaction ID" + "Upload init: missing transaction ID" } val orderId = requireNotNull(initResp.orderID) { - "Upload init phase: missing order ID" + "Upload init: missing order ID" } txLog.payload(payload, "xml") @@ -252,8 +264,8 @@ class EbicsClient( // Transfer phase for (i in 1..preparedPayload.segments.size) { val transferXml = impl.uploadTransfer(tId, preparedPayload, i) - val transferResp = impl.postBTS(client, transferXml, "Upload transfer phase", txLog.step("transfer$i")) - .okOrFail("Upload transfer phase") + val transferResp = impl.postBTS(client, transferXml, "Upload transfer", txLog.step("transfer$i")) + .okOrFail("Upload transfer") } return orderId } @@ -366,7 +378,7 @@ class DataEncryptionInfo( class EbicsResponse<T>( val technicalCode: EbicsReturnCode, val bankCode: EbicsReturnCode, - private val content: T + internal val content: T ) { /** Checks that return codes are both EBICS_OK */ fun ok(): T? { @@ -380,12 +392,12 @@ class EbicsResponse<T>( /** Checks that return codes are both EBICS_OK or throw an exception */ fun okOrFail(phase: String): T { - require(technicalCode.kind() != EbicsReturnCode.Kind.Error) { - "$phase has technical error: $technicalCode" - } - require(bankCode.kind() != EbicsReturnCode.Kind.Error) { - "$phase has bank error: $bankCode" + if (technicalCode.kind() == EbicsReturnCode.Kind.Error) { + throw EbicsError.Code("$phase has technical error: $technicalCode", technicalCode, bankCode) + } else if (bankCode.kind() == EbicsReturnCode.Kind.Error) { + throw EbicsError.Code("$phase has bank error: $bankCode", technicalCode, bankCode) + } else { + return content } - return content } } \ No newline at end of file diff --git a/nexus/src/main/kotlin/tech/libeufin/nexus/ebics/EbicsOrder.kt b/nexus/src/main/kotlin/tech/libeufin/nexus/ebics/EbicsOrder.kt @@ -27,10 +27,10 @@ sealed class EbicsOrder(val schema: String) { ): EbicsOrder("H004") data class V3( val type: String, - val name: String? = null, + val service: String? = null, val scope: String? = null, - val messageName: String? = null, - val messageVersion: String? = null, + val message: String? = null, + val version: String? = null, val container: String? = null, val option: String? = null ): EbicsOrder("H005") @@ -38,46 +38,122 @@ sealed class EbicsOrder(val schema: String) { companion object { val WSS_PARAMS = EbicsOrder.V3( type = "BTD", - name = "OTH", + service = "OTH", scope = "DE", - messageName = "wssparam" + message = "wssparam" ) } + + fun description(): String = buildString { + when (this@EbicsOrder) { + is EbicsOrder.V2_5 -> { + append(type) + append('-') + append(attribute) + } + is EbicsOrder.V3 -> { + append(type) + for (part in sequenceOf(service, message, option)) { + if (part != null) { + append('-') + append(part) + } + } + } + } + } + + fun doc(): OrderDoc? { + return when (this) { + is EbicsOrder.V2_5 -> { + when (this.type) { + "HAC" -> OrderDoc.acknowledgement + "Z01" -> OrderDoc.status + "Z52" -> OrderDoc.report + "Z53" -> OrderDoc.statement + "Z54" -> OrderDoc.notification + else -> null + } + } + is EbicsOrder.V3 -> { + when (this.type) { + "HAC" -> OrderDoc.acknowledgement + "BTD" -> when (this.message) { + "pain.002" -> OrderDoc.status + "camt.052" -> OrderDoc.report + "camt.053" -> OrderDoc.statement + "camt.054" -> OrderDoc.notification + else -> null + } + else -> null + } + } + } + } +} + +enum class OrderDoc { + /// EBICS acknowledgement - CustomerAcknowledgement HAC pain.002 + acknowledgement, + /// Payment status - CustomerPaymentStatusReport pain.002 + status, + /// Account intraday reports - BankToCustomerAccountReport camt.052 + report, + /// Account statements - BankToCustomerStatement camt.053 + statement, + /// Debit & credit notifications - BankToCustomerDebitCreditNotification camt.054 + notification; + + fun shortDescription(): String = when (this) { + acknowledgement -> "EBICS acknowledgement" + status -> "Payment status" + report -> "Account intraday reports" + statement -> "Account statements" + notification -> "Debit & credit notifications" + } + + fun fullDescription(): String = when (this) { + acknowledgement -> "EBICS acknowledgement - CustomerAcknowledgement HAC pain.002" + status -> "Payment status - CustomerPaymentStatusReport pain.002" + report -> "Account intraday reports - BankToCustomerAccountReport camt.052" + statement -> "Account statements - BankToCustomerStatement camt.053" + notification -> "Debit & credit notifications - BankToCustomerDebitCreditNotification camt.054" + } } enum class Dialect { postfinance, gls; - fun downloadDoc(doc: SupportedDocument, ebics2: Boolean): EbicsOrder { + fun downloadDoc(doc: OrderDoc, ebics2: Boolean): EbicsOrder { return when (this) { postfinance -> { // TODO test platform need EBICS2 for HAC, should we use a separate dialect ? if (ebics2) { when (doc) { - SupportedDocument.PAIN_002 -> EbicsOrder.V2_5("Z01", "DZHNN") - SupportedDocument.CAMT_052 -> EbicsOrder.V2_5("Z52", "DZHNN") - SupportedDocument.CAMT_053 -> EbicsOrder.V2_5("Z53", "DZHNN") - SupportedDocument.CAMT_054 -> EbicsOrder.V2_5("Z54", "DZHNN") - SupportedDocument.PAIN_002_LOGS -> EbicsOrder.V2_5("HAC", "DZHNN") + OrderDoc.acknowledgement -> EbicsOrder.V2_5("HAC", "DZHNN") + OrderDoc.status -> EbicsOrder.V2_5("Z01", "DZHNN") + OrderDoc.report -> EbicsOrder.V2_5("Z52", "DZHNN") + OrderDoc.statement -> EbicsOrder.V2_5("Z53", "DZHNN") + OrderDoc.notification -> EbicsOrder.V2_5("Z54", "DZHNN") } } else { when (doc) { - SupportedDocument.PAIN_002 -> EbicsOrder.V3("BTD", "PSR", "CH", "pain.002", "10", "ZIP") - SupportedDocument.CAMT_052 -> EbicsOrder.V3("BTD", "STM", "CH", "camt.052", "08", "ZIP") - SupportedDocument.CAMT_053 -> EbicsOrder.V3("BTD", "EOP", "CH", "camt.053", "08", "ZIP") - SupportedDocument.CAMT_054 -> EbicsOrder.V3("BTD", "REP", "CH", "camt.054", "08", "ZIP") - SupportedDocument.PAIN_002_LOGS -> EbicsOrder.V3("HAC") + OrderDoc.acknowledgement -> EbicsOrder.V3("HAC") + OrderDoc.status -> EbicsOrder.V3("BTD", "PSR", "CH", "pain.002", "10", "ZIP") + OrderDoc.report -> EbicsOrder.V3("BTD", "STM", "CH", "camt.052", "08", "ZIP") + OrderDoc.statement -> EbicsOrder.V3("BTD", "EOP", "CH", "camt.053", "08", "ZIP") + OrderDoc.notification -> EbicsOrder.V3("BTD", "REP", "CH", "camt.054", "08", "ZIP") } } } // TODO for GLS we might have to fetch the same kind of files from multiple orders gls -> when (doc) { - SupportedDocument.PAIN_002 -> EbicsOrder.V3("BTD", "REP", "DE", "pain.002", null, "ZIP", "SCT") - SupportedDocument.CAMT_052 -> EbicsOrder.V3("BTD", "STM", "DE", "camt.052", null, "ZIP") - SupportedDocument.CAMT_053 -> EbicsOrder.V3("BTD", "EOP", "DE", "camt.053", null, "ZIP") - SupportedDocument.CAMT_054 -> EbicsOrder.V3("BTD", "STM", "DE", "camt.054", null, "ZIP", "SCI") - SupportedDocument.PAIN_002_LOGS -> EbicsOrder.V3("HAC") + OrderDoc.acknowledgement -> EbicsOrder.V3("HAC") + OrderDoc.status -> EbicsOrder.V3("BTD", "REP", "DE", "pain.002", null, "ZIP", "SCT") + OrderDoc.report -> EbicsOrder.V3("BTD", "STM", "DE", "camt.052", null, "ZIP") + OrderDoc.statement -> EbicsOrder.V3("BTD", "EOP", "DE", "camt.053", null, "ZIP") + OrderDoc.notification -> EbicsOrder.V3("BTD", "STM", "DE", "camt.054", null, "ZIP", "SCI") } } } diff --git a/nexus/src/main/kotlin/tech/libeufin/nexus/ebics/EbicsWS.kt b/nexus/src/main/kotlin/tech/libeufin/nexus/ebics/EbicsWS.kt @@ -27,9 +27,11 @@ import io.ktor.serialization.kotlinx.* import io.ktor.websocket.* import kotlinx.serialization.Serializable import kotlinx.serialization.json.* +import kotlinx.coroutines.channels.* +import kotlinx.coroutines.* import org.slf4j.Logger import org.slf4j.LoggerFactory -import tech.libeufin.common.encodeBase64 +import tech.libeufin.common.* private val logger: Logger = LoggerFactory.getLogger("libeufin-nexus-ws") @@ -136,11 +138,70 @@ suspend fun WssParams.connect(client: HttpClient, lambda: suspend (WssNotificati } }) { while (true) { - logger.info("waiting for msg") + logger.trace("wait for ws msg") // TODO use receiveDeserialized from ktor when it works val msg = receiveJson<WssNotification>() - logger.info("msg: {}", msg) + logger.trace("received: {}", msg) + if (msg is WssGeneralInfo) { + for (info in msg.INFO) { + logger.info("info: {}", info.FREE) + } + } lambda(msg) } } +} + +suspend fun listenForNotification(client: EbicsClient): ReceiveChannel<List<EbicsOrder>>? { + // Try to get params + val params = try { + client.wssParams() + } catch (e: EbicsError) { + if ( + // Expected EBICS error + (e is EbicsError.Code && e.technicalCode == EbicsReturnCode.EBICS_INVALID_ORDER_IDENTIFIER) || + // Netzbon HTTP error + (e is EbicsError.HTTP && e.status == HttpStatusCode.BadRequest) + ) { + // Failure is expected if this wss is not supported + logger.info("Real-time EBICS notifications is not supported") + return null + } else { + throw e + } + } + logger.info("Listening to real-time EBICS notifications") + val channel = Channel<List<EbicsOrder>>() + val backoff = ExpoBackoffDecorr() + kotlin.concurrent.thread(isDaemon = true) { + runBlocking { + while (true) { + try { + val params = client.wssParams() + logger.trace("{}", params) + params.connect(client.client) { msg -> + backoff.reset() + if (msg is WssNewData) { + val orders = msg.BTF.map { + EbicsOrder.V3( + type = "BTD", + service = it.SERVICE, + scope = it.SCOPE, + message = it.MSGNAME, + version = it.VERSION, + container = it.CONTTYPE, + option = it.OPTION + ) + } + channel.send(orders) + } + } + } catch (e: Exception) { + e.fmtLog(logger) + delay(backoff.next()) + } + } + } + } + return channel } \ No newline at end of file diff --git a/nexus/src/main/kotlin/tech/libeufin/nexus/test/TxCheck.kt b/nexus/src/main/kotlin/tech/libeufin/nexus/test/TxCheck.kt @@ -20,8 +20,7 @@ package tech.libeufin.nexus.test import io.ktor.client.* -import tech.libeufin.common.fmt -import tech.libeufin.common.rand +import tech.libeufin.common.* import tech.libeufin.nexus.BankPublicKeysFile import tech.libeufin.nexus.ClientPrivateKeysFile import tech.libeufin.nexus.NexusEbicsConfig @@ -89,7 +88,7 @@ suspend fun txCheck( fetch.close(firstTxId, "Close first fetch a second time") result.idempotentClose = true } catch (e: Exception) { - logger.debug(e.fmt()) + logger.debug { e.fmt() } } return result diff --git a/nexus/src/test/kotlin/EbicsTest.kt b/nexus/src/test/kotlin/EbicsTest.kt @@ -34,14 +34,15 @@ class EbicsTest { // code, and 200. @Test fun postMessage() = conf { config -> - assertFailsWith<EbicsError.Transport> { + assertFailsWith<EbicsError.HTTP> { getMockedClient { respondError(HttpStatusCode.NotFound) }.postToBank("http://ignored.example.com/", ByteArray(0), "Test") }.run { + assertEquals(HttpStatusCode.NotFound, status) assertEquals("Test: bank HTTP error: 404 Not Found", message) } - assertFailsWith<EbicsError.Transport> { + assertFailsWith<EbicsError.Network> { getMockedClient { throw Exception("Simulate failure") }.postToBank("http://ignored.example.com/", ByteArray(0), "Test") diff --git a/testbench/src/main/kotlin/Main.kt b/testbench/src/main/kotlin/Main.kt @@ -90,10 +90,10 @@ class Cli : CliktCommand("Run integration tests on banks provider") { LIBEUFIN_NEXUS_HOME = test/$platform [nexus-fetch] - FREQUENCY = 5s + FREQUENCY = 5d [nexus-submit] - FREQUENCY = 5s + FREQUENCY = 5d [libeufin-nexusdb-postgres] CONFIG = postgres:///libeufintestbench