libeufin

Integration and sandbox testing for FinTech APIs and data formats
Log | Files | Refs | Submodules | README | LICENSE

commit 4810f43922486dfb56273e5b93222978fa07585e
parent 715ffba34e0d73c3b4419e7ea1b83c4e1e8f918d
Author: Antoine A <>
Date:   Wed, 24 Jul 2024 14:29:23 +0200

nexus: bench and add missing index

Diffstat:
MMakefile | 8++++++--
Mbank/src/test/kotlin/AmountTest.kt | 36++++++++++++++++++------------------
Mbank/src/test/kotlin/bench.kt | 173++++++++++++++++++++++++++++---------------------------------------------------
Acommon/src/main/kotlin/test/bench.kt | 101+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Mdatabase-versioning/libeufin-nexus-0006.sql | 4+++-
Anexus/src/test/kotlin/bench.kt | 160+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
6 files changed, 349 insertions(+), 133 deletions(-)

diff --git a/Makefile b/Makefile @@ -121,6 +121,10 @@ doc: echo "Open build/dokka/htmlMultiModule/index.html" .PHONY: bench-db -bench-db: install-nobuild-files - ./gradlew :bank:test --tests Bench.benchDb -i +bank-bench-db: install-nobuild-files + ./gradlew cleanTest :bank:test --tests Bench.benchDb -i --no-build-cache + +.PHONY: bench-db +nexus-bench-db: install-nobuild-files + ./gradlew cleanTest :nexus:test --tests Bench.benchDb -i --no-build-cache diff --git a/bank/src/test/kotlin/AmountTest.kt b/bank/src/test/kotlin/AmountTest.kt @@ -264,16 +264,16 @@ class AmountTest { @Test fun conversionRevert() = dbSetup { db -> db.conn { conn -> + val applyStmt = conn.prepareStatement("SELECT amount.val, amount.frac FROM conversion_apply_ratio((?, ?)::taler_amount, (?, ?)::taler_amount, (?, ?)::taler_amount, ?::rounding_mode) as amount") fun TalerAmount.apply(ratio: DecimalNumber, tiny: DecimalNumber = DecimalNumber("0.00000001"), roundingMode: String = "zero"): TalerAmount { - val stmt = conn.prepareStatement("SELECT amount.val, amount.frac FROM conversion_apply_ratio((?, ?)::taler_amount, (?, ?)::taler_amount, (?, ?)::taler_amount, ?::rounding_mode) as amount") - stmt.setLong(1, this.value) - stmt.setInt(2, this.frac) - stmt.setLong(3, ratio.value) - stmt.setInt(4, ratio.frac) - stmt.setLong(5, tiny.value) - stmt.setInt(6, tiny.frac) - stmt.setString(7, roundingMode) - return stmt.one { + applyStmt.setLong(1, this.value) + applyStmt.setInt(2, this.frac) + applyStmt.setLong(3, ratio.value) + applyStmt.setInt(4, ratio.frac) + applyStmt.setLong(5, tiny.value) + applyStmt.setInt(6, tiny.frac) + applyStmt.setString(7, roundingMode) + return applyStmt.one { TalerAmount( it.getLong(1), it.getInt(2), @@ -282,16 +282,16 @@ class AmountTest { } } + val revertStmt = conn.prepareStatement("SELECT amount.val, amount.frac FROM conversion_revert_ratio((?, ?)::taler_amount, (?, ?)::taler_amount, (?, ?)::taler_amount, ?::rounding_mode) as amount") fun TalerAmount.revert(ratio: DecimalNumber, tiny: DecimalNumber = DecimalNumber("0.00000001"), roundingMode: String = "zero"): TalerAmount { - val stmt = conn.prepareStatement("SELECT amount.val, amount.frac FROM conversion_revert_ratio((?, ?)::taler_amount, (?, ?)::taler_amount, (?, ?)::taler_amount, ?::rounding_mode) as amount") - stmt.setLong(1, this.value) - stmt.setInt(2, this.frac) - stmt.setLong(3, ratio.value) - stmt.setInt(4, ratio.frac) - stmt.setLong(5, tiny.value) - stmt.setInt(6, tiny.frac) - stmt.setString(7, roundingMode) - return stmt.one { + revertStmt.setLong(1, this.value) + revertStmt.setInt(2, this.frac) + revertStmt.setLong(3, ratio.value) + revertStmt.setInt(4, ratio.frac) + revertStmt.setLong(5, tiny.value) + revertStmt.setInt(6, tiny.frac) + revertStmt.setString(7, roundingMode) + return revertStmt.one { TalerAmount( it.getLong(1), it.getInt(2), diff --git a/bank/src/test/kotlin/bench.kt b/bank/src/test/kotlin/bench.kt @@ -23,6 +23,7 @@ import org.junit.Test import org.postgresql.jdbc.PgConnection import tech.libeufin.bank.* import tech.libeufin.common.* +import tech.libeufin.common.test.* import tech.libeufin.common.crypto.PwCrypto import java.time.Instant import java.time.LocalDateTime @@ -49,73 +50,65 @@ class Bench { // In general half of the data is for generated account and half is for customer val mid = amount / 2 - val copyManager = conn.getCopyAPI() val password = PwCrypto.hashpw("password") - fun gen(table: String, lambda: (Int) -> String) { - println("Gen rows for $table") - val full = buildString(150*amount) { - repeat(amount) { - append(lambda(it+1)) - } - } - copyManager.copyIn("COPY $table FROM STDIN", full.reader()) - } - val token32 = ByteArray(32) val token64 = ByteArray(64) - gen("customers(login, name, password_hash, cashout_payto)") { - "account_$it\t$password\tMr n°$it\t$unknownPayto\n" - } - gen("bank_accounts(internal_payto_uri, owning_customer_id, is_public)") { - "payto://x-taler-bank/localhost/account_$it\t${it+skipAccount}\t${it%3==0}\n" - } - gen("bearer_tokens(content, creation_time, expiration_time, scope, is_refreshable, bank_customer, description, last_access)") { - val account = if (it > mid) customerAccount else it+4 - val hex = token32.rand().encodeHex() - "\\\\x$hex\t0\t0\treadonly\tfalse\t$account\t\\N\t0\n" - } - gen("bank_account_transactions(creditor_payto_uri, creditor_name, debtor_payto_uri, debtor_name, subject, amount, transaction_date, direction, bank_account_id)") { - val account = if (it > mid) customerAccount else it+4 - "creditor_payto\tcreditor_name\tdebtor_payto\tdebtor_name\tsubject\t(42,0)\t0\tcredit\t$exchangeAccount\n" + - "creditor_payto\tcreditor_name\tdebtor_payto\tdebtor_name\tsubject\t(42,0)\t0\tdebit\t$account\n" - } - gen("bank_transaction_operations") { - val hex = token32.rand().encodeHex() - "\\\\x$hex\t$it\n" - } - gen("tan_challenges(body, op, code, creation_date, expiration_date, retry_counter, customer)") { - val account = if (it > mid) customerAccount else it+4 - "body\taccount_reconfig\tcode\t0\t0\t0\t$account\n" - } - gen("taler_withdrawal_operations(withdrawal_uuid, wallet_bank_account, reserve_pub, creation_date)") { - val account = if (it > mid) customerAccount else it+4 - val hex = token32.rand().encodeHex() - val uuid = UUID.randomUUID() - "$uuid\t$account\t\\\\x$hex\t0\n" - } - gen("taler_exchange_outgoing(wtid, request_uid, exchange_base_url, bank_transaction, creditor_account_id)") { - val hex32 = token32.rand().encodeHex() - val hex64 = token64.rand().encodeHex() - "\\\\x$hex32\t\\\\x$hex64\turl\t${it*2-1}\t$it\n" - } - gen("taler_exchange_incoming(reserve_pub, bank_transaction)") { - val hex = token32.rand().encodeHex() - "\\\\x$hex\t${it*2}\n" - } - gen("bank_stats(timeframe, start_time)") { - val instant = Instant.ofEpochSecond(it.toLong()) - val date = LocalDateTime.ofInstant(instant, ZoneId.of("UTC")) - "day\t$date\n" - } - gen("cashout_operations(request_uid,amount_debit,amount_credit,subject,creation_time,bank_account,local_transaction)") { - val account = if (it > mid) customerAccount else it+4 - val hex = token32.rand().encodeHex() - "\\\\x$hex\t(0,0)\t(0,0)\tsubject\t0\t$account\t$it\n" - } - - // Update database statistics for better perf - conn.execSQLUpdate("VACUUM ANALYZE"); + conn.genData(amount, sequenceOf( + "customers(login, name, password_hash, cashout_payto)" to { + "account_$it\t$password\tMr n°$it\t$unknownPayto\n" + }, + "bank_accounts(internal_payto_uri, owning_customer_id, is_public)" to { + "payto://x-taler-bank/localhost/account_$it\t${it+skipAccount}\t${it%3==0}\n" + }, + "bearer_tokens(content, creation_time, expiration_time, scope, is_refreshable, bank_customer, description, last_access)" to { + val account = if (it > mid) customerAccount else it+4 + val hex = token32.rand().encodeHex() + "\\\\x$hex\t0\t0\treadonly\tfalse\t$account\t\\N\t0\n" + }, + "bank_account_transactions(creditor_payto_uri, creditor_name, debtor_payto_uri, debtor_name, subject, amount, transaction_date, direction, bank_account_id)" to { + val account = if (it > mid) customerAccount else it+4 + "creditor_payto\tcreditor_name\tdebtor_payto\tdebtor_name\tsubject\t(42,0)\t0\tcredit\t$exchangeAccount\n" + + "creditor_payto\tcreditor_name\tdebtor_payto\tdebtor_name\tsubject\t(42,0)\t0\tdebit\t$account\n" + }, + "bank_transaction_operations" to { + val hex = token32.rand().encodeHex() + "\\\\x$hex\t$it\n" + }, + "tan_challenges(body, op, code, creation_date, expiration_date, retry_counter, customer)" to { + val account = if (it > mid) customerAccount else it+4 + "body\taccount_reconfig\tcode\t0\t0\t0\t$account\n" + }, + "taler_withdrawal_operations(withdrawal_uuid, wallet_bank_account, reserve_pub, creation_date)" to { + val account = if (it > mid) customerAccount else it+4 + val hex = token32.rand().encodeHex() + val uuid = UUID.randomUUID() + "$uuid\t$account\t\\\\x$hex\t0\n" + }, + "taler_exchange_outgoing(wtid, request_uid, exchange_base_url, bank_transaction, creditor_account_id)" to { + val hex32 = token32.rand().encodeHex() + val hex64 = token64.rand().encodeHex() + "\\\\x$hex32\t\\\\x$hex64\turl\t${it*2-1}\t$it\n" + }, + "taler_exchange_incoming(type, reserve_pub, account_pub, bank_transaction)" to { + val hex = token32.rand().encodeHex() + if (it % 2 == 0) { + "reserve\t\\\\x$hex\t\\N\t${it*2}\n" + } else { + "kyc\t\\N\t\\\\x$hex\t${it*2}\n" + } + }, + "bank_stats(timeframe, start_time)" to { + val instant = Instant.ofEpochSecond(it.toLong()) + val date = LocalDateTime.ofInstant(instant, ZoneId.of("UTC")) + "day\t$date\n" + }, + "cashout_operations(request_uid,amount_debit,amount_credit,subject,creation_time,bank_account,local_transaction)" to { + val account = if (it > mid) customerAccount else it+4 + val hex = token32.rand().encodeHex() + "\\\\x$hex\t(0,0)\t(0,0)\tsubject\t0\t$account\t$it\n" + } + )) } @Test @@ -129,45 +122,7 @@ class Bench { } println("Bench $ITER times with $AMOUNT rows") - val WARN = 4.toDuration(DurationUnit.MILLISECONDS) - val ERR = 50.toDuration(DurationUnit.MILLISECONDS) - - - suspend fun fmtMeasures(times: LongArray): List<String> { - val min: Long = times.min() - val max: Long = times.max() - val mean: Long = times.average().toLong() - val variance = times.map { (it.toDouble() - mean).pow(2) }.average() - val stdVar: Long = sqrt(variance.toDouble()).toLong() - return sequenceOf(min, mean, max, stdVar).map { - val duration = it.toDuration(DurationUnit.MICROSECONDS) - val str = duration.toString() - if (duration > ERR) { - ANSI.red(str) - } else if (duration > WARN) { - ANSI.yellow(str) - } else { - ANSI.green(str) - } - - }.toList() - } - - val measures: MutableList<List<String>> = mutableListOf() - - suspend fun <R> measureAction(name: String, lambda: suspend (Int) -> R): List<R> { - val results = mutableListOf<R>() - val times = LongArray(ITER) { idx -> - measureTime { - val result = lambda(idx) - results.add(result) - }.inWholeMicroseconds - } - measures.add(listOf(ANSI.magenta(name)) + fmtMeasures(times)) - return results - } - - bankSetup { db -> + bench(ITER) { bankSetup { db -> // Prepare custoemr accounts fillCashoutInfo("customer") setMaxDebt("customer", "KUDOS:1000000") @@ -175,6 +130,9 @@ class Bench { // Generate data db.conn { genData(it, AMOUNT) } + // Warm HTTP client + client.get("/config").assertOk() + // Accounts val paytos = measureAction("account_create") { client.post("/accounts") { @@ -385,15 +343,6 @@ class Bench { measureAction("gc") { db.gc.collect(Instant.now(), java.time.Duration.ZERO, java.time.Duration.ZERO, java.time.Duration.ZERO) } - } - - val cols = IntArray(5) { 0 } - - printTable( - listOf("benchmark", "min", "mean", "max", "std").map { ANSI.bold(it) }, - measures, - ' ', - listOf(ColumnStyle.DEFAULT) + List(5) { ColumnStyle(false) } - ) + } } } } \ No newline at end of file diff --git a/common/src/main/kotlin/test/bench.kt b/common/src/main/kotlin/test/bench.kt @@ -0,0 +1,100 @@ +/* + * This file is part of LibEuFin. + * Copyright (C) 2024 Taler Systems S.A. + + * LibEuFin is free software; you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation; either version 3, or + * (at your option) any later version. + + * LibEuFin is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General + * Public License for more details. + + * You should have received a copy of the GNU Affero General Public + * License along with LibEuFin; see the file COPYING. If not, see + * <http://www.gnu.org/licenses/> + */ + +package tech.libeufin.common.test + +import tech.libeufin.common.* +import org.postgresql.jdbc.PgConnection +import java.time.Instant +import java.time.LocalDateTime +import java.time.ZoneId +import java.util.* +import kotlin.math.max +import kotlin.math.pow +import kotlin.math.sqrt +import kotlin.time.DurationUnit +import kotlin.time.measureTime +import kotlin.time.toDuration + +fun PgConnection.genData(amount: Int, generators: Sequence<Pair<String, (Int) -> String>>) { + val copyManager = this.getCopyAPI() + + for ((table, generator) in generators) { + println("Gen rows for $table") + val full = buildString(150*amount) { + repeat(amount) { + append(generator(it+1)) + } + } + copyManager.copyIn("COPY $table FROM STDIN", full.reader()) + } + + // Update database statistics for better perf + this.execSQLUpdate("VACUUM ANALYZE"); +} + +class Benchmark(private val iter: Int) { + private val WARN = 4.toDuration(DurationUnit.MILLISECONDS) + private val ERR = 50.toDuration(DurationUnit.MILLISECONDS) + internal val measures: MutableList<List<String>> = mutableListOf() + + internal fun fmtMeasures(times: LongArray): List<String> { + val min: Long = times.min() + val max: Long = times.max() + val mean: Long = times.average().toLong() + val variance = times.map { (it.toDouble() - mean).pow(2) }.average() + val stdVar: Long = sqrt(variance.toDouble()).toLong() + return sequenceOf(min, mean, max, stdVar).map { + val duration = it.toDuration(DurationUnit.MICROSECONDS) + val str = duration.toString() + if (duration > ERR) { + ANSI.red(str) + } else if (duration > WARN) { + ANSI.yellow(str) + } else { + ANSI.green(str) + } + + }.toList() + } + + suspend fun <R> measureAction(name: String, lambda: suspend (Int) -> R): List<R> { + val results = mutableListOf<R>() + val times = LongArray(iter) { idx -> + measureTime { + val result = lambda(idx) + results.add(result) + }.inWholeMicroseconds + } + measures.add(listOf(ANSI.magenta(name)) + fmtMeasures(times)) + return results + } +} + +fun bench(iter: Int, lambda: Benchmark.() -> Unit) { + val bench = Benchmark(iter) + lambda(bench) + val cols = IntArray(5) { 0 } + printTable( + listOf("benchmark", "min", "mean", "max", "std").map { ANSI.bold(it) }, + bench.measures, + ' ', + listOf(ColumnStyle.DEFAULT) + List(5) { ColumnStyle(false) } + ) +} +\ No newline at end of file diff --git a/database-versioning/libeufin-nexus-0006.sql b/database-versioning/libeufin-nexus-0006.sql @@ -19,7 +19,6 @@ SELECT _v.register_patch('libeufin-nexus-0006', NULL, NULL); SET search_path TO libeufin_nexus; - -- Support all taler incoming transaction types CREATE TYPE taler_incoming_type AS ENUM ('reserve' ,'kyc', 'wad'); @@ -38,4 +37,7 @@ ALTER TABLE talerable_incoming_transactions ); ALTER TABLE talerable_incoming_transactions ALTER COLUMN type DROP DEFAULT; +CREATE INDEX talerable_incoming_transactions_kyc_index ON talerable_incoming_transactions (account_pub) WHERE account_pub IS NOT NULL; +COMMENT ON INDEX talerable_incoming_transactions_kyc_index IS 'for reconciling KYC transaction without bank_id'; + COMMIT; diff --git a/nexus/src/test/kotlin/bench.kt b/nexus/src/test/kotlin/bench.kt @@ -0,0 +1,159 @@ +/* + * This file is part of LibEuFin. + * Copyright (C) 2024 Taler Systems S.A. + + * LibEuFin is free software; you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation; either version 3, or + * (at your option) any later version. + + * LibEuFin is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General + * Public License for more details. + + * You should have received a copy of the GNU Affero General Public + * License along with LibEuFin; see the file COPYING. If not, see + * <http://www.gnu.org/licenses/> + */ + +import io.ktor.client.request.* +import io.ktor.http.* +import org.junit.Test +import org.postgresql.jdbc.PgConnection +import tech.libeufin.nexus.* +import tech.libeufin.nexus.cli.* +import tech.libeufin.common.* +import tech.libeufin.common.test.* +import tech.libeufin.common.crypto.PwCrypto +import java.time.Instant +import java.time.LocalDateTime +import java.time.ZoneId +import java.util.* +import kotlin.math.max +import kotlin.math.pow +import kotlin.math.sqrt +import kotlin.time.DurationUnit +import kotlin.time.measureTime +import kotlin.time.toDuration + +class Bench { + + /** Generate [amount] rows to fill the database */ + fun genData(conn: PgConnection, amount: Int) { + val amount = max(amount, 10) + val token32 = ByteArray(32) + val token64 = ByteArray(64) + + conn.genData(amount, sequenceOf( + "incoming_transactions(amount, wire_transfer_subject, execution_time, debit_payto_uri, bank_id)" to { + "(42,0)\tsubject\t0\tdebit_payto\tBANK_ID${it*2}\n" + + "(42,0)\tsubject\t0\tdebit_payto\tBANK_ID${it*2+1}\n" + + "(42,0)\tsubject\t0\tdebit_payto\t\\N\n" + }, + "outgoing_transactions(amount, wire_transfer_subject, execution_time, credit_payto_uri, message_id)" to { + "(42,0)\tsubject\t0\tcredit_payto\tMSG_ID${it*2}\n" + + "(42,0)\tsubject\t0\tcredit_payto\tMSG_ID${it*2+1}\n" + }, + "initiated_outgoing_transactions(amount, wire_transfer_subject, initiation_time, credit_payto_uri, outgoing_transaction_id, request_uid, order_id)" to { + "(42,0)\tsubject\t0\tcredit_payto\t${it*2}\tREQUEST_UID$it\tORDER_ID$it\n" + }, + "talerable_incoming_transactions(type, reserve_public_key, account_pub, incoming_transaction_id)" to { + val hex = token32.rand().encodeHex() + if (it % 2 == 0) { + "reserve\t\\\\x$hex\t\\N\t${it*2}\n" + } else { + "kyc\t\\N\t\\\\x$hex\t${it*2}\n" + } + }, + "talerable_outgoing_transactions(wtid, exchange_base_url, outgoing_transaction_id)" to { + val hex = token32.rand().encodeHex() + "\\\\x$hex\turl\t${it*2-1}\n" + }, + "transfer_operations(initiated_outgoing_transaction_id, request_uid, wtid, exchange_base_url)" to { + val hex32 = token32.rand().encodeHex() + val hex64 = token64.rand().encodeHex() + "$it\t\\\\x$hex64\t\\\\x$hex32\turl\n" + } + )) + } + + @Test + fun benchDb() { + val ITER = System.getenv("BENCH_ITER")?.toIntOrNull() ?: 0 + val AMOUNT = System.getenv("BENCH_AMOUNT")?.toIntOrNull() ?: 0 + + if (ITER == 0) { + println("Skip benchmark, missing BENCH_ITER") + return + } + println("Bench $ITER times with $AMOUNT rows") + + bench(ITER) { serverSetup { db -> + // Generate data + db.conn { genData(it, AMOUNT) } + + // Warm HTTP client + client.getA("/taler-revenue/config").assertOk() + + // Ingest + measureAction("ingest_in") { + ingestIn(db) + } + measureAction("ingest_out") { + ingestOut(db) + } + measureAction("ingest_reserve") { + talerableIn(db) + } + measureAction("ingest_kyc") { + talerableKycIn(db) + } + measureAction("ingest_reserve_missing_id") { + val incoming = genInPay("test with ${ShortHashCode.rand()} reserve pub") + ingestIncomingPayment(db, incoming.copy(bankId = null), AccountType.exchange) + ingestIncomingPayment(db, incoming, AccountType.exchange) + } + measureAction("ingest_kyc_missing_id") { + val incoming = genInPay("test with KYC:${ShortHashCode.rand()} account pub") + ingestIncomingPayment(db, incoming.copy(bankId = null), AccountType.exchange) + ingestIncomingPayment(db, incoming, AccountType.exchange) + } + + // Revenue API + measureAction("transaction_revenue") { + client.getA("/taler-revenue/history").assertOk() + } + + // Wire gateway + measureAction("wg_transfer") { + client.postA("/taler-wire-gateway/transfer") { + json { + "request_uid" to HashCode.rand() + "amount" to "CHF:0.0001" + "exchange_base_url" to "http://exchange.example.com/" + "wtid" to ShortHashCode.rand() + "credit_account" to grothoffPayto + } + }.assertOk() + } + measureAction("wg_add") { + client.postA("/taler-wire-gateway/admin/add-incoming") { + json { + "amount" to "CHF:0.0001" + "reserve_pub" to EddsaPublicKey.rand() + "debit_account" to grothoffPayto + } + }.assertOk() + } + measureAction("wg_incoming") { + client.getA("/taler-wire-gateway/history/incoming") + .assertOk() + } + measureAction("wg_outgoing") { + client.getA("/taler-wire-gateway/history/outgoing") + .assertOk() + } + }} + } +} +\ No newline at end of file