commit ce74df3f515cc5caec39381c00645714f3d7f5ba parent fd4889f4a19a7993e65af6d9bd6b0231604534a6 Author: MS <ms@taler.net> Date: Fri, 30 Dec 2022 19:37:36 +0100 Use Ktor 2.2.1 and general polishing. Diffstat:
31 files changed, 661 insertions(+), 873 deletions(-)
diff --git a/access-api-stash/AccessApiNexus.kt b/access-api-stash/AccessApiNexus.kt @@ -1,210 +0,0 @@ -package tech.libeufin.nexus.`access-api` - -import com.fasterxml.jackson.databind.JsonNode -import com.fasterxml.jackson.databind.ObjectMapper -import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper -import io.ktor.client.* -import io.ktor.client.features.* -import io.ktor.client.request.* -import io.ktor.client.statement.* -import io.ktor.http.* -import io.ktor.util.* -import org.jetbrains.exposed.sql.not -import org.jetbrains.exposed.sql.transactions.transaction -import tech.libeufin.nexus.* -import tech.libeufin.nexus.server.AccessApiNewTransport -import tech.libeufin.nexus.server.EbicsNewTransport -import tech.libeufin.nexus.server.FetchSpecJson -import tech.libeufin.nexus.server.client -import tech.libeufin.util.* -import java.net.URL -import java.nio.charset.Charset - -private fun getAccessApiClient(connId: String): AccessApiClientEntity { - val conn = NexusBankConnectionEntity.find { - NexusBankConnectionsTable.connectionId eq connId - }.firstOrNull() ?: throw notFound("Connection '$connId' not found.") - val client = AccessApiClientEntity.find { - AccessApiClientsTable.nexusBankConnection eq conn.id.value - }.firstOrNull() ?: throw notFound("Connection '$connId' has no client data.") - return client -} - -suspend fun HttpClient.accessApiReq( - method: HttpMethod, - url: String, - body: Any? = null, - // username, password - credentials: Pair<String, String>): String? { - - val reqBuilder: HttpRequestBuilder.() -> Unit = { - contentType(ContentType.Application.Json) - if (body != null) - this.body = body - - headers.apply { - this.set( - "Authorization", - "Basic " + bytesToBase64("${credentials.first}:${credentials.second}".toByteArray(Charsets.UTF_8)) - ) - } - } - return try { - when(method) { - HttpMethod.Get -> { - this.get(url, reqBuilder) - } - HttpMethod.Post -> { - this.post(url, reqBuilder) - } - else -> throw internalServerError("Method $method not supported.") - } - } catch (e: ClientRequestException) { - logger.error(e.message) - throw NexusError( - HttpStatusCode.BadGateway, - e.message - ) - } -} - -/** - * Talk to the Sandbox via native Access API. The main reason - * for this class was to still allow x-taler-bank as a wire method - * (to accommodate wallet harness tests), and therefore skip the - * Camt+Pain schemas. - */ -class JsonBankConnectionProtocol: BankConnectionProtocol { - - override fun createConnection(connId: String, user: NexusUserEntity, data: JsonNode) { - val bankConn = NexusBankConnectionEntity.new { - this.connectionId = connId - owner = user - type = "access-api" - } - val newTransportData = jacksonObjectMapper( - ).treeToValue(data, AccessApiNewTransport::class.java) ?: throw NexusError( - HttpStatusCode.BadRequest, "Access Api details not found in request" - ) - AccessApiClientEntity.new { - username = newTransportData.username - bankURL = newTransportData.bankURL - remoteBankAccountLabel = newTransportData.remoteBankAccountLabel - nexusBankConnection = bankConn - password = newTransportData.password - } - } - - override fun getConnectionDetails(conn: NexusBankConnectionEntity): JsonNode { - val details = transaction { getAccessApiClient(conn.connectionId) } - val ret = ObjectMapper().createObjectNode() - ret.put("username", details.username) - ret.put("bankURL", details.bankURL) - ret.put("passwordHash", CryptoUtil.hashpw(details.password)) - ret.put("remoteBankAccountLabel", details.remoteBankAccountLabel) - return ret - } - - override suspend fun submitPaymentInitiation( - httpClient: HttpClient, - paymentInitiationId: Long // must refer to an x-taler-bank payto://-instruction. - ) { - val payInit = XTalerBankPaymentInitiationEntity.findById(paymentInitiationId) ?: throw notFound( - "Payment initiation '$paymentInitiationId' not found." - ) - val conn = payInit.defaultBankConnection ?: throw notFound( - "No default bank connection for payment initiation '${paymentInitiationId}' was found." - ) - val details = getAccessApiClient(conn.connectionId) - - client.accessApiReq( - method = HttpMethod.Post, - url = urlJoinNoDrop( - details.bankURL, - "accounts/${details.remoteBankAccountLabel}/transactions" - ), - body = object { - val paytoUri = payInit.paytoUri - val amount = payInit.amount - val subject = payInit.subject - }, - credentials = Pair(details.username, details.password) - ) - } - /** - * This function gets always the fresh transactions from - * the bank. Any other Wire Gateway API policies will be - * implemented by the respective facade (XTalerBank.kt) */ - override suspend fun fetchTransactions( - fetchSpec: FetchSpecJson, - client: HttpClient, - bankConnectionId: String, - /** - * Label of the local bank account that mirrors - * the remote bank account pointed to by 'bankConnectionId' */ - accountId: String - ) { - val details = getAccessApiClient(bankConnectionId) - val txsRaw = client.accessApiReq( - method = HttpMethod.Get, - url = urlJoinNoDrop( - details.bankURL, - "accounts/${details.remoteBankAccountLabel}/transactions" - ), - credentials = Pair(details.username, details.password) - ) - // What format does Access API communicates the records in? - /** - * NexusXTalerBankTransactions.new { - * - * .. details .. - * } - */ - } - - override fun exportBackup(bankConnectionId: String, passphrase: String): JsonNode { - throw NexusError( - HttpStatusCode.NotImplemented, - "Operation not needed." - ) - } - - override fun exportAnalogDetails(conn: NexusBankConnectionEntity): ByteArray { - throw NexusError( - HttpStatusCode.NotImplemented, - "Operation not needed." - ) - } - - override suspend fun fetchAccounts(client: HttpClient, connId: String) { - throw NexusError( - HttpStatusCode.NotImplemented, - "access-api connections assume that remote and local bank" + - " accounts are called the same. No need to 'fetch'" - ) - } - - override fun createConnectionFromBackup( - connId: String, - user: NexusUserEntity, - passphrase: String?, - backup: JsonNode - ) { - throw NexusError( - HttpStatusCode.NotImplemented, - "Operation not needed." - ) - } - - override suspend fun connect(client: HttpClient, connId: String) { - /** - * Future versions might create a bank account at this step. - * Right now, all the tests do create those accounts beforehand. - */ - throw NexusError( - HttpStatusCode.NotImplemented, - "Operation not needed." - ) - } -} - diff --git a/build.gradle b/build.gradle @@ -1,7 +1,8 @@ import org.apache.tools.ant.filters.ReplaceTokens plugins { - id 'org.jetbrains.kotlin.jvm' version '1.5.30' + // id 'org.jetbrains.kotlin.jvm' version '1.5.30' + id 'org.jetbrains.kotlin.jvm' version '1.7.22' id 'idea' } @@ -12,6 +13,10 @@ if (!JavaVersion.current().isJava11Compatible()){ } allprojects { + ext.set("ktor_version", "2.2.1") + ext.set("ktor_auth_version", "1.6.8") + ext.set("exposed_version", "0.32.1") + repositories { mavenCentral() jcenter() diff --git a/nexus/build.gradle b/nexus/build.gradle @@ -3,14 +3,6 @@ plugins { id 'java' id 'application' id 'org.jetbrains.kotlin.jvm' - /** - * Design choice: native installation logic doesn't provide one - * single command to generate a unique jar, and even by generating - * a unique jar manually, then the usual gradle wrappers wouldn't be - * able to run those. Therefore, the dependency below ('shadow') - * was added as it provides _both_: unique jar packaging _and_ a - * suitable launch script. - */ id "com.github.johnrengelman.shadow" version "5.2.0" } @@ -51,12 +43,8 @@ compileTestKotlin { } } -def ktor_version = '1.6.1' -def exposed_version = '0.32.1' - dependencies { // Core language libraries - implementation 'org.jetbrains.kotlin:kotlin-stdlib-jdk8:1.5.21' implementation 'org.jetbrains.kotlinx:kotlinx-coroutines-core:1.5.1-native-mt' // LibEuFin util library @@ -78,7 +66,6 @@ dependencies { implementation('com.github.ajalt:clikt:2.8.0') // Exposed, an SQL library - implementation "org.jetbrains.exposed:exposed-core:$exposed_version" implementation "org.jetbrains.exposed:exposed-dao:$exposed_version" implementation "org.jetbrains.exposed:exposed-jdbc:$exposed_version" @@ -88,12 +75,15 @@ dependencies { // Ktor, an HTTP client and server library implementation "io.ktor:ktor-server-core:$ktor_version" + implementation "io.ktor:ktor-server-content-negotiation:$ktor_version" + implementation "io.ktor:ktor-server-status-pages:$ktor_version" implementation "io.ktor:ktor-client-apache:$ktor_version" implementation "io.ktor:ktor-client-auth:$ktor_version" implementation "io.ktor:ktor-server-netty:$ktor_version" + // Brings the call-logging library too. implementation "io.ktor:ktor-server-test-host:$ktor_version" - implementation "io.ktor:ktor-auth:$ktor_version" - implementation "io.ktor:ktor-jackson:$ktor_version" + implementation "io.ktor:ktor-auth:$ktor_auth_version" + implementation "io.ktor:ktor-serialization-jackson:$ktor_version" // PDF generation implementation 'com.itextpdf:itext7-core:7.1.16' diff --git a/nexus/src/main/kotlin/tech/libeufin/nexus/Anastasis.kt b/nexus/src/main/kotlin/tech/libeufin/nexus/Anastasis.kt @@ -1,9 +1,7 @@ package tech.libeufin.nexus -import io.ktor.application.* import io.ktor.client.* import io.ktor.http.* -import io.ktor.response.* import org.jetbrains.exposed.sql.transactions.transaction import tech.libeufin.nexus.iso20022.TransactionDetails import tech.libeufin.nexus.server.PermissionQuery @@ -13,7 +11,9 @@ import tech.libeufin.util.EbicsProtocolError import kotlin.math.abs import kotlin.math.min import io.ktor.content.TextContent -import io.ktor.routing.* +import io.ktor.server.application.* +import io.ktor.server.response.* +import io.ktor.server.routing.* import tech.libeufin.util.buildIbanPaytoUri data class AnastasisIncomingBankTransaction( @@ -107,7 +107,7 @@ private suspend fun historyIncoming(call: ApplicationCall) { return call.respond(TextContent(customConverter(history), ContentType.Application.Json)) } -fun anastasisFacadeRoutes(route: Route, httpClient: HttpClient) { +fun anastasisFacadeRoutes(route: Route) { route.get("/history/incoming") { historyIncoming(call) return@get diff --git a/nexus/src/main/kotlin/tech/libeufin/nexus/Auth.kt b/nexus/src/main/kotlin/tech/libeufin/nexus/Auth.kt @@ -1,9 +1,8 @@ package tech.libeufin.nexus import UtilError -import io.ktor.application.* import io.ktor.http.* -import io.ktor.request.* +import io.ktor.server.request.* import org.jetbrains.exposed.sql.and import org.jetbrains.exposed.sql.transactions.transaction import tech.libeufin.nexus.server.Permission diff --git a/nexus/src/main/kotlin/tech/libeufin/nexus/Main.kt b/nexus/src/main/kotlin/tech/libeufin/nexus/Main.kt @@ -40,7 +40,7 @@ import java.io.File import kotlin.system.exitProcess val logger: Logger = LoggerFactory.getLogger("tech.libeufin.nexus") -val NEXUS_DB_ENV_VAR_NAME = "LIBEUFIN_NEXUS_DB_CONNECTION" +const val NEXUS_DB_ENV_VAR_NAME = "LIBEUFIN_NEXUS_DB_CONNECTION" class NexusCommand : CliktCommand() { init { diff --git a/nexus/src/main/kotlin/tech/libeufin/nexus/Taler.kt b/nexus/src/main/kotlin/tech/libeufin/nexus/Taler.kt @@ -20,20 +20,20 @@ package tech.libeufin.nexus import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper -import io.ktor.application.ApplicationCall -import io.ktor.application.call +import io.ktor.server.application.ApplicationCall +import io.ktor.server.application.call import io.ktor.client.* -import io.ktor.client.features.* +import io.ktor.client.plugins.* import io.ktor.client.request.* import io.ktor.content.TextContent import io.ktor.http.* -import io.ktor.request.receive -import io.ktor.response.respond -import io.ktor.response.respondText -import io.ktor.routing.Route -import io.ktor.routing.get -import io.ktor.routing.post -import io.ktor.util.* +import io.ktor.server.request.receive +import io.ktor.server.response.respond +import io.ktor.server.response.respondText +import io.ktor.server.routing.Route +import io.ktor.server.routing.get +import io.ktor.server.routing.post +import io.ktor.server.util.* import org.jetbrains.exposed.dao.Entity import org.jetbrains.exposed.dao.id.IdTable import org.jetbrains.exposed.sql.* @@ -486,10 +486,10 @@ private suspend fun addIncoming(call: ApplicationCall) { val currentBody = call.receive<String>() val fromDb = transaction { val f = FacadeEntity.findByName(facadeId) ?: throw notFound("facade $facadeId not found") - val state = FacadeStateEntity.find { + val facadeState = FacadeStateEntity.find { FacadeStateTable.facade eq f.id }.firstOrNull() ?: throw internalServerError("facade $facadeId has no state!") - val conn = NexusBankConnectionEntity.findByName(state.bankConnection) ?: throw internalServerError( + val conn = NexusBankConnectionEntity.findByName(facadeState.bankConnection) ?: throw internalServerError( "state of facade $facadeId has no bank connection!" ) val ebicsData = NexusEbicsSubscribersTable.select { @@ -500,10 +500,12 @@ private suspend fun addIncoming(call: ApplicationCall) { // Resort Sandbox URL from EBICS endpoint. val sandboxUrl = URL(ebicsData[NexusEbicsSubscribersTable.ebicsURL]) // NOTE: the exchange username must be 'exchange', at the Sandbox. - return@transaction Pair(url { - protocol = URLProtocol(sandboxUrl.protocol, 80) - host = sandboxUrl.host - if (sandboxUrl.port != 80) port = sandboxUrl.port + return@transaction Pair( + url { + protocol = URLProtocol(sandboxUrl.protocol, 80) + host = sandboxUrl.host + if (sandboxUrl.port != 80) + port = sandboxUrl.port path( "demobanks", "default", @@ -512,22 +514,17 @@ private suspend fun addIncoming(call: ApplicationCall) { "admin", "add-incoming" ) - }, state.bankAccount + }, + facadeState.bankAccount ) } val client = HttpClient { followRedirects = true } try { - client.post<String>( - urlString = fromDb.first, - block = { - this.body = currentBody - this.header( - "Authorization", - buildBasicAuthLine("exchange", "x") - ) - this.header("Content-Type", "application/json") - } - ) + client.post(fromDb.first) { + setBody(currentBody) + basicAuth("exchange", "x") + contentType(ContentType.Application.Json) + } } catch (e: ClientRequestException) { logger.error("Proxying /admin/add/incoming to the Sandbox failed: $e") } catch (e: Exception) { diff --git a/nexus/src/main/kotlin/tech/libeufin/nexus/bankaccount/BankAccount.kt b/nexus/src/main/kotlin/tech/libeufin/nexus/bankaccount/BankAccount.kt @@ -20,7 +20,7 @@ package tech.libeufin.nexus.bankaccount import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper -import io.ktor.application.ApplicationCall +import io.ktor.server.application.ApplicationCall import io.ktor.client.HttpClient import io.ktor.http.HttpStatusCode import org.jetbrains.exposed.sql.* @@ -183,13 +183,14 @@ fun processCamtMessage( "Nexus hit a report or statement of a wrong IBAN!" ) it.balances.forEach { b -> - var clbdCount = 0 if (b.type == "CLBD") { - clbdCount++ val lastBalance = NexusBankBalanceEntity.all().lastOrNull() /** * Store balances different from the one that came from the bank, - * or the very first balance. + * or the very first balance. This approach has the following inconvenience: + * the 'balance' held at Nexus does not differentiate between one + * coming from a statement and one coming from a report. As a consequence, + * the two types of balances may override each other without notice. */ if ((lastBalance == null) || (b.amount.toPlainString() != lastBalance.balance)) { diff --git a/nexus/src/main/kotlin/tech/libeufin/nexus/ebics/EbicsClient.kt b/nexus/src/main/kotlin/tech/libeufin/nexus/ebics/EbicsClient.kt @@ -23,8 +23,9 @@ package tech.libeufin.nexus.ebics import io.ktor.client.HttpClient -import io.ktor.client.features.* +import io.ktor.client.plugins.* import io.ktor.client.request.* +import io.ktor.client.statement.* import io.ktor.http.* import org.slf4j.Logger import org.slf4j.LoggerFactory @@ -39,13 +40,10 @@ private suspend inline fun HttpClient.postToBank(url: String, body: String): Str HttpStatusCode.InternalServerError, "EBICS (outgoing) document is invalid" ) - val response: String = try { - this.post( - urlString = url, - block = { - this.body = body - } - ) + val response: HttpResponse = try { + this.post(urlString = url) { + setBody(body) + } } catch (e: ClientRequestException) { logger.error(e.message) throw NexusError( @@ -60,8 +58,17 @@ private suspend inline fun HttpClient.postToBank(url: String, body: String): Str e.message ?: "Could not reach the bank" ) } - // logger.debug("Receiving: $response") - return response + /** + * EBICS should be expected only after a 200 OK response + * (including problematic ones); throw exception in all the other cases, + * by echoing what the bank said. + */ + if (response.status.value != HttpStatusCode.OK.value) + throw NexusError( + HttpStatusCode.BadGateway, + "bank says: ${response.bodyAsText()}" + ) + return response.bodyAsText() } sealed class EbicsDownloadResult @@ -227,7 +234,7 @@ suspend fun doEbicsUploadTransaction( logger.debug("Bank acknowledges EBICS upload initialization. Transaction ID: $transactionID.") /* now send actual payload */ - val payload = createEbicsRequestForUploadTransferPhase( + val ebicsPayload = createEbicsRequestForUploadTransferPhase( subscriberDetails, transactionID, preparedUploadData, @@ -235,7 +242,7 @@ suspend fun doEbicsUploadTransaction( ) val txRespStr = client.postToBank( subscriberDetails.ebicsUrl, - payload + ebicsPayload ) val txResp = parseAndValidateEbicsResponse(subscriberDetails, txRespStr) when (txResp.technicalReturnCode) { @@ -303,4 +310,4 @@ suspend fun doEbicsHpbRequest( "Cannot find data in a HPB response" ) return parseEbicsHpbOrder(orderData) -} +} +\ No newline at end of file diff --git a/nexus/src/main/kotlin/tech/libeufin/nexus/ebics/EbicsNexus.kt b/nexus/src/main/kotlin/tech/libeufin/nexus/ebics/EbicsNexus.kt @@ -31,15 +31,15 @@ import com.itextpdf.kernel.pdf.PdfWriter import com.itextpdf.layout.Document import com.itextpdf.layout.element.AreaBreak import com.itextpdf.layout.element.Paragraph -import io.ktor.application.call +import io.ktor.server.application.call import io.ktor.client.HttpClient import io.ktor.http.ContentType import io.ktor.http.HttpStatusCode -import io.ktor.request.receiveOrNull -import io.ktor.response.respond -import io.ktor.response.respondText -import io.ktor.routing.Route -import io.ktor.routing.post +import io.ktor.server.request.* +import io.ktor.server.response.respond +import io.ktor.server.response.respondText +import io.ktor.server.routing.Route +import io.ktor.server.routing.post import org.jetbrains.exposed.sql.and import org.jetbrains.exposed.sql.insert import org.jetbrains.exposed.sql.select @@ -196,14 +196,13 @@ fun getEbicsSubscriberDetails(bankConnectionId: String): EbicsClientSubscriberDe fun Route.ebicsBankProtocolRoutes(client: HttpClient) { post("test-host") { - val r = call.receiveJson<EbicsHostTestRequest>() + val r = call.receive<EbicsHostTestRequest>() val qr = doEbicsHostVersionQuery(client, r.ebicsBaseUrl, r.ebicsHostId) call.respond(qr) return@post } } - fun Route.ebicsBankConnectionRoutes(client: HttpClient) { post("/send-ini") { val subscriber = transaction { @@ -320,12 +319,8 @@ fun Route.ebicsBankConnectionRoutes(client: HttpClient) { if (orderType.length != 3) { throw NexusError(HttpStatusCode.BadRequest, "ebics order type must be three characters") } - val paramsJson = call.receiveOrNull<EbicsStandardOrderParamsDateJson>() - val orderParams = if (paramsJson == null) { - EbicsStandardOrderParams() - } else { - paramsJson.toOrderParams() - } + val paramsJson = call.receiveNullable<EbicsStandardOrderParamsDateJson>() + val orderParams = paramsJson?.toOrderParams() ?: EbicsStandardOrderParams() val subscriberDetails = transaction { val conn = requireBankConnection(call, "connid") if (conn.type != "ebics") { diff --git a/nexus/src/main/kotlin/tech/libeufin/nexus/server/NexusServer.kt b/nexus/src/main/kotlin/tech/libeufin/nexus/server/NexusServer.kt @@ -20,6 +20,8 @@ package tech.libeufin.nexus.server import UtilError +import io.ktor.serialization.jackson.* +import io.ktor.server.plugins.contentnegotiation.* import com.fasterxml.jackson.core.util.DefaultIndenter import com.fasterxml.jackson.core.util.DefaultPrettyPrinter import com.fasterxml.jackson.databind.JsonNode @@ -28,20 +30,19 @@ import com.fasterxml.jackson.databind.DeserializationFeature import com.fasterxml.jackson.databind.JsonMappingException import com.fasterxml.jackson.databind.SerializationFeature import com.fasterxml.jackson.databind.exc.MismatchedInputException -import com.fasterxml.jackson.module.kotlin.KotlinModule -import com.fasterxml.jackson.module.kotlin.MissingKotlinParameterException -import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper -import io.ktor.application.* +import com.fasterxml.jackson.module.kotlin.* import io.ktor.client.* -import io.ktor.features.* import io.ktor.http.* -import io.ktor.jackson.* import io.ktor.network.sockets.* -import io.ktor.request.* -import io.ktor.response.* -import io.ktor.routing.* +import io.ktor.server.application.* import io.ktor.server.engine.* import io.ktor.server.netty.* +import io.ktor.server.plugins.* +import io.ktor.server.plugins.callloging.* +import io.ktor.server.plugins.statuspages.* +import io.ktor.server.request.* +import io.ktor.server.response.* +import io.ktor.server.routing.* import io.ktor.util.* import org.jetbrains.exposed.exceptions.ExposedSQLException import org.jetbrains.exposed.sql.and @@ -51,11 +52,12 @@ import tech.libeufin.nexus.* import tech.libeufin.nexus.bankaccount.* import tech.libeufin.nexus.ebics.* import tech.libeufin.nexus.iso20022.CamtBankAccountEntry +import tech.libeufin.sandbox.SandboxErrorDetailJson +import tech.libeufin.sandbox.SandboxErrorJson import tech.libeufin.util.* import java.net.BindException import java.net.URLEncoder import kotlin.system.exitProcess -import java.net.URL /** * Return facade state depending on the type. @@ -121,18 +123,6 @@ fun ApplicationCall.expectUrlParameter(name: String): String { ?: throw NexusError(HttpStatusCode.BadRequest, "Parameter '$name' not provided in URI") } -suspend inline fun <reified T : Any> ApplicationCall.receiveJson(): T { - try { - return this.receive() - } catch (e: MissingKotlinParameterException) { - throw NexusError(HttpStatusCode.BadRequest, "Missing value for ${e.pathReference}") - } catch (e: MismatchedInputException) { - throw NexusError(HttpStatusCode.BadRequest, "Invalid value for '${e.pathReference}'") - } catch (e: JsonParseException) { - throw NexusError(HttpStatusCode.BadRequest, "Invalid JSON") - } -} - fun requireBankConnectionInternal(connId: String): NexusBankConnectionEntity { return transaction { NexusBankConnectionEntity.find { NexusBankConnectionsTable.connectionId eq connId }.firstOrNull() @@ -157,6 +147,7 @@ val nexusApp: Application.() -> Unit = { this.level = Level.DEBUG this.logger = tech.libeufin.nexus.logger } + install(LibeufinDecompressionPlugin) install(ContentNegotiation) { jackson { enable(SerializationFeature.INDENT_OUTPUT) @@ -164,12 +155,21 @@ val nexusApp: Application.() -> Unit = { indentArraysWith(DefaultPrettyPrinter.FixedSpaceIndenter.instance) indentObjectsWith(DefaultIndenter(" ", "\n")) }) - registerModule(KotlinModule(nullisSameAsDefault = true)) + registerModule( + KotlinModule.Builder() + .withReflectionCacheSize(512) + .configure(KotlinFeature.NullToEmptyCollection, false) + .configure(KotlinFeature.NullToEmptyMap, false) + .configure(KotlinFeature.NullIsSameAsDefault, enabled = true) + .configure(KotlinFeature.SingletonSupport, enabled = false) + .configure(KotlinFeature.StrictNullChecks, false) + .build() + ) configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false) } } install(StatusPages) { - exception<NexusError> { cause -> + exception<NexusError> { call, cause -> logger.error("Caught exception while handling '${call.request.uri} (${cause.reason})") call.respond( status = cause.statusCode, @@ -180,7 +180,7 @@ val nexusApp: Application.() -> Unit = { ) ) } - exception<JsonMappingException> { cause -> + exception<JsonMappingException> { call, cause -> logger.error("Exception while handling '${call.request.uri}'", cause) call.respond( HttpStatusCode.BadRequest, @@ -191,7 +191,7 @@ val nexusApp: Application.() -> Unit = { ) ) } - exception<UtilError> { cause -> + exception<UtilError> { call, cause -> logger.error("Exception while handling '${call.request.uri}'", cause) call.respond( cause.statusCode, @@ -202,7 +202,7 @@ val nexusApp: Application.() -> Unit = { ) ) } - exception<EbicsProtocolError> { cause -> + exception<EbicsProtocolError> { call, cause -> logger.error("Caught exception while handling '${call.request.uri}' (${cause.reason})") call.respond( cause.httpStatusCode, @@ -213,7 +213,19 @@ val nexusApp: Application.() -> Unit = { ) ) } - exception<Exception> { cause -> + exception<BadRequestException> { call, cause -> + tech.libeufin.sandbox.logger.error("Exception while handling '${call.request.uri}', ${cause.message}") + call.respond( + HttpStatusCode.BadRequest, + SandboxErrorJson( + error = SandboxErrorDetailJson( + type = "util-error", + description = cause.message ?: "Bad request but did not find exact cause." + ) + ) + ) + } + exception<Exception> { call, cause -> logger.error("Uncaught exception while handling '${call.request.uri}'") cause.printStackTrace() call.respond( @@ -226,7 +238,6 @@ val nexusApp: Application.() -> Unit = { ) } } - install(RequestBodyDecompression) intercept(ApplicationCallPipeline.Fallback) { if (this.call.response.status() == null) { call.respondText("Not found (no route matched).\n", ContentType.Text.Plain, HttpStatusCode.NotFound) @@ -332,7 +343,7 @@ val nexusApp: Application.() -> Unit = { // change a user's password post("/users/{username}/password") { - val body = call.receiveJson<ChangeUserPassword>() + val body = call.receive<ChangeUserPassword>() val targetUsername = ensureNonNull(call.parameters["username"]) transaction { requireSuperuser(call.request) @@ -351,7 +362,7 @@ val nexusApp: Application.() -> Unit = { // Add a new ordinary user in the system (requires superuser privileges) post("/users") { - val body = call.receiveJson<CreateUserRequest>() + val body = call.receive<CreateUserRequest>() val requestedUsername = requireValidResourceName(body.username) transaction { requireSuperuser(call.request) @@ -896,7 +907,7 @@ val nexusApp: Application.() -> Unit = { name = f.facadeName, type = f.type, baseUrl = URLBuilder(call.request.getBaseUrl()).apply { - pathComponents("facades", f.facadeName, f.type) + this.appendPathSegments(listOf("facades", f.facadeName, f.type)) encodedPath += "/" }.buildString(), config = getFacadeState(f.type, f) @@ -921,7 +932,7 @@ val nexusApp: Application.() -> Unit = { name = it.facadeName, type = it.type, baseUrl = URLBuilder(call.request.getBaseUrl()).apply { - pathComponents("facades", it.facadeName, it.type) + this.appendPathSegments(listOf("facades", it.facadeName, it.type)) encodedPath += "/" }.buildString(), config = getFacadeState(it.type, it) @@ -1041,7 +1052,7 @@ val nexusApp: Application.() -> Unit = { talerFacadeRoutes(this) } route("/facades/{fcid}/anastasis") { - anastasisFacadeRoutes(this, client) + anastasisFacadeRoutes(this) } // Hello endpoint. diff --git a/nexus/src/main/kotlin/tech/libeufin/nexus/server/RequestBodyDecompression.kt b/nexus/src/main/kotlin/tech/libeufin/nexus/server/RequestBodyDecompression.kt @@ -19,9 +19,9 @@ package tech.libeufin.nexus.server -import io.ktor.application.* -import io.ktor.features.* -import io.ktor.request.* +import io.ktor.http.* +import io.ktor.server.application.* +import io.ktor.server.request.* import io.ktor.util.* import io.ktor.util.pipeline.* import io.ktor.utils.io.* @@ -30,37 +30,18 @@ import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.withContext import java.util.zip.InflaterInputStream -/** - * Decompress request bodies. - */ -class RequestBodyDecompression private constructor() { - companion object Feature : - ApplicationFeature<Application, RequestBodyDecompression.Configuration, RequestBodyDecompression> { - override val key: AttributeKey<RequestBodyDecompression> = AttributeKey("Request Body Decompression") - override fun install( - pipeline: Application, - configure: RequestBodyDecompression.Configuration.() -> Unit - ): RequestBodyDecompression { - pipeline.receivePipeline.intercept(ApplicationReceivePipeline.Before) { - if (this.context.request.headers["Content-Encoding"] == "deflate") { - val deflated = this.subject.value as ByteReadChannel - val brc = withContext(Dispatchers.IO) { - val inflated = InflaterInputStream(deflated.toInputStream()) - // False positive in current Kotlin version, we're already in Dispatchers.IO! - @Suppress("BlockingMethodInNonBlockingContext") val bytes = inflated.readAllBytes() - ByteReadChannel(bytes) - } - proceedWith(ApplicationReceiveRequest(this.subject.typeInfo, brc)) - return@intercept +val LibeufinDecompressionPlugin = createApplicationPlugin("RequestingBodyDecompression") { + onCallReceive { call -> + transformBody { data -> + if (call.request.headers[HttpHeaders.ContentEncoding] == "deflate") { + val brc = withContext(Dispatchers.IO) { + val inflated = InflaterInputStream(data.toInputStream()) + @Suppress("BlockingMethodInNonBlockingContext") + val bytes = inflated.readAllBytes() + ByteReadChannel(bytes) } - proceed() - return@intercept - } - return RequestBodyDecompression() + brc + } else data } } - - class Configuration { - - } } \ No newline at end of file diff --git a/nexus/src/test/kotlin/DownloadAndSubmit.kt b/nexus/src/test/kotlin/DownloadAndSubmit.kt @@ -1,12 +1,12 @@ import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper -import io.ktor.application.* +import io.ktor.server.application.* import io.ktor.client.* import io.ktor.client.request.* -import io.ktor.features.* import io.ktor.http.* -import io.ktor.request.* -import io.ktor.response.* -import io.ktor.routing.* +import io.ktor.server.plugins.contentnegotiation.* +import io.ktor.server.request.* +import io.ktor.server.response.* +import io.ktor.server.routing.* import io.ktor.server.testing.* import kotlinx.coroutines.runBlocking import org.jetbrains.exposed.sql.transactions.transaction @@ -112,7 +112,8 @@ class DownloadAndSubmit { "Exist in logging!", "TESTKUDOS:5" ) - withTestApplication(sandboxApp) { + testApplication { + application(sandboxApp) runBlocking { fetchBankAccountTransactions( client, @@ -139,7 +140,8 @@ class DownloadAndSubmit { @Test fun upload() { withNexusAndSandboxUser { - withTestApplication(sandboxApp) { + testApplication { + application(sandboxApp) val conn = EbicsBankConnectionProtocol() runBlocking { // Create Pain.001 to be submitted. @@ -181,7 +183,8 @@ class DownloadAndSubmit { @Test fun unallowedDebtorIban() { withNexusAndSandboxUser { - withTestApplication(sandboxApp) { + testApplication { + application(sandboxApp) runBlocking { val bar = transaction { NexusBankAccountEntity.findByName("bar") } val painMessage = createPain001document( @@ -233,7 +236,8 @@ class DownloadAndSubmit { @Test fun invalidPain001() { withNexusAndSandboxUser { - withTestApplication(sandboxApp) { + testApplication { + application(sandboxApp) runBlocking { // Create Pain.001 to be submitted. addPaymentInitiation( @@ -246,9 +250,7 @@ class DownloadAndSubmit { currency = "TESTKUDOS" ), transaction { - NexusBankAccountEntity.findByName( - "foo" - ) ?: throw Exception("Test failed") + NexusBankAccountEntity.findByName("foo") ?: throw Exception("Test failed") } ) // Encounters errors. @@ -270,7 +272,8 @@ class DownloadAndSubmit { @Test fun unsupportedCurrency() { withNexusAndSandboxUser { - withTestApplication(sandboxApp) { + testApplication { + application(sandboxApp) runBlocking { // Create Pain.001 to be submitted. addPaymentInitiation( @@ -283,9 +286,7 @@ class DownloadAndSubmit { currency = "EUR" ), transaction { - NexusBankAccountEntity.findByName( - "foo" - ) ?: throw Exception("Test failed") + NexusBankAccountEntity.findByName("foo") ?: throw Exception("Test failed") } ) var thrown = false diff --git a/nexus/src/test/kotlin/JsonTest.kt b/nexus/src/test/kotlin/JsonTest.kt @@ -2,8 +2,13 @@ import com.fasterxml.jackson.databind.JsonNode import org.junit.Test import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper import com.fasterxml.jackson.module.kotlin.readValue +import io.ktor.client.plugins.* +import io.ktor.client.request.* +import io.ktor.http.* +import io.ktor.server.testing.* import tech.libeufin.nexus.server.CreateBankConnectionFromBackupRequestJson import tech.libeufin.nexus.server.CreateBankConnectionFromNewRequestJson +import tech.libeufin.sandbox.sandboxApp class JsonTest { @@ -23,5 +28,15 @@ class JsonTest { assert(roundTripNew.data.toString() == "{}" && roundTripNew.type == "ebics" && roundTripNew.name == "new-connection") } - + /*@Test + fun testSandboxJsonParsing() { + testApplication { + application(sandboxApp) + client.post("/admin/ebics/subscribers") { + basicAuth("admin", "foo") + contentType(ContentType.Application.Json) + setBody("{}") + } + } + }*/ } \ No newline at end of file diff --git a/nexus/src/test/kotlin/MakeEnv.kt b/nexus/src/test/kotlin/MakeEnv.kt @@ -70,7 +70,8 @@ fun prepNexusDb() { type = "ebics" } tech.libeufin.nexus.EbicsSubscriberEntity.new { - ebicsURL = "http://localhost:5000/ebicsweb" + // ebicsURL = "http://localhost:5000/ebicsweb" + ebicsURL = "http://localhost/ebicsweb" hostID = "eufinSandbox" partnerID = "foo" userID = "foo" @@ -84,7 +85,7 @@ fun prepNexusDb() { bankEncryptionPublicKey = ExposedBlob(bankKeys.enc.public.encoded) bankAuthenticationPublicKey = ExposedBlob(bankKeys.auth.public.encoded) } - val a = NexusBankAccountEntity.new { + NexusBankAccountEntity.new { bankAccountName = "foo" iban = FOO_USER_IBAN bankCode = "SANDBOXX" @@ -92,7 +93,7 @@ fun prepNexusDb() { highestSeenBankMessageSerialId = 0 accountHolder = "foo" } - val b = NexusBankAccountEntity.new { + NexusBankAccountEntity.new { bankAccountName = "bar" iban = BAR_USER_IBAN bankCode = "SANDBOXX" @@ -141,7 +142,7 @@ fun prepSandboxDb() { this.demoBank = demoBank isPublic = false } - val otherBankAccount = BankAccountEntity.new { + BankAccountEntity.new { iban = BAR_USER_IBAN /** * For now, keep same semantics of Pybank: a username diff --git a/nexus/src/test/kotlin/SandboxAccessApiTest.kt b/nexus/src/test/kotlin/SandboxAccessApiTest.kt @@ -1,16 +1,12 @@ import com.fasterxml.jackson.databind.ObjectMapper -import io.ktor.client.features.* +import io.ktor.client.plugins.* import io.ktor.client.request.* import io.ktor.client.statement.* -import io.ktor.client.utils.* import io.ktor.http.* import io.ktor.server.testing.* -import io.netty.handler.codec.http.HttpResponseStatus import kotlinx.coroutines.runBlocking -import org.jetbrains.exposed.sql.transactions.transaction import org.junit.Test import tech.libeufin.sandbox.* -import tech.libeufin.util.buildBasicAuthLine class SandboxAccessApiTest { val mapper = ObjectMapper() @@ -19,39 +15,24 @@ class SandboxAccessApiTest { fun debitWithdraw() { withTestDatabase { prepSandboxDb() - withTestApplication(sandboxApp) { + testApplication { + this.application(sandboxApp) runBlocking { // Normal, successful withdrawal. - client.post<Any>("/demobanks/default/access-api/accounts/foo/withdrawals") { + client.post("/demobanks/default/access-api/accounts/foo/withdrawals") { expectSuccess = true - headers { - append( - HttpHeaders.ContentType, - ContentType.Application.Json - ) - append( - HttpHeaders.Authorization, - buildBasicAuthLine("foo", "foo") - ) - } - this.body = "{\"amount\": \"TESTKUDOS:1\"}" + setBody("{\"amount\": \"TESTKUDOS:1\"}") + contentType(ContentType.Application.Json) + basicAuth("foo", "foo") } // Withdrawal over the debit threshold. - val r: HttpStatusCode = client.post("/demobanks/default/access-api/accounts/foo/withdrawals") { + val r: HttpResponse = client.post("/demobanks/default/access-api/accounts/foo/withdrawals") { expectSuccess = false - headers { - append( - HttpHeaders.ContentType, - ContentType.Application.Json - ) - append( - HttpHeaders.Authorization, - buildBasicAuthLine("foo", "foo") - ) - } - this.body = "{\"amount\": \"TESTKUDOS:99999999999\"}" + contentType(ContentType.Application.Json) + basicAuth("foo", "foo") + setBody("{\"amount\": \"TESTKUDOS:99999999999\"}") } - assert(HttpStatusCode.Forbidden.value == r.value) + assert(HttpStatusCode.Forbidden.value == r.status.value) } } } @@ -61,12 +42,13 @@ class SandboxAccessApiTest { * Tests that 'admin' and 'bank' are not possible to register * and that after 'admin' logs in it gets access to the bank's * main account. - */ + */ // FIXME: avoid giving Content-Type at every request. @Test fun adminRegisterAndLoginTest() { withTestDatabase { prepSandboxDb() - withTestApplication(sandboxApp) { + testApplication { + application(sandboxApp) runBlocking { val registerAdmin = mapper.writeValueAsString(object { val username = "admin" @@ -77,17 +59,10 @@ class SandboxAccessApiTest { val password = "y" }) for (b in mutableListOf<String>(registerAdmin, registerBank)) { - val r = client.post<HttpResponse>( - urlString = "/demobanks/default/access-api/testing/register", - ) { - this.body = b + val r = client.post("/demobanks/default/access-api/testing/register") { + setBody(b) + contentType(ContentType.Application.Json) expectSuccess = false - headers { - append( - HttpHeaders.ContentType, - ContentType.Application.Json - ) - } } assert(r.status.value == HttpStatusCode.Forbidden.value) } @@ -99,17 +74,11 @@ class SandboxAccessApiTest { "setting the balance", "TESTKUDOS:99" ) - // Get admin's balance. - val r = client.get<String>( - urlString = "/demobanks/default/access-api/accounts/admin", - ) { + // Get admin's balance. Not asserting; it + // fails on != 200 responses. + val r = client.get("/demobanks/default/access-api/accounts/admin") { expectSuccess = true - headers { - append( - HttpHeaders.Authorization, - buildBasicAuthLine("admin", "foo") - ) - } + basicAuth("admin", "foo") } println(r) } @@ -121,7 +90,8 @@ class SandboxAccessApiTest { fun registerTest() { // Test IBAN conflict detection. withSandboxTestDatabase { - withTestApplication(sandboxApp) { + testApplication { + application(sandboxApp) runBlocking { val bodyFoo = mapper.writeValueAsString(object { val username = "x" @@ -138,58 +108,24 @@ class SandboxAccessApiTest { val password = "y" val iban = BAR_USER_IBAN }) - // The following block would allow to save many LOC, - // but gets somehow ignored. - /*client.config { - this.defaultRequest { - headers { - append( - HttpHeaders.ContentType, - ContentType.Application.Json - ) - } - expectSuccess = false - } - }*/ // Succeeds. - client.post<HttpResponse>( - urlString = "/demobanks/default/access-api/testing/register", - ) { - this.body = bodyFoo + client.post("/demobanks/default/access-api/testing/register") { + setBody(bodyFoo) + contentType(ContentType.Application.Json) expectSuccess = true - headers { - append( - HttpHeaders.ContentType, - ContentType.Application.Json - ) - } } // Hits conflict, because of the same IBAN. - val r = client.post<HttpResponse>( - "/demobanks/default/access-api/testing/register" - ) { - this.body = bodyBar + val r = client.post("/demobanks/default/access-api/testing/register") { + setBody(bodyBar) expectSuccess = false - headers { - append( - HttpHeaders.ContentType, - ContentType.Application.Json - ) - } + contentType(ContentType.Application.Json) } - assert(r.status.value == HttpResponseStatus.CONFLICT.code()) + assert(r.status.value == HttpStatusCode.Conflict.value) // Succeeds, because of a new IBAN. - client.post<HttpResponse>( - "/demobanks/default/access-api/testing/register" - ) { - this.body = bodyBaz + client.post("/demobanks/default/access-api/testing/register") { + setBody(bodyBaz) expectSuccess = true - headers { - append( - HttpHeaders.ContentType, - ContentType.Application.Json - ) - } + contentType(ContentType.Application.Json) } } } diff --git a/nexus/src/test/kotlin/SandboxBankAccountTest.kt b/nexus/src/test/kotlin/SandboxBankAccountTest.kt @@ -1,9 +1,4 @@ -import io.ktor.client.features.* -import io.ktor.client.request.* -import io.ktor.client.statement.* import io.ktor.http.* -import io.ktor.server.testing.* -import kotlinx.coroutines.runBlocking import org.junit.Test import tech.libeufin.sandbox.SandboxError import tech.libeufin.sandbox.getBalance diff --git a/nexus/src/test/kotlin/SandboxCircuitApiTest.kt b/nexus/src/test/kotlin/SandboxCircuitApiTest.kt @@ -1,22 +1,42 @@ -import io.ktor.client.features.* -import io.ktor.client.features.get +import io.ktor.client.plugins.auth.* +import io.ktor.client.plugins.auth.providers.* import io.ktor.client.request.* +import io.ktor.client.statement.* import io.ktor.server.testing.* import kotlinx.coroutines.runBlocking import org.junit.Test import tech.libeufin.sandbox.sandboxApp class SandboxCircuitApiTest { - // Get /config + // Get /config, fails if != 200. @Test fun config() { withSandboxTestDatabase { - withTestApplication(sandboxApp) { + testApplication { + application(sandboxApp) runBlocking { - val r: String = client.get("/demobanks/default/circuit-api/config") - println(r) + val r= client.get("/demobanks/default/circuit-api/config") + println(r.bodyAsText()) } } } } + + // Tests the registration logic. Triggers + // any error code, following at least one execution + // path. + @Test + fun registration() { + withSandboxTestDatabase { + testApplication { + application(sandboxApp) + runBlocking { + client.post("/demobanks/default/circuit-api/accounts") { + basicAuth("admin", "foo") + } + } + } + } + + } } \ No newline at end of file diff --git a/nexus/src/test/kotlin/SandboxLegacyApiTest.kt b/nexus/src/test/kotlin/SandboxLegacyApiTest.kt @@ -1,9 +1,7 @@ -import com.fasterxml.jackson.databind.JsonNode import com.fasterxml.jackson.databind.ObjectMapper -import com.fasterxml.jackson.databind.util.ByteBufferBackedInputStream -import io.ktor.client.features.* +import io.ktor.client.plugins.* import io.ktor.client.request.* -import io.ktor.client.statement.HttpResponse +import io.ktor.client.statement.* import io.ktor.http.* import io.ktor.server.testing.* import io.ktor.util.* @@ -16,12 +14,7 @@ import tech.libeufin.sandbox.sandboxApp import tech.libeufin.util.buildBasicAuthLine import tech.libeufin.util.getIban import java.io.ByteArrayOutputStream -import java.io.InputStream -import java.nio.ByteBuffer -/** - * Mostly checking legacy API's access control. - */ class SandboxLegacyApiTest { fun dbHelper (f: () -> Unit) { withTestDatabase { @@ -31,12 +24,12 @@ class SandboxLegacyApiTest { } val mapper = ObjectMapper() - // EBICS Subscribers API. @Test - fun adminEbiscSubscribers() { + fun adminEbicsSubscribers() { dbHelper { - withTestApplication(sandboxApp) { + testApplication { + application(sandboxApp) runBlocking { /** * Create a EBICS subscriber. That conflicts because @@ -51,79 +44,42 @@ class SandboxLegacyApiTest { }) var r: HttpResponse = client.post("/admin/ebics/subscribers") { expectSuccess = false - headers { - append( - HttpHeaders.Authorization, - buildBasicAuthLine("admin", "foo") - ) - append( - HttpHeaders.ContentType, - ContentType.Application.Json - ) - } - this.body = body + contentType(ContentType.Application.Json) + basicAuth("admin", "foo") + setBody(body) } assert(r.status.value == HttpStatusCode.Conflict.value) - /** - * Check that EBICS subscriber indeed exists. - */ + + // Check that EBICS subscriber indeed exists. r = client.get("/admin/ebics/subscribers") { - headers { - append( - HttpHeaders.Authorization, - buildBasicAuthLine("admin", "foo") - ) - } + basicAuth("admin", "foo") } assert(r.status.value == HttpStatusCode.OK.value) - val buf = ByteArrayOutputStream() - r.content.read { buf.write(it.array()) } - val respObj = mapper.readTree(buf.toString()) + val respObj = mapper.readTree(r.bodyAsText()) assert("foo" == respObj.get("subscribers").get(0).get("userID").asText()) - /** - * Try same operations as above, with wrong admin credentials - */ + + // Try same operations as above, with wrong admin credentials r = client.get("/admin/ebics/subscribers") { expectSuccess = false - headers { - append( - HttpHeaders.Authorization, - buildBasicAuthLine("admin", "wrong") - ) - } + basicAuth("admin", "wrong") } assert(r.status.value == HttpStatusCode.Unauthorized.value) r = client.post("/admin/ebics/subscribers") { expectSuccess = false - headers { - append( - HttpHeaders.Authorization, - buildBasicAuthLine("admin", "wrong") - ) - } + basicAuth("admin", "wrong") } assert(r.status.value == HttpStatusCode.Unauthorized.value) - // Good credentials, but unauthorized user. + // Good credentials, but insufficient rights. r = client.get("/admin/ebics/subscribers") { expectSuccess = false - headers { - append( - HttpHeaders.Authorization, - buildBasicAuthLine("foo", "foo") - ) - } + basicAuth("foo", "foo") } - assert(r.status.value == HttpStatusCode.Unauthorized.value) + assert(r.status.value == HttpStatusCode.Forbidden.value) r = client.post("/admin/ebics/subscribers") { expectSuccess = false - headers { - append( - HttpHeaders.Authorization, - buildBasicAuthLine("foo", "foo") - ) - } + basicAuth("foo", "foo") } - assert(r.status.value == HttpStatusCode.Unauthorized.value) + assert(r.status.value == HttpStatusCode.Forbidden.value) /** * Give a bank account to the existing subscriber. Bank account * is (implicitly / hard-coded) hosted at default demobank. @@ -135,19 +91,11 @@ class SandboxLegacyApiTest { val partnerID = "baz" val systemID = "foo" }) - client.post<HttpResponse>("/admin/ebics/subscribers") { + client.post("/admin/ebics/subscribers") { expectSuccess = true - headers { - append( - HttpHeaders.Authorization, - buildBasicAuthLine("admin", "foo") - ) - append( - HttpHeaders.ContentType, - ContentType.Application.Json - ) - } - this.body = body + contentType(ContentType.Application.Json) + basicAuth("admin", "foo") + setBody(body) } // Associate new bank account to it. body = mapper.writeValueAsString(object { @@ -162,65 +110,42 @@ class SandboxLegacyApiTest { val name = "Now Have Account" val label = "baz" }) - client.post<HttpResponse>("/admin/ebics/bank-accounts") { + client.post("/admin/ebics/bank-accounts") { expectSuccess = true - headers { - append( - HttpHeaders.Authorization, - buildBasicAuthLine("admin", "foo") - ) - append( - HttpHeaders.ContentType, - ContentType.Application.Json - ) - } - this.body = body + expectSuccess = true + contentType(ContentType.Application.Json) + basicAuth("admin", "foo") + setBody(body) } r = client.get("/admin/ebics/subscribers") { - headers { - append( - HttpHeaders.Authorization, - buildBasicAuthLine("admin", "foo") - ) - } + basicAuth("admin", "foo") } assert(r.status.value == HttpStatusCode.OK.value) - val buf_ = ByteArrayOutputStream() - r.content.read { buf_.write(it.array()) } - val respObj_ = mapper.readTree(buf_.toString()) + val respObj_ = mapper.readTree(r.bodyAsText()) val bankAccountLabel = respObj_.get("subscribers").get(1).get("demobankAccountLabel").asText() assert("baz" == bankAccountLabel) // Same operation, wrong/unauth credentials. r = client.post("/admin/ebics/bank-accounts") { expectSuccess = false - headers { - append( - HttpHeaders.Authorization, - buildBasicAuthLine("admin", "wrong") - ) - } + basicAuth("admin", "wrong") } assert(r.status.value == HttpStatusCode.Unauthorized.value) r = client.post("/admin/ebics/bank-accounts") { expectSuccess = false - headers { - append( - HttpHeaders.Authorization, - buildBasicAuthLine("foo", "foo") - ) - } + basicAuth("foo", "foo") } - assert(r.status.value == HttpStatusCode.Unauthorized.value) + assert(r.status.value == HttpStatusCode.Forbidden.value) } } } } // EBICS Hosts API. - @Ignore + @Test fun adminEbicsCreateHost() { dbHelper { - withTestApplication(sandboxApp) { + testApplication { + application(sandboxApp) runBlocking { val body = mapper.writeValueAsString( object { @@ -229,73 +154,38 @@ class SandboxLegacyApiTest { } ) // Valid request, good credentials. - var r = client.post<HttpResponse>("/admin/ebics/hosts") { - this.body = body - this.headers { - append( - HttpHeaders.Authorization, - buildBasicAuthLine("admin", "foo") - ) - append( - HttpHeaders.ContentType, - ContentType.Application.Json - ) - } - } - assert(r.status.value == HttpResponseStatus.OK.code()) - r = client.get("/admin/ebics/hosts") { - expectSuccess = false - + client.post("/admin/ebics/hosts") { + expectSuccess = true + setBody(body) + contentType(ContentType.Application.Json) + basicAuth("admin", "foo") } + var r = client.get("/admin/ebics/hosts") { expectSuccess = false } assert(r.status.value == HttpResponseStatus.UNAUTHORIZED.code()) - r = client.get("/admin/ebics/hosts") { - this.headers { - append( - HttpHeaders.Authorization, - buildBasicAuthLine("admin", "foo") - ) - } + client.get("/admin/ebics/hosts") { + basicAuth("admin", "foo") + expectSuccess = true } - assert(r.status.value == HttpResponseStatus.OK.code()) // Invalid, with good credentials. r = client.post("/admin/ebics/hosts") { expectSuccess = false - this.body = "invalid" - this.headers { - append( - io.ktor.http.HttpHeaders.Authorization, - buildBasicAuthLine("admin", "foo") - ) - append( - io.ktor.http.HttpHeaders.ContentType, - ContentType.Application.Json - ) - } + setBody("invalid") + contentType(ContentType.Application.Json) + basicAuth("admin", "foo") } - assert(r.status.value == HttpResponseStatus.BAD_REQUEST.code()) + assert(r.status.value == HttpStatusCode.BadRequest.value) // Unauth: admin with wrong password. r = client.post("/admin/ebics/hosts") { expectSuccess = false - this.headers { - append( - io.ktor.http.HttpHeaders.Authorization, - buildBasicAuthLine("admin", "bar") - ) - } + basicAuth("admin", "bar") } - assert(r.status.value == HttpResponseStatus.UNAUTHORIZED.code()) + assert(r.status.value == HttpStatusCode.Unauthorized.value) // Auth & forbidden resource. r = client.post("/admin/ebics/hosts") { expectSuccess = false - this.headers { - append( - io.ktor.http.HttpHeaders.Authorization, - // Exist, but no rights over the EBICS host. - buildBasicAuthLine("foo", "foo") - ) - } + basicAuth("foo", "foo") } - assert(r.status.value == HttpResponseStatus.UNAUTHORIZED.code()) + assert(r.status.value == HttpStatusCode.Forbidden.value) } } } diff --git a/sandbox/build.gradle b/sandbox/build.gradle @@ -1,7 +1,8 @@ plugins { + id 'kotlin' id 'java' - id 'org.jetbrains.kotlin.jvm' id 'application' + id 'org.jetbrains.kotlin.jvm' id "com.github.johnrengelman.shadow" version "5.2.0" } @@ -15,6 +16,12 @@ compileKotlin { } } +compileTestKotlin { + kotlinOptions { + jvmTarget = "11" + } +} + task installToPrefix(type: Copy) { dependsOn(installShadowDist) from("build/install/sandbox-shadow") { @@ -29,29 +36,20 @@ task installToPrefix(type: Copy) { */ into "${project.findProperty('prefix') ?: '/tmp'}" } - -compileTestKotlin { - kotlinOptions { - jvmTarget = "11" - } -} +apply plugin: 'kotlin-kapt' sourceSets { - main.java.srcDirs = ['src/main/java', 'src/main/kotlin'] + main.java.srcDirs = [ + 'src/main/java', + 'src/main/kotlin' + ] } -def ktor_version = '1.6.1' -/** - * Exposed 0.38.2 caused a SQLITE_BUSY error at test-auditor.sh. - * The error was caused by a concurrent handling of a CCT EBICS - * message (see handleCct()). - */ -def exposed_version = '0.32.1' - dependencies { + implementation 'org.jetbrains.kotlinx:kotlinx-coroutines-core:1.5.1-native-mt' implementation "com.hubspot.jinjava:jinjava:2.5.9" - implementation 'org.jetbrains.kotlin:kotlin-stdlib-jdk8:1.5.21' implementation 'ch.qos.logback:logback-classic:1.2.5' + implementation project(":util") // XML: implementation "javax.xml.bind:jaxb-api:2.3.0" @@ -68,19 +66,22 @@ dependencies { implementation "org.jetbrains.exposed:exposed-jdbc:$exposed_version" implementation "io.ktor:ktor-server-core:$ktor_version" + implementation "io.ktor:ktor-server-call-logging:$ktor_version" + implementation "io.ktor:ktor-server-cors:$ktor_version" + implementation "io.ktor:ktor-server-content-negotiation:$ktor_version" + implementation "io.ktor:ktor-server-status-pages:$ktor_version" implementation "io.ktor:ktor-client-apache:$ktor_version" + implementation "io.ktor:ktor-client-auth:$ktor_version" implementation "io.ktor:ktor-server-netty:$ktor_version" - implementation "io.ktor:ktor-jackson:$ktor_version" - implementation "io.ktor:ktor-auth:$ktor_version" + implementation "io.ktor:ktor-server-test-host:$ktor_version" + implementation "io.ktor:ktor-auth:$ktor_auth_version" + implementation "io.ktor:ktor-serialization-jackson:$ktor_version" testImplementation 'org.jetbrains.kotlin:kotlin-test-junit:1.5.21' testImplementation 'org.jetbrains.kotlin:kotlin-test:1.5.21' testImplementation group: "junit", name: "junit", version: '4.13.2' - - implementation project(":util") } - application { mainClassName = "tech.libeufin.sandbox.MainKt" applicationName = "libeufin-sandbox" diff --git a/sandbox/src/main/kotlin/tech/libeufin/sandbox/CircuitApi.kt b/sandbox/src/main/kotlin/tech/libeufin/sandbox/CircuitApi.kt @@ -1,8 +1,14 @@ package tech.libeufin.sandbox -import io.ktor.application.* -import io.ktor.response.* -import io.ktor.routing.* +import io.ktor.server.application.* +import io.ktor.http.* +import io.ktor.server.request.* +import io.ktor.server.response.* +import io.ktor.server.routing.* +import org.jetbrains.exposed.sql.transactions.transaction +import tech.libeufin.util.InvalidPaytoError +import tech.libeufin.util.conflict +import tech.libeufin.util.parsePayto // CIRCUIT API TYPES @@ -22,11 +28,78 @@ class RatioAndFees( val sell_out_fee: Float = 0F ) +// User registration request +class CircuitAccountRequest( + val username: String, + val password: String, + val contact_data: CircuitAccountData, + val name: String, + val cashout_address: String, // payto + val internal_iban: String? // Shall be "= null" ? +) +// User contact data to send the TAN. +class CircuitAccountData( + val email: String?, + val phone: String? +) + +/** + * Allows only the administrator to add new accounts. + */ fun circuitApi(circuitRoute: Route) { + circuitRoute.post("/accounts") { + call.request.basicAuth(onlyAdmin = true) + val req = call.receive<CircuitAccountRequest>() + // Validity and availability check on the input data. + if (req.contact_data.email != null) { + val maybeEmailConflict = DemobankCustomerEntity.find { + DemobankCustomersTable.email eq req.contact_data.email + }.firstOrNull() + if (maybeEmailConflict != null) { + // Warning since two individuals claimed one same e-mail address. + logger.warn("Won't register user ${req.username}: e-mail conflict on ${req.contact_data.email}") + throw conflict("E-mail address already in use!") + } + // Syntactic validation. Warn on error, since UI could avoid this. + // FIXME + // From Taler TypeScript: + // /^(([^<>()[\]\\.,;:\s@"]+(\.[^<>()[\]\\.,;:\s@"]+)*)|(".+"))@((\[[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}])|(([a-zA-Z\-0-9]+\.)+[a-zA-Z]{2,}))$/; + } + if (req.contact_data.phone != null) { + val maybePhoneConflict = DemobankCustomerEntity.find { + DemobankCustomersTable.phone eq req.contact_data.phone + }.firstOrNull() + if (maybePhoneConflict != null) { + // Warning since two individuals claimed one same phone number. + logger.warn("Won't register user ${req.username}: phone conflict on ${req.contact_data.email}") + throw conflict("Phone number already in use!") + } + // Syntactic validation. Warn on error, since UI could avoid this. + // FIXME + // From Taler TypeScript + // /^\+[0-9 ]*$/; + } + // Check that cash-out address parses. + try { + parsePayto(req.cashout_address) + } catch (e: InvalidPaytoError) { + // Warning because the UI could avoid this. + logger.warn("Won't register account ${req.username}: invalid cash-out address: ${req.cashout_address}") + } + transaction { + val newAccount = insertNewAccount( + username = req.username, + password = req.password, + name = req.name + ) + newAccount.customer.phone = req.contact_data.phone + newAccount.customer.email = req.contact_data.email + } + call.respond(HttpStatusCode.NoContent) + return@post + } circuitRoute.get("/config") { - call.respond(ConfigResp( - ratios_and_fees = RatioAndFees() - )) + call.respond(ConfigResp(ratios_and_fees = RatioAndFees())) return@get } } \ No newline at end of file diff --git a/sandbox/src/main/kotlin/tech/libeufin/sandbox/DB.kt b/sandbox/src/main/kotlin/tech/libeufin/sandbox/DB.kt @@ -120,6 +120,8 @@ object DemobankCustomersTable : LongIdTable() { val username = text("username") val passwordHash = text("passwordHash") val name = text("name").nullable() + val email = text("email").nullable() + val phone = text("phone").nullable() } class DemobankCustomerEntity(id: EntityID<Long>) : LongEntity(id) { @@ -127,6 +129,8 @@ class DemobankCustomerEntity(id: EntityID<Long>) : LongEntity(id) { var username by DemobankCustomersTable.username var passwordHash by DemobankCustomersTable.passwordHash var name by DemobankCustomersTable.name + var email by DemobankCustomersTable.email + var phone by DemobankCustomersTable.phone } /** @@ -370,7 +374,9 @@ class BankAccountTransactionEntity(id: EntityID<Long>) : LongEntity(id) { /** * Table that keeps information about which bank accounts (iban+bic+name) - * are active in the system. + * are active in the system. In the current version, 'label' and 'owner' + * are always equal; future versions may change this, when one customer can + * own multiple bank accounts. */ object BankAccountsTable : IntIdTable() { val iban = text("iban") diff --git a/sandbox/src/main/kotlin/tech/libeufin/sandbox/EbicsProtocolBackend.kt b/sandbox/src/main/kotlin/tech/libeufin/sandbox/EbicsProtocolBackend.kt @@ -20,12 +20,12 @@ package tech.libeufin.sandbox -import io.ktor.application.* +import io.ktor.server.application.* import io.ktor.http.ContentType import io.ktor.http.HttpStatusCode -import io.ktor.request.* -import io.ktor.response.respond -import io.ktor.response.respondText +import io.ktor.server.request.* +import io.ktor.server.response.respond +import io.ktor.server.response.respondText import io.ktor.util.AttributeKey import org.apache.xml.security.binding.xmldsig.RSAKeyValueType import org.jetbrains.exposed.sql.* diff --git a/sandbox/src/main/kotlin/tech/libeufin/sandbox/Helpers.kt b/sandbox/src/main/kotlin/tech/libeufin/sandbox/Helpers.kt @@ -19,9 +19,9 @@ package tech.libeufin.sandbox -import io.ktor.application.* +import io.ktor.server.application.* import io.ktor.http.HttpStatusCode -import io.ktor.request.* +import io.ktor.server.request.* import org.jetbrains.exposed.sql.SqlExpressionBuilder.eq import org.jetbrains.exposed.sql.and import org.jetbrains.exposed.sql.transactions.transaction @@ -45,6 +45,89 @@ data class SandboxCamt( ) /** + * DB helper inserting a new "account" into the database. + * The account is made of a 'customer' and 'bank account' + * object. The helper checks first that the username is + * acceptable (chars, no institutional names, available + * names); then checks that IBAN is available and then adds + * the two database objects under the given demobank. This + * function contains the common logic shared by the Access + * and Circuit API. Additional data that is peculiar to one + * API should be added separately. + * + * It returns a AccountPair type. That contains the customer + * object and the bank account; the caller may this way add custom + * values to them. */ +data class AccountPair( + val customer: DemobankCustomerEntity, + val bankAccount: BankAccountEntity +) +fun insertNewAccount(username: String, + password: String, + name: String? = null, // tests do not usually give one. + iban: String? = null, + isPublic: Boolean = false, + demobank: String = "default"): AccountPair { + requireValidResourceName(username) + // Forbid institutional usernames. + if (username == "bank" || username == "admin") { + logger.info("Username: $username not allowed.") + throw forbidden("Username: $username is not allowed.") + } + + val demobankFromDb = getDemobank(demobank) + // Bank's fault, because when this function gets + // called, the demobank must exist. + if (demobankFromDb == null) { + logger.error("Demobank '$demobank' not found. Won't add account $username") + throw internalServerError("Demobank $demobank not found. Won't add account $username") + } + // Generate a IBAN if the caller didn't provide one. + val newIban = iban ?: getIban() + // Check IBAN collisions. + val checkIbanExist = BankAccountEntity.find(BankAccountsTable.iban eq newIban).firstOrNull() + if (checkIbanExist != null) { + logger.info("IBAN $newIban not available. Won't register username $username") + throw conflict("IBAN $iban not available.") + } + // Check username availability. + val checkCustomerExist = transaction { + DemobankCustomerEntity.find { + DemobankCustomersTable.username eq username + }.firstOrNull() + } + if (checkCustomerExist != null) { + throw SandboxError( + HttpStatusCode.Conflict, + "Username $username not available." + ) + } + val newCustomer = DemobankCustomerEntity.new { + this.username = username + passwordHash = CryptoUtil.hashpw(password) + this.name = name // nullable + } + // Actual account creation. + val newBankAccount = BankAccountEntity.new { + this.iban = newIban + /** + * For now, keep same semantics of Pybank: a username + * is AS WELL a bank account label. In other words, it + * identifies a customer AND a bank account. The reason + * to have the two values (label and owner) is to allow + * multiple bank accounts being owned by one customer. + */ + label = username + owner = username + this.demoBank = demobankFromDb + this.isPublic = isPublic + } + if (demobankFromDb.withSignupBonus) + newBankAccount.bonus("${demobankFromDb.currency}:100") + return AccountPair(customer = newCustomer, bankAccount = newBankAccount) +} + +/** * * Return true if access to the bank account can be granted, * false otherwise. @@ -90,10 +173,7 @@ fun ApplicationRequest.basicAuth(onlyAdmin: Boolean = false): String? { ) return credentials.first } - /** - * If only admin auth was allowed, here it failed already, - * hence throw 401. */ - if (onlyAdmin) throw unauthorized("Only admin allowed.") + if (onlyAdmin) throw forbidden("Only admin allowed.") val passwordHash = transaction { val customer = getCustomer(credentials.first) customer.passwordHash diff --git a/sandbox/src/main/kotlin/tech/libeufin/sandbox/Main.kt b/sandbox/src/main/kotlin/tech/libeufin/sandbox/Main.kt @@ -21,11 +21,15 @@ package tech.libeufin.sandbox import UtilError import com.fasterxml.jackson.core.JsonParseException +import com.fasterxml.jackson.core.JsonProcessingException import com.fasterxml.jackson.core.util.DefaultIndenter import com.fasterxml.jackson.core.util.DefaultPrettyPrinter +import com.fasterxml.jackson.databind.DeserializationFeature +import com.fasterxml.jackson.databind.JsonMappingException import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.databind.SerializationFeature import com.fasterxml.jackson.databind.exc.MismatchedInputException +import com.fasterxml.jackson.module.kotlin.KotlinFeature import com.fasterxml.jackson.module.kotlin.KotlinModule import com.fasterxml.jackson.module.kotlin.MissingKotlinParameterException import com.github.ajalt.clikt.core.CliktCommand @@ -36,15 +40,21 @@ import com.github.ajalt.clikt.parameters.arguments.argument import com.github.ajalt.clikt.parameters.options.* import com.github.ajalt.clikt.parameters.types.int import execThrowableOrTerminate -import io.ktor.application.* -import io.ktor.features.* +import io.ktor.server.application.* import io.ktor.http.* -import io.ktor.jackson.* -import io.ktor.request.* -import io.ktor.response.* -import io.ktor.routing.* +import io.ktor.serialization.* +import io.ktor.serialization.jackson.* import io.ktor.server.engine.* import io.ktor.server.netty.* +import io.ktor.server.plugins.* +import io.ktor.server.plugins.contentnegotiation.* +import io.ktor.server.plugins.statuspages.* +import io.ktor.server.request.* +import io.ktor.server.response.* +import io.ktor.server.routing.* +import io.ktor.server.util.* +import io.ktor.server.plugins.callloging.* +import io.ktor.server.plugins.cors.routing.* import io.ktor.util.* import io.ktor.util.date.* import kotlinx.coroutines.* @@ -446,26 +456,22 @@ fun main(args: Array<String>) { ).main(args) } -suspend inline fun <reified T : Any> ApplicationCall.receiveJson(): T { - try { - return this.receive() - } catch (e: MissingKotlinParameterException) { - throw SandboxError(HttpStatusCode.BadRequest, "Missing value for ${e.pathReference}") - } catch (e: MismatchedInputException) { - // Note: POSTing "[]" gets here but e.pathReference is blank. - throw SandboxError(HttpStatusCode.BadRequest, "Invalid value for '${e.pathReference}'") - } catch (e: JsonParseException) { - throw SandboxError(HttpStatusCode.BadRequest, "Invalid JSON") - } -} - fun setJsonHandler(ctx: ObjectMapper) { ctx.enable(SerializationFeature.INDENT_OUTPUT) ctx.setDefaultPrettyPrinter(DefaultPrettyPrinter().apply { indentArraysWith(DefaultPrettyPrinter.FixedSpaceIndenter.instance) indentObjectsWith(DefaultIndenter(" ", "\n")) }) - ctx.registerModule(KotlinModule(nullisSameAsDefault = true)) + ctx.registerModule( + KotlinModule.Builder() + .withReflectionCacheSize(512) + .configure(KotlinFeature.NullToEmptyCollection, false) + .configure(KotlinFeature.NullToEmptyMap, false) + .configure(KotlinFeature.NullIsSameAsDefault, enabled = true) + .configure(KotlinFeature.SingletonSupport, enabled = false) + .configure(KotlinFeature.StrictNullChecks, false) + .build() + ) } val sandboxApp: Application.() -> Unit = { @@ -478,14 +484,14 @@ val sandboxApp: Application.() -> Unit = { } install(CORS) { anyHost() - header(HttpHeaders.Authorization) - header(HttpHeaders.ContentType) - method(HttpMethod.Options) - // logger.info("Enabling CORS (assuming no endpoint uses cookies).") + allowHeader(HttpHeaders.Authorization) + allowHeader(HttpHeaders.ContentType) + allowMethod(HttpMethod.Options) allowCredentials = true } install(IgnoreTrailingSlash) install(ContentNegotiation) { + register(ContentType.Text.Xml, XMLEbicsConverter()) /** * Content type "text" must go to the XML parser @@ -502,7 +508,7 @@ val sandboxApp: Application.() -> Unit = { } install(StatusPages) { // Bank's fault: it should check the operands. Respond 500 - exception<ArithmeticException> { cause -> + exception<ArithmeticException> { call, cause -> logger.error("Exception while handling '${call.request.uri}', ${cause.stackTraceToString()}") call.respond( HttpStatusCode.InternalServerError, @@ -514,8 +520,9 @@ val sandboxApp: Application.() -> Unit = { ) ) } - exception<SandboxError> { cause -> - logger.error("Exception while handling '${call.request.uri}', ${cause.reason}") + // Not necessarily the bank's fault. + exception<SandboxError> { call, cause -> + logger.debug("Exception while handling '${call.request.uri}', ${cause.reason}") call.respond( cause.statusCode, SandboxErrorJson( @@ -526,8 +533,9 @@ val sandboxApp: Application.() -> Unit = { ) ) } - exception<UtilError> { cause -> - logger.error("Exception while handling '${call.request.uri}', ${cause.reason}") + // Not necessarily the bank's fault. + exception<UtilError> { call, cause -> + logger.debug("Exception while handling '${call.request.uri}', ${cause.reason}") call.respond( cause.statusCode, SandboxErrorJson( @@ -538,9 +546,30 @@ val sandboxApp: Application.() -> Unit = { ) ) } + // Happens when a request fails to parse. + exception<BadRequestException> { call, wrapper -> + var errorMessage: String? = wrapper.message // default, if no further details can be found. + if (errorMessage == null) { + logger.error("The bank didn't detect the cause of a bad request, fail.") + throw SandboxError( + HttpStatusCode.InternalServerError, + "Did not find bad request details." + ) + } + logger.debug(errorMessage) + call.respond( + HttpStatusCode.BadRequest, + SandboxErrorJson( + error = SandboxErrorDetailJson( + type = "util-error", + description = errorMessage + ) + ) + ) + } // Catch-all error, respond 500 because the bank didn't handle it. - exception<Throwable> { cause -> - logger.error("Exception while handling '${call.request.uri}'", cause.stackTrace) + exception<Throwable> { call, cause -> + logger.error("Unhandled exception while handling '${call.request.uri}'\n${cause.stackTraceToString()}") call.respond( HttpStatusCode.InternalServerError, SandboxErrorJson( @@ -551,9 +580,9 @@ val sandboxApp: Application.() -> Unit = { ) ) } - exception<EbicsRequestError> { e -> - logger.info("Handling EbicsRequestError: ${e.message}") - respondEbicsTransfer(call, e.errorText, e.errorCode) + exception<EbicsRequestError> { call, cause -> + logger.info("Handling EbicsRequestError: ${cause.message}") + respondEbicsTransfer(call, cause.errorText, cause.errorCode) } } intercept(ApplicationCallPipeline.Setup) { @@ -568,10 +597,7 @@ val sandboxApp: Application.() -> Unit = { ) } - ac.attributes.put( - ADMIN_PASSWORD_ATTRIBUTE_KEY, - adminPassword - ) + ac.attributes.put(ADMIN_PASSWORD_ATTRIBUTE_KEY, adminPassword) } return@intercept } @@ -586,7 +612,6 @@ val sandboxApp: Application.() -> Unit = { } } routing { - get("/") { call.respondText( "Hello, this is the Sandbox\n", @@ -597,7 +622,7 @@ val sandboxApp: Application.() -> Unit = { // Query details in the body. post("/admin/payments/camt") { val username = call.request.basicAuth() - val body = call.receiveJson<CamtParams>() + val body = call.receive<CamtParams>() if (body.type != 53) throw SandboxError( HttpStatusCode.NotFound, "Only Camt.053 documents can be generated." @@ -630,7 +655,7 @@ val sandboxApp: Application.() -> Unit = { */ post("/admin/bank-accounts/{label}") { val username = call.request.basicAuth() - val body = call.receiveJson<BankAccountInfo>() + val body = call.receive<BankAccountInfo>() if (!allowOwnerOrAdmin(username, body.label)) throw unauthorized("User '$username' has no rights over" + " bank account '${body.label}'" @@ -688,7 +713,7 @@ val sandboxApp: Application.() -> Unit = { // The debtor is not required to have a customer account at this Sandbox. post("/admin/bank-accounts/{label}/simulate-incoming-transaction") { call.request.basicAuth(onlyAdmin = true) - val body = call.receiveJson<IncomingPaymentInfo>() + val body = call.receive<IncomingPaymentInfo>() val accountLabel = ensureNonNull(call.parameters["label"]) val reqDebtorBic = body.debtorBic if (reqDebtorBic != null && !validateBic(reqDebtorBic)) { @@ -734,7 +759,7 @@ val sandboxApp: Application.() -> Unit = { // Associates a new bank account with an existing Ebics subscriber. post("/admin/ebics/bank-accounts") { call.request.basicAuth(onlyAdmin = true) - val body = call.receiveJson<EbicsBankAccountRequest>() + val body = call.receive<EbicsBankAccountRequest>() if (!validateBic(body.bic)) { throw SandboxError(HttpStatusCode.BadRequest, "invalid BIC (${body.bic})") } @@ -907,7 +932,7 @@ val sandboxApp: Application.() -> Unit = { */ post("/admin/ebics/subscribers") { call.request.basicAuth(onlyAdmin = true) - val body = call.receiveJson<EbicsSubscriberObsoleteApi>() + val body = call.receive<EbicsSubscriberObsoleteApi>() transaction { // Check it exists first. val maybeSubscriber = EbicsSubscriberEntity.find { @@ -984,11 +1009,18 @@ val sandboxApp: Application.() -> Unit = { // Create a new EBICS host post("/admin/ebics/hosts") { call.request.basicAuth(onlyAdmin = true) - val req = call.receiveJson<EbicsHostCreateRequest>() + val req = call.receive<EbicsHostCreateRequest>() val pairA = CryptoUtil.generateRsaKeyPair(2048) val pairB = CryptoUtil.generateRsaKeyPair(2048) val pairC = CryptoUtil.generateRsaKeyPair(2048) transaction { + val maybeHost = EbicsHostEntity.find { + EbicsHostsTable.hostID eq req.hostID + }.firstOrNull() + if (maybeHost != null) { + logger.info("EBICS host '${req.hostID}' exists already, this request conflicts.") + throw conflict("EBICS host '${req.hostID}' exists already") + } EbicsHostEntity.new { this.ebicsVersion = req.ebicsVersion this.hostId = req.hostID @@ -1091,7 +1123,7 @@ val sandboxApp: Application.() -> Unit = { } logger.debug("TWG add-incoming passed authentication") val body = try { - call.receiveJson<TWGAdminAddIncoming>() + call.receive<TWGAdminAddIncoming>() } catch (e: Exception) { logger.error("/admin/add-incoming failed at parsing the request body") throw SandboxError( @@ -1133,7 +1165,7 @@ val sandboxApp: Application.() -> Unit = { } post("/withdrawal-operation/{wopid}") { val wopid: String = ensureNonNull(call.parameters["wopid"]) - val body = call.receiveJson<TalerWithdrawalSelection>() + val body = call.receive<TalerWithdrawalSelection>() val transferDone = transaction { val wo = TalerWithdrawalEntity.find { TalerWithdrawalsTable.wopid eq java.util.UUID.fromString(wopid) @@ -1201,7 +1233,7 @@ val sandboxApp: Application.() -> Unit = { route("/access-api") { post("/accounts/{account_name}/transactions") { val bankAccount = getBankAccountWithAuth(call) - val req = call.receiveJson<NewTransactionReq>() + val req = call.receive<NewTransactionReq>() val payto = parsePayto(req.paytoUri) val amount: String? = payto.amount ?: req.amount if (amount == null) throw badRequest("Amount is missing") @@ -1259,7 +1291,7 @@ val sandboxApp: Application.() -> Unit = { if (maybeOwnedAccount.owner != username && WITH_AUTH) throw unauthorized( "Customer '$username' has no rights over bank account '${maybeOwnedAccount.label}'" ) - val req = call.receiveJson<WithdrawalRequest>() + val req = call.receive<WithdrawalRequest>() // Check for currency consistency val amount = parseAmount(req.amount) if (amount.currency != demobank.currency) @@ -1294,21 +1326,23 @@ val sandboxApp: Application.() -> Unit = { -1 ) host = "withdraw" - pathComponents( - /** - * encodes the hostname(+port) of the actual - * bank that will serve the withdrawal request. - */ - baseUrl.host.plus( - if (baseUrl.port != -1) - ":${baseUrl.port}" - else "" - ), - baseUrl.path, // has x-forwarded-prefix, or single slash. - "demobanks", - demobank.name, - "integration-api", - wo.wopid.toString() + this.appendPathSegments( + listOf( + /** + * encodes the hostname(+port) of the actual + * bank that will serve the withdrawal request. + */ + baseUrl.host.plus( + if (baseUrl.port != -1) + ":${baseUrl.port}" + else "" + ), + baseUrl.path, // has x-forwarded-prefix, or single slash. + "demobanks", + demobank.name, + "integration-api", + wo.wopid.toString() + ) ) } call.respond(object { @@ -1516,69 +1550,28 @@ val sandboxApp: Application.() -> Unit = { "The bank doesn't allow new registrations at the moment." ) } - val req = call.receiveJson<CustomerRegistration>() - // Forbid 'admin' or 'bank' usernames. - if (req.username == "bank" || req.username == "admin") - throw forbidden("Unallowed username: ${req.username}") - val checkCustomerExist = transaction { - DemobankCustomerEntity.find { - DemobankCustomersTable.username eq req.username - }.firstOrNull() - } - /** - * Not allowing 'bank' username, as it's been assigned - * to the default bank's bank account. - */ - if (checkCustomerExist != null) { - throw SandboxError( - HttpStatusCode.Conflict, - "Username ${req.username} not available." - ) - } - val newIban = req.iban ?: getIban() - // Double-check if IBAN was taken already. - val checkIbanExist = transaction { - BankAccountEntity.find(BankAccountsTable.iban eq newIban).firstOrNull() - } - if (checkIbanExist != null) - throw conflict("Proposed IBAN not available.") - - // Create new customer. - requireValidResourceName(req.username) - val bankAccount = transaction { - val bankAccount = BankAccountEntity.new { - iban = newIban - /** - * For now, keep same semantics of Pybank: a username - * is AS WELL a bank account label. In other words, it - * identifies a customer AND a bank account. - */ - label = req.username - owner = req.username - this.demoBank = demobank + val req = call.receive<CustomerRegistration>() + val newAccount = transaction { + insertNewAccount( + req.username, + req.password, + name = req.name, + iban = req.iban, isPublic = req.isPublic - } - DemobankCustomerEntity.new { - username = req.username - passwordHash = CryptoUtil.hashpw(req.password) - name = req.name // nullable - } - if (demobank.withSignupBonus) - bankAccount.bonus("${demobank.currency}:100") - bankAccount + ) } - val balance = getBalance(bankAccount, withPending = true) + val balance = getBalance(newAccount.bankAccount, withPending = true) call.respond(object { val balance = object { - val amount = "${demobank.currency}:$balance" - val credit_debit_indicator = "CRDT" + val amount = "${demobank.currency}:${balance.abs()}" + val credit_debit_indicator = if (balance < BigDecimal.ZERO) "DBIT" else "CRDT" } val paytoUri = buildIbanPaytoUri( - iban = bankAccount.iban, - bic = bankAccount.bic, + iban = newAccount.bankAccount.iban, + bic = newAccount.bankAccount.bic, receiverName = getPersonNameFromCustomer(req.username) ) - val iban = bankAccount.iban + val iban = newAccount.bankAccount.iban }) return@post } @@ -1592,7 +1585,7 @@ val sandboxApp: Application.() -> Unit = { // Only the admin can create Ebics subscribers. val user = call.request.basicAuth() if (user != "admin") throw forbidden("Only the Admin can create Ebics subscribers.") - val body = call.receiveJson<EbicsSubscriberInfo>() + val body = call.receive<EbicsSubscriberInfo>() // Create or get the Ebics subscriber that is found. transaction { val subscriber: EbicsSubscriberEntity = EbicsSubscriberEntity.find { @@ -1633,7 +1626,7 @@ fun serverMain(port: Int, localhostOnly: Boolean, ipv4Only: Boolean) { this.port = port this.host = if (localhostOnly) "[::1]" else "[::]" } - parentCoroutineContext = Dispatchers.Main + // parentCoroutineContext = Dispatchers.Main module(sandboxApp) }, configure = { diff --git a/sandbox/src/main/kotlin/tech/libeufin/sandbox/XMLEbicsConverter.kt b/sandbox/src/main/kotlin/tech/libeufin/sandbox/XMLEbicsConverter.kt @@ -1,27 +1,25 @@ package tech.libeufin.sandbox -import io.ktor.application.* -import io.ktor.features.* import io.ktor.http.* import io.ktor.http.content.* -import io.ktor.request.* -import io.ktor.response.* -import io.ktor.util.pipeline.* +import io.ktor.serialization.* +import io.ktor.util.reflect.* import io.ktor.utils.io.* +import io.ktor.utils.io.charsets.* import io.ktor.utils.io.jvm.javaio.* import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.withContext import tech.libeufin.util.XMLUtil -import java.io.OutputStream -import java.nio.channels.ByteChannel class XMLEbicsConverter : ContentConverter { - override suspend fun convertForReceive( - context: PipelineContext<ApplicationReceiveRequest, ApplicationCall>): Any? { - val value = context.subject.value as? ByteReadChannel ?: return null + override suspend fun deserialize( + charset: io.ktor.utils.io.charsets.Charset, + typeInfo: io.ktor.util.reflect.TypeInfo, + content: ByteReadChannel + ): Any? { return withContext(Dispatchers.IO) { try { - receiveEbicsXmlInternal(value.toInputStream().reader().readText()) + receiveEbicsXmlInternal(content.toInputStream().reader().readText()) } catch (e: Exception) { throw SandboxError( HttpStatusCode.BadRequest, @@ -30,11 +28,28 @@ class XMLEbicsConverter : ContentConverter { } } } - override suspend fun convertForSend( - context: PipelineContext<Any, ApplicationCall>, + + // The following annotation was suggested by Intellij. + @Deprecated( + "Please override and use serializeNullable instead", + replaceWith = ReplaceWith("serializeNullable(charset, typeInfo, contentType, value)"), + level = DeprecationLevel.WARNING + ) + override suspend fun serialize( contentType: ContentType, + charset: Charset, + typeInfo: TypeInfo, value: Any - ): Any? { + ): OutgoingContent? { + return super.serializeNullable(contentType, charset, typeInfo, value) + } + + override suspend fun serializeNullable( + contentType: ContentType, + charset: Charset, + typeInfo: TypeInfo, + value: Any? + ): OutgoingContent? { val conv = try { XMLUtil.convertJaxbToString(value) } catch (e: Exception) { @@ -42,7 +57,6 @@ class XMLEbicsConverter : ContentConverter { * Not always a error: the content negotiation might have * only checked if this handler could convert the response. */ - // logger.info("Could not use XML custom converter for this response.") return null } return OutputStreamContent({ @@ -50,7 +64,7 @@ class XMLEbicsConverter : ContentConverter { withContext(Dispatchers.IO) { out.write(conv.toByteArray()) }}, - contentType.withCharset(context.call.suitableCharset()) + contentType.withCharset(charset) ) } } \ No newline at end of file diff --git a/util/build.gradle b/util/build.gradle @@ -26,12 +26,9 @@ sourceSets { main.java.srcDirs = ['src/main/java', 'src/main/kotlin'] } -def exposed_version = '0.32.1' def netty_version = '4.1.68.Final' -def ktor_version = '1.6.1' dependencies { - implementation 'org.jetbrains.kotlin:kotlin-stdlib-jdk8:1.5.21' implementation 'io.ktor:ktor-server-netty:1.6.1' implementation 'ch.qos.logback:logback-classic:1.2.5' @@ -53,8 +50,9 @@ dependencies { implementation "io.netty:netty-all:$netty_version" implementation "io.netty:netty-transport-native-epoll:$netty_version" implementation "io.ktor:ktor-server-test-host:$ktor_version" - implementation "io.ktor:ktor-jackson:$ktor_version" + implementation "io.ktor:ktor-serialization-jackson:$ktor_version" + testImplementation "io.ktor:ktor-server-content-negotiation:$ktor_version" testImplementation group: 'junit', name: 'junit', version: '4.13.2' testImplementation 'org.jetbrains.kotlin:kotlin-test-junit:1.5.21' testImplementation 'org.jetbrains.kotlin:kotlin-test:1.5.21' diff --git a/util/src/main/kotlin/Config.kt b/util/src/main/kotlin/Config.kt @@ -4,7 +4,7 @@ import ch.qos.logback.classic.Level import ch.qos.logback.classic.LoggerContext import ch.qos.logback.classic.util.ContextInitializer import ch.qos.logback.core.util.Loader -import io.ktor.application.* +import io.ktor.server.application.* import io.ktor.util.* import org.slf4j.LoggerFactory import printLnErr @@ -63,12 +63,6 @@ fun setLogLevel(logLevel: String?) { } } -internal fun <T : Any>ApplicationCall.maybeAttribute(name: String): T? { - val key = AttributeKey<T>("name") - if (!this.attributes.contains(key)) return null - return this.attributes[key] -} - /** * Retun the attribute, or throw 500 Internal server error. */ diff --git a/util/src/main/kotlin/HTTP.kt b/util/src/main/kotlin/HTTP.kt @@ -1,12 +1,12 @@ package tech.libeufin.util import UtilError -import io.ktor.application.* import io.ktor.http.* -import io.ktor.request.* +import io.ktor.server.application.* +import io.ktor.server.request.* +import io.ktor.server.util.* import io.ktor.util.* import logger -import org.jetbrains.exposed.sql.transactions.transaction import java.net.URLDecoder fun unauthorized(msg: String): UtilError { @@ -85,9 +85,8 @@ fun conflict(msg: String): UtilError { fun ApplicationRequest.getBaseUrl(): String { return if (this.headers.contains("X-Forwarded-Host")) { logger.info("Building X-Forwarded- base URL") - /** - * FIXME: should tolerate a missing X-Forwarded-Prefix. - */ + + // FIXME: should tolerate a missing X-Forwarded-Prefix. var prefix: String = this.headers.get("X-Forwarded-Prefix") ?: throw internalServerError("Reverse proxy did not define X-Forwarded-Prefix") if (!prefix.endsWith("/")) @@ -100,8 +99,8 @@ fun ApplicationRequest.getBaseUrl(): String { host = this.headers.get("X-Forwarded-Host") ?: throw internalServerError( "Reverse proxy did not define X-Forwarded-Host" ), - encodedPath = prefix ).apply { + encodedPath = prefix // Gets dropped otherwise. if (!encodedPath.endsWith("/")) encodedPath += "/" @@ -138,7 +137,7 @@ fun expectAdmin(username: String?) { if (username != "admin") throw unauthorized("Only admin allowed: $username is not.") } -fun getHTTPBasicAuthCredentials(request: ApplicationRequest): Pair<String, String> { +fun getHTTPBasicAuthCredentials(request: io.ktor.server.request.ApplicationRequest): Pair<String, String> { val authHeader = getAuthorizationHeader(request) return extractUserAndPassword(authHeader) } @@ -168,7 +167,6 @@ fun buildBasicAuthLine(username: String, password: String): String { * will then be compared with the one kept into the database. */ fun extractUserAndPassword(authorizationHeader: String): Pair<String, String> { - // logger.debug("Authenticating: $authorizationHeader") val (username, password) = try { // FIXME/note: line below doesn't check for "Basic" presence. val split = authorizationHeader.split(" ") @@ -181,7 +179,7 @@ fun extractUserAndPassword(authorizationHeader: String): Pair<String, String> { } catch (e: Exception) { throw UtilError( HttpStatusCode.BadRequest, - "invalid Authorization:-header received: ${e.message}", + "invalid Authorization header received: ${e.message}", LibeufinErrorCode.LIBEUFIN_EC_AUTHENTICATION_FAILED ) } diff --git a/util/src/main/kotlin/UnixDomainSocket.kt b/util/src/main/kotlin/UnixDomainSocket.kt @@ -1,4 +1,6 @@ -import io.ktor.application.* +import io.ktor.client.plugins.* +import io.ktor.client.request.* +import io.ktor.server.application.* import io.ktor.client.statement.* import io.ktor.http.* import io.ktor.http.HttpHeaders @@ -67,30 +69,24 @@ class LibeufinHttpInit( class LibeufinHttpHandler( private val app: Application.() -> Unit ) : SimpleChannelInboundHandler<FullHttpRequest>() { - @OptIn(EngineAPI::class) + // @OptIn(EngineAPI::class) override fun channelRead0(ctx: ChannelHandlerContext, msg: FullHttpRequest) { - withTestApplication(app) { + testApplication { + application(app) val httpVersion = msg.protocolVersion() // Proxying the request to Ktor API. - val call = handleRequest { - msg.headers().forEach { addHeader(it.key, it.value) } + val r = client.request(msg.uri()) { + expectSuccess = false method = HttpMethod(msg.method().name()) - uri = msg.uri() - version = httpVersion.text() setBody(ByteBufInputStream(msg.content()).readAllBytes()) } - val statusCode: Int = call.response.status()?.value ?: throw UtilError( - HttpStatusCode.InternalServerError, - "app proxied via Unix domain socket did not include a response status code", - ec = null // FIXME: to be defined. - ) // Responding to Netty API. val response = DefaultHttpResponse( httpVersion, - HttpResponseStatus.valueOf(statusCode) + HttpResponseStatus.valueOf(r.status.value) ) var chunked = false - call.response.headers.allValues().forEach { s, list -> + r.headers.forEach { s, list -> if (s == HttpHeaders.TransferEncoding && list.contains("chunked")) chunked = true response.headers().set(s, list.joinToString()) @@ -100,12 +96,12 @@ class LibeufinHttpHandler( ctx.writeAndFlush( HttpChunkedInput( ChunkedStream( - ByteArrayInputStream(call.response.byteContent) + ByteArrayInputStream(r.readBytes()) ) ) ) } else { - ctx.writeAndFlush(Unpooled.wrappedBuffer(call.response.byteContent)) + ctx.writeAndFlush(Unpooled.wrappedBuffer(r.readBytes())) } } } diff --git a/util/src/test/kotlin/DomainSocketTest.kt b/util/src/test/kotlin/DomainSocketTest.kt @@ -3,16 +3,16 @@ import com.fasterxml.jackson.core.util.DefaultPrettyPrinter import com.fasterxml.jackson.databind.DeserializationFeature import com.fasterxml.jackson.databind.SerializationFeature import com.fasterxml.jackson.module.kotlin.KotlinModule -import io.ktor.application.* -import io.ktor.features.* +import io.ktor.server.application.* import io.ktor.http.* -import io.ktor.response.* -import io.ktor.routing.* +import io.ktor.server.response.* +import io.ktor.server.routing.* import org.junit.Test -import io.ktor.jackson.jackson -import io.ktor.request.* +import io.ktor.serialization.jackson.* +import io.ktor.server.request.* import org.junit.Assert import org.junit.Ignore +import io.ktor.server.plugins.contentnegotiation.* class DomainSocketTest { @Test @Ignore