commit 69f7cd208c41ba0436984b8455f8652a4f537f76
parent 3eb59a3e6e9505fd6786310dd4e6eb5810ae7091
Author: Marcello Stanisci <stanisci.m@gmail.com>
Date: Mon, 20 Apr 2020 21:18:15 +0200
Get bank testcases up to 'credit-1' to pass.
The following changes were needed:
* Implement decompression of upload data.
* Wrap timestamps within 'ms_t'.
* Allow x-taler-bank as Payto type.
* Introduce NEXUS_PRODUCTION env variable to allow
EBICS-free tests.
* Avoid Gson serializer upon respond, as it includes
the "charset" token into the Content-Type response
header, and this latter makes the exchange parser unhappy.
* Store booking date in Long format (Raw payment table).
* Avoid the "204 No Content" HTTP status code, as it makes
the exchange unhappy; return "200 OK" on empty histories
instead.
Diffstat:
4 files changed, 159 insertions(+), 57 deletions(-)
diff --git a/nexus/src/main/kotlin/tech/libeufin/nexus/DB.kt b/nexus/src/main/kotlin/tech/libeufin/nexus/DB.kt
@@ -96,7 +96,7 @@ object EbicsRawBankTransactionsTable : LongIdTable() {
val debitorIban = text("debitorIban")
val debitorName = text("debitorName")
val counterpartBic = text("counterpartBic")
- val bookingDate = text("bookingDate")
+ val bookingDate = long("bookingDate")
val status = text("status") // BOOK, ..
}
diff --git a/nexus/src/main/kotlin/tech/libeufin/nexus/Helpers.kt b/nexus/src/main/kotlin/tech/libeufin/nexus/Helpers.kt
@@ -4,6 +4,8 @@ import io.ktor.application.ApplicationCall
import io.ktor.http.HttpStatusCode
import org.jetbrains.exposed.sql.and
import org.jetbrains.exposed.sql.transactions.transaction
+import org.joda.time.DateTime
+import org.joda.time.format.DateTimeFormat
import tech.libeufin.util.CryptoUtil
import tech.libeufin.util.base64ToBytes
import javax.sql.rowset.serial.SerialBlob
@@ -103,10 +105,9 @@ fun extractUserAndHashedPassword(authorizationHeader: String): Pair<String, Byte
* @return subscriber id
*/
fun authenticateRequest(authorization: String?): String {
- val headerLine = authorization ?: throw NexusError(
+ val headerLine = if (authorization == null) throw NexusError(
HttpStatusCode.BadRequest, "Authentication:-header line not found"
- )
- logger.debug("Checking for authorization: $headerLine")
+ ) else authorization
val subscriber = transaction {
val (user, pass) = extractUserAndHashedPassword(headerLine)
EbicsSubscriberEntity.find {
@@ -115,3 +116,7 @@ fun authenticateRequest(authorization: String?): String {
} ?: throw NexusError(HttpStatusCode.Forbidden, "Wrong password")
return subscriber.id.value
}
+
+fun parseDate(date: String): DateTime {
+ return DateTime.parse(date, DateTimeFormat.forPattern("YYYY-MM-DD"))
+}
diff --git a/nexus/src/main/kotlin/tech/libeufin/nexus/Main.kt b/nexus/src/main/kotlin/tech/libeufin/nexus/Main.kt
@@ -22,15 +22,13 @@ package tech.libeufin.nexus
import io.ktor.application.ApplicationCallPipeline
import io.ktor.application.call
import io.ktor.application.install
-import io.ktor.auth.Authentication
-import io.ktor.auth.basic
import io.ktor.client.HttpClient
-import io.ktor.features.CallLogging
-import io.ktor.features.ContentNegotiation
-import io.ktor.features.StatusPages
+import io.ktor.features.*
import io.ktor.gson.gson
import io.ktor.http.ContentType
import io.ktor.http.HttpStatusCode
+import io.ktor.request.ApplicationReceivePipeline
+import io.ktor.request.ApplicationReceiveRequest
import io.ktor.request.receive
import io.ktor.request.uri
import io.ktor.response.respond
@@ -40,6 +38,11 @@ import io.ktor.routing.post
import io.ktor.routing.routing
import io.ktor.server.engine.embeddedServer
import io.ktor.server.netty.Netty
+import io.ktor.util.KtorExperimentalAPI
+import kotlinx.coroutines.io.ByteReadChannel
+import kotlinx.coroutines.io.jvm.javaio.toByteReadChannel
+import kotlinx.coroutines.io.jvm.javaio.toInputStream
+import kotlinx.io.core.ExperimentalIoApi
import org.jetbrains.exposed.sql.SizedIterable
import org.jetbrains.exposed.sql.StdOutSqlLogger
import org.jetbrains.exposed.sql.addLogger
@@ -60,6 +63,8 @@ import java.time.ZoneId
import java.time.ZonedDateTime
import java.time.format.DateTimeFormatter
import java.util.*
+import java.util.zip.Inflater
+import java.util.zip.InflaterInputStream
import javax.crypto.EncryptedPrivateKeyInfo
import javax.sql.rowset.serial.SerialBlob
@@ -86,6 +91,10 @@ data class NexusError(val statusCode: HttpStatusCode, val reason: String) : Exce
val logger: Logger = LoggerFactory.getLogger("tech.libeufin.nexus")
+fun isProduction(): Boolean {
+ return System.getenv("NEXUS_PRODUCTION") != null
+}
+
fun getSubscriberEntityFromId(id: String): EbicsSubscriberEntity {
return transaction {
EbicsSubscriberEntity.findById(id) ?: throw NexusError(
@@ -146,14 +155,33 @@ fun getSubscriberDetailsFromBankAccount(bankAccountId: String): EbicsClientSubsc
* is guaranteed to be non empty.
*/
fun getBankAccountsInfoFromId(id: String): SizedIterable<EbicsAccountInfoEntity> {
+ logger.debug("Looking up bank account of user '$id'")
val list = transaction {
EbicsAccountInfoEntity.find {
EbicsAccountsInfoTable.subscriber eq id
}
}
- if (list.empty()) throw NexusError(
- HttpStatusCode.NotFound, "This subscriber '$id' did never fetch its own bank accounts, request HTD first."
- )
+ if (list.empty()) {
+ if (!isProduction()) {
+ /* make up a bank account info object */
+ transaction {
+ EbicsAccountInfoEntity.new("mocked-bank-account") {
+ subscriber = EbicsSubscriberEntity.findById(id) ?: throw NexusError(
+ HttpStatusCode.NotFound, "Please create subscriber '${id}' first."
+ )
+ accountHolder = "Tests runner"
+ iban = "IBAN-FOR-TESTS"
+ bankCode = "BIC-FOR-TESTS"
+ }
+ }
+ logger.debug("Faked bank account info object for user '$id'")
+ } else throw NexusError(
+ HttpStatusCode.NotFound,
+ "This subscriber '$id' did never fetch its own bank accounts, request HTD first."
+ )
+ // call this function again now that the database is augmented with the mocked information.
+ return getBankAccountsInfoFromId(id)
+ }
return list
}
@@ -336,6 +364,8 @@ fun createPain001entity(entry: Pain001Data, debtorAccountId: String): Pain001Ent
}
}
+@ExperimentalIoApi
+@KtorExperimentalAPI
fun main() {
dbCreateTables()
testData()
@@ -343,6 +373,7 @@ fun main() {
expectSuccess = false // this way, it does not throw exceptions on != 200 responses.
}
val server = embeddedServer(Netty, port = 5001) {
+
install(CallLogging) {
this.level = Level.DEBUG
this.logger = tech.libeufin.nexus.logger
@@ -370,12 +401,13 @@ fun main() {
cause.statusCode
)
}
- exception<javax.xml.bind.UnmarshalException> { cause ->
- logger.error("Exception while handling '${call.request.uri}'", cause)
+ exception<Exception> { cause ->
+ logger.error("Uncaught exception while handling '${call.request.uri}'", cause)
+ logger.error(cause.toString())
call.respondText(
- "Could not convert string into JAXB\n",
+ "Internal server error",
ContentType.Text.Plain,
- HttpStatusCode.NotFound
+ HttpStatusCode.InternalServerError
)
}
}
@@ -387,6 +419,18 @@ fun main() {
}
}
+ receivePipeline.intercept(ApplicationReceivePipeline.Before) {
+ if (this.context.request.headers["Content-Encoding"] == "deflate") {
+ logger.debug("About to inflate received data")
+ val deflated = this.subject.value as ByteReadChannel
+ val inflated = InflaterInputStream(deflated.toInputStream())
+ proceedWith(ApplicationReceiveRequest(this.subject.typeInfo, inflated.toByteReadChannel()))
+ return@intercept
+ }
+ proceed()
+ return@intercept
+ }
+
routing {
get("/") {
call.respondText("Hello by Nexus!\n")
@@ -677,12 +721,10 @@ fun main() {
currency = camt53doc.pickString("//*[local-name()='Ntry']//*[local-name()='Amt']/@Ccy")
amount = camt53doc.pickString("//*[local-name()='Ntry']//*[local-name()='Amt']")
status = camt53doc.pickString("//*[local-name()='Ntry']//*[local-name()='Sts']")
- bookingDate = camt53doc.pickString("//*[local-name()='BookgDt']//*[local-name()='Dt']")
+ bookingDate = parseDate(camt53doc.pickString("//*[local-name()='BookgDt']//*[local-name()='Dt']")).millis
nexusSubscriber = getSubscriberEntityFromId(id)
- creditorName =
- camt53doc.pickString("//*[local-name()='RltdPties']//*[local-name()='Dbtr']//*[local-name()='Nm']")
- creditorIban =
- camt53doc.pickString("//*[local-name()='CdtrAcct']//*[local-name()='IBAN']")
+ creditorName = camt53doc.pickString("//*[local-name()='RltdPties']//*[local-name()='Dbtr']//*[local-name()='Nm']")
+ creditorIban = camt53doc.pickString("//*[local-name()='CdtrAcct']//*[local-name()='IBAN']")
debitorName = camt53doc.pickString("//*[local-name()='RltdPties']//*[local-name()='Dbtr']//*[local-name()='Nm']")
debitorIban = camt53doc.pickString("//*[local-name()='DbtrAcct']//*[local-name()='IBAN']")
counterpartBic = camt53doc.pickString("//*[local-name()='RltdAgts']//*[local-name()='BIC']")
@@ -1415,8 +1457,12 @@ fun main() {
call.respondText("Bank keys stored in database\n", ContentType.Text.Plain, HttpStatusCode.OK)
return@post
}
+ post("/test/intercept") {
+ call.respondText(call.receive<String>() + "\n")
+ return@post
+ }
}
}
logger.info("Up and running")
server.start(wait = true)
-}
+}
+\ No newline at end of file
diff --git a/nexus/src/main/kotlin/tech/libeufin/nexus/taler.kt b/nexus/src/main/kotlin/tech/libeufin/nexus/taler.kt
@@ -1,6 +1,8 @@
package tech.libeufin.nexus
+import com.google.gson.Gson
import io.ktor.application.call
+import io.ktor.content.TextContent
import io.ktor.http.ContentType
import io.ktor.http.HttpStatusCode
import io.ktor.request.receive
@@ -18,6 +20,7 @@ import tech.libeufin.util.Amount
import tech.libeufin.util.CryptoUtil
import tech.libeufin.util.toZonedString
import kotlin.math.abs
+import kotlin.math.min
class Taler(app: Route) {
@@ -73,8 +76,11 @@ class Taler(app: Route) {
val debit_account: String
)
+ private data class GnunetTimestamp(
+ val t_ms: Long
+ )
private data class TalerAddIncomingResponse(
- val timestamp: Long,
+ val timestamp: GnunetTimestamp,
val row_id: Long
)
@@ -94,13 +100,36 @@ class Taler(app: Route) {
/**
* Helper functions
*/
-
fun parsePayto(paytoUri: String): Payto {
- // payto://iban/BIC?/IBAN?name=<name>
- val match = Regex("payto://iban/([A-Z0-9]+/)?([A-Z0-9]+)\\?name=(\\w+)").find(paytoUri) ?: throw
- NexusError(HttpStatusCode.BadRequest, "invalid payto URI ($paytoUri)")
- val (bic, iban, name) = match.destructured
- return Payto(name, iban, bic.replace("/", ""))
+ /**
+ * First try to parse a "iban"-type payto URI. If that fails,
+ * then assume a test is being run under the "x-taler-bank" type.
+ * If that one fails too, throw exception.
+ *
+ * Note: since the Nexus doesn't have the notion of "x-taler-bank",
+ * such URIs must yield a iban-compatible tuple of values. Therefore,
+ * the plain bank account number maps to a "iban", and the <bank hostname>
+ * maps to a "bic".
+ */
+
+
+ /**
+ * payto://iban/BIC?/IBAN?name=<name>
+ * payto://x-taler-bank/<bank hostname>/<plain account number>
+ */
+
+ val ibanMatch = Regex("payto://iban/([A-Z0-9]+/)?([A-Z0-9]+)\\?name=(\\w+)").find(paytoUri)
+ if (ibanMatch != null) {
+ val (bic, iban, name) = ibanMatch.destructured
+ return Payto(name, iban, bic.replace("/", ""))
+ }
+ val xTalerBankMatch = Regex("payto://x-taler-bank/localhost/([0-9])?").find(paytoUri)
+ if (xTalerBankMatch != null) {
+ val xTalerBankAcctNo = xTalerBankMatch.destructured.component1()
+ return Payto("Taler Exchange", xTalerBankAcctNo, "localhost")
+ }
+
+ throw NexusError(HttpStatusCode.BadRequest, "invalid payto URI ($paytoUri)")
}
fun parseAmount(amount: String): AmountWithCurrency {
@@ -123,9 +152,6 @@ class Taler(app: Route) {
private fun getPaytoUri(iban: String, bic: String): String {
return "payto://iban/$iban/$bic"
}
- private fun parseDate(date: String): DateTime {
- return DateTime.parse(date, DateTimeFormat.forPattern("YYYY-MM-DD"))
- }
/** Builds the comparison operator for history entries based on the sign of 'delta' */
private fun getComparisonOperator(delta: Int, start: Long): Op<Boolean> {
@@ -158,6 +184,18 @@ class Taler(app: Route) {
}
}
+ /**
+ * The Taler layer cannot rely on the ktor-internal JSON-converter/responder,
+ * because this one adds a "charset" extra information in the Content-Type header
+ * that makes the GNUnet JSON parser unhappy.
+ *
+ * The workaround is to explicitly convert the 'data class'-object into a JSON
+ * string (what this function does), and use the simpler respondText method.
+ */
+ private fun customConverter(body: Any): String {
+ return Gson().toJson(body)
+ }
+
/** Attach Taler endpoints to the main Web server */
init {
@@ -201,7 +239,8 @@ class Taler(app: Route) {
),
exchangeBankAccount.id.value
)
- val rawEbics = if (System.getenv("NEXUS_PRODUCTION") == null) {
+
+ val rawEbics = if (!isProduction()) {
EbicsRawBankTransactionEntity.new {
sourceFileName = "test"
unstructuredRemittanceInformation = transferRequest.wtid
@@ -213,7 +252,7 @@ class Taler(app: Route) {
creditorName = creditorObj.name
creditorIban = creditorObj.iban
counterpartBic = creditorObj.bic
- bookingDate = DateTime.now().toString("Y-MM-dd")
+ bookingDate = DateTime.now().millis
nexusSubscriber = exchangeBankAccount.subscriber
status = "BOOK"
}
@@ -264,20 +303,29 @@ class Taler(app: Route) {
debitorIban = debtor.iban
debitorName = debtor.name
counterpartBic = debtor.bic
- bookingDate = DateTime.now().toZonedString()
+ bookingDate = DateTime.now().millis
status = "BOOK"
+ nexusSubscriber = getSubscriberEntityFromId(exchangeId)
}
/** This payment is "valid by default" and will be returned
* as soon as the exchange will ask for new payments. */
val row = TalerIncomingPaymentEntity.new {
payment = rawPayment
+ valid = true
}
Pair(rawPayment.bookingDate, row.id.value)
}
- call.respond(HttpStatusCode.OK, TalerAddIncomingResponse(
- timestamp = parseDate(bookingDate).millis / 1000,
- row_id = opaque_row_id
- ))
+ call.respond(
+ TextContent(
+ customConverter(
+ TalerAddIncomingResponse(
+ timestamp = GnunetTimestamp(bookingDate/ 1000),
+ row_id = opaque_row_id
+ )
+ ),
+ ContentType.Application.Json
+ )
+ )
return@post
}
@@ -398,9 +446,8 @@ class Taler(app: Route) {
row_id = it.id.value,
amount = it.amount,
wtid = it.wtid,
- date = parseDate(it.rawConfirmed?.bookingDate ?: throw NexusError(
- HttpStatusCode.InternalServerError, "Null value met after check, VERY strange.")
- ).millis / 1000,
+ date = it.rawConfirmed?.bookingDate?.div(1000) ?: throw NexusError(
+ HttpStatusCode.InternalServerError, "Null value met after check, VERY strange."),
credit_account = it.creditAccount,
debit_account = getPaytoUri(subscriberBankAccount.iban, subscriberBankAccount.bankCode),
exchange_base_url = "FIXME-to-request-along-subscriber-registration"
@@ -423,26 +470,29 @@ class Taler(app: Route) {
val startCmpOp = getComparisonOperator(delta, start)
transaction {
val subscriberBankAccount = getBankAccountsInfoFromId(subscriberId)
- TalerIncomingPaymentEntity.find {
+ val orderedPayments = TalerIncomingPaymentEntity.find {
TalerIncomingPayments.valid eq true and startCmpOp
- }.orderTaler(delta).subList(0, abs(delta)).forEach {
- history.incoming_transactions.add(
- TalerIncomingBankTransaction(
- date = parseDate(it.payment.bookingDate).millis / 1000, // timestamp in seconds
- row_id = it.id.value,
- amount = "${it.payment.currency}:${it.payment.amount}",
- reserve_pub = it.payment.unstructuredRemittanceInformation,
- debit_account = getPaytoUri(
- it.payment.debitorName, it.payment.debitorIban, it.payment.counterpartBic
- ),
- credit_account = getPaytoUri(
- it.payment.creditorName, it.payment.creditorIban, subscriberBankAccount.first().bankCode
+ }.orderTaler(delta)
+ if (orderedPayments.isNotEmpty()) {
+ orderedPayments.subList(0, min(abs(delta), orderedPayments.size)).forEach {
+ history.incoming_transactions.add(
+ TalerIncomingBankTransaction(
+ date = it.payment.bookingDate / 1000, // timestamp in seconds
+ row_id = it.id.value,
+ amount = "${it.payment.currency}:${it.payment.amount}",
+ reserve_pub = it.payment.unstructuredRemittanceInformation,
+ debit_account = getPaytoUri(
+ it.payment.debitorName, it.payment.debitorIban, it.payment.counterpartBic
+ ),
+ credit_account = getPaytoUri(
+ it.payment.creditorName, it.payment.creditorIban, subscriberBankAccount.first().bankCode
+ )
)
)
- )
+ }
}
}
- call.respond(history)
+ call.respond(TextContent(customConverter(history), ContentType.Application.Json))
return@get
}
}
