diff options
author | Antoine A <> | 2024-02-13 17:20:33 +0100 |
---|---|---|
committer | Antoine A <> | 2024-02-13 17:20:33 +0100 |
commit | 3046601e239d774716527d4a0a34f585ac5ade79 (patch) | |
tree | 093ef827ed28be1c32ab25d3342b1a0e324a6ab4 | |
parent | 5837035a2e5679f529196ae85bc041d695a69176 (diff) | |
download | libeufin-0.9.4-dev.19.tar.gz libeufin-0.9.4-dev.19.tar.bz2 libeufin-0.9.4-dev.19.zip |
Reduce memory usage by using InputStream as much as possiblev0.9.4-dev.19
21 files changed, 191 insertions, 180 deletions
diff --git a/bank/src/main/kotlin/tech/libeufin/bank/db/WithdrawalDAO.kt b/bank/src/main/kotlin/tech/libeufin/bank/db/WithdrawalDAO.kt index cc952691..12977610 100644 --- a/bank/src/main/kotlin/tech/libeufin/bank/db/WithdrawalDAO.kt +++ b/bank/src/main/kotlin/tech/libeufin/bank/db/WithdrawalDAO.kt @@ -152,7 +152,6 @@ class WithdrawalDAO(private val db: Database) { now: Instant, is2fa: Boolean ): WithdrawalConfirmationResult = db.serializable { conn -> - // TODO login check val stmt = conn.prepareStatement(""" SELECT out_no_op, diff --git a/bank/src/test/kotlin/SecurityTest.kt b/bank/src/test/kotlin/SecurityTest.kt index b5559983..16d2b3f8 100644 --- a/bank/src/test/kotlin/SecurityTest.kt +++ b/bank/src/test/kotlin/SecurityTest.kt @@ -30,19 +30,12 @@ import kotlinx.serialization.json.* import org.junit.Test import tech.libeufin.bank.* import tech.libeufin.common.* -import tech.libeufin.common.* -import java.io.ByteArrayOutputStream -import java.util.zip.DeflaterOutputStream inline fun <reified B> HttpRequestBuilder.jsonDeflate(b: B) { val json = Json.encodeToString(kotlinx.serialization.serializer<B>(), b); contentType(ContentType.Application.Json) headers.set(HttpHeaders.ContentEncoding, "deflate") - val bos = ByteArrayOutputStream() - val ios = DeflaterOutputStream(bos) - ios.write(json.toByteArray()) - ios.finish() - setBody(bos.toByteArray()) + setBody(json.toByteArray().inputStream().deflate().readBytes()) } class SecurityTest { diff --git a/bank/src/test/kotlin/helpers.kt b/bank/src/test/kotlin/helpers.kt index e6c53565..f4d3a4a6 100644 --- a/bank/src/test/kotlin/helpers.kt +++ b/bank/src/test/kotlin/helpers.kt @@ -23,7 +23,6 @@ import io.ktor.client.statement.* import io.ktor.http.* import io.ktor.server.testing.* import java.io.ByteArrayOutputStream -import java.util.zip.DeflaterOutputStream import java.nio.file.* import kotlin.test.* import kotlin.io.path.* diff --git a/common/src/main/kotlin/CryptoUtil.kt b/common/src/main/kotlin/CryptoUtil.kt index bcf19ae1..3729627e 100644 --- a/common/src/main/kotlin/CryptoUtil.kt +++ b/common/src/main/kotlin/CryptoUtil.kt @@ -21,6 +21,7 @@ package tech.libeufin.common import org.bouncycastle.jce.provider.BouncyCastleProvider import java.io.ByteArrayOutputStream +import java.io.InputStream import java.math.BigInteger import java.security.* import java.security.interfaces.RSAPrivateCrtKey @@ -176,18 +177,18 @@ object CryptoUtil { } fun decryptEbicsE002(enc: EncryptionResult, privateKey: RSAPrivateCrtKey): ByteArray { - return decryptEbicsE002( + return CryptoUtil.decryptEbicsE002( enc.encryptedTransactionKey, - enc.encryptedData, + enc.encryptedData.inputStream(), privateKey - ) + ).readBytes() } fun decryptEbicsE002( encryptedTransactionKey: ByteArray, - encryptedData: ByteArray, + encryptedData: InputStream, privateKey: RSAPrivateCrtKey - ): ByteArray { + ): CipherInputStream { val asymmetricCipher = Cipher.getInstance( "RSA/None/PKCS1Padding", bouncyCastleProvider @@ -201,8 +202,7 @@ object CryptoUtil { ) val ivParameterSpec = IvParameterSpec(ByteArray(16)) symmetricCipher.init(Cipher.DECRYPT_MODE, secretKeySpec, ivParameterSpec) - val data = symmetricCipher.doFinal(encryptedData) - return data + return CipherInputStream(encryptedData, symmetricCipher) } /** diff --git a/common/src/main/kotlin/Stream.kt b/common/src/main/kotlin/Stream.kt new file mode 100644 index 00000000..7b9689b5 --- /dev/null +++ b/common/src/main/kotlin/Stream.kt @@ -0,0 +1,55 @@ +/* + * This file is part of LibEuFin. + * Copyright (C) 2024 Taler Systems S.A. + + * LibEuFin is free software; you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation; either version 3, or + * (at your option) any later version. + + * LibEuFin is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General + * Public License for more details. + + * You should have received a copy of the GNU Affero General Public + * License along with LibEuFin; see the file COPYING. If not, see + * <http://www.gnu.org/licenses/> + */ + +package tech.libeufin.common + +import java.io.InputStream +import java.io.FilterInputStream +import java.util.zip.DeflaterInputStream +import java.util.zip.InflaterInputStream +import java.util.zip.* +import java.util.Base64 + +/** Unzip an input stream and run [lambda] over each entry */ +fun InputStream.unzipEach(lambda: (String, InputStream) -> Unit) { + ZipInputStream(this).use { zip -> + while (true) { + val entry = zip.getNextEntry() + if (entry == null) break; + val entryStream = object: FilterInputStream(zip) { + override fun close() { + zip.closeEntry(); + } + } + lambda(entry.name, entryStream) + } + } +} + +/** Decode a base64 an input stream */ +fun InputStream.decodeBase64(): InputStream + = Base64.getDecoder().wrap(this) + +/** Deflate an input stream */ +fun InputStream.deflate(): DeflaterInputStream + = DeflaterInputStream(this) + +/** Inflate an input stream */ +fun InputStream.inflate(): InflaterInputStream + = InflaterInputStream(this)
\ No newline at end of file diff --git a/ebics/src/main/kotlin/Ebics.kt b/ebics/src/main/kotlin/Ebics.kt index 91a41022..7651be74 100644 --- a/ebics/src/main/kotlin/Ebics.kt +++ b/ebics/src/main/kotlin/Ebics.kt @@ -36,9 +36,11 @@ import java.time.Instant import java.time.ZoneId import java.time.ZonedDateTime import java.util.* +import java.io.InputStream import javax.xml.bind.JAXBElement import javax.xml.datatype.DatatypeFactory import javax.xml.datatype.XMLGregorianCalendar +import org.w3c.dom.Document data class EbicsProtocolError( val httpStatusCode: HttpStatusCode, @@ -308,9 +310,9 @@ class HpbResponseData( val authenticationVersion: String ) -fun parseEbicsHpbOrder(orderDataRaw: ByteArray): HpbResponseData { +fun parseEbicsHpbOrder(orderDataRaw: InputStream): HpbResponseData { val resp = try { - XMLUtil.convertBytesToJaxb<HPBResponseOrderData>(orderDataRaw) + XMLUtil.convertToJaxb<HPBResponseOrderData>(orderDataRaw) } catch (e: Exception) { throw EbicsProtocolError(HttpStatusCode.InternalServerError, "Invalid XML (as HPB response) received from bank") } @@ -331,10 +333,10 @@ fun parseEbicsHpbOrder(orderDataRaw: ByteArray): HpbResponseData { ) } -fun ebics3toInternalRepr(response: ByteArray): EbicsResponseContent { +fun ebics3toInternalRepr(response: Document): EbicsResponseContent { // logger.debug("Converting bank resp to internal repr.: $response") val resp: JAXBElement<Ebics3Response> = try { - XMLUtil.convertBytesToJaxb(response) + XMLUtil.convertDomToJaxb(response) } catch (e: Exception) { throw EbicsProtocolError( HttpStatusCode.InternalServerError, @@ -368,9 +370,9 @@ fun ebics3toInternalRepr(response: ByteArray): EbicsResponseContent { ) } -fun ebics25toInternalRepr(response: ByteArray): EbicsResponseContent { +fun ebics25toInternalRepr(response: Document): EbicsResponseContent { val resp: JAXBElement<EbicsResponse> = try { - XMLUtil.convertBytesToJaxb(response) + XMLUtil.convertDomToJaxb(response) } catch (e: Exception) { throw EbicsProtocolError( HttpStatusCode.InternalServerError, diff --git a/ebics/src/main/kotlin/EbicsOrderUtil.kt b/ebics/src/main/kotlin/EbicsOrderUtil.kt index c056ddde..79ff829a 100644 --- a/ebics/src/main/kotlin/EbicsOrderUtil.kt +++ b/ebics/src/main/kotlin/EbicsOrderUtil.kt @@ -21,33 +21,23 @@ package tech.libeufin.ebics import java.lang.IllegalArgumentException import java.security.SecureRandom -import java.util.zip.DeflaterInputStream -import java.util.zip.InflaterInputStream +import java.io.InputStream +import tech.libeufin.common.* /** * Helpers for dealing with order compression, encryption, decryption, chunking and re-assembly. */ object EbicsOrderUtil { - // Decompression only, no XML involved. - fun decodeOrderData(encodedOrderData: ByteArray): ByteArray { - return InflaterInputStream(encodedOrderData.inputStream()).use { - it.readAllBytes() - } - } - inline fun <reified T> decodeOrderDataXml(encodedOrderData: ByteArray): T { - return InflaterInputStream(encodedOrderData.inputStream()).use { - val bytes = it.readAllBytes() - XMLUtil.convertBytesToJaxb<T>(bytes).value + return encodedOrderData.inputStream().inflate().use { + XMLUtil.convertToJaxb<T>(it).value } } inline fun <reified T> encodeOrderDataXml(obj: T): ByteArray { val bytes = XMLUtil.convertJaxbToBytes(obj) - return DeflaterInputStream(bytes.inputStream()).use { - it.readAllBytes() - } + return bytes.inputStream().deflate().readAllBytes() } @kotlin.ExperimentalStdlibApi diff --git a/ebics/src/main/kotlin/XMLUtil.kt b/ebics/src/main/kotlin/XMLUtil.kt index 5947341a..c27afbc5 100644 --- a/ebics/src/main/kotlin/XMLUtil.kt +++ b/ebics/src/main/kotlin/XMLUtil.kt @@ -288,9 +288,7 @@ class XMLUtil private constructor() { * @return InputStream object, as wanted by the validator. */ fun validateFromBytes(xml: ByteArray): Boolean { - val xmlInputStream: InputStream = ByteArrayInputStream(xml) - val xmlSource = StreamSource(xmlInputStream) - return validate(xmlSource) + return validate(StreamSource(xml.inputStream())) } inline fun <reified T> convertJaxbToBytes( @@ -333,11 +331,11 @@ class XMLUtil private constructor() { * @param documentBytes the bytes to convert into JAXB. * @return the JAXB object reflecting the original XML document. */ - inline fun <reified T> convertBytesToJaxb(documentBytes: ByteArray): JAXBElement<T> { + inline fun <reified T> convertToJaxb(documentBytes: InputStream): JAXBElement<T> { val jc = JAXBContext.newInstance(T::class.java) val u = jc.createUnmarshaller() return u.unmarshal( /* Marshalling the object into the document. */ - StreamSource(ByteArrayInputStream(documentBytes)), + StreamSource(documentBytes), T::class.java ) } @@ -374,25 +372,25 @@ class XMLUtil private constructor() { /** * Convert a DOM document to the JAXB representation. * - * @param finalType class type of the output * @param document the document to convert into JAXB. * @return the JAXB object reflecting the original XML document. */ - fun <T> convertDomToJaxb(finalType: Class<T>, document: Document): JAXBElement<T> { - val jc = JAXBContext.newInstance(finalType) + inline fun <reified T> convertDomToJaxb(document: Document): JAXBElement<T> { + val jc = JAXBContext.newInstance(T::class.java) /* Marshalling the object into the document. */ val m = jc.createUnmarshaller() - return m.unmarshal(document, finalType) // document "went" into Jaxb + return m.unmarshal(document, T::class.java) // document "went" into Jaxb } /** Parse [xml] into a XML DOM */ - fun parseBytesIntoDom(xml: ByteArray): Document { + fun parseIntoDom(xml: InputStream): Document { val factory = DocumentBuilderFactory.newInstance().apply { isNamespaceAware = true } - val xmlInputStream = ByteArrayInputStream(xml) val builder = factory.newDocumentBuilder() - return builder.parse(InputSource(xmlInputStream)) + return xml.use { + builder.parse(InputSource(it)) + } } fun signEbicsResponse(ebicsResponse: EbicsResponse, privateKey: RSAPrivateCrtKey): ByteArray { diff --git a/ebics/src/main/kotlin/XmlCombinators.kt b/ebics/src/main/kotlin/XmlCombinators.kt index d2d4d25e..ee295010 100644 --- a/ebics/src/main/kotlin/XmlCombinators.kt +++ b/ebics/src/main/kotlin/XmlCombinators.kt @@ -25,6 +25,7 @@ import org.w3c.dom.Element import java.io.StringWriter import javax.xml.stream.XMLOutputFactory import javax.xml.stream.XMLStreamWriter +import java.io.InputStream import java.time.format.* import java.time.* @@ -133,8 +134,8 @@ class XmlDestructor internal constructor(private val el: Element) { fun attr(index: String): String = el.getAttribute(index) } -fun <T> destructXml(xml: ByteArray, root: String, f: XmlDestructor.() -> T): T { - val doc = XMLUtil.parseBytesIntoDom(xml) +fun <T> destructXml(xml: InputStream, root: String, f: XmlDestructor.() -> T): T { + val doc = XMLUtil.parseIntoDom(xml) if (doc.documentElement.tagName != root) { throw DestructionError("expected root '$root' got '${doc.documentElement.tagName}'") } diff --git a/ebics/src/test/kotlin/EbicsMessagesTest.kt b/ebics/src/test/kotlin/EbicsMessagesTest.kt index 2b8d63ca..7dbdc350 100644 --- a/ebics/src/test/kotlin/EbicsMessagesTest.kt +++ b/ebics/src/test/kotlin/EbicsMessagesTest.kt @@ -40,8 +40,8 @@ class EbicsMessagesTest { @Test fun testImportNonRoot() { val classLoader = ClassLoader.getSystemClassLoader() - val ini = classLoader.getResource("ebics_ini_inner_key.xml") - val jaxb = XMLUtil.convertBytesToJaxb<SignatureTypes.SignaturePubKeyOrderData>(ini.readBytes()) + val ini = classLoader.getResourceAsStream("ebics_ini_inner_key.xml") + val jaxb = XMLUtil.convertToJaxb<SignatureTypes.SignaturePubKeyOrderData>(ini) assertEquals("A006", jaxb.value.signaturePubKeyInfo.signatureVersion) } @@ -51,8 +51,8 @@ class EbicsMessagesTest { @Test fun testStringToJaxb() { val classLoader = ClassLoader.getSystemClassLoader() - val ini = classLoader.getResource("ebics_ini_request_sample.xml") - val jaxb = XMLUtil.convertBytesToJaxb<EbicsUnsecuredRequest>(ini.readBytes()) + val ini = classLoader.getResourceAsStream("ebics_ini_request_sample.xml") + val jaxb = XMLUtil.convertToJaxb<EbicsUnsecuredRequest>(ini) println("jaxb loaded") assertEquals( "INI", @@ -81,10 +81,9 @@ class EbicsMessagesTest { @Test fun testDomToJaxb() { val classLoader = ClassLoader.getSystemClassLoader() - val ini = classLoader.getResource("ebics_ini_request_sample.xml")!! - val iniDom = XMLUtil.parseBytesIntoDom(ini.readBytes()) + val ini = classLoader.getResourceAsStream("ebics_ini_request_sample.xml") + val iniDom = XMLUtil.parseIntoDom(ini) XMLUtil.convertDomToJaxb<EbicsUnsecuredRequest>( - EbicsUnsecuredRequest::class.java, iniDom ) } @@ -114,15 +113,15 @@ class EbicsMessagesTest { @Test fun testParseHiaRequestOrderData() { val classLoader = ClassLoader.getSystemClassLoader() - val hia = classLoader.getResource("hia_request_order_data.xml")!!.readBytes() - XMLUtil.convertBytesToJaxb<HIARequestOrderData>(hia) + val hia = classLoader.getResourceAsStream("hia_request_order_data.xml") + XMLUtil.convertToJaxb<HIARequestOrderData>(hia) } @Test fun testHiaLoad() { val classLoader = ClassLoader.getSystemClassLoader() - val hia = classLoader.getResource("hia_request.xml")!! - val hiaDom = XMLUtil.parseBytesIntoDom(hia.readBytes()) + val hia = classLoader.getResourceAsStream("hia_request.xml") + val hiaDom = XMLUtil.parseIntoDom(hia) val x: Element = hiaDom.getElementsByTagNameNS( "urn:org:ebics:H004", "OrderDetails" @@ -135,7 +134,6 @@ class EbicsMessagesTest { ) XMLUtil.convertDomToJaxb<EbicsUnsecuredRequest>( - EbicsUnsecuredRequest::class.java, hiaDom ) } @@ -144,11 +142,11 @@ class EbicsMessagesTest { fun testLoadInnerKey() { val jaxbKey = run { val classLoader = ClassLoader.getSystemClassLoader() - val file = classLoader.getResource( + val file = classLoader.getResourceAsStream( "ebics_ini_inner_key.xml" ) assertNotNull(file) - XMLUtil.convertBytesToJaxb<SignatureTypes.SignaturePubKeyOrderData>(file.readBytes()) + XMLUtil.convertToJaxb<SignatureTypes.SignaturePubKeyOrderData>(file) } val modulus = jaxbKey.value.signaturePubKeyInfo.pubKeyValue.rsaKeyValue.modulus @@ -159,8 +157,8 @@ class EbicsMessagesTest { @Test fun testLoadIniMessage() { val classLoader = ClassLoader.getSystemClassLoader() - val text = classLoader.getResource("ebics_ini_request_sample.xml")!!.readBytes() - XMLUtil.convertBytesToJaxb<EbicsUnsecuredRequest>(text) + val text = classLoader.getResourceAsStream("ebics_ini_request_sample.xml")!! + XMLUtil.convertToJaxb<EbicsUnsecuredRequest>(text) } @Test @@ -189,8 +187,8 @@ class EbicsMessagesTest { @Test fun testLoadHpb() { val classLoader = ClassLoader.getSystemClassLoader() - val text = classLoader.getResource("hpb_request.xml")!!.readBytes() - XMLUtil.convertBytesToJaxb<EbicsNpkdRequest>(text) + val text = classLoader.getResourceAsStream("hpb_request.xml")!! + XMLUtil.convertToJaxb<EbicsNpkdRequest>(text) } @Test @@ -358,7 +356,7 @@ class EbicsMessagesTest { } val str = XMLUtil.convertJaxbToBytes(ebicsRequestObj) - val doc = XMLUtil.parseBytesIntoDom(str) + val doc = XMLUtil.parseIntoDom(str.inputStream()) val pair = CryptoUtil.generateRsaKeyPair(1024) XMLUtil.signEbicsDocument(doc, pair.private) val bytes = XMLUtil.convertDomToBytes(doc) diff --git a/ebics/src/test/kotlin/EbicsOrderUtilTest.kt b/ebics/src/test/kotlin/EbicsOrderUtilTest.kt index c78da738..3d8ed821 100644 --- a/ebics/src/test/kotlin/EbicsOrderUtilTest.kt +++ b/ebics/src/test/kotlin/EbicsOrderUtilTest.kt @@ -302,7 +302,7 @@ class EbicsOrderUtilTest { </Permission> </UserInfo> </HTDResponseOrderData> - """.trimIndent().toByteArray() - XMLUtil.convertBytesToJaxb<HTDResponseOrderData>(orderDataXml); + """.trimIndent().toByteArray().inputStream() + XMLUtil.convertToJaxb<HTDResponseOrderData>(orderDataXml); } }
\ No newline at end of file diff --git a/ebics/src/test/kotlin/XmlUtilTest.kt b/ebics/src/test/kotlin/XmlUtilTest.kt index 93ca8bf2..647918d5 100644 --- a/ebics/src/test/kotlin/XmlUtilTest.kt +++ b/ebics/src/test/kotlin/XmlUtilTest.kt @@ -26,7 +26,7 @@ import tech.libeufin.ebics.ebics_h004.EbicsKeyManagementResponse import tech.libeufin.ebics.ebics_h004.EbicsResponse import tech.libeufin.ebics.ebics_h004.EbicsTypes import tech.libeufin.ebics.ebics_h004.HTDResponseOrderData -import tech.libeufin.common.CryptoUtil +import tech.libeufin.common.* import tech.libeufin.ebics.XMLUtil import java.security.KeyPairGenerator import java.util.* @@ -37,7 +37,7 @@ class XmlUtilTest { @Test fun deserializeConsecutiveLists() { - val tmp = XMLUtil.convertBytesToJaxb<HTDResponseOrderData>(""" + val tmp = XMLUtil.convertToJaxb<HTDResponseOrderData>(""" <?xml version="1.0" encoding="UTF-8" standalone="yes"?> <HTDResponseOrderData xmlns="urn:org:ebics:H004"> <PartnerInfo> @@ -80,7 +80,7 @@ class XmlUtilTest { <OrderTypes>C54 C53 C52 CCC</OrderTypes> </Permission> </UserInfo> - </HTDResponseOrderData>""".trimIndent().toByteArray() + </HTDResponseOrderData>""".trimIndent().toByteArray().inputStream() ) println(tmp.value.partnerInfo.orderInfoList[0].description) @@ -89,7 +89,7 @@ class XmlUtilTest { @Test fun exceptionOnConversion() { try { - XMLUtil.convertBytesToJaxb<EbicsKeyManagementResponse>("<malformed xml>".toByteArray()) + XMLUtil.convertToJaxb<EbicsKeyManagementResponse>("<malformed xml>".toByteArray().inputStream()) } catch (e: javax.xml.bind.UnmarshalException) { // just ensuring this is the exception println("caught") @@ -114,12 +114,12 @@ class XmlUtilTest { @Test fun basicSigningTest() { - val doc = XMLUtil.parseBytesIntoDom(""" + val doc = XMLUtil.parseIntoDom(""" <myMessage xmlns:ebics="urn:org:ebics:H004"> <ebics:AuthSignature /> <foo authenticate="true">Hello World</foo> </myMessage> - """.trimIndent().toByteArray()) + """.trimIndent().toByteArray().inputStream()) val kpg = KeyPairGenerator.getInstance("RSA") kpg.initialize(2048) val pair = kpg.genKeyPair() @@ -154,7 +154,7 @@ class XmlUtilTest { } val signature = signEbicsResponse(response, pair.private) - val signatureJaxb = XMLUtil.convertBytesToJaxb<EbicsResponse>(signature) + val signatureJaxb = XMLUtil.convertToJaxb<EbicsResponse>(signature.inputStream()) assertTrue( XMLUtil.verifyEbicsDocument( @@ -166,13 +166,13 @@ class XmlUtilTest { @Test fun multiAuthSigningTest() { - val doc = XMLUtil.parseBytesIntoDom(""" + val doc = XMLUtil.parseIntoDom(""" <myMessage xmlns:ebics="urn:org:ebics:H004"> <ebics:AuthSignature /> <foo authenticate="true">Hello World</foo> <bar authenticate="true">Another one!</bar> </myMessage> - """.trimIndent().toByteArray()) + """.trimIndent().toByteArray().inputStream()) val kpg = KeyPairGenerator.getInstance("RSA") kpg.initialize(2048) val pair = kpg.genKeyPair() @@ -183,10 +183,10 @@ class XmlUtilTest { @Test fun testRefSignature() { val classLoader = ClassLoader.getSystemClassLoader() - val docText = classLoader.getResourceAsStream("signature1/doc.xml")!!.readAllBytes() - val doc = XMLUtil.parseBytesIntoDom(docText) - val keyText = classLoader.getResourceAsStream("signature1/public_key.txt")!!.readAllBytes() - val keyBytes = Base64.getDecoder().decode(keyText) + val docText = classLoader.getResourceAsStream("signature1/doc.xml") + val doc = XMLUtil.parseIntoDom(docText) + val keyStream = classLoader.getResourceAsStream("signature1/public_key.txt") + val keyBytes = keyStream.decodeBase64().readAllBytes() val key = CryptoUtil.loadRsaPublicKey(keyBytes) assertTrue(XMLUtil.verifyEbicsDocument(doc, key)) } diff --git a/nexus/build.gradle b/nexus/build.gradle index 3d6c6c7c..b510dba6 100644 --- a/nexus/build.gradle +++ b/nexus/build.gradle @@ -27,9 +27,6 @@ dependencies { // XML parsing/binding and encryption implementation("jakarta.xml.bind:jakarta.xml.bind-api:2.3.3") - // Compression - implementation("org.apache.commons:commons-compress:1.25.0") - // Command line parsing implementation("com.github.ajalt.clikt:clikt:$clikt_version") implementation("org.postgresql:postgresql:$postgres_version") diff --git a/nexus/src/main/kotlin/tech/libeufin/nexus/EbicsFetch.kt b/nexus/src/main/kotlin/tech/libeufin/nexus/EbicsFetch.kt index 03d37e76..d4234cd1 100644 --- a/nexus/src/main/kotlin/tech/libeufin/nexus/EbicsFetch.kt +++ b/nexus/src/main/kotlin/tech/libeufin/nexus/EbicsFetch.kt @@ -31,6 +31,7 @@ import tech.libeufin.common.* import tech.libeufin.ebics.* import tech.libeufin.ebics.ebics_h005.Ebics3Request import java.io.IOException +import java.io.InputStream import java.time.Instant import java.time.LocalDate import java.time.ZoneId @@ -78,12 +79,12 @@ data class FetchContext( * length is zero. It returns null, if the bank assigned an * error to the EBICS transaction. */ -private suspend fun <T> downloadHelper( +private suspend fun downloadHelper( ctx: FetchContext, lastExecutionTime: Instant? = null, doc: SupportedDocument, - processing: (ByteArray) -> T -): T? { + processing: (InputStream) -> Unit +) { val isEbics3 = doc != SupportedDocument.PAIN_002_LOGS val initXml = if (isEbics3) { createEbics3DownloadInitialization( @@ -246,7 +247,7 @@ suspend fun ingestIncomingPayment( private fun ingestDocument( db: Database, currency: String, - xml: ByteArray, + xml: InputStream, whichDocument: SupportedDocument ) { when (whichDocument) { @@ -308,7 +309,7 @@ private fun ingestDocument( private fun ingestDocuments( db: Database, currency: String, - content: ByteArray, + content: InputStream, whichDocument: SupportedDocument ) { when (whichDocument) { @@ -317,7 +318,7 @@ private fun ingestDocuments( SupportedDocument.CAMT_053, SupportedDocument.CAMT_052 -> { try { - content.unzipForEach { fileName, xmlContent -> + content.unzipEach { fileName, xmlContent -> logger.trace("parse $fileName") ingestDocument(db, currency, xmlContent, whichDocument) } @@ -364,14 +365,12 @@ private suspend fun fetchDocuments( } val doc = doc.doc() // downloading the content - downloadHelper(ctx, lastExecutionTime, doc) { content -> - if (!content.isEmpty()) { - ctx.fileLogger.logFetch( - content, - doc == SupportedDocument.PAIN_002_LOGS - ) - ingestDocuments(db, ctx.cfg.currency, content, doc) - } + downloadHelper(ctx, lastExecutionTime, doc) { stream -> + val loggedStream = ctx.fileLogger.logFetch( + stream, + doc == SupportedDocument.PAIN_002_LOGS + ) + ingestDocuments(db, ctx.cfg.currency, loggedStream, doc) } true } catch (e: Exception) { diff --git a/nexus/src/main/kotlin/tech/libeufin/nexus/EbicsSetup.kt b/nexus/src/main/kotlin/tech/libeufin/nexus/EbicsSetup.kt index bd95543b..a5c40f35 100644 --- a/nexus/src/main/kotlin/tech/libeufin/nexus/EbicsSetup.kt +++ b/nexus/src/main/kotlin/tech/libeufin/nexus/EbicsSetup.kt @@ -110,7 +110,7 @@ private fun handleHpbResponse( throw Exception("HPB content not found in a EBICS response with successful return codes.") } val hpbObj = try { - parseEbicsHpbOrder(hpbBytes) + parseEbicsHpbOrder(hpbBytes.inputStream()) } catch (e: Exception) { throw Exception("HPB response content seems invalid", e) } diff --git a/nexus/src/main/kotlin/tech/libeufin/nexus/Iso20022.kt b/nexus/src/main/kotlin/tech/libeufin/nexus/Iso20022.kt index 97ba52f6..f6562c2e 100644 --- a/nexus/src/main/kotlin/tech/libeufin/nexus/Iso20022.kt +++ b/nexus/src/main/kotlin/tech/libeufin/nexus/Iso20022.kt @@ -23,6 +23,7 @@ import tech.libeufin.ebics.* import java.net.URLEncoder import java.time.* import java.time.format.* +import java.io.InputStream /** @@ -145,7 +146,7 @@ data class CustomerAck( * * @param xml pain.002 input document */ -fun parseCustomerAck(xml: ByteArray): List<CustomerAck> { +fun parseCustomerAck(xml: InputStream): List<CustomerAck> { return destructXml(xml, "Document") { one("CstmrPmtStsRpt").map("OrgnlPmtInfAndSts") { val actionType = one("OrgnlPmtInfId").enum<HacAction>() @@ -214,7 +215,7 @@ data class Reason ( * * @param xml pain.002 input document */ -fun parseCustomerPaymentStatusReport(xml: ByteArray): PaymentStatus { +fun parseCustomerPaymentStatusReport(xml: InputStream): PaymentStatus { fun XmlDestructor.reasons(): List<Reason> { return map("StsRsnInf") { val code = one("Rsn").one("Cd").enum<ExternalStatusReasonCode>() @@ -255,7 +256,7 @@ fun parseCustomerPaymentStatusReport(xml: ByteArray): PaymentStatus { * @param outgoing list of outgoing payments */ fun parseTxNotif( - notifXml: ByteArray, + notifXml: InputStream, acceptedCurrency: String, incoming: MutableList<IncomingPayment>, outgoing: MutableList<OutgoingPayment> @@ -324,7 +325,7 @@ fun parseTxNotif( * @param xml the input document. */ private fun notificationForEachTx( - xml: ByteArray, + xml: InputStream, directionLambda: XmlDestructor.(Instant) -> Unit ) { destructXml(xml, "Document") { diff --git a/nexus/src/main/kotlin/tech/libeufin/nexus/Log.kt b/nexus/src/main/kotlin/tech/libeufin/nexus/Log.kt index 3299a16e..88e3481a 100644 --- a/nexus/src/main/kotlin/tech/libeufin/nexus/Log.kt +++ b/nexus/src/main/kotlin/tech/libeufin/nexus/Log.kt @@ -19,7 +19,6 @@ package tech.libeufin.nexus -import tech.libeufin.nexus.ebics.unzipForEach import tech.libeufin.common.* import java.io.* import java.nio.file.* @@ -50,12 +49,12 @@ class FileLogger(path: String?) { /** * Logs EBICS fetch content if EBICS debug logging is enabled * - * @param content EBICS fetch content + * @param stream EBICS fetch content * @param hac only true when downloading via HAC (EBICS 2) */ - fun logFetch(content: ByteArray, hac: Boolean = false) { - if (dir == null) return; - + fun logFetch(stream: InputStream, hac: Boolean = false): InputStream { + if (dir == null) return stream; + val content = stream.readBytes() // Subdir based on current day. val now = Instant.now() val asUtcDate = LocalDate.ofInstant(now, ZoneId.of("UTC")) @@ -67,10 +66,13 @@ class FileLogger(path: String?) { subDir.resolve("${nowMs}_HAC_response.pain.002.xml").writeBytes(content) } else { // Write each ZIP entry in the combined dir. - content.unzipForEach { fileName, xmlContent -> - subDir.resolve("${nowMs}_$fileName").writeBytes(xmlContent) + content.inputStream().unzipEach { fileName, xmlContent -> + xmlContent.use { + Files.copy(it, subDir.resolve("${nowMs}_$fileName")) + } } } + return content.inputStream() } /** diff --git a/nexus/src/main/kotlin/tech/libeufin/nexus/ebics/Ebics2.kt b/nexus/src/main/kotlin/tech/libeufin/nexus/ebics/Ebics2.kt index 9f8a5d9f..1d96e93d 100644 --- a/nexus/src/main/kotlin/tech/libeufin/nexus/ebics/Ebics2.kt +++ b/nexus/src/main/kotlin/tech/libeufin/nexus/ebics/Ebics2.kt @@ -36,6 +36,7 @@ import java.security.interfaces.RSAPrivateCrtKey import java.time.Instant import java.time.ZoneId import java.util.* +import java.io.InputStream import javax.xml.datatype.DatatypeFactory private val logger: Logger = LoggerFactory.getLogger("libeufin-nexus-ebics2") @@ -151,10 +152,11 @@ fun createEbics25DownloadTransferPhase( */ fun parseKeysMgmtResponse( clientEncryptionKey: RSAPrivateCrtKey, - xml: ByteArray + xml: InputStream ): EbicsKeyManagementResponseContent? { + // TODO throw instead of null val jaxb = try { - XMLUtil.convertBytesToJaxb<EbicsKeyManagementResponse>(xml) + XMLUtil.convertToJaxb<EbicsKeyManagementResponse>(xml) } catch (e: Exception) { tech.libeufin.nexus.logger.error("Could not parse the raw response from bank into JAXB.") return null @@ -172,7 +174,7 @@ fun parseKeysMgmtResponse( clientEncryptionKey, DataEncryptionInfo(this.transactionKey, this.encryptionPubKeyDigest.value), listOf(encOrderData) - ) + ).readBytes() } } val bankReturnCode = EbicsReturnCode.lookup(jaxb.value.body.returnCode.value) // business error diff --git a/nexus/src/main/kotlin/tech/libeufin/nexus/ebics/EbicsCommon.kt b/nexus/src/main/kotlin/tech/libeufin/nexus/ebics/EbicsCommon.kt index 58718824..66a38b62 100644 --- a/nexus/src/main/kotlin/tech/libeufin/nexus/ebics/EbicsCommon.kt +++ b/nexus/src/main/kotlin/tech/libeufin/nexus/ebics/EbicsCommon.kt @@ -42,20 +42,20 @@ import io.ktor.client.plugins.* import io.ktor.client.request.* import io.ktor.client.statement.* import io.ktor.http.* -import org.apache.commons.compress.archivers.zip.ZipFile -import org.apache.commons.compress.utils.SeekableInMemoryByteChannel +import io.ktor.utils.io.jvm.javaio.* import org.slf4j.Logger import org.slf4j.LoggerFactory import tech.libeufin.nexus.* import tech.libeufin.common.* import tech.libeufin.ebics.* import tech.libeufin.ebics.ebics_h005.Ebics3Request +import java.io.SequenceInputStream import java.io.ByteArrayOutputStream +import java.io.InputStream import java.security.interfaces.RSAPrivateCrtKey import java.time.LocalDateTime import java.time.format.DateTimeFormatter import java.util.* -import java.util.zip.DeflaterInputStream /** * Available EBICS versions. @@ -73,24 +73,6 @@ enum class SupportedDocument { CAMT_054 } - -/** - * Unzips the ByteArray and runs the lambda over each entry. - * - * @param lambda function that gets the (fileName, fileContent) pair - * for each entry in the ZIP archive as input. - */ -fun ByteArray.unzipForEach(lambda: (String, ByteArray) -> Unit) { - val mem = SeekableInMemoryByteChannel(this) - ZipFile(mem).use { file -> - file.getEntriesInPhysicalOrder().iterator().forEach { - lambda( - it.name, file.getInputStream(it).readAllBytes() - ) - } - } -} - /** * Decrypts and decompresses the business payload that was * transported within an EBICS message from the bank @@ -106,21 +88,16 @@ fun decryptAndDecompressPayload( clientEncryptionKey: RSAPrivateCrtKey, encryptionInfo: DataEncryptionInfo, chunks: List<String> -): ByteArray { - val buf = StringBuilder() - chunks.forEach { buf.append(it) } - val decoded = Base64.getDecoder().decode(buf.toString()) - val er = CryptoUtil.EncryptionResult( - encryptionInfo.transactionKey, - encryptionInfo.bankPubDigest, - decoded - ) - val dataCompr = CryptoUtil.decryptEbicsE002( - er, - clientEncryptionKey - ) - return EbicsOrderUtil.decodeOrderData(dataCompr) -} +): InputStream = + SequenceInputStream(Collections.enumeration(chunks.map { it.toByteArray().inputStream() })) // Aggregate + .decodeBase64() + .run { + CryptoUtil.decryptEbicsE002( + encryptionInfo.transactionKey, + this, + clientEncryptionKey + ) + }.inflate() /** * POSTs the EBICS message to the bank. @@ -129,7 +106,7 @@ fun decryptAndDecompressPayload( * @param msg EBICS message as raw bytes. * @return the raw bank response. */ -suspend fun HttpClient.postToBank(bankUrl: String, msg: ByteArray): ByteArray { +suspend fun HttpClient.postToBank(bankUrl: String, msg: ByteArray): InputStream { logger.debug("POSTing EBICS to '$bankUrl'") val res = post(urlString = bankUrl) { contentType(ContentType.Text.Xml) @@ -138,7 +115,7 @@ suspend fun HttpClient.postToBank(bankUrl: String, msg: ByteArray): ByteArray { if (res.status != HttpStatusCode.OK) { throw Exception("Invalid response status: ${res.status}") } - return res.readBytes() // TODO input stream + return res.bodyAsChannel().toInputStream() } /** @@ -278,19 +255,19 @@ private fun areCodesOk(ebicsResponseContent: EbicsResponseContent) = * @param bankKeys bank EBICS public keys. * @param reqXml raw EBICS XML request of the init phase. * @param isEbics3 true for EBICS 3, false otherwise. - * @param processing processing lambda receiving EBICS files as bytes or empty bytes if nothing to download. - * @return T if the transaction was successful and null if the transaction was empty. If the failure is at the EBICS + * @param processing processing lambda receiving EBICS files as a byte stream if the transaction was not empty. + * @return T if the transaction was successful. If the failure is at the EBICS * level EbicsSideException is thrown else ités the expection of the processing lambda. */ -suspend fun <T> ebicsDownload( +suspend fun ebicsDownload( client: HttpClient, cfg: EbicsSetupConfig, clientKeys: ClientPrivateKeysFile, bankKeys: BankPublicKeysFile, reqXml: ByteArray, isEbics3: Boolean, - processing: (ByteArray) -> T -): T { + processing: (InputStream) -> Unit +) { val initResp = postEbics(client, cfg, bankKeys, reqXml, isEbics3) logger.debug("Download init phase done. EBICS- and bank-technical codes are: ${initResp.technicalReturnCode}, ${initResp.bankReturnCode}") if (initResp.technicalReturnCode != EbicsReturnCode.EBICS_OK) { @@ -298,7 +275,7 @@ suspend fun <T> ebicsDownload( } if (initResp.bankReturnCode == EbicsReturnCode.EBICS_NO_DOWNLOAD_DATA_AVAILABLE) { logger.debug("Download content is empty") - return processing(ByteArray(0)) + return } if (initResp.bankReturnCode != EbicsReturnCode.EBICS_OK) { throw Exception("Download init phase has bank-technical error: ${initResp.bankReturnCode}") @@ -419,11 +396,11 @@ class EbicsSideException( */ fun parseAndValidateEbicsResponse( bankKeys: BankPublicKeysFile, - resp: ByteArray, + resp: InputStream, withEbics3: Boolean ): EbicsResponseContent { - val responseDocument = try { - XMLUtil.parseBytesIntoDom(resp) + val doc = try { + XMLUtil.parseIntoDom(resp) } catch (e: Exception) { throw EbicsSideException( "Bank response apparently invalid", @@ -431,9 +408,9 @@ fun parseAndValidateEbicsResponse( ) } if (!XMLUtil.verifyEbicsDocument( - responseDocument, - bankKeys.bank_authentication_public_key, - withEbics3 + doc, + bankKeys.bank_authentication_public_key, + withEbics3 )) { throw EbicsSideException( "Bank signature did not verify", @@ -441,8 +418,8 @@ fun parseAndValidateEbicsResponse( ) } if (withEbics3) - return ebics3toInternalRepr(resp) - return ebics25toInternalRepr(resp) + return ebics3toInternalRepr(doc) + return ebics25toInternalRepr(doc) } /** @@ -490,9 +467,7 @@ fun prepareUploadPayload( val plainTransactionKey = encryptionResult.plainTransactionKey ?: throw Exception("Could not generate the transaction key, cannot encrypt the payload!") // Then only E002 symmetric (with ephemeral key) encrypt. - val compressedInnerPayload = DeflaterInputStream( - payload.inputStream() - ).use { it.readAllBytes() } + val compressedInnerPayload = payload.inputStream().deflate().readAllBytes() val encryptedPayload = CryptoUtil.encryptEbicsE002withTransactionKey( compressedInnerPayload, bankKeys.bank_encryption_public_key, diff --git a/nexus/src/test/kotlin/Ebics.kt b/nexus/src/test/kotlin/Ebics.kt index 507820ce..f3fad224 100644 --- a/nexus/src/test/kotlin/Ebics.kt +++ b/nexus/src/test/kotlin/Ebics.kt @@ -35,7 +35,7 @@ class Ebics { @Test fun iniMessage() = conf { config -> val msg = generateIniMessage(config, clientKeys) - val ini = XMLUtil.convertBytesToJaxb<EbicsUnsecuredRequest>(msg) // ensures is valid + val ini = XMLUtil.convertToJaxb<EbicsUnsecuredRequest>(msg.inputStream()) // ensures is valid assertEquals(ini.value.header.static.orderDetails.orderType, "INI") // ensures is INI } @@ -43,7 +43,7 @@ class Ebics { @Test fun hiaMessage() = conf { config -> val msg = generateHiaMessage(config, clientKeys) - val ini = XMLUtil.convertBytesToJaxb<EbicsUnsecuredRequest>(msg) // ensures is valid + val ini = XMLUtil.convertToJaxb<EbicsUnsecuredRequest>(msg.inputStream()) // ensures is valid assertEquals(ini.value.header.static.orderDetails.orderType, "HIA") // ensures is HIA } @@ -51,7 +51,7 @@ class Ebics { @Test fun hpbMessage() = conf { config -> val msg = generateHpbMessage(config, clientKeys) - val ini = XMLUtil.convertBytesToJaxb<EbicsUnsecuredRequest>(msg) // ensures is valid + val ini = XMLUtil.convertToJaxb<EbicsUnsecuredRequest>(msg.inputStream()) // ensures is valid assertEquals(ini.value.header.static.orderDetails.orderType, "HPB") // ensures is HPB } // POSTs an EBICS message to the mock bank. Tests diff --git a/testbench/src/test/kotlin/Iso20022Test.kt b/testbench/src/test/kotlin/Iso20022Test.kt index 7d5077ad..27914c78 100644 --- a/testbench/src/test/kotlin/Iso20022Test.kt +++ b/testbench/src/test/kotlin/Iso20022Test.kt @@ -30,15 +30,15 @@ class Iso20022Test { val fetch = file.resolve("fetch") if (file.isDirectory() && fetch.exists()) { for (log in fetch.listDirectoryEntries()) { - val str = log.readBytes() + val content = Files.newInputStream(log) val name = log.toString() println(name) if (name.contains("HAC")) { - parseCustomerAck(str) + parseCustomerAck(content) } else if (name.contains("pain.002")) { - parseCustomerPaymentStatusReport(str) + parseCustomerPaymentStatusReport(content) } else { - parseTxNotif(str, "CHF", mutableListOf(), mutableListOf()) + parseTxNotif(content, "CHF", mutableListOf(), mutableListOf()) } } } |