Reduce memory usage by using InputStream as much as possiblev0.9.4-dev.19

21 files changed, 191 insertions, 180 deletions

diff --git a/bank/src/main/kotlin/tech/libeufin/bank/db/WithdrawalDAO.kt b/bank/src/main/kotlin/tech/libeufin/bank/db/WithdrawalDAO.kt
index cc952691..12977610 100644
--- a/bank/src/main/kotlin/tech/libeufin/bank/db/WithdrawalDAO.kt
+++ b/bank/src/main/kotlin/tech/libeufin/bank/db/WithdrawalDAO.kt
@@ -152,7 +152,6 @@ class WithdrawalDAO(private val db: Database) {
         now: Instant,
         is2fa: Boolean
     ): WithdrawalConfirmationResult = db.serializable { conn ->
-         // TODO login check
         val stmt = conn.prepareStatement("""
             SELECT
               out_no_op,
diff --git a/bank/src/test/kotlin/SecurityTest.kt b/bank/src/test/kotlin/SecurityTest.kt
index b5559983..16d2b3f8 100644
--- a/bank/src/test/kotlin/SecurityTest.kt
+++ b/bank/src/test/kotlin/SecurityTest.kt
@@ -30,19 +30,12 @@ import kotlinx.serialization.json.*
 import org.junit.Test
 import tech.libeufin.bank.*
 import tech.libeufin.common.*
-import tech.libeufin.common.*
-import java.io.ByteArrayOutputStream
-import java.util.zip.DeflaterOutputStream
 
 inline fun <reified B> HttpRequestBuilder.jsonDeflate(b: B) {
     val json = Json.encodeToString(kotlinx.serialization.serializer<B>(), b);
     contentType(ContentType.Application.Json)
     headers.set(HttpHeaders.ContentEncoding, "deflate")
-    val bos = ByteArrayOutputStream()
-    val ios = DeflaterOutputStream(bos)
-    ios.write(json.toByteArray())
-    ios.finish()
-    setBody(bos.toByteArray())
+    setBody(json.toByteArray().inputStream().deflate().readBytes())
 }
 
 class SecurityTest {
diff --git a/bank/src/test/kotlin/helpers.kt b/bank/src/test/kotlin/helpers.kt
index e6c53565..f4d3a4a6 100644
--- a/bank/src/test/kotlin/helpers.kt
+++ b/bank/src/test/kotlin/helpers.kt
@@ -23,7 +23,6 @@ import io.ktor.client.statement.*
 import io.ktor.http.*
 import io.ktor.server.testing.*
 import java.io.ByteArrayOutputStream
-import java.util.zip.DeflaterOutputStream
 import java.nio.file.*
 import kotlin.test.*
 import kotlin.io.path.*
diff --git a/common/src/main/kotlin/CryptoUtil.kt b/common/src/main/kotlin/CryptoUtil.kt
index bcf19ae1..3729627e 100644
--- a/common/src/main/kotlin/CryptoUtil.kt
+++ b/common/src/main/kotlin/CryptoUtil.kt
@@ -21,6 +21,7 @@ package tech.libeufin.common
 
 import org.bouncycastle.jce.provider.BouncyCastleProvider
 import java.io.ByteArrayOutputStream
+import java.io.InputStream
 import java.math.BigInteger
 import java.security.*
 import java.security.interfaces.RSAPrivateCrtKey
@@ -176,18 +177,18 @@ object CryptoUtil {
     }
 
     fun decryptEbicsE002(enc: EncryptionResult, privateKey: RSAPrivateCrtKey): ByteArray {
-        return decryptEbicsE002(
+        return CryptoUtil.decryptEbicsE002(
             enc.encryptedTransactionKey,
-            enc.encryptedData,
+            enc.encryptedData.inputStream(),
             privateKey
-        )
+        ).readBytes()
     }
 
     fun decryptEbicsE002(
         encryptedTransactionKey: ByteArray,
-        encryptedData: ByteArray,
+        encryptedData: InputStream,
         privateKey: RSAPrivateCrtKey
-    ): ByteArray {
+    ): CipherInputStream {
         val asymmetricCipher = Cipher.getInstance(
             "RSA/None/PKCS1Padding",
             bouncyCastleProvider
@@ -201,8 +202,7 @@ object CryptoUtil {
         )
         val ivParameterSpec = IvParameterSpec(ByteArray(16))
         symmetricCipher.init(Cipher.DECRYPT_MODE, secretKeySpec, ivParameterSpec)
-        val data = symmetricCipher.doFinal(encryptedData)
-        return data
+        return CipherInputStream(encryptedData, symmetricCipher)
     }
 
     /**
diff --git a/common/src/main/kotlin/Stream.kt b/common/src/main/kotlin/Stream.kt
new file mode 100644
index 00000000..7b9689b5
--- /dev/null
+++ b/common/src/main/kotlin/Stream.kt
@@ -0,0 +1,55 @@
+/*
+ * This file is part of LibEuFin.
+ * Copyright (C) 2024 Taler Systems S.A.
+
+ * LibEuFin is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation; either version 3, or
+ * (at your option) any later version.
+
+ * LibEuFin is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+ * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Affero General
+ * Public License for more details.
+
+ * You should have received a copy of the GNU Affero General Public
+ * License along with LibEuFin; see the file COPYING.  If not, see
+ * <http://www.gnu.org/licenses/>
+ */
+
+package tech.libeufin.common
+
+import java.io.InputStream
+import java.io.FilterInputStream
+import java.util.zip.DeflaterInputStream
+import java.util.zip.InflaterInputStream
+import java.util.zip.*
+import java.util.Base64
+
+/** Unzip an input stream and run [lambda] over each entry */
+fun InputStream.unzipEach(lambda: (String, InputStream) -> Unit) {
+    ZipInputStream(this).use { zip ->
+        while (true) {
+            val entry = zip.getNextEntry()
+            if (entry == null) break;
+            val entryStream = object: FilterInputStream(zip) {
+                override fun close() {
+                    zip.closeEntry();
+                }
+            }
+            lambda(entry.name, entryStream)
+        }
+    }
+}
+
+/** Decode a base64 an input stream */
+fun InputStream.decodeBase64(): InputStream 
+    = Base64.getDecoder().wrap(this)
+
+/** Deflate an input stream */
+fun InputStream.deflate(): DeflaterInputStream 
+    = DeflaterInputStream(this)
+
+/** Inflate an input stream */
+fun InputStream.inflate(): InflaterInputStream 
+    = InflaterInputStream(this)
\ No newline at end of file
diff --git a/ebics/src/main/kotlin/Ebics.kt b/ebics/src/main/kotlin/Ebics.kt
index 91a41022..7651be74 100644
--- a/ebics/src/main/kotlin/Ebics.kt
+++ b/ebics/src/main/kotlin/Ebics.kt
@@ -36,9 +36,11 @@ import java.time.Instant
 import java.time.ZoneId
 import java.time.ZonedDateTime
 import java.util.*
+import java.io.InputStream
 import javax.xml.bind.JAXBElement
 import javax.xml.datatype.DatatypeFactory
 import javax.xml.datatype.XMLGregorianCalendar
+import org.w3c.dom.Document
 
 data class EbicsProtocolError(
     val httpStatusCode: HttpStatusCode,
@@ -308,9 +310,9 @@ class HpbResponseData(
     val authenticationVersion: String
 )
 
-fun parseEbicsHpbOrder(orderDataRaw: ByteArray): HpbResponseData {
+fun parseEbicsHpbOrder(orderDataRaw: InputStream): HpbResponseData {
     val resp = try {
-        XMLUtil.convertBytesToJaxb<HPBResponseOrderData>(orderDataRaw)
+        XMLUtil.convertToJaxb<HPBResponseOrderData>(orderDataRaw)
     } catch (e: Exception) {
         throw EbicsProtocolError(HttpStatusCode.InternalServerError, "Invalid XML (as HPB response) received from bank")
     }
@@ -331,10 +333,10 @@ fun parseEbicsHpbOrder(orderDataRaw: ByteArray): HpbResponseData {
     )
 }
 
-fun ebics3toInternalRepr(response: ByteArray): EbicsResponseContent {
+fun ebics3toInternalRepr(response: Document): EbicsResponseContent {
     // logger.debug("Converting bank resp to internal repr.: $response")
     val resp: JAXBElement<Ebics3Response> = try {
-        XMLUtil.convertBytesToJaxb(response)
+        XMLUtil.convertDomToJaxb(response)
     } catch (e: Exception) {
         throw EbicsProtocolError(
             HttpStatusCode.InternalServerError,
@@ -368,9 +370,9 @@ fun ebics3toInternalRepr(response: ByteArray): EbicsResponseContent {
     )
 }
 
-fun ebics25toInternalRepr(response: ByteArray): EbicsResponseContent {
+fun ebics25toInternalRepr(response: Document): EbicsResponseContent {
     val resp: JAXBElement<EbicsResponse> = try {
-        XMLUtil.convertBytesToJaxb(response)
+        XMLUtil.convertDomToJaxb(response)
     } catch (e: Exception) {
         throw EbicsProtocolError(
             HttpStatusCode.InternalServerError,
diff --git a/ebics/src/main/kotlin/EbicsOrderUtil.kt b/ebics/src/main/kotlin/EbicsOrderUtil.kt
index c056ddde..79ff829a 100644
--- a/ebics/src/main/kotlin/EbicsOrderUtil.kt
+++ b/ebics/src/main/kotlin/EbicsOrderUtil.kt
@@ -21,33 +21,23 @@ package tech.libeufin.ebics
 
 import java.lang.IllegalArgumentException
 import java.security.SecureRandom
-import java.util.zip.DeflaterInputStream
-import java.util.zip.InflaterInputStream
+import java.io.InputStream
+import tech.libeufin.common.*
 
 /**
  * Helpers for dealing with order compression, encryption, decryption, chunking and re-assembly.
  */
 object EbicsOrderUtil {
 
-    // Decompression only, no XML involved.
-    fun decodeOrderData(encodedOrderData: ByteArray): ByteArray {
-        return InflaterInputStream(encodedOrderData.inputStream()).use {
-            it.readAllBytes()
-        }
-    }
-
     inline fun <reified T> decodeOrderDataXml(encodedOrderData: ByteArray): T {
-        return InflaterInputStream(encodedOrderData.inputStream()).use {
-            val bytes = it.readAllBytes()
-            XMLUtil.convertBytesToJaxb<T>(bytes).value
+        return encodedOrderData.inputStream().inflate().use {
+            XMLUtil.convertToJaxb<T>(it).value
         }
     }
 
     inline fun <reified T> encodeOrderDataXml(obj: T): ByteArray {
         val bytes = XMLUtil.convertJaxbToBytes(obj)
-        return DeflaterInputStream(bytes.inputStream()).use {
-            it.readAllBytes()
-        }
+        return bytes.inputStream().deflate().readAllBytes()
     }
 
     @kotlin.ExperimentalStdlibApi
diff --git a/ebics/src/main/kotlin/XMLUtil.kt b/ebics/src/main/kotlin/XMLUtil.kt
index 5947341a..c27afbc5 100644
--- a/ebics/src/main/kotlin/XMLUtil.kt
+++ b/ebics/src/main/kotlin/XMLUtil.kt
@@ -288,9 +288,7 @@ class XMLUtil private constructor() {
          * @return InputStream object, as wanted by the validator.
          */
         fun validateFromBytes(xml: ByteArray): Boolean {
-            val xmlInputStream: InputStream = ByteArrayInputStream(xml)
-            val xmlSource = StreamSource(xmlInputStream)
-            return validate(xmlSource)
+            return validate(StreamSource(xml.inputStream()))
         }
 
         inline fun <reified T> convertJaxbToBytes(
@@ -333,11 +331,11 @@ class XMLUtil private constructor() {
          * @param documentBytes the bytes to convert into JAXB.
          * @return the JAXB object reflecting the original XML document.
          */
-        inline fun <reified T> convertBytesToJaxb(documentBytes: ByteArray): JAXBElement<T> {
+        inline fun <reified T> convertToJaxb(documentBytes: InputStream): JAXBElement<T> {
             val jc = JAXBContext.newInstance(T::class.java)
             val u = jc.createUnmarshaller()
             return u.unmarshal(            /* Marshalling the object into the document.  */
-                StreamSource(ByteArrayInputStream(documentBytes)),
+                StreamSource(documentBytes),
                 T::class.java
             )
         }
@@ -374,25 +372,25 @@ class XMLUtil private constructor() {
         /**
          * Convert a DOM document to the JAXB representation.
          *
-         * @param finalType class type of the output
          * @param document the document to convert into JAXB.
          * @return the JAXB object reflecting the original XML document.
          */
-        fun <T> convertDomToJaxb(finalType: Class<T>, document: Document): JAXBElement<T> {
-            val jc = JAXBContext.newInstance(finalType)
+        inline fun <reified T> convertDomToJaxb(document: Document): JAXBElement<T> {
+            val jc = JAXBContext.newInstance(T::class.java)
             /* Marshalling the object into the document.  */
             val m = jc.createUnmarshaller()
-            return m.unmarshal(document, finalType) // document "went" into Jaxb
+            return m.unmarshal(document, T::class.java) // document "went" into Jaxb
         }
 
         /** Parse [xml] into a XML DOM */
-        fun parseBytesIntoDom(xml: ByteArray): Document {
+        fun parseIntoDom(xml: InputStream): Document {
             val factory = DocumentBuilderFactory.newInstance().apply {
                 isNamespaceAware = true
             }
-            val xmlInputStream = ByteArrayInputStream(xml)
             val builder = factory.newDocumentBuilder()
-            return builder.parse(InputSource(xmlInputStream))
+            return xml.use { 
+                builder.parse(InputSource(it))
+            }
         }
 
         fun signEbicsResponse(ebicsResponse: EbicsResponse, privateKey: RSAPrivateCrtKey): ByteArray {
diff --git a/ebics/src/main/kotlin/XmlCombinators.kt b/ebics/src/main/kotlin/XmlCombinators.kt
index d2d4d25e..ee295010 100644
--- a/ebics/src/main/kotlin/XmlCombinators.kt
+++ b/ebics/src/main/kotlin/XmlCombinators.kt
@@ -25,6 +25,7 @@ import org.w3c.dom.Element
 import java.io.StringWriter
 import javax.xml.stream.XMLOutputFactory
 import javax.xml.stream.XMLStreamWriter
+import java.io.InputStream
 import java.time.format.*
 import java.time.*
 
@@ -133,8 +134,8 @@ class XmlDestructor internal constructor(private val el: Element) {
     fun attr(index: String): String = el.getAttribute(index)
 }
 
-fun <T> destructXml(xml: ByteArray, root: String, f: XmlDestructor.() -> T): T {
-    val doc = XMLUtil.parseBytesIntoDom(xml)
+fun <T> destructXml(xml: InputStream, root: String, f: XmlDestructor.() -> T): T {
+    val doc = XMLUtil.parseIntoDom(xml)
     if (doc.documentElement.tagName != root) {
         throw DestructionError("expected root '$root' got '${doc.documentElement.tagName}'")
     }
diff --git a/ebics/src/test/kotlin/EbicsMessagesTest.kt b/ebics/src/test/kotlin/EbicsMessagesTest.kt
index 2b8d63ca..7dbdc350 100644
--- a/ebics/src/test/kotlin/EbicsMessagesTest.kt
+++ b/ebics/src/test/kotlin/EbicsMessagesTest.kt
@@ -40,8 +40,8 @@ class EbicsMessagesTest {
     @Test
     fun testImportNonRoot() {
         val classLoader = ClassLoader.getSystemClassLoader()
-        val ini = classLoader.getResource("ebics_ini_inner_key.xml")
-        val jaxb = XMLUtil.convertBytesToJaxb<SignatureTypes.SignaturePubKeyOrderData>(ini.readBytes())
+        val ini = classLoader.getResourceAsStream("ebics_ini_inner_key.xml")
+        val jaxb = XMLUtil.convertToJaxb<SignatureTypes.SignaturePubKeyOrderData>(ini)
         assertEquals("A006", jaxb.value.signaturePubKeyInfo.signatureVersion)
     }
 
@@ -51,8 +51,8 @@ class EbicsMessagesTest {
     @Test
     fun testStringToJaxb() {
         val classLoader = ClassLoader.getSystemClassLoader()
-        val ini = classLoader.getResource("ebics_ini_request_sample.xml")
-        val jaxb = XMLUtil.convertBytesToJaxb<EbicsUnsecuredRequest>(ini.readBytes())
+        val ini = classLoader.getResourceAsStream("ebics_ini_request_sample.xml")
+        val jaxb = XMLUtil.convertToJaxb<EbicsUnsecuredRequest>(ini)
         println("jaxb loaded")
         assertEquals(
             "INI",
@@ -81,10 +81,9 @@ class EbicsMessagesTest {
     @Test
     fun testDomToJaxb() {
         val classLoader = ClassLoader.getSystemClassLoader()
-        val ini = classLoader.getResource("ebics_ini_request_sample.xml")!!
-        val iniDom = XMLUtil.parseBytesIntoDom(ini.readBytes())
+        val ini = classLoader.getResourceAsStream("ebics_ini_request_sample.xml")
+        val iniDom = XMLUtil.parseIntoDom(ini)
         XMLUtil.convertDomToJaxb<EbicsUnsecuredRequest>(
-            EbicsUnsecuredRequest::class.java,
             iniDom
         )
     }
@@ -114,15 +113,15 @@ class EbicsMessagesTest {
     @Test
     fun testParseHiaRequestOrderData() {
         val classLoader = ClassLoader.getSystemClassLoader()
-        val hia = classLoader.getResource("hia_request_order_data.xml")!!.readBytes()
-        XMLUtil.convertBytesToJaxb<HIARequestOrderData>(hia)
+        val hia = classLoader.getResourceAsStream("hia_request_order_data.xml")
+        XMLUtil.convertToJaxb<HIARequestOrderData>(hia)
     }
 
     @Test
     fun testHiaLoad() {
         val classLoader = ClassLoader.getSystemClassLoader()
-        val hia = classLoader.getResource("hia_request.xml")!!
-        val hiaDom = XMLUtil.parseBytesIntoDom(hia.readBytes())
+        val hia = classLoader.getResourceAsStream("hia_request.xml")
+        val hiaDom = XMLUtil.parseIntoDom(hia)
         val x: Element = hiaDom.getElementsByTagNameNS(
             "urn:org:ebics:H004",
             "OrderDetails"
@@ -135,7 +134,6 @@ class EbicsMessagesTest {
         )
 
         XMLUtil.convertDomToJaxb<EbicsUnsecuredRequest>(
-            EbicsUnsecuredRequest::class.java,
             hiaDom
         )
     }
@@ -144,11 +142,11 @@ class EbicsMessagesTest {
     fun testLoadInnerKey() {
         val jaxbKey = run {
             val classLoader = ClassLoader.getSystemClassLoader()
-            val file = classLoader.getResource(
+            val file = classLoader.getResourceAsStream(
                 "ebics_ini_inner_key.xml"
             )
             assertNotNull(file)
-            XMLUtil.convertBytesToJaxb<SignatureTypes.SignaturePubKeyOrderData>(file.readBytes())
+            XMLUtil.convertToJaxb<SignatureTypes.SignaturePubKeyOrderData>(file)
         }
 
         val modulus = jaxbKey.value.signaturePubKeyInfo.pubKeyValue.rsaKeyValue.modulus
@@ -159,8 +157,8 @@ class EbicsMessagesTest {
     @Test
     fun testLoadIniMessage() {
         val classLoader = ClassLoader.getSystemClassLoader()
-        val text = classLoader.getResource("ebics_ini_request_sample.xml")!!.readBytes()
-        XMLUtil.convertBytesToJaxb<EbicsUnsecuredRequest>(text)
+        val text = classLoader.getResourceAsStream("ebics_ini_request_sample.xml")!!
+        XMLUtil.convertToJaxb<EbicsUnsecuredRequest>(text)
     }
 
     @Test
@@ -189,8 +187,8 @@ class EbicsMessagesTest {
     @Test
     fun testLoadHpb() {
         val classLoader = ClassLoader.getSystemClassLoader()
-        val text = classLoader.getResource("hpb_request.xml")!!.readBytes()
-        XMLUtil.convertBytesToJaxb<EbicsNpkdRequest>(text)
+        val text = classLoader.getResourceAsStream("hpb_request.xml")!!
+        XMLUtil.convertToJaxb<EbicsNpkdRequest>(text)
     }
 
     @Test
@@ -358,7 +356,7 @@ class EbicsMessagesTest {
         }
 
         val str = XMLUtil.convertJaxbToBytes(ebicsRequestObj)
-        val doc = XMLUtil.parseBytesIntoDom(str)
+        val doc = XMLUtil.parseIntoDom(str.inputStream())
         val pair = CryptoUtil.generateRsaKeyPair(1024)
         XMLUtil.signEbicsDocument(doc, pair.private)
         val bytes = XMLUtil.convertDomToBytes(doc)
diff --git a/ebics/src/test/kotlin/EbicsOrderUtilTest.kt b/ebics/src/test/kotlin/EbicsOrderUtilTest.kt
index c78da738..3d8ed821 100644
--- a/ebics/src/test/kotlin/EbicsOrderUtilTest.kt
+++ b/ebics/src/test/kotlin/EbicsOrderUtilTest.kt
@@ -302,7 +302,7 @@ class EbicsOrderUtilTest {
                     </Permission>
                 </UserInfo>
             </HTDResponseOrderData>
-        """.trimIndent().toByteArray()
-        XMLUtil.convertBytesToJaxb<HTDResponseOrderData>(orderDataXml);
+        """.trimIndent().toByteArray().inputStream()
+        XMLUtil.convertToJaxb<HTDResponseOrderData>(orderDataXml);
     }
 }
\ No newline at end of file
diff --git a/ebics/src/test/kotlin/XmlUtilTest.kt b/ebics/src/test/kotlin/XmlUtilTest.kt
index 93ca8bf2..647918d5 100644
--- a/ebics/src/test/kotlin/XmlUtilTest.kt
+++ b/ebics/src/test/kotlin/XmlUtilTest.kt
@@ -26,7 +26,7 @@ import tech.libeufin.ebics.ebics_h004.EbicsKeyManagementResponse
 import tech.libeufin.ebics.ebics_h004.EbicsResponse
 import tech.libeufin.ebics.ebics_h004.EbicsTypes
 import tech.libeufin.ebics.ebics_h004.HTDResponseOrderData
-import tech.libeufin.common.CryptoUtil
+import tech.libeufin.common.*
 import tech.libeufin.ebics.XMLUtil
 import java.security.KeyPairGenerator
 import java.util.*
@@ -37,7 +37,7 @@ class XmlUtilTest {
 
     @Test
     fun deserializeConsecutiveLists() {
-        val tmp = XMLUtil.convertBytesToJaxb<HTDResponseOrderData>("""
+        val tmp = XMLUtil.convertToJaxb<HTDResponseOrderData>("""
             <?xml version="1.0" encoding="UTF-8" standalone="yes"?>
             <HTDResponseOrderData xmlns="urn:org:ebics:H004">
               <PartnerInfo>
@@ -80,7 +80,7 @@ class XmlUtilTest {
                   <OrderTypes>C54 C53 C52 CCC</OrderTypes>
                 </Permission>
               </UserInfo>
-            </HTDResponseOrderData>""".trimIndent().toByteArray()
+            </HTDResponseOrderData>""".trimIndent().toByteArray().inputStream()
         )
 
         println(tmp.value.partnerInfo.orderInfoList[0].description)
@@ -89,7 +89,7 @@ class XmlUtilTest {
     @Test
     fun exceptionOnConversion() {
         try {
-            XMLUtil.convertBytesToJaxb<EbicsKeyManagementResponse>("<malformed xml>".toByteArray())
+            XMLUtil.convertToJaxb<EbicsKeyManagementResponse>("<malformed xml>".toByteArray().inputStream())
         } catch (e: javax.xml.bind.UnmarshalException) {
             // just ensuring this is the exception
             println("caught")
@@ -114,12 +114,12 @@ class XmlUtilTest {
 
     @Test
     fun basicSigningTest() {
-        val doc = XMLUtil.parseBytesIntoDom("""
+        val doc = XMLUtil.parseIntoDom("""
             <myMessage xmlns:ebics="urn:org:ebics:H004">
                 <ebics:AuthSignature />
                 <foo authenticate="true">Hello World</foo>
             </myMessage>
-        """.trimIndent().toByteArray())
+        """.trimIndent().toByteArray().inputStream())
         val kpg = KeyPairGenerator.getInstance("RSA")
         kpg.initialize(2048)
         val pair = kpg.genKeyPair()
@@ -154,7 +154,7 @@ class XmlUtilTest {
         }
 
         val signature = signEbicsResponse(response, pair.private)
-        val signatureJaxb = XMLUtil.convertBytesToJaxb<EbicsResponse>(signature)
+        val signatureJaxb = XMLUtil.convertToJaxb<EbicsResponse>(signature.inputStream())
 
         assertTrue(
             XMLUtil.verifyEbicsDocument(
@@ -166,13 +166,13 @@ class XmlUtilTest {
 
     @Test
     fun multiAuthSigningTest() {
-        val doc = XMLUtil.parseBytesIntoDom("""
+        val doc = XMLUtil.parseIntoDom("""
             <myMessage xmlns:ebics="urn:org:ebics:H004">
                 <ebics:AuthSignature />
                 <foo authenticate="true">Hello World</foo>
                 <bar authenticate="true">Another one!</bar>
             </myMessage>
-        """.trimIndent().toByteArray())
+        """.trimIndent().toByteArray().inputStream())
         val kpg = KeyPairGenerator.getInstance("RSA")
         kpg.initialize(2048)
         val pair = kpg.genKeyPair()
@@ -183,10 +183,10 @@ class XmlUtilTest {
     @Test
     fun testRefSignature() {
         val classLoader = ClassLoader.getSystemClassLoader()
-        val docText = classLoader.getResourceAsStream("signature1/doc.xml")!!.readAllBytes()
-        val doc = XMLUtil.parseBytesIntoDom(docText)
-        val keyText = classLoader.getResourceAsStream("signature1/public_key.txt")!!.readAllBytes()
-        val keyBytes = Base64.getDecoder().decode(keyText)
+        val docText = classLoader.getResourceAsStream("signature1/doc.xml")
+        val doc = XMLUtil.parseIntoDom(docText)
+        val keyStream = classLoader.getResourceAsStream("signature1/public_key.txt")
+        val keyBytes = keyStream.decodeBase64().readAllBytes()
         val key = CryptoUtil.loadRsaPublicKey(keyBytes)
         assertTrue(XMLUtil.verifyEbicsDocument(doc, key))
     }
diff --git a/nexus/build.gradle b/nexus/build.gradle
index 3d6c6c7c..b510dba6 100644
--- a/nexus/build.gradle
+++ b/nexus/build.gradle
@@ -27,9 +27,6 @@ dependencies {
     // XML parsing/binding and encryption
     implementation("jakarta.xml.bind:jakarta.xml.bind-api:2.3.3")
 
-    // Compression
-    implementation("org.apache.commons:commons-compress:1.25.0")
-
     // Command line parsing
     implementation("com.github.ajalt.clikt:clikt:$clikt_version")
     implementation("org.postgresql:postgresql:$postgres_version")
diff --git a/nexus/src/main/kotlin/tech/libeufin/nexus/EbicsFetch.kt b/nexus/src/main/kotlin/tech/libeufin/nexus/EbicsFetch.kt
index 03d37e76..d4234cd1 100644
--- a/nexus/src/main/kotlin/tech/libeufin/nexus/EbicsFetch.kt
+++ b/nexus/src/main/kotlin/tech/libeufin/nexus/EbicsFetch.kt
@@ -31,6 +31,7 @@ import tech.libeufin.common.*
 import tech.libeufin.ebics.*
 import tech.libeufin.ebics.ebics_h005.Ebics3Request
 import java.io.IOException
+import java.io.InputStream
 import java.time.Instant
 import java.time.LocalDate
 import java.time.ZoneId
@@ -78,12 +79,12 @@ data class FetchContext(
  *         length is zero.  It returns null, if the bank assigned an
  *         error to the EBICS transaction.
  */
-private suspend fun <T> downloadHelper(
+private suspend fun downloadHelper(
     ctx: FetchContext,
     lastExecutionTime: Instant? = null,
     doc: SupportedDocument,
-    processing: (ByteArray) -> T
-): T? {
+    processing: (InputStream) -> Unit
+) {
     val isEbics3 = doc != SupportedDocument.PAIN_002_LOGS
     val initXml = if (isEbics3) {
         createEbics3DownloadInitialization(
@@ -246,7 +247,7 @@ suspend fun ingestIncomingPayment(
 private fun ingestDocument(
     db: Database,
     currency: String,
-    xml: ByteArray,
+    xml: InputStream,
     whichDocument: SupportedDocument
 ) {
     when (whichDocument) {
@@ -308,7 +309,7 @@ private fun ingestDocument(
 private fun ingestDocuments(
     db: Database,
     currency: String,
-    content: ByteArray,
+    content: InputStream,
     whichDocument: SupportedDocument
 ) {
     when (whichDocument) {
@@ -317,7 +318,7 @@ private fun ingestDocuments(
         SupportedDocument.CAMT_053, 
         SupportedDocument.CAMT_052 -> {
             try {
-                content.unzipForEach { fileName, xmlContent ->
+                content.unzipEach { fileName, xmlContent ->
                     logger.trace("parse $fileName")
                     ingestDocument(db, currency, xmlContent, whichDocument)
                 }
@@ -364,14 +365,12 @@ private suspend fun fetchDocuments(
             }
             val doc = doc.doc()
             // downloading the content
-            downloadHelper(ctx, lastExecutionTime, doc) { content ->
-                if (!content.isEmpty()) {
-                    ctx.fileLogger.logFetch(
-                        content,
-                        doc == SupportedDocument.PAIN_002_LOGS
-                    )
-                    ingestDocuments(db, ctx.cfg.currency, content, doc)
-                }
+            downloadHelper(ctx, lastExecutionTime, doc) { stream ->
+                val loggedStream = ctx.fileLogger.logFetch(
+                    stream,
+                    doc == SupportedDocument.PAIN_002_LOGS
+                )
+                ingestDocuments(db, ctx.cfg.currency, loggedStream, doc)
             }
             true
         } catch (e: Exception) {
diff --git a/nexus/src/main/kotlin/tech/libeufin/nexus/EbicsSetup.kt b/nexus/src/main/kotlin/tech/libeufin/nexus/EbicsSetup.kt
index bd95543b..a5c40f35 100644
--- a/nexus/src/main/kotlin/tech/libeufin/nexus/EbicsSetup.kt
+++ b/nexus/src/main/kotlin/tech/libeufin/nexus/EbicsSetup.kt
@@ -110,7 +110,7 @@ private fun handleHpbResponse(
         throw Exception("HPB content not found in a EBICS response with successful return codes.")
     }
     val hpbObj = try {
-        parseEbicsHpbOrder(hpbBytes)
+        parseEbicsHpbOrder(hpbBytes.inputStream())
     } catch (e: Exception) {
         throw Exception("HPB response content seems invalid", e)
     }
diff --git a/nexus/src/main/kotlin/tech/libeufin/nexus/Iso20022.kt b/nexus/src/main/kotlin/tech/libeufin/nexus/Iso20022.kt
index 97ba52f6..f6562c2e 100644
--- a/nexus/src/main/kotlin/tech/libeufin/nexus/Iso20022.kt
+++ b/nexus/src/main/kotlin/tech/libeufin/nexus/Iso20022.kt
@@ -23,6 +23,7 @@ import tech.libeufin.ebics.*
 import java.net.URLEncoder
 import java.time.*
 import java.time.format.*
+import java.io.InputStream
 
 
 /**
@@ -145,7 +146,7 @@ data class CustomerAck(
  *
  * @param xml pain.002 input document
  */
-fun parseCustomerAck(xml: ByteArray): List<CustomerAck> {
+fun parseCustomerAck(xml: InputStream): List<CustomerAck> {
     return destructXml(xml, "Document") {
         one("CstmrPmtStsRpt").map("OrgnlPmtInfAndSts") {
             val actionType = one("OrgnlPmtInfId").enum<HacAction>()
@@ -214,7 +215,7 @@ data class Reason (
  *
  * @param xml pain.002 input document
  */
-fun parseCustomerPaymentStatusReport(xml: ByteArray): PaymentStatus {
+fun parseCustomerPaymentStatusReport(xml: InputStream): PaymentStatus {
     fun XmlDestructor.reasons(): List<Reason> {
         return map("StsRsnInf") {
             val code = one("Rsn").one("Cd").enum<ExternalStatusReasonCode>()
@@ -255,7 +256,7 @@ fun parseCustomerPaymentStatusReport(xml: ByteArray): PaymentStatus {
  * @param outgoing list of outgoing payments
  */
 fun parseTxNotif(
-    notifXml: ByteArray,
+    notifXml: InputStream,
     acceptedCurrency: String,
     incoming: MutableList<IncomingPayment>,
     outgoing: MutableList<OutgoingPayment>
@@ -324,7 +325,7 @@ fun parseTxNotif(
  * @param xml the input document.
  */
 private fun notificationForEachTx(
-    xml: ByteArray,
+    xml: InputStream,
     directionLambda: XmlDestructor.(Instant) -> Unit
 ) {
     destructXml(xml, "Document") {
diff --git a/nexus/src/main/kotlin/tech/libeufin/nexus/Log.kt b/nexus/src/main/kotlin/tech/libeufin/nexus/Log.kt
index 3299a16e..88e3481a 100644
--- a/nexus/src/main/kotlin/tech/libeufin/nexus/Log.kt
+++ b/nexus/src/main/kotlin/tech/libeufin/nexus/Log.kt
@@ -19,7 +19,6 @@
 
 package tech.libeufin.nexus
 
-import tech.libeufin.nexus.ebics.unzipForEach
 import tech.libeufin.common.*
 import java.io.*
 import java.nio.file.*
@@ -50,12 +49,12 @@ class FileLogger(path: String?) {
     /**
      * Logs EBICS fetch content if EBICS debug logging is enabled
      *
-     * @param content EBICS fetch content
+     * @param stream EBICS fetch content
      * @param hac only true when downloading via HAC (EBICS 2)
      */
-    fun logFetch(content: ByteArray, hac: Boolean = false) {
-        if (dir == null) return;
-
+    fun logFetch(stream: InputStream, hac: Boolean = false): InputStream {
+        if (dir == null) return stream;
+        val content = stream.readBytes()
         // Subdir based on current day.
         val now = Instant.now()
         val asUtcDate = LocalDate.ofInstant(now, ZoneId.of("UTC"))
@@ -67,10 +66,13 @@ class FileLogger(path: String?) {
             subDir.resolve("${nowMs}_HAC_response.pain.002.xml").writeBytes(content)
         } else {
             // Write each ZIP entry in the combined dir.
-            content.unzipForEach { fileName, xmlContent ->
-                subDir.resolve("${nowMs}_$fileName").writeBytes(xmlContent)
+            content.inputStream().unzipEach { fileName, xmlContent ->
+                xmlContent.use {
+                    Files.copy(it, subDir.resolve("${nowMs}_$fileName"))
+                }
             }
         }
+        return content.inputStream()
     }
 
     /**
diff --git a/nexus/src/main/kotlin/tech/libeufin/nexus/ebics/Ebics2.kt b/nexus/src/main/kotlin/tech/libeufin/nexus/ebics/Ebics2.kt
index 9f8a5d9f..1d96e93d 100644
--- a/nexus/src/main/kotlin/tech/libeufin/nexus/ebics/Ebics2.kt
+++ b/nexus/src/main/kotlin/tech/libeufin/nexus/ebics/Ebics2.kt
@@ -36,6 +36,7 @@ import java.security.interfaces.RSAPrivateCrtKey
 import java.time.Instant
 import java.time.ZoneId
 import java.util.*
+import java.io.InputStream
 import javax.xml.datatype.DatatypeFactory
 
 private val logger: Logger = LoggerFactory.getLogger("libeufin-nexus-ebics2")
@@ -151,10 +152,11 @@ fun createEbics25DownloadTransferPhase(
  */
 fun parseKeysMgmtResponse(
     clientEncryptionKey: RSAPrivateCrtKey,
-    xml: ByteArray
+    xml: InputStream
 ): EbicsKeyManagementResponseContent? {
+    // TODO throw instead of null
     val jaxb = try {
-        XMLUtil.convertBytesToJaxb<EbicsKeyManagementResponse>(xml)
+        XMLUtil.convertToJaxb<EbicsKeyManagementResponse>(xml)
     } catch (e: Exception) {
         tech.libeufin.nexus.logger.error("Could not parse the raw response from bank into JAXB.")
         return null
@@ -172,7 +174,7 @@ fun parseKeysMgmtResponse(
                 clientEncryptionKey,
                 DataEncryptionInfo(this.transactionKey, this.encryptionPubKeyDigest.value),
                 listOf(encOrderData)
-            )
+            ).readBytes()
         }
     }
     val bankReturnCode = EbicsReturnCode.lookup(jaxb.value.body.returnCode.value) // business error
diff --git a/nexus/src/main/kotlin/tech/libeufin/nexus/ebics/EbicsCommon.kt b/nexus/src/main/kotlin/tech/libeufin/nexus/ebics/EbicsCommon.kt
index 58718824..66a38b62 100644
--- a/nexus/src/main/kotlin/tech/libeufin/nexus/ebics/EbicsCommon.kt
+++ b/nexus/src/main/kotlin/tech/libeufin/nexus/ebics/EbicsCommon.kt
@@ -42,20 +42,20 @@ import io.ktor.client.plugins.*
 import io.ktor.client.request.*
 import io.ktor.client.statement.*
 import io.ktor.http.*
-import org.apache.commons.compress.archivers.zip.ZipFile
-import org.apache.commons.compress.utils.SeekableInMemoryByteChannel
+import io.ktor.utils.io.jvm.javaio.*
 import org.slf4j.Logger
 import org.slf4j.LoggerFactory
 import tech.libeufin.nexus.*
 import tech.libeufin.common.*
 import tech.libeufin.ebics.*
 import tech.libeufin.ebics.ebics_h005.Ebics3Request
+import java.io.SequenceInputStream
 import java.io.ByteArrayOutputStream
+import java.io.InputStream
 import java.security.interfaces.RSAPrivateCrtKey
 import java.time.LocalDateTime
 import java.time.format.DateTimeFormatter
 import java.util.*
-import java.util.zip.DeflaterInputStream
 
 /**
  * Available EBICS versions.
@@ -73,24 +73,6 @@ enum class SupportedDocument {
     CAMT_054
 }
 
-
-/**
- * Unzips the ByteArray and runs the lambda over each entry.
- *
- * @param lambda function that gets the (fileName, fileContent) pair
- *        for each entry in the ZIP archive as input.
- */
-fun ByteArray.unzipForEach(lambda: (String, ByteArray) -> Unit) {
-    val mem = SeekableInMemoryByteChannel(this)
-    ZipFile(mem).use { file ->
-        file.getEntriesInPhysicalOrder().iterator().forEach {
-            lambda(
-                it.name, file.getInputStream(it).readAllBytes()
-            )
-        }
-    }
-}
-
 /**
  * Decrypts and decompresses the business payload that was
  * transported within an EBICS message from the bank
@@ -106,21 +88,16 @@ fun decryptAndDecompressPayload(
     clientEncryptionKey: RSAPrivateCrtKey,
     encryptionInfo: DataEncryptionInfo,
     chunks: List<String>
-): ByteArray {
-    val buf = StringBuilder()
-    chunks.forEach { buf.append(it) }
-    val decoded = Base64.getDecoder().decode(buf.toString())
-    val er = CryptoUtil.EncryptionResult(
-        encryptionInfo.transactionKey,
-        encryptionInfo.bankPubDigest,
-        decoded
-    )
-    val dataCompr = CryptoUtil.decryptEbicsE002(
-        er,
-        clientEncryptionKey
-    )
-    return EbicsOrderUtil.decodeOrderData(dataCompr)
-}
+): InputStream =
+    SequenceInputStream(Collections.enumeration(chunks.map { it.toByteArray().inputStream() })) // Aggregate
+        .decodeBase64()
+        .run {
+            CryptoUtil.decryptEbicsE002(
+                encryptionInfo.transactionKey,
+                this,
+                clientEncryptionKey
+            )
+        }.inflate()
 
 /**
  * POSTs the EBICS message to the bank.
@@ -129,7 +106,7 @@ fun decryptAndDecompressPayload(
  * @param msg EBICS message as raw bytes.
  * @return the raw bank response.
  */
-suspend fun HttpClient.postToBank(bankUrl: String, msg: ByteArray): ByteArray {
+suspend fun HttpClient.postToBank(bankUrl: String, msg: ByteArray): InputStream {
     logger.debug("POSTing EBICS to '$bankUrl'")
     val res = post(urlString = bankUrl) {
         contentType(ContentType.Text.Xml)
@@ -138,7 +115,7 @@ suspend fun HttpClient.postToBank(bankUrl: String, msg: ByteArray): ByteArray {
     if (res.status != HttpStatusCode.OK) {
         throw Exception("Invalid response status: ${res.status}")
     }
-    return res.readBytes() // TODO input stream
+    return res.bodyAsChannel().toInputStream()
 }
 
 /**
@@ -278,19 +255,19 @@ private fun areCodesOk(ebicsResponseContent: EbicsResponseContent) =
  * @param bankKeys bank EBICS public keys.
  * @param reqXml raw EBICS XML request of the init phase.
  * @param isEbics3 true for EBICS 3, false otherwise.
- * @param processing processing lambda receiving EBICS files as bytes or empty bytes if nothing to download.
- * @return T if the transaction was successful and null if the transaction was empty. If the failure is at the EBICS 
+ * @param processing processing lambda receiving EBICS files as a byte stream if the transaction was not empty.
+ * @return T if the transaction was successful. If the failure is at the EBICS 
  *         level EbicsSideException is thrown else ités the expection of the processing lambda.
  */
-suspend fun <T> ebicsDownload(
+suspend fun ebicsDownload(
     client: HttpClient,
     cfg: EbicsSetupConfig,
     clientKeys: ClientPrivateKeysFile,
     bankKeys: BankPublicKeysFile,
     reqXml: ByteArray,
     isEbics3: Boolean,
-    processing: (ByteArray) -> T
-): T {
+    processing: (InputStream) -> Unit
+) {
     val initResp = postEbics(client, cfg, bankKeys, reqXml, isEbics3)
     logger.debug("Download init phase done.  EBICS- and bank-technical codes are: ${initResp.technicalReturnCode}, ${initResp.bankReturnCode}")
     if (initResp.technicalReturnCode != EbicsReturnCode.EBICS_OK) {
@@ -298,7 +275,7 @@ suspend fun <T> ebicsDownload(
     }
     if (initResp.bankReturnCode == EbicsReturnCode.EBICS_NO_DOWNLOAD_DATA_AVAILABLE) {
         logger.debug("Download content is empty")
-        return processing(ByteArray(0))
+        return
     }
     if (initResp.bankReturnCode != EbicsReturnCode.EBICS_OK) {
         throw Exception("Download init phase has bank-technical error: ${initResp.bankReturnCode}")
@@ -419,11 +396,11 @@ class EbicsSideException(
  */
 fun parseAndValidateEbicsResponse(
     bankKeys: BankPublicKeysFile,
-    resp: ByteArray,
+    resp: InputStream,
     withEbics3: Boolean
 ): EbicsResponseContent {
-    val responseDocument = try {
-        XMLUtil.parseBytesIntoDom(resp)
+    val doc = try {
+        XMLUtil.parseIntoDom(resp)
     } catch (e: Exception) {
         throw EbicsSideException(
             "Bank response apparently invalid",
@@ -431,9 +408,9 @@ fun parseAndValidateEbicsResponse(
         )
     }
     if (!XMLUtil.verifyEbicsDocument(
-            responseDocument,
-            bankKeys.bank_authentication_public_key,
-            withEbics3
+        doc,
+        bankKeys.bank_authentication_public_key,
+        withEbics3
     )) {
         throw EbicsSideException(
             "Bank signature did not verify",
@@ -441,8 +418,8 @@ fun parseAndValidateEbicsResponse(
         )
     }
     if (withEbics3)
-        return ebics3toInternalRepr(resp)
-    return ebics25toInternalRepr(resp)
+        return ebics3toInternalRepr(doc)
+    return ebics25toInternalRepr(doc)
 }
 
 /**
@@ -490,9 +467,7 @@ fun prepareUploadPayload(
     val plainTransactionKey = encryptionResult.plainTransactionKey
         ?: throw Exception("Could not generate the transaction key, cannot encrypt the payload!")
     // Then only E002 symmetric (with ephemeral key) encrypt.
-    val compressedInnerPayload = DeflaterInputStream(
-        payload.inputStream()
-    ).use { it.readAllBytes() }
+    val compressedInnerPayload = payload.inputStream().deflate().readAllBytes()
     val encryptedPayload = CryptoUtil.encryptEbicsE002withTransactionKey(
         compressedInnerPayload,
         bankKeys.bank_encryption_public_key,
diff --git a/nexus/src/test/kotlin/Ebics.kt b/nexus/src/test/kotlin/Ebics.kt
index 507820ce..f3fad224 100644
--- a/nexus/src/test/kotlin/Ebics.kt
+++ b/nexus/src/test/kotlin/Ebics.kt
@@ -35,7 +35,7 @@ class Ebics {
     @Test
     fun iniMessage() = conf { config -> 
         val msg = generateIniMessage(config, clientKeys)
-        val ini = XMLUtil.convertBytesToJaxb<EbicsUnsecuredRequest>(msg) // ensures is valid
+        val ini = XMLUtil.convertToJaxb<EbicsUnsecuredRequest>(msg.inputStream()) // ensures is valid
         assertEquals(ini.value.header.static.orderDetails.orderType, "INI") // ensures is INI
     }
 
@@ -43,7 +43,7 @@ class Ebics {
     @Test
     fun hiaMessage() = conf { config -> 
         val msg = generateHiaMessage(config, clientKeys)
-        val ini = XMLUtil.convertBytesToJaxb<EbicsUnsecuredRequest>(msg) // ensures is valid
+        val ini = XMLUtil.convertToJaxb<EbicsUnsecuredRequest>(msg.inputStream()) // ensures is valid
         assertEquals(ini.value.header.static.orderDetails.orderType, "HIA") // ensures is HIA
     }
 
@@ -51,7 +51,7 @@ class Ebics {
     @Test
     fun hpbMessage() = conf { config -> 
         val msg = generateHpbMessage(config, clientKeys)
-        val ini = XMLUtil.convertBytesToJaxb<EbicsUnsecuredRequest>(msg) // ensures is valid
+        val ini = XMLUtil.convertToJaxb<EbicsUnsecuredRequest>(msg.inputStream()) // ensures is valid
         assertEquals(ini.value.header.static.orderDetails.orderType, "HPB") // ensures is HPB
     }
     // POSTs an EBICS message to the mock bank.  Tests
diff --git a/testbench/src/test/kotlin/Iso20022Test.kt b/testbench/src/test/kotlin/Iso20022Test.kt
index 7d5077ad..27914c78 100644
--- a/testbench/src/test/kotlin/Iso20022Test.kt
+++ b/testbench/src/test/kotlin/Iso20022Test.kt
@@ -30,15 +30,15 @@ class Iso20022Test {
                 val fetch = file.resolve("fetch")
                 if (file.isDirectory() && fetch.exists()) {
                     for (log in fetch.listDirectoryEntries()) {
-                        val str = log.readBytes()
+                        val content = Files.newInputStream(log)
                         val name = log.toString()
                         println(name)
                         if (name.contains("HAC")) {
-                            parseCustomerAck(str)
+                            parseCustomerAck(content)
                         } else if (name.contains("pain.002")) {
-                            parseCustomerPaymentStatusReport(str)
+                            parseCustomerPaymentStatusReport(content)
                         } else {
-                            parseTxNotif(str, "CHF", mutableListOf(), mutableListOf())
+                            parseTxNotif(content, "CHF", mutableListOf(), mutableListOf())
                         }
                     }
                 }