config parsing: directives and brace substitution

2 files changed, 228 insertions, 23 deletions

diff --git a/bank/src/main/kotlin/tech/libeufin/bank/Main.kt b/bank/src/main/kotlin/tech/libeufin/bank/Main.kt
index b0e6d174..d630bfaf 100644
--- a/bank/src/main/kotlin/tech/libeufin/bank/Main.kt
+++ b/bank/src/main/kotlin/tech/libeufin/bank/Main.kt
@@ -24,23 +24,23 @@ import ConfigSource
 import TalerConfig
 import TalerConfigError
 import com.github.ajalt.clikt.core.CliktCommand
-import com.github.ajalt.clikt.parameters.options.*
 import com.github.ajalt.clikt.core.context
 import com.github.ajalt.clikt.core.subcommands
 import com.github.ajalt.clikt.output.CliktHelpFormatter
 import com.github.ajalt.clikt.parameters.arguments.argument
+import com.github.ajalt.clikt.parameters.options.flag
+import com.github.ajalt.clikt.parameters.options.option
 import com.github.ajalt.clikt.parameters.options.versionOption
 import io.ktor.http.*
-import io.ktor.server.application.*
-import io.ktor.server.plugins.*
-import io.ktor.server.plugins.requestvalidation.*
-import io.ktor.server.plugins.contentnegotiation.*
 import io.ktor.serialization.kotlinx.json.*
+import io.ktor.server.application.*
 import io.ktor.server.engine.*
 import io.ktor.server.netty.*
+import io.ktor.server.plugins.*
 import io.ktor.server.plugins.callloging.*
-import kotlinx.serialization.*
+import io.ktor.server.plugins.contentnegotiation.*
 import io.ktor.server.plugins.cors.routing.*
+import io.ktor.server.plugins.requestvalidation.*
 import io.ktor.server.plugins.statuspages.*
 import io.ktor.server.request.*
 import io.ktor.server.response.*
@@ -49,6 +49,8 @@ import io.ktor.utils.io.*
 import io.ktor.utils.io.jvm.javaio.*
 import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.withContext
+import kotlinx.serialization.ExperimentalSerializationApi
+import kotlinx.serialization.KSerializer
 import kotlinx.serialization.descriptors.*
 import kotlinx.serialization.encoding.Decoder
 import kotlinx.serialization.encoding.Encoder
@@ -58,7 +60,8 @@ import net.taler.common.errorcodes.TalerErrorCode
 import org.slf4j.Logger
 import org.slf4j.LoggerFactory
 import org.slf4j.event.Level
-import tech.libeufin.util.*
+import tech.libeufin.util.CryptoUtil
+import tech.libeufin.util.getVersion
 import java.time.Duration
 import java.time.Instant
 import java.time.temporal.ChronoUnit
@@ -116,7 +119,7 @@ data class BankApplicationContext(
     val suggestedWithdrawalExchange: String?,
     /**
      * URL where the user should be redirected to complete the captcha.
-     * It can contain the substring "{woid}" that is going to be replaced 
+     * It can contain the substring "{woid}" that is going to be replaced
      * with the withdrawal operation id and should point where the bank
      * SPA is located.
      */
@@ -143,7 +146,8 @@ object TalerProtocolTimestampSerializer : KSerializer<TalerProtocolTimestamp> {
     }
 
     override fun deserialize(decoder: Decoder): TalerProtocolTimestamp {
-        val jsonInput = decoder as? JsonDecoder ?: throw internalServerError("TalerProtocolTimestamp had no JsonDecoder")
+        val jsonInput =
+            decoder as? JsonDecoder ?: throw internalServerError("TalerProtocolTimestamp had no JsonDecoder")
         val json = try {
             jsonInput.decodeJsonElement().jsonObject
         } catch (e: Exception) {
@@ -441,11 +445,25 @@ fun durationFromPretty(s: String): Long {
         }
         val n = currentNum.toInt(10)
         durationUs += when (c) {
-            's' -> { n * 1000000 }
-            'm' -> { n * 1000000 * 60 }
-            'h' -> { n * 1000000 * 60 * 60 }
-            'd' -> { n * 1000000 * 60 * 60 * 24 }
-            else -> { throw Error("invalid duration, unsupported unit '$c'") }
+            's' -> {
+                n * 1000000
+            }
+
+            'm' -> {
+                n * 1000000 * 60
+            }
+
+            'h' -> {
+                n * 1000000 * 60 * 60
+            }
+
+            'd' -> {
+                n * 1000000 * 60 * 60 * 24
+            }
+
+            else -> {
+                throw Error("invalid duration, unsupported unit '$c'")
+            }
         }
         parsingNum = true
         currentNum = ""
@@ -525,6 +543,7 @@ class ServeBank : CliktCommand("Run libeufin-bank HTTP server", name = "serve")
         "--config", "-c",
         help = "set the configuration file"
     )
+
     init {
         context {
             helpFormatter = CliktHelpFormatter(showDefaultValues = true)
@@ -565,6 +584,7 @@ class ChangePw : CliktCommand("Change account password", name = "passwd") {
     )
     private val account by argument("account")
     private val password by argument("password")
+
     init {
         context {
             helpFormatter = CliktHelpFormatter(showDefaultValues = true)
@@ -590,11 +610,12 @@ class ChangePw : CliktCommand("Change account password", name = "passwd") {
     }
 }
 
-class BankConfig : CliktCommand("Dump the configuration", name = "debug-config-dump") {
+class BankConfigDump : CliktCommand("Dump the configuration", name = "dump") {
     private val configFile by option(
         "--config", "-c",
         help = "set the configuration file"
     )
+
     init {
         context {
             helpFormatter = CliktHelpFormatter(showDefaultValues = true)
@@ -609,6 +630,77 @@ class BankConfig : CliktCommand("Dump the configuration", name = "debug-config-d
     }
 }
 
+class BankConfigPathsub : CliktCommand("Substitute variables in a path", name = "pathsub") {
+    private val configFile by option(
+        "--config", "-c",
+        help = "set the configuration file"
+    )
+
+    private val pathExpr by argument()
+
+    init {
+        context {
+            helpFormatter = CliktHelpFormatter(showDefaultValues = true)
+        }
+    }
+
+    override fun run() {
+        val config = TalerConfig(BANK_CONFIG_SOURCE)
+        config.load(this.configFile)
+        println(config.pathsub(pathExpr))
+    }
+}
+
+class BankConfigGet : CliktCommand("Lookup config value", name = "get") {
+    private val configFile by option(
+        "--config", "-c",
+        help = "set the configuration file"
+    )
+
+    private val isPath by option(
+        "--filename", "-f",
+        help = "interpret value as path with dollar-expansion"
+    ).flag()
+
+    private val sectionName by argument()
+    private val optionName by argument()
+
+
+    init {
+        context {
+            helpFormatter = CliktHelpFormatter(showDefaultValues = true)
+        }
+    }
+
+    override fun run() {
+        val config = TalerConfig(BANK_CONFIG_SOURCE)
+        config.load(this.configFile)
+        if (isPath) {
+            val res = config.lookupValuePath(sectionName, optionName)
+            if (res == null) {
+                logger.info("value not found in config")
+                exitProcess(2)
+            }
+            println(res)
+        } else {
+            val res = config.lookupValueString(sectionName, optionName)
+            if (res == null) {
+                logger.info("value not found in config")
+                exitProcess(2)
+            }
+            println(res)
+        }
+    }
+}
+
+class BankConfig : CliktCommand("Dump the configuration", name = "config") {
+    init {
+        subcommands(BankConfigDump(), BankConfigPathsub(), BankConfigGet())
+    }
+
+    override fun run() = Unit
+}
+
 fun main(args: Array<String>) {
     LibeufinBankCommand().main(args)
 }
diff --git a/util/src/main/kotlin/TalerConfig.kt b/util/src/main/kotlin/TalerConfig.kt
index a3f8e22a..7df9c036 100644
--- a/util/src/main/kotlin/TalerConfig.kt
+++ b/util/src/main/kotlin/TalerConfig.kt
@@ -20,6 +20,7 @@
 import java.io.File
 import java.nio.file.Paths
 import kotlin.io.path.Path
+import kotlin.io.path.isReadable
 import kotlin.io.path.listDirectoryEntries
 
 private data class Entry(val value: String)
@@ -50,14 +51,14 @@ data class ConfigSource(
  * @param configSource information about where to load configuration defaults from
  */
 class TalerConfig(
-    private val configSource: ConfigSource
+    private val configSource: ConfigSource,
 ) {
     private val sectionMap: MutableMap<String, Section> = mutableMapOf()
 
     private val componentName = configSource.componentName
     private val installPathBinary = configSource.installPathBinary
 
-    private fun internalLoadFromString(s: String) {
+    private fun internalLoadFromString(s: String, sourceFilename: String?) {
         val lines = s.lines()
         var lineNum = 0
         var currentSection: String? = null
@@ -72,7 +73,38 @@ class TalerConfig(
 
             val directiveMatch = reDirective.matchEntire(line)
             if (directiveMatch != null) {
-                throw NotImplementedError("config directives are not implemented yet")
+                if (sourceFilename == null) {
+                    throw TalerConfigError("Directives are only supported when loading from file")
+                }
+                val directiveName = directiveMatch.groups[1]!!.value.lowercase()
+                val directiveArg = directiveMatch.groups[2]!!.value
+                when (directiveName) {
+                    "inline" -> {
+                        val innerFilename = normalizeInlineFilename(sourceFilename, directiveArg.trim())
+                        this.loadFromFilename(innerFilename)
+                    }
+
+                    "inline-matching" -> {
+                        val glob = directiveArg.trim()
+                        this.loadFromGlob(sourceFilename, glob)
+                    }
+
+                    "inline-secret" -> {
+                        val arg = directiveArg.trim()
+                        val sp = arg.split(" ")
+                        if (sp.size != 2) {
+                            throw TalerConfigError("invalid configuration, @inline-secret@ directive requires exactly two arguments")
+                        }
+                        val sectionName = sp[0]
+                        val secretFilename = normalizeInlineFilename(sourceFilename, sp[1])
+                        loadSecret(sectionName, secretFilename)
+                    }
+
+                    else -> {
+                        throw TalerConfigError("unsupported directive '$directiveName'")
+                    }
+                }
+                continue
             }
 
             val secMatch = reSection.matchEntire(line)
@@ -98,7 +130,41 @@ class TalerConfig(
                 )
                 continue
             }
-            throw TalerConfigError("expected section header, option assignment or directive in line $lineNum")
+            throw TalerConfigError("expected section header, option assignment or directive in line $lineNum file ${sourceFilename ?: "<input>"}")
+        }
+    }
+
+    private fun loadFromGlob(parentFilename: String, glob: String) {
+        val fullFileglob: String
+        val parentDir = Path(parentFilename).parent!!.toString()
+        if (glob.startsWith("/")) {
+            fullFileglob = glob
+        } else {
+            fullFileglob = Paths.get(parentDir, glob).toString()
+        }
+
+        val head = Path(fullFileglob).parent.toString()
+        val tail = Path(fullFileglob).fileName.toString()
+
+        // FIXME: Check that the Kotlin glob matches the glob from our spec
+        for (entry in Path(head).listDirectoryEntries(tail)) {
+            loadFromFilename(entry.toString())
+        }
+    }
+
+    private fun normalizeInlineFilename(parentFilename: String, f: String): String {
+        if (f[0] == '/') {
+            return f
+        }
+        val parentDir = Path(parentFilename).parent!!.toString()
+        return Paths.get(parentDir, f).toRealPath().toString()
+    }
+
+    private fun loadSecret(sectionName: String, secretFilename: String) {
+        if (!Path(secretFilename).isReadable()) {
+            logger.warn("unable to read secrets from $secretFilename")
+        } else {
+            this.loadFromFilename(secretFilename)
         }
     }
 
@@ -114,7 +180,7 @@ class TalerConfig(
     }
 
     fun loadFromString(s: String) {
-        internalLoadFromString(s)
+        internalLoadFromString(s, null)
     }
 
     private fun lookupEntry(section: String, option: String): Entry? {
@@ -214,7 +280,7 @@ class TalerConfig(
     fun loadFromFilename(filename: String) {
         val f = File(filename)
         val contents = f.readText()
-        loadFromString(contents)
+        internalLoadFromString(contents, filename)
     }
 
     private fun loadDefaultsFromDir(dirname: String) {
@@ -237,7 +303,7 @@ class TalerConfig(
         loadDefaultsFromDir(baseConfigDir)
     }
 
-    fun variableLookup(x: String, recursionDepth: Int = 0): String? {
+    private fun variableLookup(x: String, recursionDepth: Int = 0): String? {
         val pathRes = this.lookupValueString("PATHS", x)
         if (pathRes != null) {
             return pathsub(pathRes, recursionDepth + 1)
@@ -265,7 +331,54 @@ class TalerConfig(
             }
             if (l + 1 < s.length && s[l + 1] == '{') {
                 // ${var}
-                throw NotImplementedError("bracketed variables not yet supported")
+                var depth = 1
+                val start = l
+                var p = start + 2;
+                var hasDefault = false
+                var insideNamePath = true
+                // Find end of the ${...} expression
+                while (p < s.length) {
+                    if (s[p] == '}') {
+                        insideNamePath = false
+                        depth--
+                    } else if (s.length > p + 1 && s[p] == '$' && s[p + 1] == '{') {
+                        depth++
+                        insideNamePath = false
+                    } else if (s.length > p + 1 && insideNamePath && s[p] == ':' && s[p + 1] == '-') {
+                        hasDefault = true
+                    }
+                    p++
+                    if (depth == 0) {
+                        break
+                    }
+                }
+                if (depth == 0) {
+                    val inner = s.substring(start + 2, p - 1)
+                    val varName: String
+                    val varDefault: String?
+                    if (hasDefault) {
+                        val res = inner.split(":-", limit = 2)
+                        varName = res[0]
+                        varDefault = res[1]
+                    } else {
+                        varName = inner
+                        varDefault = null
+                    }
+                    val r = variableLookup(varName, recursionDepth + 1)
+                    if (r != null) {
+                        result.append(r)
+                        l = p
+                        continue
+                    } else if (varDefault != null) {
+                        val resolvedDefault = pathsub(varDefault, recursionDepth + 1)
+                        result.append(resolvedDefault)
+                        l = p
+                        continue
+                    } else {
+                        throw TalerConfigError("malformed variable expression can't resolve variable '$varName'")
+                    }
+                }
+                throw TalerConfigError("malformed variable expression (unbalanced)")
             } else {
                 // $var
                 var varEnd = l + 1