libeufin

commit 4610a0e5bba4f264b3c26b9018ed0fd3663f68ca
parent 11bed004ab69a5f3d065fe636cb39d2cd9fca6ef
Author: Antoine A <>
Date:   Mon,  4 Dec 2023 23:22:48 +0000

Exponential backoff on failure in NotificationWatcher

Diffstat:
M
bank/src/main/kotlin/tech/libeufin/bank/db/NotificationWatcher.kt
 | 
5
+++++
A
util/src/main/kotlin/Backoff.kt
 | 
42
++++++++++++++++++++++++++++++++++++++++++

2 files changed, 47 insertions(+), 0 deletions(-)
diff --git a/bank/src/main/kotlin/tech/libeufin/bank/db/NotificationWatcher.kt b/bank/src/main/kotlin/tech/libeufin/bank/db/NotificationWatcher.kt
@@ -43,6 +43,8 @@ internal class NotificationWatcher(private val pgSource: PGSimpleDataSource) {
     // Withdrawal confirmation flow, the key is the public withdrawal UUID
     private val withdrawalFlow = ConcurrentHashMap<UUID, CountedSharedFlow<WithdrawalStatus>>()
 
+    private val backoff = ExpoBackoffDecorr()
+
     init {
         // Run notification logic in a separated thread
         kotlin.concurrent.thread(isDaemon = true) { 
@@ -57,6 +59,8 @@ internal class NotificationWatcher(private val pgSource: PGSimpleDataSource) {
                         conn.execSQLUpdate("LISTEN incoming_tx")
                         conn.execSQLUpdate("LISTEN withdrawal_status")
 
+                        backoff.reset()
+
                         while (true) {
                             conn.getNotifications(0) // Block until we receive at least one notification
                                 .forEach {
@@ -99,6 +103,7 @@ internal class NotificationWatcher(private val pgSource: PGSimpleDataSource) {
                         }
                     } catch (e: Exception) {
                         logger.warn("notification_watcher failed: $e")
+                        delay(backoff.next())
                     }
                 }
             }
diff --git a/util/src/main/kotlin/Backoff.kt b/util/src/main/kotlin/Backoff.kt
@@ -0,0 +1,41 @@
+/*
+ * This file is part of LibEuFin.
+ * Copyright (C) 2023 Taler Systems S.A.
+
+ * LibEuFin is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation; either version 3, or
+ * (at your option) any later version.
+
+ * LibEuFin is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+ * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Affero General
+ * Public License for more details.
+
+ * You should have received a copy of the GNU Affero General Public
+ * License along with LibEuFin; see the file COPYING.  If not, see
+ * <http://www.gnu.org/licenses/>
+ */
+
+package tech.libeufin.util
+
+import kotlin.random.Random
+
+/** Infinite exponential backoff with decorrelated jitter */
+class ExpoBackoffDecorr(
+    private val base: Long = 100, // 0.1 second
+    private val max: Long = 5000, // 5 second
+    private val factor: Double = 2.0,
+) {
+    private var sleep: Long = base
+
+    public fun next() : Long {
+        sleep = Random.nextDouble(base.toDouble(), sleep.toDouble() * factor)
+                        .toLong().coerceAtMost(max)
+        return sleep
+    }
+
+    public fun reset() {
+        sleep = base
+    }
+}
+\ No newline at end of file