From 0ccb604a22f4134c682a5399aedfc239de2f608b Mon Sep 17 00:00:00 2001 From: Christian Grothoff Date: Thu, 14 Nov 2019 18:51:46 +0100 Subject: wip --- src/Makefile.am | 2 +- src/include/sync_database_plugin.h | 173 +++++++++++++++++++++++++------- src/syncdb/Makefile.am | 6 +- src/syncdb/plugin_sync_postgres.c | 196 ++++++++++++++++++++++++++++++++----- src/syncdb/sync_db_plugin.c | 22 ++--- src/util/Makefile.am | 28 ++++++ src/util/os_installation.c | 71 ++++++++++++++ src/util/sync-config | 17 ++++ 8 files changed, 443 insertions(+), 72 deletions(-) create mode 100644 src/util/Makefile.am create mode 100644 src/util/os_installation.c create mode 100644 src/util/sync-config diff --git a/src/Makefile.am b/src/Makefile.am index 0746ad4..cd274f1 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -1,3 +1,3 @@ # This Makefile is in the public domain AM_CPPFLAGS = -I$(top_srcdir)/src/include -SUBDIRS = include syncdb sync lib +SUBDIRS = include util syncdb sync lib diff --git a/src/include/sync_database_plugin.h b/src/include/sync_database_plugin.h index 5563cf3..b847888 100644 --- a/src/include/sync_database_plugin.h +++ b/src/include/sync_database_plugin.h @@ -1,25 +1,25 @@ /* - This file is part of Sync + This file is part of GNU Taler Copyright (C) 2019 Taler Systems SA - Sync is free software; you can redistribute it and/or modify it under the + Taler is free software; you can redistribute it and/or modify it under the terms of the GNU Lesser General Public License as published by the Free Software Foundation; either version 3, or (at your option) any later version. - Sync is distributed in the hope that it will be useful, but WITHOUT ANY + Taler is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with - Sync; see the file COPYING.GPL. If not, see + Taler; see the file COPYING.GPL. If not, see */ /** * @file include/sync_database_plugin.h * @brief database access for Sync * @author Christian Grothoff */ -#ifndef TALER_SYNC_DATABASE_PLUGIN_H -#define TALER_SYNC_DATABASE_PLUGIN_H +#ifndef SYNC_DATABASE_PLUGIN_H +#define SYNC_DATABASE_PLUGIN_H #include #include @@ -27,6 +27,79 @@ #include #include +/** + * Private key identifying an account. + */ +struct SYNC_AccountPrivateKey +{ + /** + * We use EdDSA. + */ + struct GNUNET_EdDSAPrivateKey eddsa_priv; +}; + + +/** + * Public key identifying an account. + */ +struct SYNC_AccountPublicKey +{ + /** + * We use EdDSA. + */ + struct GNUNET_EdDSAPrivateKey eddsa_priv; +}; + + +/** + * Signature made with an account's public key. + */ +struct SYNC_AccountSignature +{ + /** + * We use EdDSA. + */ + struct GNUNET_EdDSASignature eddsa_sig; +}; + + +/** + * Possible status codes returned from the SYNC database. + */ +enum SYNC_DB_QueryStatus +{ + /** + * Update failed because the old backup hash does not match what we previously had in the DB. + */ + SYNC_DB_OLD_BACKUP_MISSMATCH = -4, + + /** + * Account is unpaid. + */ + SYNC_DB_QS_PAYMENT_REQUIRED = -3, + + /** + * Hard database issue, retries futile. + */ + SYNC_DB_HARD_ERROR = -2, + + /** + * Soft database error, retrying may help. + */ + SYNC_DB_SOFT_ERROR = -1, + + /** + * Database succeeded, but no results. + */ + SYNC_DB_NO_RESULTS = 0, + + /** + * Database succeeded, one change or result. + */ + SYNC_DB_ONE_RESULT = 1 +}; + + /** * Handle to interact with the database. * @@ -67,51 +140,85 @@ struct SYNC_DatabasePlugin * truth and financial records older than @a fin_expire. * * @param cls closure - * @param fin_expire financial records older than the given - * time stamp should be garbage collected (usual - * values might be something like 6-10 years in the past) + * @param expire backups older than the given time stamp should be garbage collected * @return transaction status */ enum SYNC_DB_QueryStatus (*gc)(void *cls, - struct GNUNET_TIME_Absolute fin_expire); + struct GNUNET_TIME_Absolute expire); /** - * Do a pre-flight check that we are not in an uncommitted transaction. - * If we are, try to commit the previous transaction and output a warning. - * Does not return anything, as we will continue regardless of the outcome. - * - * @param cls the `struct PostgresClosure` with the plugin-specific state - */ - void - (*preflight) (void *cls); - - /** - * Check that the database connection is still up. - * - * @param pg connection to check - */ - void - (*check_connection) (void *cls); + * Store backup. Only applicable for the FIRST backup under + * an @a account_pub. Use @e update_backup_TR to update an + * existing backup. + * + * @param cls closure + * @param account_pub account to store @a backup under + * @param account_sig signature affirming storage request + * @param backup_hash hash of @a backup + * @param backup_size number of bytes in @a backup + * @param backup raw data to backup + * @return transaction status + */ + enum SYNC_DB_QueryStatus + (*store_backup_TR)(void *cls, + const struct SYNC_AccountPublicKey *account_pub, + const struct SYNC_AccountSignature *account_sig, + const struct GNUNET_HashCode *backup_hash, + size_t backup_size, + const void *backup); /** - * Store backup. + * Update backup. * * @param cls closure + * @param account_pub account to store @a backup under + * @param account_sig signature affirming storage request + * @param old_backup_hash hash of the previous backup (must match) + * @param backup_hash hash of @a backup + * @param backup_size number of bytes in @a backup + * @param backup raw data to backup * @return transaction status */ - enum GNUNET_DB_QueryStatus - (*store_backup)(void *cls, - ...); + enum SYNC_DB_QueryStatus + (*update_backup_TR)(void *cls, + const struct SYNC_AccountPublicKey *account_pub, + const struct GNUNET_HashCode *old_backup_hash, + const struct SYNC_AccountSignature *account_sig, + const struct GNUNET_HashCode *backup_hash, + size_t backup_size, + const void *backup); /** * Obtain backup. * * @param cls closure + * @param account_pub account to store @a backup under + * @param account_sig[OUT] set to signature affirming storage request + * @param backup_hash[OUT] set to hash of @a backup + * @param backup_size[OUT] set to number of bytes in @a backup + * @param backup[OUT] set to raw data to backup, caller MUST FREE */ - enum GNUNET_DB_QueryStatus - (*lookup_backup)(void *cls, - ...); + enum SYNC_DB_QueryStatus + (*lookup_backup_TR)(void *cls, + const struct SYNC_AccountPublicKey *account_pub, + struct SYNC_AccountSignature *account_sig, + struct GNUNET_HashCode *backup_hash, + size_t *backup_size, + void **backup); + + /** + * Increment account lifetime. + * + * @param cls closure + * @param account_pub which account received a payment + * @param lifetime for how long is the account now paid (increment) + * @return transaction status + */ + enum SYNC_DB_QueryStatus + (*increment_lifetime_TR)(void *cls, + const struct SYNC_AccountPublicKey *account_pub, + struct GNUNET_TIME_Relative lifetime); }; #endif diff --git a/src/syncdb/Makefile.am b/src/syncdb/Makefile.am index 56db964..9b3d4b0 100644 --- a/src/syncdb/Makefile.am +++ b/src/syncdb/Makefile.am @@ -1,7 +1,7 @@ # This Makefile.am is in the public domain AM_CPPFLAGS = -I$(top_srcdir)/src/include -plugindir = $(libdir)/taler +plugindir = $(libdir)/sync if HAVE_POSTGRESQL if HAVE_GNUNETPQ @@ -27,11 +27,11 @@ libsyncdb_la_LIBADD = \ -lgnunetutil libsyncdb_la_LDFLAGS = \ $(POSTGRESQL_LDFLAGS) \ - -version-info 2:0:0 \ + -version-info 0:0:0 \ -no-undefined libsync_plugin_db_postgres_la_SOURCES = \ - plugin_sync_postgres.c + plugin_syncdb_postgres.c libsync_plugin_db_postgres_la_LIBADD = \ $(LTLIBINTL) libsync_plugin_db_postgres_la_LDFLAGS = \ diff --git a/src/syncdb/plugin_sync_postgres.c b/src/syncdb/plugin_sync_postgres.c index 4e63198..59df4d5 100644 --- a/src/syncdb/plugin_sync_postgres.c +++ b/src/syncdb/plugin_sync_postgres.c @@ -15,10 +15,8 @@ */ /** * @file sync/plugin_syncdb_postgres.c - * @brief database helper functions for postgres used by the sync - * @author Sree Harsha Totakura + * @brief database helper functions for postgres used by sync * @author Christian Grothoff - * @author Marcello Stanisci */ #include "platform.h" #include @@ -27,12 +25,6 @@ #include "sync_database_plugin.h" #include "sync_database_lib.h" -/** - * How often do we re-try if we run into a DB serialization error? - */ -#define MAX_RETRIES 3 - - /** * Type of the "cls" argument given to each of the functions in * our API. @@ -69,6 +61,7 @@ postgres_drop_tables (void *cls) { struct PostgresClosure *pg = cls; struct GNUNET_PQ_ExecuteStatement es[] = { + GNUNET_PQ_make_try_execute ("DROP TABLE IF EXISTS accounts CASCADE;"), GNUNET_PQ_make_try_execute ("DROP TABLE IF EXISTS backups;"), GNUNET_PQ_EXECUTE_STATEMENT_END }; @@ -125,6 +118,7 @@ postgres_preflight (void *cls) pg->transaction_name = NULL; } + /** * Start a transaction. * @@ -157,12 +151,13 @@ begin_transaction (void *cls, return GNUNET_OK; } + /** -* Roll back the current transaction of a database connection. -* -* @param cls the `struct PostgresClosure` with the plugin-specific state -* @return #GNUNET_OK on success -*/ + * Roll back the current transaction of a database connection. + * + * @param cls the `struct PostgresClosure` with the plugin-specific state + * @return #GNUNET_OK on success + */ static void rollback (void *cls) { @@ -182,14 +177,14 @@ rollback (void *cls) pg->transaction_name = NULL; } + /** * Commit the current transaction of a database connection. * * @param cls the `struct PostgresClosure` with the plugin-specific state * @return transaction status code */ - -static enum SYNC_DB_QueryStatus +static enum GNUNET_DB_QueryStatus commit_transaction (void *cls) { struct PostgresClosure *pg = cls; @@ -206,6 +201,108 @@ commit_transaction (void *cls) } +/** + * Function called to perform "garbage collection" on the + * database, expiring records we no longer require. Deletes + * all user records that are not paid up (and by cascade deletes + * the associated recovery documents). Also deletes expired + * truth and financial records older than @a fin_expire. + * + * @param cls closure + * @param expire backups older than the given time stamp should be garbage collected + * @return transaction status + */ +static enum SYNC_DB_QueryStatus +postgres_gc (void *cls, + struct GNUNET_TIME_Absolute expire) +{ +} + + +/** + * Store backup. Only applicable for the FIRST backup under + * an @a account_pub. Use @e update_backup_TR to update an + * existing backup. + * + * @param cls closure + * @param account_pub account to store @a backup under + * @param account_sig signature affirming storage request + * @param backup_hash hash of @a backup + * @param backup_size number of bytes in @a backup + * @param backup raw data to backup + * @return transaction status + */ +static enum SYNC_DB_QueryStatus +postgres_store_backup (void *cls, + const struct SYNC_AccountPublicKey *account_pub, + const struct SYNC_AccountSignature *account_sig, + const struct GNUNET_HashCode *backup_hash, + size_t backup_size, + const void *backup) +{ +} + + +/** + * Update backup. + * + * @param cls closure + * @param account_pub account to store @a backup under + * @param account_sig signature affirming storage request + * @param old_backup_hash hash of the previous backup (must match) + * @param backup_hash hash of @a backup + * @param backup_size number of bytes in @a backup + * @param backup raw data to backup + * @return transaction status + */ +static enum SYNC_DB_QueryStatus +postgres_update_backup (void *cls, + const struct SYNC_AccountPublicKey *account_pub, + const struct GNUNET_HashCode *old_backup_hash, + const struct SYNC_AccountSignature *account_sig, + const struct GNUNET_HashCode *backup_hash, + size_t backup_size, + const void *backup) +{ +} + + +/** + * Obtain backup. + * + * @param cls closure + * @param account_pub account to store @a backup under + * @param account_sig[OUT] set to signature affirming storage request + * @param backup_hash[OUT] set to hash of @a backup + * @param backup_size[OUT] set to number of bytes in @a backup + * @param backup[OUT] set to raw data to backup, caller MUST FREE + */ +static enum SYNC_DB_QueryStatus +postgres_lookup_backup (void *cls, + const struct SYNC_AccountPublicKey *account_pub, + struct SYNC_AccountSignature *account_sig, + struct GNUNET_HashCode *backup_hash, + size_t *backup_size, + void **backup) +{ +} + + +/** + * Increment account lifetime. + * + * @param cls closure + * @param account_pub which account received a payment + * @param lifetime for how long is the account now paid (increment) + * @return transaction status + */ +static enum SYNC_DB_QueryStatus +postgres_increment_lifetime (void *cls, + const struct SYNC_AccountPublicKey *account_pub, + struct GNUNET_TIME_Relative lifetime) +{ +} + /** * Initialize Postgres database subsystem. @@ -223,19 +320,68 @@ libsync_plugin_db_postgres_init (void *cls) /* Orders created by the frontend, not signed or given a nonce yet. The contract terms will change (nonce will be added) when moved to the contract terms table */ + GNUNET_PQ_make_execute ("CREATE TABLE IF NOT EXISTS accounts" + "(" + "account_pub BYTEA PRIMARY KEY CHECK (length(account_pub)=32)," + "expiration_date INT8 NOT NULL" + ");"), GNUNET_PQ_make_execute ("CREATE TABLE IF NOT EXISTS backups" - "(" - "data BYTEA NOT NULL," + "(" + "account_pub BYTEA PRIMARY KEY REFERENCES accounts (account_pub)," + "account_sig BYTEA NOT NULL CHECK (length(account_sig)=64)," + "backup_hash BYTEA NOT NULL CHECK (length(backup_hash)=64)," + "data BYTEA NOT NULL" ");"), + /* index for gc */ + GNUNET_PQ_make_try_execute ( + "CREATE INDEX accounts_expire ON " + "accounts (expiration_date);"), GNUNET_PQ_EXECUTE_STATEMENT_END }; struct GNUNET_PQ_PreparedStatement ps[] = { + GNUNET_PQ_make_prepare ("account_insert", + "INSERT INTO accounts " + "(" + "account_pub," + "expiration_date" + ") VALUES " + "($1,$2);", + 2), + GNUNET_PQ_make_prepare ("account_update", + "UPDATE accounts " + "SET" + " expiration_date=expiration_data+$1 " + "WHERE" + " account_pub=$2;", + 2), + GNUNET_PQ_make_prepare ("gc", + "DELETE FROM accounts " + "WHERE" + " expiration_data<$1;", + 1), GNUNET_PQ_make_prepare ("backup_insert", "INSERT INTO backups " - "(data" + "(" + "account_pub," + "account_sig," + "backup_hash," + "data" ") VALUES " - "($1);", - 1), + "($1,$2,$3,$4);", + 4), + GNUNET_PQ_make_prepare ("backup_update", + "UPDATE backups " + " SET" + " backup_hash=$1," + " account_sig=$2," + " data=$3" + " WHERE" + " account_pub=$4" + " AND" + " backup_hash=$5" + ") VALUES " + "($1,$2,$3,$4,$5);", + 5), GNUNET_PQ_make_prepare ("do_commit", "COMMIT", 0), @@ -256,9 +402,10 @@ libsync_plugin_db_postgres_init (void *cls) plugin = GNUNET_new (struct SYNC_DatabasePlugin); plugin->cls = pg; plugin->drop_tables = &postgres_drop_tables; - plugin->preflight = &postgres_preflight; - plugin->rollback = &rollback; - plugin->commit = &commit_transaction; + plugin->gc = &postgres_gc; + plugin->store_backup_TR = &postgres_store_backup; + plugin->update_backup_TR = &postgres_update_backup; + plugin->increment_lifetime_TR = &postgres_increment_lifetime; return plugin; } @@ -281,4 +428,5 @@ libsync_plugin_db_postgres_done (void *cls) return NULL; } + /* end of plugin_syncdb_postgres.c */ diff --git a/src/syncdb/sync_db_plugin.c b/src/syncdb/sync_db_plugin.c index 6b2c6e0..6c4f8d5 100644 --- a/src/syncdb/sync_db_plugin.c +++ b/src/syncdb/sync_db_plugin.c @@ -1,6 +1,6 @@ /* This file is part of TALER - Copyright (C) 2015, 2016 GNUnet e.V. and INRIA + Copyright (C) 2019 Taler Systems SA TALER is free software; you can redistribute it and/or modify it under the terms of the GNU Lesser General Public License as published by the Free Software @@ -14,13 +14,13 @@ TALER; see the file COPYING. If not, see */ /** - * @file merchantdb/merchantdb_plugin.c + * @file syncdb/sync_db_plugin.c * @brief Logic to load database plugin * @author Christian Grothoff * @author Sree Harsha Totakura */ #include "platform.h" -#include "anastasis_database_plugin.h" +#include "sync_database_plugin.h" #include @@ -30,27 +30,27 @@ * @param cfg configuration to use * @return #GNUNET_OK on success */ -struct ANASTASIS_DatabasePlugin * -ANASTASIS_DB_plugin_load (const struct GNUNET_CONFIGURATION_Handle *cfg) +struct SYNC_DatabasePlugin * +SYNC_DB_plugin_load (const struct GNUNET_CONFIGURATION_Handle *cfg) { char *plugin_name; char *lib_name; struct GNUNET_CONFIGURATION_Handle *cfg_dup; - struct ANASTASIS_DatabasePlugin *plugin; + struct SYNC_DatabasePlugin *plugin; if (GNUNET_SYSERR == GNUNET_CONFIGURATION_get_value_string (cfg, - "anastasis", + "sync", "db", &plugin_name)) { GNUNET_log_config_missing (GNUNET_ERROR_TYPE_ERROR, - "anastasis", + "sync", "db"); return NULL; } (void) GNUNET_asprintf (&lib_name, - "libanastasis_plugin_db_%s", + "libsync_plugin_db_%s", plugin_name); GNUNET_free (plugin_name); cfg_dup = GNUNET_CONFIGURATION_dup (cfg); @@ -70,7 +70,7 @@ ANASTASIS_DB_plugin_load (const struct GNUNET_CONFIGURATION_Handle *cfg) * @param plugin the plugin to unload */ void -ANASTASIS_DB_plugin_unload (struct ANASTASIS_DatabasePlugin *plugin) +SYNC_DB_plugin_unload (struct SYNC_DatabasePlugin *plugin) { char *lib_name; @@ -146,4 +146,4 @@ plugin_fini () } -/* end of anastasis_db_plugin.c */ +/* end of sync_db_plugin.c */ diff --git a/src/util/Makefile.am b/src/util/Makefile.am new file mode 100644 index 0000000..bc1291c --- /dev/null +++ b/src/util/Makefile.am @@ -0,0 +1,28 @@ +# This Makefile.am is in the public domain +AM_CPPFLAGS = -I$(top_srcdir)/src/include $(LIBGCRYPT_CFLAGS) + +if USE_COVERAGE + AM_CFLAGS = --coverage -O0 + XLIB = -lgcov +endif + + +pkgcfgdir = $(prefix)/share/sync/config.d/ + +pkgcfg_DATA = \ + paths.conf + +bin_SCRIPTS = \ + sync-config + +lib_LTLIBRARIES = \ + libsyncutil.la + +libsyncutil_la_SOURCES = \ + os_installation.c +libsyncutil_la_LIBADD = \ + -lgnunetutil \ + $(XLIB) +libsyncutil_la_LDFLAGS = \ + -version-info 0:0:0 \ + -export-dynamic -no-undefined diff --git a/src/util/os_installation.c b/src/util/os_installation.c new file mode 100644 index 0000000..85092d1 --- /dev/null +++ b/src/util/os_installation.c @@ -0,0 +1,71 @@ +/* + This file is part of GNU Taler. + Copyright (C) 2019 Taler Systems SA + + Sync is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + Sync is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with Sync; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + Boston, MA 02110-1301, USA. +*/ + +/** + * @file os_installation.c + * @brief initialize libgnunet OS subsystem for Sync. + * @author Christian Grothoff + */ +#include "platform.h" +#include + + +/** + * Default project data used for installation path detection + * for GNU Sync. + */ +static const struct GNUNET_OS_ProjectData sync_pd = { + .libname = "libsyncutil", + .project_dirname = "sync", + .binary_name = "sync-httpd", + .env_varname = "SYNC_PREFIX", + .base_config_varname = "SYNC_BASE_CONFIG", + .bug_email = "taler@lists.gnu.org", + .homepage = "http://www.gnu.org/s/taler/", + .config_file = "sync.conf", + .user_config_file = "~/.config/sync.conf", + .version = PACKAGE_VERSION, + .is_gnu = 1, + .gettext_domain = "sync", + .gettext_path = NULL, +}; + + +/** + * Return default project data used by Sync. + */ +const struct GNUNET_OS_ProjectData * +SYNC_project_data_default (void) +{ + return &sync_pd; +} + + +/** + * Initialize libsyncutil. + */ +void __attribute__ ((constructor)) +SYNC_OS_init () +{ + GNUNET_OS_init (&sync_pd); +} + + +/* end of os_installation.c */ diff --git a/src/util/sync-config b/src/util/sync-config new file mode 100644 index 0000000..6184167 --- /dev/null +++ b/src/util/sync-config @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +set -eu + +if ! type gnunet-config >/dev/null; then + echo "$0 needs gnunet-config to be installed" + exit 1 +fi + +# FIXME: not very portable ... +# FIXME: should use "libdir" instead of prefix/lib, but somehow +# the recursive expansion does not work ;-(. +GC=`which gnunet-config` +ASAN="" +A=`ldd $GC | grep libasan` && ASAN=`echo $A | awk '{print $3 ":"}'` +export LD_PRELOAD=${ASAN}${LD_PRELOAD:-}:/home/grothoff/lib/libsyncutil.so +exec gnunet-config "$@" -- cgit v1.2.3