exchange

Base system with REST service to issue digital coins, run by the payment service provider
Log | Files | Refs | Submodules | README | LICENSE

commit af7b61bbe698b9c5aabbc176426d1e2d012121d6
parent 93c947b3d705bcedc1cfb4459d64abae3438dbd9
Author: Christian Grothoff <grothoff@gnunet.org>
Date:   Mon, 24 Mar 2025 15:48:02 +0100

partition statistics tables

Diffstat:
Msrc/exchangedb/exchange-xx10.sql | 566++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++-------------------
1 file changed, 430 insertions(+), 136 deletions(-)

diff --git a/src/exchangedb/exchange-xx10.sql b/src/exchangedb/exchange-xx10.sql @@ -18,7 +18,7 @@ -- @brief Tables for statistics -- @author Christian Grothoff --- FIXME: add partitions and figure out if we do need NULL fo h_payto! +-- FIXME: figure out if we do need NULL fo h_payto! BEGIN; @@ -70,50 +70,122 @@ CREATE INDEX exchange_statistic_bucket_meta_by_origin (origin); -CREATE TABLE exchange_statistic_bucket_counter - (bmeta_serial_id INT8 NOT NULL - REFERENCES exchange_statistic_bucket_meta (bmeta_serial_id) ON DELETE CASCADE - ,h_payto BYTEA CHECK (LENGTH(h_payto)=32) - ,bucket_start INT8 NOT NULL - ,bucket_range statistic_range NOT NULL - ,cumulative_number INT8 NOT NULL - ,UNIQUE (bmeta_serial_id,h_payto,bucket_start,bucket_range) - ); -COMMENT ON TABLE exchange_statistic_bucket_counter - IS 'various numeric statistics (cumulative counters) being tracked by bucket into which they fall'; -COMMENT ON COLUMN exchange_statistic_bucket_counter.bmeta_serial_id - IS 'identifies what the statistic is about'; -COMMENT ON COLUMN exchange_statistic_bucket_counter.h_payto - IS 'identifies an account (hash of normalized payto) for which the statistic is kept, NULL for global statistics'; -COMMENT ON COLUMN exchange_statistic_bucket_counter.bucket_start - IS 'start date for the bucket in seconds since the epoch'; -COMMENT ON COLUMN exchange_statistic_bucket_counter.bucket_range - IS 'range of the bucket'; -COMMENT ON COLUMN exchange_statistic_bucket_counter.cumulative_number - IS 'aggregate (sum) of tracked by the statistic; what exactly is tracked is determined by the keyword'; - - -CREATE TABLE exchange_statistic_bucket_amount - (bmeta_serial_id INT8 NOT NULL - REFERENCES exchange_statistic_bucket_meta (bmeta_serial_id) ON DELETE CASCADE - ,h_payto BYTEA CHECK (LENGTH(h_payto)=32) - ,bucket_start INT8 NOT NULL - ,bucket_range statistic_range NOT NULL - ,cumulative_value taler_amount NOT NULL - ,UNIQUE (bmeta_serial_id,h_payto,bucket_start,bucket_range) - ); -COMMENT ON TABLE exchange_statistic_bucket_amount - IS 'various amount statistics (in various currencies) being tracked'; -COMMENT ON COLUMN exchange_statistic_bucket_amount.bmeta_serial_id - IS 'identifies what the statistic is about'; -COMMENT ON COLUMN exchange_statistic_bucket_amount.h_payto - IS 'identifies an account (hash of normalized payto) for which the statistic is kept, NULL for global statistics'; -COMMENT ON COLUMN exchange_statistic_bucket_amount.bucket_start - IS 'start date for the bucket in seconds since the epoch'; -COMMENT ON COLUMN exchange_statistic_bucket_amount.bucket_range - IS 'range of the bucket'; -COMMENT ON COLUMN exchange_statistic_bucket_amount.cumulative_value - IS 'amount being tracked'; +CREATE FUNCTION create_table_exchange_statistic_bucket_counter ( + IN partition_suffix TEXT DEFAULT NULL +) +RETURNS VOID +LANGUAGE plpgsql +AS $$ +BEGIN + PERFORM create_partitioned_table ( + 'CREATE TABLE %I' + '(bmeta_serial_id INT8 NOT NULL' + ' REFERENCES exchange_statistic_bucket_meta (bmeta_serial_id) ON DELETE CASCADE' + ',h_payto BYTEA CHECK (LENGTH(h_payto)=32)' + ',bucket_start INT8 NOT NULL' + ',bucket_range statistic_range NOT NULL' + ',cumulative_number INT8 NOT NULL' + ',UNIQUE (h_payto,bmeta_serial_id,bucket_start,bucket_range)' + ') %s;' + ,'exchange_statistic_bucket_counter' + ,'PARTITION BY HASH (h_payto)' + ,partition_suffix + ); + PERFORM comment_partitioned_table( + 'various numeric statistics (cumulative counters) being tracked by bucket into which they fall' + ,'exchange_statistic_bucket_counter' + ,partition_suffix + ); + PERFORM comment_partitioned_column( + 'identifies what the statistic is about' + ,'bmeta_serial_id' + ,'exchange_statistic_bucket_counter' + ,partition_suffix + ); + PERFORM comment_partitioned_column( + 'identifies an account (hash of normalized payto) for which the statistic is kept, NULL for global statistics' + ,'h_payto' + ,'exchange_statistic_bucket_counter' + ,partition_suffix + ); + PERFORM comment_partitioned_column( + 'start date for the bucket in seconds since the epoch' + ,'bucket_start' + ,'exchange_statistic_bucket_counter' + ,partition_suffix + ); + PERFORM comment_partitioned_column( + 'range of the bucket' + ,'bucket_range' + ,'exchange_statistic_bucket_counter' + ,partition_suffix + ); + PERFORM comment_partitioned_column( + 'aggregate (sum) of tracked by the statistic; what exactly is tracked is determined by the keyword' + ,'cumulative_number' + ,'exchange_statistic_bucket_counter' + ,partition_suffix + ); +END $$; + + +CREATE FUNCTION create_table_exchange_statistic_bucket_amount ( + IN partition_suffix TEXT DEFAULT NULL +) +RETURNS VOID +LANGUAGE plpgsql +AS $$ +BEGIN + PERFORM create_partitioned_table ( + 'CREATE TABLE %I' + '(bmeta_serial_id INT8 NOT NULL' + ' REFERENCES exchange_statistic_bucket_meta (bmeta_serial_id) ON DELETE CASCADE' + ',h_payto BYTEA CHECK (LENGTH(h_payto)=32)' + ',bucket_start INT8 NOT NULL' + ',bucket_range statistic_range NOT NULL' + ',cumulative_value taler_amount NOT NULL' + ',UNIQUE (h_payto,bmeta_serial_id,bucket_start,bucket_range)' + ') %s;' + ,'exchange_statistic_bucket_amount' + ,'PARTITION BY HASH(h_payto)' + ,partition_suffix + ); + PERFORM comment_partitioned_table ( + 'various amount statistics (in various currencies) being tracked', + ,'exchange_statistic_bucket_amount' + ,partition_suffix + ); + PERFORM comment_partitioned_column( + 'identifies what the statistic is about' + ,'bmeta_serial_id' + ,'exchange_statistic_bucket_amount' + ,partition_suffix + ); + PERFORM comment_partitioned_column ( + 'identifies an account (hash of normalized payto) for which the statistic is kept, NULL for global statistics', + ,'h_payto' + ,'exchange_statistic_bucket_amount' + ,partition_suffix + ); + PERFORM comment_partitioned_column( + 'start date for the bucket in seconds since the epoch' + ,'bucket_start' + ,'exchange_statistic_bucket_amount' + ,partition_suffix + ); + PERFORM comment_partitioned_column( + 'range of the bucket' + ,'bucket_range' + ,'exchange_statistic_bucket_amount' + ,partition_suffix + ); + PERFORM comment_partitioned_column( + 'amount being tracked' + ,'cumulative_value' + ,'exchange_statistic_bucket_amount' + ,partition_suffix + ); +END $$; -- -------------- Interval statistics --------------------- @@ -152,95 +224,269 @@ CREATE INDEX exchange_statistic_interval_meta_by_origin ON exchange_statistic_interval_meta (origin); -CREATE TABLE exchange_statistic_counter_event - (nevent_serial_id INT8 GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY - ,imeta_serial_id INT8 - REFERENCES exchange_statistic_interval_meta (imeta_serial_id) ON DELETE CASCADE - ,h_payto BYTEA CHECK (LENGTH(h_payto)=32) - ,slot INT8 NOT NULL - ,delta INT8 NOT NULL - ,UNIQUE (imeta_serial_id, h_payto, slot) - ); -COMMENT ON TABLE exchange_statistic_counter_event - IS 'number to decrement an interval statistic by when a certain time value is reached'; -COMMENT ON COLUMN exchange_statistic_counter_event.nevent_serial_id - IS 'unique identifier for this number event'; -COMMENT ON COLUMN exchange_statistic_counter_event.imeta_serial_id - IS 'identifies what the statistic is about; must be of stype number'; -COMMENT ON COLUMN exchange_statistic_counter_event.h_payto - IS 'identifies an account (hash of normalized payto) for which the statistic is kept, NULL for global statistics'; -COMMENT ON COLUMN exchange_statistic_counter_event.slot - IS 'identifies the time slot at which the given event(s) happened, rounded down by the respective precisions value'; -COMMENT ON COLUMN exchange_statistic_counter_event.delta - IS 'total cumulative number that was added at the time identified by slot'; - -CREATE TABLE exchange_statistic_interval_counter - (imeta_serial_id INT8 NOT NULL - REFERENCES exchange_statistic_interval_meta (imeta_serial_id) ON DELETE CASCADE - ,h_payto BYTEA CHECK (LENGTH(h_payto)=32) - ,range INT8 NOT NULL - ,event_delimiter INT8 NOT NULL - REFERENCES exchange_statistic_counter_event (nevent_serial_id) ON DELETE RESTRICT - ,cumulative_number INT8 NOT NULL - ,UNIQUE (imeta_serial_id,h_payto,range) - ); -COMMENT ON TABLE exchange_statistic_interval_counter - IS 'various numeric statistics (cumulative counters) being tracked'; -COMMENT ON COLUMN exchange_statistic_interval_counter.imeta_serial_id - IS 'identifies what the statistic is about'; -COMMENT ON COLUMN exchange_statistic_interval_counter.h_payto - IS 'identifies an account (hash of normalized payto) for which the statistic is kept, NULL for global statistics'; -COMMENT ON COLUMN exchange_statistic_interval_counter.range - IS 'for which range is this the counter; note that the cumulative_number excludes the values already stored in smaller ranges'; -COMMENT ON COLUMN exchange_statistic_interval_counter.event_delimiter - IS 'determines the last event currently included in the interval'; -COMMENT ON COLUMN exchange_statistic_interval_counter.cumulative_number - IS 'aggregate (sum) of tracked by the statistic; what exactly is tracked is determined by the keyword'; - - -CREATE TABLE exchange_statistic_amount_event - (aevent_serial_id INT8 GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY - ,imeta_serial_id INT8 - REFERENCES exchange_statistic_interval_meta (imeta_serial_id) ON DELETE CASCADE - ,h_payto BYTEA CHECK (LENGTH(h_payto)=32) - ,slot INT8 NOT NULL - ,delta taler_amount NOT NULL - ,CONSTRAINT event_key UNIQUE (imeta_serial_id, h_payto, slot) - ); -COMMENT ON TABLE exchange_statistic_amount_event - IS 'amount to decrement an interval statistic by when a certain time value is reached'; -COMMENT ON COLUMN exchange_statistic_amount_event.aevent_serial_id - IS 'unique identifier for this amount event'; -COMMENT ON COLUMN exchange_statistic_amount_event.imeta_serial_id - IS 'identifies what the statistic is about; must be of clazz interval and of stype amount'; -COMMENT ON COLUMN exchange_statistic_amount_event.h_payto - IS 'identifies an account (hash of normalized payto) for which the statistic is kept, NULL for global statistics'; -COMMENT ON COLUMN exchange_statistic_amount_event.slot - IS 'identifies the time slot at which the given event(s) happened'; -COMMENT ON COLUMN exchange_statistic_amount_event.delta - IS 'total cumulative amount that was added at the time identified by slot'; - - -CREATE TABLE exchange_statistic_interval_amount - (imeta_serial_id INT8 NOT NULL - REFERENCES exchange_statistic_interval_meta (imeta_serial_id) ON DELETE CASCADE - ,h_payto BYTEA CHECK (LENGTH(h_payto)=32) - ,event_delimiter INT8 NOT NULL - REFERENCES exchange_statistic_amount_event (aevent_serial_id) ON DELETE RESTRICT - ,range INT8 NOT NULL - ,cumulative_value taler_amount NOT NULL - ,UNIQUE (imeta_serial_id,h_payto,range) - ); -COMMENT ON TABLE exchange_statistic_interval_amount - IS 'various amount statistics being tracked'; -COMMENT ON COLUMN exchange_statistic_interval_amount.imeta_serial_id - IS 'identifies what the statistic is about'; -COMMENT ON COLUMN exchange_statistic_interval_amount.h_payto - IS 'identifies an account (hash of normalized payto) for which the statistic is kept, NULL for global statistics'; -COMMENT ON COLUMN exchange_statistic_interval_amount.range - IS 'for which range is this the counter; note that the cumulative_number excludes the values already stored in smaller ranges'; -COMMENT ON COLUMN exchange_statistic_interval_amount.cumulative_value - IS 'amount affected by the event'; + +CREATE FUNCTION create_table_exchange_statistic_counter_event ( + IN partition_suffix TEXT DEFAULT NULL +) +RETURNS VOID +LANGUAGE plpgsql +AS $$ +BEGIN + PERFORM create_partitioned_table( + 'CREATE TABLE %I' + '(nevent_serial_id INT8 GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY' + ',imeta_serial_id INT8' + ' REFERENCES exchange_statistic_interval_meta (imeta_serial_id) ON DELETE CASCADE' + ',h_payto BYTEA CHECK (LENGTH(h_payto)=32)' + ',slot INT8 NOT NULL' + ',delta INT8 NOT NULL' + ',UNIQUE (h_payto,imeta_serial_id,slot)' + ') %s ;' + ,'exchange_statistic_counter_event' + ,'PARTITION BY HASH(h_payto)' + ,partition_suffix + ); + PERFORM comment_partitioned_table( + 'number to decrement an interval statistic by when a certain time value is reached'; + ,'exchange_statistic_counter_event' + ,partition_suffix + ); + PERFORM comment_partitioned_column( + 'unique identifier for this number event' + ,'nevent_serial_id' + ,'exchange_statistic_counter_event' + ,partition_suffix + ); + PERFORM comment_partitioned_column( + 'identifies what the statistic is about; must be of stype number' + ,'imeta_serial_id' + ,'exchange_statistic_counter_event' + ,partition_suffix + ); + PERFORM comment_partitioned_column( + 'identifies an account (hash of normalized payto) for which the statistic is kept, NULL for global statistics' + ,'h_payto' + ,'exchange_statistic_counter_event' + ,partition_suffix + ); + PERFORM comment_partitioned_column( + 'identifies the time slot at which the given event(s) happened, rounded down by the respective precisions value' + ,'slot' + ,'exchange_statistic_counter_event' + ,partition_suffix + ); + PERFORM comment_partitioned_column( + 'total cumulative number that was added at the time identified by slot' + ,'delta' + ,'exchange_statistic_counter_event' + ,partition_suffix + ); +END $$; + + +CREATE FUNCTION create_table_exchange_statistic_interval_counter ( + IN partition_suffix TEXT DEFAULT NULL +) +RETURNS VOID +LANGUAGE plpgsql +AS $$ +BEGIN + PERFORM create_partitioned_table( + 'CREATE TABLE %I' + '(imeta_serial_id INT8 NOT NULL' + ' REFERENCES exchange_statistic_interval_meta (imeta_serial_id) ON DELETE CASCADE' + ',h_payto BYTEA CHECK (LENGTH(h_payto)=32)' + ',range INT8 NOT NULL' + ',event_delimiter INT8 NOT NULL' + ',cumulative_number INT8 NOT NULL' + ',UNIQUE (h_payto,imeta_serial_id,range)' + ') %s ;' + ,'exchange_statistic_interval_counter' + ,'PARTITION BY HASH(h_payto)' + ,partition_suffix + ); + PERFORM comment_partitioned_table( + 'various numeric statistics (cumulative counters) being tracked' + ,'exchange_statistic_interval_counter' + ,partition_suffix + ); + PERFORM comment_partitioned_column( + 'identifies what the statistic is about' + ,'imeta_serial_id' + ,'exchange_statistic_interval_counter' + ,partition_suffix + ); + PERFORM comment_partitioned_column( + 'identifies an account (hash of normalized payto) for which the statistic is kept, NULL for global statistics' + ,'h_payto' + ,'exchange_statistic_interval_counter' + ,partition_suffix + ); + PERFORM comment_partitioned_column( + 'for which range is this the counter; note that the cumulative_number excludes the values already stored in smaller ranges' + ,'range' + ,'exchange_statistic_interval_counter' + ,partition_suffix + ); + PERFORM comment_partitioned_column( + 'determines the last event currently included in the interval' + ,'event_delimiter' + ,'exchange_statistic_interval_counter' + ,partition_suffix + ); + PERFORM comment_partitioned_column( + 'aggregate (sum) of tracked by the statistic; what exactly is tracked is determined by the keyword' + ,'cumulative_number' + ,'exchange_statistic_interval_counter' + ,partition_suffix + ); +END $$; + + +CREATE FUNCTION foreign_table_exchange_statistic_interval_counter() +RETURNS VOID +LANGUAGE plpgsql +AS $$ +DECLARE + table_name TEXT DEFAULT 'exchange_statistic_interval_counter'; +BEGIN + EXECUTE FORMAT ( + 'ALTER TABLE ' || table_name || + ' ADD CONSTRAINT ' || table_name || '_event_delimiter_foreign_key' + ' FOREIGN KEY (event_delimiter) ' + ' REFERENCES exchange_statistic_counter_event (nevent_serial_id) ON DELETE RESTRICT' + ); +END $$; + + +CREATE FUNCTION create_table_exchange_statistic_amount_event ( + IN partition_suffix TEXT DEFAULT NULL +) +RETURNS VOID +LANGUAGE plpgsql +AS $$ +BEGIN + PERFORM create_partitioned_table( + 'CREATE TABLE %I' + '(aevent_serial_id INT8 GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY' + ',imeta_serial_id INT8' + ' REFERENCES exchange_statistic_interval_meta (imeta_serial_id) ON DELETE CASCADE' + ',h_payto BYTEA CHECK (LENGTH(h_payto)=32)' + ',slot INT8 NOT NULL' + ',delta taler_amount NOT NULL' + ',CONSTRAINT event_key UNIQUE (h_payto,imeta_serial_id,slot)' + ') %s ;' + ,'exchange_statistic_amount_event' + ,'PARTITION BY HASH(h_payto)' + ,partition_suffix + ); + PERFORM comment_partitioned_table( + 'amount to decrement an interval statistic by when a certain time value is reached' + ,'exchange_statistic_amount_event' + ,partition_suffix + ); +PERFORM comment_partitioned_column( + 'unique identifier for this amount event' + ,'aevent_serial_id' + ,'exchange_statistic_amount_event' + ,partition_suffix + ); +PERFORM comment_partitioned_column + 'identifies what the statistic is about; must be of clazz interval and of stype amount' + ,'imeta_serial_id' + ,'exchange_statistic_amount_event' + ,partition_suffix + ); +PERFORM comment_partitioned_column + 'identifies an account (hash of normalized payto) for which the statistic is kept, NULL for global statistics' + ,'h_payto' + ,'exchange_statistic_amount_event' + ,partition_suffix + ); +PERFORM comment_partitioned_column + 'identifies the time slot at which the given event(s) happened' + ,'slot' + ,'exchange_statistic_amount_event' + ,partition_suffix + ); +PERFORM comment_partitioned_column + 'total cumulative amount that was added at the time identified by slot' + ,'delta' + ,'exchange_statistic_amount_event' + ,partition_suffix + ); +END $$; + + + +CREATE FUNCTION create_table_exchange_statistic_interval_amount ( + IN partition_suffix TEXT DEFAULT NULL +) +RETURNS VOID +LANGUAGE plpgsql +AS $$ +BEGIN + PERFORM create_partitioned_table( + 'CREATE TABLE %I' + '(imeta_serial_id INT8 NOT NULL' + ' REFERENCES exchange_statistic_interval_meta (imeta_serial_id) ON DELETE CASCADE' + ',h_payto BYTEA CHECK (LENGTH(h_payto)=32)' + ',event_delimiter INT8 NOT NULL' + ',range INT8 NOT NULL' + ',cumulative_value taler_amount NOT NULL' + ',UNIQUE (h_payto,imeta_serial_id,range)' + ') %s ;' + ,'exchange_statistic_interval_amount' + ,'PARTITION BY HASH(h_payto)' + ,partition_suffix + ); + PERFORM comment_partitioned_table( + 'various amount statistics being tracked' + ,'exchange_statistic_interval_amount' + ,partition_suffix + ); + PERFORM comment_partitioned_column( + 'identifies what the statistic is about' + ,'imeta_serial_id' + ,'exchange_statistic_interval_amount' + ,partition_suffix + ); + PERFORM comment_partitioned_column( + 'identifies an account (hash of normalized payto) for which the statistic is kept, NULL for global statistics' + ,'h_payto' + ,'exchange_statistic_interval_amount' + ,partition_suffix + ); + PERFORM comment_partitioned_column( + 'for which range is this the counter; note that the cumulative_number excludes the values already stored in smaller ranges' + ,'range' + ,'exchange_statistic_interval_amount' + ,partition_suffix + ); + PERFORM comment_partitioned_column( + 'amount affected by the event' + ,'cumulative_value' + ,'exchange_statistic_interval_amount' + ,partition_suffix + ); +END $$; + + +CREATE FUNCTION foreign_table_exchange_statistic_interval_amount() +RETURNS VOID +LANGUAGE plpgsql +AS $$ +DECLARE + table_name TEXT DEFAULT 'exchange_statistic_interval_amount'; +BEGIN + EXECUTE FORMAT ( + 'ALTER TABLE ' || table_name || + ' ADD CONSTRAINT ' || table_name || '_event_delimiter_foreign_key' + ' FOREIGN KEY (event_delimiter) ' + ' REFERENCES exchange_statistic_amount_event (aevent_serial_id) ON DELETE RESTRICT' + ); +END $$; + CREATE TYPE exchange_statistic_interval_number_get_return_value AS @@ -248,7 +494,7 @@ CREATE TYPE exchange_statistic_interval_number_get_return_value ,rvalue INT8 ); COMMENT ON TYPE exchange_statistic_interval_number_get_return_value - IS 'Return type for exchange_statistic_interval_number_get stored procedure'; + 'Return type for exchange_statistic_interval_number_get stored procedure'; CREATE TYPE exchange_statistic_interval_amount_get_return_value AS @@ -256,8 +502,56 @@ CREATE TYPE exchange_statistic_interval_amount_get_return_value ,rvalue taler_amount ); COMMENT ON TYPE exchange_statistic_interval_amount_get_return_value - IS 'Return type for exchange_statistic_interval_amount_get stored procedure'; + 'Return type for exchange_statistic_interval_amount_get stored procedure'; + +INSERT INTO exchange_tables + (name + ,version + ,action + ,partitioned + ,by_range) + VALUES + ('exchange_statistic_bucket_counter' + ,'exchange-xx10' + ,'create' + ,TRUE + ,FALSE), + ('exchange_statistic_bucket_amount' + ,'exchange-xx10' + ,'create' + ,TRUE + ,FALSE), + ('exchange_statistic_counter_event' + ,'exchange-xx10' + ,'create' + ,TRUE + ,FALSE), + ('exchange_statistic_interval_counter' + ,'exchange-xx10' + ,'create' + ,TRUE + ,FALSE), + ('exchange_statistic_interval_counter' + ,'exchange-xx10' + ,'foreign' + ,TRUE + ,FALSE), + ('exchange_statistic_amount_event' + ,'exchange-xx10' + ,'create' + ,TRUE + ,FALSE), + ('exchange_statistic_interval_amount' + ,'exchange-xx10' + ,'create' + ,TRUE + ,FALSE), + ('exchange_statistic_interval_amount' + ,'exchange-xx10' + ,'foreign' + ,TRUE + ,FALSE); COMMIT;