commit 9d1b609be4ef6f6977b5a19a3835cb650c03e75f parent 546704cf4d80dae6b0c611b26333e65ed3d96a32 Author: Marcello Stanisci <stanisci.m@gmail.com> Date: Thu, 29 Nov 2018 01:56:17 +0100 Auditor deployment. Putting reports generating logic. Diffstat:
| A | buildbot/auditor-report.tex.j2 | | | 870 | +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ |
| A | buildbot/make_auditor_reports.sh | | | 38 | ++++++++++++++++++++++++++++++++++++++ |
| A | buildbot/render_auditor_reports.py | | | 46 | ++++++++++++++++++++++++++++++++++++++++++++++ |
3 files changed, 954 insertions(+), 0 deletions(-)
diff --git a/buildbot/auditor-report.tex.j2 b/buildbot/auditor-report.tex.j2 @@ -0,0 +1,870 @@ +% This file is part of TALER +% Copyright (C) 2016, 2017 Taler Systems SA +% +% TALER is free software; you can redistribute it and/or modify it under the +% terms of the GNU Affero General Public License as published by the Free Software +% Foundation; either version 3, or (at your option) any later version. +% +% TALER is distributed in the hope that it will be useful, but WITHOUT ANY +% WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +% A PARTICULAR PURPOSE. See the GNU Affero General Public License for more details. +% +% You should have received a copy of the GNU Affero General Public License along with +% TALER; see the file COPYING. If not, see <http://www.gnu.org/licenses/> +% +% +% With respect to this file, our interpretation of the license is +% that publishing an audit report (i.e. in TeX or PDF) requires +% publishing the corresponding j2 template sources under AGPL, and +% linking to them from the report. (This file _is_ source code, +% the generated PDF is the service under definition of the AGPL.) +% +\documentclass{article} % {acmart} +\usepackage{url} +\usepackage[T1]{fontenc} +\usepackage[utf8]{inputenc} +\usepackage{multirow} +\usepackage{longtable} + +\begin{document} + +\title{Taler Auditor Report} + +% You must also credit the original author. +\author{Christian Grothoff} +\maketitle + +% If you update this template, complying with the license requires +% stating the license and publishing the J2 source and linking to it +% from the generated PDF. So if you change this outside of the Taler +% Git repository, you must update this link (and the link must remain +% available to the receiver of the result from the generated TeX, PDF +% or other format). + +This report is based on a template licensed under the Affero General Public +License, either version 3, or (at your option) any later version. +The source code for the template is available at \url{https://git.taler.net/}. + + +\section{Operations} + +The balance of the escrow account should +be {\bf {{ data.total_escrow_balance }}}. + +\noindent +The active operational risk stands at +{\bf {{ data.total_active_risk }}}. + + +\section{Income} + +This section analyzes the income of the exchange operator from fees. + +\begin{table}[h!] +\begin{center} + \caption{Fee revenue summary} + \label{table:revenue} + \begin{tabular}{l|r} + Category & Amount \\ \hline \hline + Withdraw fees & {{ data.total_withdraw_fee_income }} \\ + Deposit fees & {{ data.total_deposit_fee_income }} \\ + Melt fees & {{ data.total_melt_fee_income }} \\ + Refund fees & {{ data.total_refund_fee_income }} \\ + Aggregation fees & {{ data.total_aggregation_fee_income }} \\ + {\bf Total} & {{ data.income_fee_total }} \\ +\end{tabular} +\end{center} +\end{table} + + +\section{Lag} + +This section analyzes the lag, which is by how much the exchange's aggregator is behind in +making wire transfers that have been due. + +The total amount the exchange currently lags behind is +{\bf {{ wire.total_amount_lag }}}. + +Note that some lag is perfectly normal, as tiny amounts that are too small to be wired +are deferred beyond the due date, hoping that additional transfers will push them above +the tiny threshold. Below, we report {\em non-tiny} wire transfers that are lagging behind. + +{% if wire.lag_details|length() == 0 %} + {\bf No non-tiny wire transfers that are lagging behind detected.} +{% else %} + \begin{longtable}{p{1.5cm}|r|c|rl} + \multicolumn{4}{l}{\bf Coin} \\ + {\bf Deadline} & {\bf Amount} & {\bf Row} & {\bf Claimed done} \\ + \multicolumn{4}{l}{\bf Target account} \\ \hline \hline +\endfirsthead + \multicolumn{4}{l}{\bf Coin} \\ + {\bf Deadline} & {\bf Amount} & {\bf Row} & {\bf Claimed done} \\ + \multicolumn{4}{l}{\bf Target account} \\ \hline \hline +\endhead + \hline \hline + \multicolumn{4}{l}{\bf Coin} \\ + {\bf Deadline} & {\bf Amount} & {\bf Row} & {\bf Claimed done} \\ + \multicolumn{4}{l}{\bf Target account} \\ +\endfoot + \hline \hline + \multicolumn{4}{l}{\bf Coin} \\ + {\bf Deadline} & {\bf Amount} & {\bf Row} & {\bf Claimed done} \\ + \multicolumn{4}{l}{\bf Target account} \\ + \caption{Lagging non-tiny transactions.} + \label{table:lag} +\endlastfoot +{% for item in wire.lag_details %} + \multicolumn{4}{l}{ {\tt {{ item.coin_pub }} } } \\ +\nopagebreak + & + {{ item.deadline }} & + {{ item.amount }} & + {{ item.row }} & + {{ item.claimed_done }} \\ +\nopagebreak + \multicolumn{4}{l}{ {\tt {{ item.account }} } } \\ \hline +{% endfor %} + \end{longtable} +{% endif %} + + + +\section{Deposit confirmation lag} + +This section analyzes the lag, which is by how much the exchange's +database reporting is behind in providing us with information about +deposit confirmations. Merchants probabilisitcally report deposit +confirmations to the auditor directly, so if the exchange is slow at +synchronizing its database with the auditor, some deposit +confirmations may be known at the auditor only directly. However, any +delta not accounted for by database synchronization delays is an +indicator of a malicious exchange (or online singing key compromise) +and should be answered by revoking the exchange's online siging keys. +% FIXME: reference PhD thesis? + +The total amount the exchange currently lags behind is +{\bf {{ data.missing_deposit_confirmation_total }} } or +{\bf {{ data.missing_deposit_confirmation_count }} } deposit confirmations. + +Note that some lag is perfectly normal. +Below, we report {\em all} deposit confirmations that are lagging behind. + +{% if data.deposit_confirmation_inconsistencies|length() == 0 %} + {\bf No deposit confirmations that are lagging behind detected.} +{% else %} + \begin{longtable}{p{1.5cm}|r|c|r} + {\bf Timestamp} & {\bf Amount} & {\bf Row} \\ + \multicolumn{3}{l}{\bf Target account} \\ \hline \hline +\endfirsthead + {\bf Timestamp} & {\bf Amount} & {\bf Row} \\ + \multicolumn{3}{l}{\bf Target account} \\ \hline \hline +\endhead + \hline \hline + {\bf Timestamp} & {\bf Amount} & {\bf Row} \\ + \multicolumn{3}{l}{\bf Target account} \\ +\endfoot + \hline \hline + {\bf Timestamp} & {\bf Amount} & {\bf Row} \\ + \multicolumn{3}{l}{\bf Target account} \\ + \caption{Missing deposit confirmations.} + \label{table:missing_dc} +\endlastfoot +{% for item in data.deposit_confirmation_inconsistencies %} + & + {{ item.timestamp }} & + {{ item.amount }} & + {{ item.row }} \\ +\nopagebreak + \multicolumn{3}{l}{ {\tt {{ item.account }} } } \\ \hline +{% endfor %} + \end{longtable} +{% endif %} + + +\section{Major irregularities} + +This section describes the possible major irregularities that the +auditor has checked, and lists all of the actual irregularities +encountered in detail. + +\subsection{Emergencies} + +Emergencies are errors where more coins were deposited than the +exchange remembers issuing. This usually means that the private keys +of the exchange were compromised (stolen or factored) and subsequently +used to sign coins off the books. If this happens, all coins of the +respective denomination that the exchange has redeemed so far may have +been created by the attacker, and the exchange would have to refund +all of the outstanding coins from ordinary users. Thus, the {\bf risk +exposure} is the amount of coins in circulation for a particular +denominatin and the maximum loss for the exchange from this type of +compromise. + +{% if data.emergencies|length() == 0 %} + {\bf No emergencies detected.} +{% else %} + \begin{longtable}{p{1.5cm}|r|c|r} + {\bf Public key hash} & {\bf Denomination} & {\bf Lifetime} & {\bf Risk exposure} \\ \hline \hline +\endfirsthead + {\bf Public key hash} & {\bf Denomination} & {\bf Lifetime} & {\bf Risk exposure} \\ \hline \hline +\endhead + \hline \hline + {\bf Public key hash} & {\bf Denomination} & {\bf Lifetime} & {\bf Risk exposure} \\ +\endfoot + \hline +% FIXME: replace these with the summary column adding up the amounts! + \multicolumn{4}{|c|}{ {\bf Total risk from emergencies}} & {{ data.emergencies_risk_total }} \\ + \caption{Emergencies.} + \label{table:emergencies} +\endlastfoot +{% for item in data.reserve_inconsistencies %} + \multicolumn{4}{l}{ {\tt {{ item.denompub_hash }} } } \\ +\nopagebreak + & + {{ item.value }} & + {{ item.start }} - {{ item.deposit_end }} & + {{ item.denom_risk }} \\ \hline +{% endfor %} + \end{longtable} +{% endif %} + + +\subsection{Arithmetic problems} + +This section lists cases where the arithmetic of the exchange +involving amounts disagrees with the arithmetic of the auditor. +Disagreements imply that either the exchange made a loss (sending out +too much money), or screwed a customer (and thus at least needs to fix +the financial damage done to the customer). + +{% if data.amount_arithmetic_inconsistencies|length() == 0 %} + {\bf No arithmetic problems detected.} +{% else %} + \begin{longtable}{p{3.5cm}|l|r|r} + {\bf Operation} & {\bf Table row} & \multicolumn{2}{|c|}{ {\bf Exchange}} & \multicolumn{2}{|c}{ {\bf Auditor}} \\ + \hline \hline +\endfirsthead + {\bf Operation} & {\bf Table row} & \multicolumn{2}{|c|}{ {\bf Exchange}} & \multicolumn{2}{|c}{ {\bf Auditor}} \\ \hline \hline +\endhead + \hline \hline + {\bf Operation} & {\bf Table row} & \multicolumn{2}{|c|}{ {\bf Exchange}} & \multicolumn{2}{|c}{ {\bf Auditor}} \\ +\endfoot + \hline \hline + \multicolumn{2}{l|}{ {\bf $\sum$ Deltas (Auditor-Exchange)} } & + + {{ data.total_arithmetic_delta_plus }} & + - {{ data.total_arithmetic_delta_minus }} \\ + \caption{Arithmetic inconsistencies.} + \label{table:amount:arithmetic:inconsistencies} +\endlastfoot +{% for item in data.amount_arithmetic_inconsistencies %} + {{ item.operation }} & + {{ item.rowid }} & + {{ item.exchange }} & + {{ item.auditor }} \\ \hline +{% endfor %} + \end{longtable} +{% endif %} + +\subsection{Reserve withdrawals exceeding balance} + +This section highlights cases where more coins were withdrawn from a +reserve than the reserve contained funding for. This is a serious +compromise resulting in proportional financial losses to the exchange. + + +{% if data.reserve_balance_insufficient_inconsistencies|length() == 0 %} + {\bf All withdrawals were covered by sufficient reserve funding.} +{% else %} + \begin{longtable}{p{4.5cm}|r} + {\bf Reserve} & \multicolumn{2}{|c|}{ {\bf Loss}} \\ \hline \hline +\endfirsthead + {\bf Reserve} & \multicolumn{2}{|c|}{ {\bf Loss}} \\ \hline \hline +\endhead + \hline \hline + {\bf Reserve} & \multicolumn{2}{|c|}{ {\bf Loss}} +\endfoot + \hline + {\bf Total loss} & & + {{ data.total_loss_balance_insufficient }} \\ + \caption{Reserves with withdrawals higher than reserve funding.} + \label{table:reserve:balance_insufficient} +\endlastfoot +{% for item in data.reserve_balance_insufficient_inconsistencies %} + \multicolumn{3}{l}{ {\tt {{ item.reserve_pub }} } } \\ +\nopagebreak + & + {{ item.loss }} \\ \hline +{% endfor %} + \end{longtable} +{% endif %} + + +\subsection{Claimed outgoing wire transfer inconsistencies} + +This section is about the exchange's database containing a +justification to make an outgoing wire transfer for an aggregated +amount for various deposits. It is reported as an inconsistency if the +amount claimed for the wire transfer does not match up the deposits +aggregated. This is about a {\em claimed} outgoing wire transfer as +violations do not imply that the wire transfer was actually made (as +that is a separate check). Note that not making the wire transfer +would be reported separately in Section~\ref{sec:wire_check_out}. + + +{% if data.wire_out_inconsistencies|length() == 0 %} + {\bf All aggregations matched up.} +{% else %} + \begin{longtable}{p{1.5cm}|l|r|r} + {\bf Destination account} & {\bf Database row} & {\bf Expected} & {\bf Claimed} \\ \hline \hline +\endfirsthead + {\bf Destination account} & {\bf Database row} & {\bf Expected} & {\bf Claimed} \\ \hline \hline +\endhead + \hline \hline + {\bf Destination account} & {\bf Database row} & {\bf Expected} & {\bf Claimed} \\ +\endfoot + \hline + {\bf Total deltas} & & + {{ data.total_wire_out_delta_plus}} & + - {{ data.total_wire_out_delta_minus}} \\ + \caption{Claimed wire out aggregate totals not matching up.} + \label{table:reserve:wire_out_balance_inconsistencies} +\endlastfoot +{% for item in data.wire_out_inconsistencies %} + \multicolumn{6}{l}{ {\tt {{ item.destination_account }} } } \\ +\nopagebreak + & {{ item.rowid }} & + {{ item.expected }} & + {{ item.claimed }} \\ \hline +{% endfor %} + \end{longtable} +{% endif %} + + +\subsection{Coin history inconsistencies} + +This section lists cases where the exchange made arithmetic errors found when +looking at the transaction history of a coin. The totals sum up the differences +in amounts that matter for profit/loss calculations of the exchange. When an +exchange merely shifted money from customers to merchants (or vice versa) without +any effects on its own balance, those entries are excluded from the total. + +{% if data.coin_inconsistencies|length() == 0 %} + {\bf All coin histories were unproblematic.} +{% else %} + \begin{longtable}{p{1.8cm}|p{3cm}|r|r} + {\bf Operation} & {\bf Coin public key} & {\bf Exchange } & {\bf Auditor} \\ + \hline \hline +\endfirsthead + {\bf Operation} & {\bf Coin public key} & {\bf Exchange} & {\bf Auditor} \\ \hline \hline +\endhead + \hline \hline + {\bf Operation} & {\bf Coin public key} & {\bf Exchange} & {\bf Auditor} \\ +\endfoot + \hline + \multicolumn{2}{l|}{ $\sum$ {\bf Delta (Auditor-Exchange)} } & + {{ data.total_coin_delta_plus }} & + - {{ data.total_coin_delta_minus }} \\ + \caption{Arithmetic inconsistencies of amount calculations involving a coin.} + \label{table:amount:arithmetic:coin:inconsistencies} +\endlastfoot +{% for item in data.coin_inconsistencies %} + {{ item.operation }} & + \multicolumn{5}{l}{ {\tt {{ item.coin_pub }} } } \\ +\nopagebreak & & + {{ item.exchange }} & + {{ item.auditor }} \\ \hline +{% endfor %} + \end{longtable} +{% endif %} + + +\subsection{Operations with bad signatures} + +This section lists operations that the exchange performed, but for +which the signatures provided are invalid. Hence the operations were +invalid and the amount involved should be considered lost. + +The key given is always the key for which the signature verification +step failed. This is the reserve public key for ``withdraw'' +operations, the coin public key for ``deposit'' and ``melt'' +operations, the merchant's public key for ``melt'' operations, +the (hash of the) denomination public key for +``payback-verify'' and ``deposit-verify'' operations, and the master +public key for ``payback-master'' operations. + +{% if data.bad_sig_losses|length() == 0 %} + {\bf All signatures were valid.} +{% else %} + \begin{longtable}{p{1.5cm}|c|l|r} + {\bf Public key} & {\bf Operation type} & Database row & {\bf Loss amount} \\ + \hline \hline +\endfirsthead + {\bf Public key} & {\bf Operation type} & Database row & {\bf Loss amount} \\ \hline \hline +\endhead + \hline \hline + {\bf Public key} & {\bf Operation type} & Database row & {\bf Loss amount} \\ +\endfoot + \hline + {\bf Total losses} & & & + {{ data.total_bad_sig_loss}} \\ + \caption{Losses from operations performed on coins without proper signatures.} + \label{table:bad_signature_losses} +\endlastfoot +{% for item in data.bad_sig_losses %} + \multicolumn{5}{l}{ {\tt {{ item.key_pub }} } } \\ +\nopagebreak + & {{ item.operation }} & {{ item.rowid }} & + {{ item.loss }} \\ \hline +{% endfor %} + \end{longtable} +{% endif %} + + + +\subsection{Actual incoming wire transfers} + +This section highlights cases where the exchange's record about +incoming wire transfers does not match with that of the bank. + +{% if wire.reserve_in_amount_inconsistencies|length() == 0 %} + {\bf All incoming wire transfer amounts and subjects matched up.} +{% else %} + \begin{longtable}{p{5.5cm}|r|r} + \multicolumn{5}{l}{ {\bf Wire transfer identifier} ({\bf Row}) } \\ + {\bf Diagnostic} & {\bf Wired} & {\bf Expected} \\ + \hline \hline +\endfirsthead + \multicolumn{5}{l}{ {\bf Wire transfer identifier} ({\bf Row}) } \\ + {\bf Diagnostic} & {\bf Wired} & {\bf Expected} \\ + \hline \hline +\endhead + \hline \hline + \multicolumn{5}{l}{ {\bf Wire transfer identifier} ({\bf Row}) } \\ + {\bf Diagnostic} & {\bf Wired} & {\bf Expected} \\ +\endfoot + \hline \hline + {\bf Total deltas} & + {{ wire.total_wire_in_delta_plus }} & + - {{ wire.total_wire_in_delta_minus}} \\ + \caption{Incoming wire transfer amounts not matching up.} + \label{table:wire_in:transfer_amount_inconsistencies} +\endlastfoot +{% for item in wire.reserve_in_amount_inconsistencies %} + \multicolumn{5}{l}{ {\tt {{ item.wtid }} } ({{ item.row }}) } \\ +\nopagebreak + {{ item.timestamp }}: & + {{ item.amount_wired }} & + {{ item.amount_expected }} \\ + {{ item.diagnostic }} & & & & \\ +%\nopagebreak +% & \multicolumn{4}{|c}{ {{ item.timestamp }} } + \hline +{% endfor %} + \end{longtable} +{% endif %} + + +\subsection{Missattributed incoming wire transfers} + +This section lists cases where the sender account record of an +incoming wire transfer differs between the exchange and the bank. +This will cause funds to be sent to the wrong account when the reserve +is closed and the remaining balance is refunded to the original +account. + + +{% if wire.missattribution_in_inconsistencies|length() == 0 %} + {\bf All incoming wire transfer sender accounts matched up.} +{% else %} + \begin{longtable}{p{6.5cm}|r} + {\bf Wire transfer identifier} & {\bf Amount} \\ + \hline \hline +\endfirsthead + {\bf Wire transfer identifier} & {\bf Amount} \\ + \hline \hline +\endhead + \hline \hline + {\bf Wire transfer identifier} & {\bf Amount} \\ +\endfoot + \hline + {\bf Total amount} & + {{ wire.total_missattribution_in_plus}} \\ + \caption{Incoming wire transfer sender accounts not matching up.} + \label{table:wire_in:sender_account_inconsistencies} +\endlastfoot +{% for item in wire.missattribution_in_inconsistencies %} + {\tt {{ item.wtid }} } & + {{ item.amount }} \\ hline +{% endfor %} + \end{longtable} +{% endif %} + + + + +\subsection{Actual outgoing wire transfers} \label{sec:wire_check_out} + +This section highlights cases where the exchange missbehaved +with respect to outgoing wire transfers. + +{% if wire.wire_out_amount_inconsistencies|length() == 0 %} + {\bf All outgoing wire transfers matched up.} +{% else %} + \begin{longtable}{p{6.5cm}|r|r} + {\bf Wire transfer identifier} & {\bf Wired} & {\bf Justified}} \\ + {\bf Diagnostic} & {\bf Row} & {\bf Timestamp} \\ + \hline \hline +\endfirsthead + {\bf Wire transfer identifier} & {\bf Wired} & {\bf Justified} \\ + {\bf Diagnostic} & {\bf Row} & {\bf Timestamp} \\ + \hline \hline +\endhead + \hline \hline + {\bf Wire transfer identifier} & {\bf Wired} & {\bf Justified} \\ + {\bf Diagnostic} & {\bf Row}} & {\bf Timestamp} \\ +\endfoot + \hline + {\bf Total deltas} & & + {{ wire.total_wire_out_delta_plus }} & + - {{ wire.total_wire_out_delta_minus }} \\ + \caption{Outgoing wire transfer amounts not matching up.} + \label{table:wire_out:transfer_amount_inconsistencies} +\endlastfoot +{% for item in wire.wire_out_amount_inconsistencies %} + {\tt {{ item.wtid }} } & + {{ item.amount_wired }} & + {{ item.amount_justified }} \\ \hline +\nopagebreak + & + {{ item.diagnostic }} & + {{ item.row }} & + {{ item.timestmap }} \\ \hline +{% endfor %} + \end{longtable} +{% endif %} + + +\section{Minor irregularities} + +\subsection{Incorrect reserve balance summary in database} + +This section highlights cases where the reserve balance summary +in the database does not match the calculations made by the auditor. +Deltas may indicate a corrupt database, but do not necessarily +translate into a financial loss (yet). + + +{% if data.reserve_balance_summary_wrong_inconsistencies|length() == 0 %} + {\bf All balances matched up.} +{% else %} + \begin{longtable}{p{1.5cm}|r|r} + {\bf Reserve} & {\bf Auditor} & {\bf Exchange} \\ \hline \hline +\endfirsthead + {\bf Reserve} & {\bf Auditor} & {\bf Exchange} \\ \hline \hline +\endhead + \hline \hline + {\bf Reserve} & {\bf Auditor} & {\bf Exchange} +\endfoot + \hline + {\bf Total deltas} & & + {{ data.total_balance_summary_delta_plus}} & + - {{ data.total_balance_summary_delta_minus}} \\ + \caption{Reserves balances not matching up.} + \label{table:reserve:balance_inconsistencies} +\endlastfoot +{% for item in data.reserve_balance_summary_wrong_inconsistencies %} + \multicolumn{5}{l}{ {\tt {{ item.reserve_pub }} } } \\ +\nopagebreak + & + {{ item.auditor }} & + {{ item.exchange }} \\ \hline +{% endfor %} + \end{longtable} +{% endif %} + + +\subsection{Wire table issues} + +This section describes issues found by the wire auditor that do not +have a clear financial impact. + +{% if wire.row_inconsistencies|length() == 0 %} + {\bf No wire row inconsistencies found.} +{% else %} + \begin{longtable}{p{5.5cm}|l|p{4.5cm}} + {\bf Table} & {\bf Row} & {\bf Diagnostic} \\ + \hline \hline +\endfirsthead + {\bf Table} & {\bf Row} & {\bf Diagnostic} \\ + \hline \hline +\endhead + \hline + {\bf Table} & {\bf Row} & {\bf Diagnostic} \\ +\endfoot + \hline + {\bf Table} & {\bf Row} & {\bf Diagnostic} \\ + \caption{Other wire table issues found (by table and row).} + \label{table:wire:misc} +\endlastfoot +{% for item in wire.row_inconsistencies %} + \verb! {{ item.table }} ! & + {{ item.row }} & + {{ item.diagnostic }} \\ +\nopagebreak + \multicolumn{3}{l}{ {\tiny {\tt {{ item.wire_offset_hash }} } } } \\ \hline +{% endfor %} + \end{longtable} +{% endif %} + + +\subsection{Outgoing wire transfer subject issues} + +This section describes issues found by the wire auditor that +relate to outgoing wire transfers being malformed. +This happens if the exchange somehow creates wire transfers +with duplicate or malformed wire transfer subjects. + +{% if wire.wire_format_inconsistencies|length() == 0 %} + {\bf No wire format inconsistencies found.} +{% else %} + \begin{longtable}{p{4.5cm}|r} + \multicolumn{3}{c}{ {\bf Row hash} } \\ + {\bf Diagnostic} & {\bf Amount} \\ + \hline \hline +\endfirsthead + \multicolumn{3}{c}{ {\bf Row hash} } \\ + {\bf Diagnostic} & {\bf Amount} \\ + \hline \hline +\endhead + \hline + \multicolumn{3}{c}{ {\bf Row hash} } \\ + {\bf Diagnostic} & {\bf Amount} \\ +\endfoot + \hline + \hline + {\bf Total} & + {{ wire.total_wire_format_amount }} \\ + \caption{Outgoing wire transfer subject issues found.} + \label{table:outgoing:wtid} +\endlastfoot +{% for item in wire.wire_format_inconsistencies %} + \multicolumn{3}{l}{ \verb! {{ item.wire_offset_hash }} ! } \\ +\nopagebreak + {{ item.diagnostic }} & + {{ item.amount }} \\ +{% endfor %} + \end{longtable} +{% endif %} + + +\subsection{Wire fee structure inconsistencies} + +This section lists cases where the exchange's database may be ambiguous +with respect to what wire fee it charges at what time. + +{% if data.wire_fee_time_inconsistencies|length() == 0 %} + {\bf No wire fee timing issues detected.} +{% else %} + \begin{longtable}{p{1.5cm}|r|p{5.5}} + {\bf Wire format} & {\bf Timestamp} & {\bf Diagnostic} + \\ \hline \hline +\endfirsthead + {\bf Wire format} & {\bf Timestamp} & {\bf Diagnostic} + \\ \hline \hline +\endhead + \hline \hline + {\bf Wire format} & {\bf Timestamp} & {\bf Diagnostic} \\ +\endfoot + \hline \hline + {\bf Wire format} & {\bf Timestamp} & {\bf Diagnostic} \\ + \caption{Wire fees with ambiguous timestamps.} + \label{table:wire_fee:ambiguity} +\endlastfoot +{% for item in data.wire_fee_time_inconsistencies %} + {\tt {{ item.type }} } & {{ item.time }} & {{ item.diagnostic }} \\ \hline +{% endfor %} + \end{longtable} +{% endif %} + + + +\subsection{Other issues} + +This section describes issues found that do not have a clear financial +impact. + +{% if data.row_inconsistencies|length() == 0 %} + {\bf No row inconsistencies found.} +{% else %} + \begin{longtable}{p{1.5cm}|l|p{5.5cm}} + {\bf Table} & {\bf Row} & {\bf Diagnostic} \\ + \hline \hline +\endfirsthead + {\bf Table} & {\bf Row} & {\bf Diagnostic} \\ + \hline \hline +\endhead + \hline \hline + {\bf Table} & {\bf Row} & {\bf Diagnostic} \\ +\endfoot + \hline \hline + {\bf Table} & {\bf Row} & {\bf Diagnostic} \\ + \caption{Other issues found (by table and row).} + \label{table:misc} +\endlastfoot +{% for item in data.row_inconsistencies %} + \verb! {{ item.table }} ! & + {{ item.row }} & + {{ item.diagnostic }} \\ \hline +{% endfor %} + \end{longtable} +{% endif %} + + +\section{Delays and timing} + +This section describes issues that are likely caused simply by +some job process of the exchange not running properly or not having +caught up with the work load yet. + +\subsection{Delayed closure of reserves} + +This section describes cases where the exchange did not +close a reserve and wire back the remaining funds when the +reserve expired. + + +{% if data.reserve_not_closed_inconsistencies|length() == 0 %} + {\bf All expired reserves were closed.} +{% else %} + \begin{longtable}{p{1.5cm}|c|r} + {\bf Reserve} & {\bf Expired} & {\bf Balance} \\ \hline \hline +\endfirsthead + {\bf Reserve} & {\bf Expired} & {\bf Balance} \\ \hline \hline +\endhead + \hline \hline + {\bf Reserve} & {\bf Expired} & {\bf Balance} +\endfoot + \hline + {\bf Sum} & & + {{ data.total_balance_reserve_not_closed}} \\ + \caption{Reserves not closed on time.} + \label{table:reserve:not_closed} +\endlastfoot +{% for item in data.reserve_not_closed_inconsistencies %} + \multicolumn{4}{l}{ {\tt {{ item.reserve_pub }} } } \\ +\nopagebreak + & + {{ item.expiration_time }} & + {{ item.balance }} \\ \hline +{% endfor %} + \end{longtable} +{% endif %} + + +\subsection{Hanging refresh operations} + +This section describes cases where the exchange booked a +coin as spent from {\tt /refresh/melt} but where the +wallet did not yet complete {\tt /refresh/reveal}. This +may happen even if the exchange is correct. + + +{% if data.refresh_hanging|length() == 0 %} + {\bf All melted coins were refreshed.} +{% else %} + \begin{longtable}{p{1.5cm}|c|r} + {\bf Key} & {\bf row} & {\bf Amount} \\ + \hline \hline +\endfirsthead + {\bf Key} & {\bf row} & {\bf Amount} \\ + \hline \hline +\endhead + \hline \hline + {\bf Key} & {\bf row} & {\bf Amount} \\ +\endfoot + \hline + {\bf Sum} & & + {{ data.total_refresh_hanging}} \\ + \caption{Refresh operations hanging.} + \label{table:refresh:hanging} +\endlastfoot +{% for item in data.refresh_hanging %} + \multicolumn{4}{l}{ {\tt {{ item.coin_pub }} } } \\ +\nopagebreak + & + {{ item.row }} & + {{ item.amount }} \\ \hline +{% endfor %} + \end{longtable} +{% endif %} + + +\subsection{Denomination key invalid at time of withdrawal} + +This section lists cases where a denomination key was not valid for +withdrawal at the time when the exchange claims to have signed a coin +with it. This would be irregular, but has no obvious financial +implications. + + +{% if data.denomination_key_validity_withdraw_inconsistencies|length() == 0 %} + {\bf All denomination keys were valid at the time of withdrawals.} +{% else %} + \begin{longtable}{p{7.5cm}|c} + {\bf Reserve} & {\bf Table row} \\ + {\bf Denomination key hash} & {\bf Execution time} \\ \hline \hline +\endfirsthead + {\bf Reserve} & {\bf Table row} \\ + {\bf Denomination key hash} & {\bf Execution time} \\ \hline \hline +\endhead + \hline \hline + {\bf Reserve} & {\bf Table row} \\ + {\bf Denomination key hash} & {\bf Execution time} \\ +\endfoot + \hline + {\bf Reserve} & {\bf Table row} \\ + {\bf Denomination key hash} & {\bf Execution time} \\ + \caption{Execution times not matching denomination key validity period.} + \label{table:withdraw:bad_time} +\endlastfoot +{% for item in data.denomination_key_validity_withdraw_inconsistencies %} + {\tt {{ item.reserve_pub }} } & {{ item.row }} \\ +\nopagebreak + & + {\tt {{ item.denompub_h }} } & {{ item.execution_date }} \\ \hline +{% endfor %} + \end{longtable} +{% endif %} + + +\subsection{Wire transfer timestamp issues} + +This section lists issues with wire transfers related to timestamps. + + +{% if wire.row_minor_inconsistencies|length() == 0 %} + {\bf No timestamp issues detected.} +{% else %} + \begin{longtable}{p{1.5cm}|r|p{5.5}} + {\bf Table} & {\bf Table row} & {\bf Diagnostic} + \\ \hline \hline +\endfirsthead + {\bf Table} & {\bf Table row} & {\bf Diagnostic} + \\ \hline \hline +\endhead + \hline \hline + {\bf Table} & {\bf Table row} & {\bf Diagnostic} \\ +\endfoot + \hline \hline + {\bf Table} & {\bf Table row} & {\bf Diagnostic} \\ + \caption{Execution times not matching in wire transfers.} + \label{table:wire:bad_time} +\endlastfoot +{% for item in wire.row_minor_inconsistencies %} + \verb! {{ item.table }} ! & {{ item.row }} & {{ item.diagnostic }} \\ \hline +{% endfor %} + \end{longtable} +{% endif %} + + +\end{document} diff --git a/buildbot/make_auditor_reports.sh b/buildbot/make_auditor_reports.sh @@ -0,0 +1,38 @@ +#!/bin/sh + + +# Remove once ready! +echo "Auditor report generation is work in progress.." +return 1 + +# 1 generate reports. + +if test -z ${TALER_DEPLOYMENT}; then + echo "Please export env variable TALER_DEPLOYMENT=(test|demo) before running this script" + exit 1 +fi + +ACTIVE_COLOR=$(sudo -u ${TALER_DEPLOYMENT} cat /home/${TALER_DEPLOYMENT}/active) +DATE=$(date +%Y-%m-%d_%H-%M) +REPORT_FILE=/home/${ACTIVE_COLOR}/audit_report.${DATE} +SUDO_COLOR="sudo -u ${ACTIVE_COLOR}" +JINJA_TEMPLATE="auditor-report.tex.j2" +REPORTS_DIRECTORY="/home/${ACTIVE_COLOR}/fixme" + +${SUDO_COLOR} taler-auditor > ${REPORT_FILE}.txt + +# 2 convert them into PDF. +# 3 put them into nginx-served directory. + +${SUDO_COLOR} ./render_auditor_reports.py \ + "${REPORT_FILE}.txt" > "${REPORT_FILE}.tex" \ + < ${JINJA_TEMPLATE} + +${SUDO_COLOR} pdflatex \ + --output-directory $REPORTS_DIRECTORY \ + "${REPORT_FILE}.tex" + +${SUDO_COLOR} rm + "$REPORTS_DIRECTORY/${REPORT_FILE}.log" + +echo "The file ${REPORT_FILE}.pdf" has been correctly generated. diff --git a/buildbot/render_auditor_reports.py b/buildbot/render_auditor_reports.py @@ -0,0 +1,46 @@ +#!/usr/bin/python +# This file is in the public domain. + +"""Expand Jinja2 templates based on JSON input. + +First command-line argument must be the JSON input from taler-auditor. +Second command-line argument must be the JSON input from the +taler-wire-auditor. + +The tool then reads the template from stdin and writes the expanded +output to stdout. + +TODO: proper installation, man page, error handling, --help option. + +@author Christian Grothoff + +""" + +import sys +import json +import jinja2 +from jinja2 import BaseLoader + + +class StdinLoader(BaseLoader): + def __init__ (self): + self.path = '-' + def get_source(self, environment, template): + source = sys.stdin.read().decode('utf-8') + return source, self.path, lambda: false + + +jsonFile1 = open (sys.argv[1], 'r') +jsonData1 = json.load(jsonFile1) + +jsonFile2 = open (sys.argv[2], 'r') +jsonData2 = json.load(jsonFile2) + +jinjaEnv = jinja2.Environment(loader=StdinLoader(), + lstrip_blocks=True, + trim_blocks=True, + undefined=jinja2.StrictUndefined, + autoescape=False) +tmpl = jinjaEnv.get_template('stdin'); + +print(tmpl.render(data = jsonData1, wire = jsonData2))