commit 5cfb497eacc02a732aa289a4de5352e9c3f79794
parent cd0a876306673dcf9677d4f5ae41af7e195a062e
Author: Iván Ávalos <avalos@disroot.org>
Date: Fri, 31 Jul 2026 12:37:39 +0200
DD92: document the sync API as implemented
The object references are now covered by the block signatures, the object
store endpoints are no longer stubs, and a request the server rejects
without changing anything answers 409 instead of 500.
Diffstat:
1 file changed, 171 insertions(+), 38 deletions(-)
diff --git a/design-documents/092-incremental-backup-sync.rst b/design-documents/092-incremental-backup-sync.rst
@@ -169,6 +169,10 @@ EdDSA public key that identifies the backup account. All upload
requests must be signed by the corresponding private key; the signature
is transmitted in the request body.
+Binary values in URLs, headers and JSON bodies (nonces, UIDs, hashes,
+signatures and the encrypted payloads themselves) are all base32-encoded
+in Crockford representation, as is usual for Taler.
+
Signatures use EdDSA with the account private key. Each signature
payload follows the common Taler signing structure with a
``purpose`` field (see :ref:`Signatures` in the API common conventions
@@ -182,12 +186,13 @@ for the general format). The specific payloads are:
* For appends, old_hash is all-zeros.
*/
struct SyncBlockUploadSignaturePS {
- struct GNUNET_CRYPTO_EccSignaturePurpose purpose;
+ struct GNUNET_CRYPTO_SignaturePurpose purpose;
struct SYNC_BlockNonce prev_nonce; ///< all-zeros if first block
struct SYNC_BlockNonce next_nonce; ///< all-zeros if last block
struct SYNC_BlockNonce nonce;
struct GNUNET_HashCode old_hash; ///< all-zeros for appends
struct GNUNET_HashCode new_hash;
+ struct GNUNET_HashCode refs_hash; ///< over object_refs, see below
};
/**
@@ -195,11 +200,12 @@ for the general format). The specific payloads are:
* Authorizes the deletion of a block.
*/
struct SyncBlockDeleteSignaturePS {
- struct GNUNET_CRYPTO_EccSignaturePurpose purpose;
+ struct GNUNET_CRYPTO_SignaturePurpose purpose;
struct SYNC_BlockNonce nonce;
struct SYNC_BlockNonce prev_nonce; ///< all-zeros if first block
struct SYNC_BlockNonce next_nonce; ///< all-zeros if last block
struct GNUNET_HashCode hash;
+ struct GNUNET_HashCode refs_hash; ///< over object_refs, see below
};
/**
@@ -207,7 +213,7 @@ for the general format). The specific payloads are:
* Authorizes the upload of a blob object.
*/
struct SyncObjectUploadSignaturePS {
- struct GNUNET_CRYPTO_EccSignaturePurpose purpose;
+ struct GNUNET_CRYPTO_SignaturePurpose purpose;
struct SYNC_ObjectUID uid;
struct GNUNET_HashCode hash;
};
@@ -215,6 +221,55 @@ for the general format). The specific payloads are:
Absent optional nonces (``prev_nonce`` / ``next_nonce``) are
treated as all-zeros in the signed data.
+The ``refs_hash`` field covers the ``object_refs`` of the request, so
+that the reference-count adjustments cannot be altered in transit. It
+is the SHA-512 hash over a canonical *binary* encoding of the
+references — not over their JSON representation.
+
+Each reference is laid out as the 64 raw UID bytes followed by the
+adjustment as a signed 16-bit integer in network byte order, and the
+resulting 66-byte records are concatenated in ascending order of UID:
+
+.. code-block:: text
+
+ +----------------------------+
+ | uid (64 byte) |
+ +----------------------------+
+ | adjustment (2 byte, int16) |
+ +----------------------------+
+
+Sorting by UID is required because ``object_refs`` travels as a JSON
+object, whose member order is not preserved. A request without any
+references hashes the empty byte string.
+
+A UID may appear at most once, since the wire format keys the references
+by UID and could not otherwise transmit them faithfully.
+
+.. http:get:: /config
+
+ Return the server's protocol version and terms. Requires no account
+ and no signature.
+
+ **Response**
+
+ :http:statuscode:`200 OK`:
+ The body is a `SyncConfig` object.
+
+ .. ts:def:: SyncConfig
+
+ interface SyncConfig {
+ name: "sync";
+ implementation: string;
+ storage_limit_in_megabytes: number;
+ liability_limit: AmountString;
+ annual_fee: AmountString;
+ version: string;
+ }
+
+ ``storage_limit_in_megabytes`` is the per-upload limit enforced for
+ both blocks and objects; exceeding it yields ``413``. ``version``
+ follows the Taler ``current:revision:age`` convention.
+
.. http:get:: /backups/${ACCOUNT_KEY}/blocks
List blocks from the account's block chain with pagination.
@@ -231,10 +286,12 @@ treated as all-zeros in the signed data.
**Response**
:http:statuscode:`200 OK`:
- The body is a JSON array of `BlockEntry` objects.
+ The body is a JSON array of `BlockEntry` objects. The array is
+ empty if the account has no blocks.
:http:statuscode:`400 Bad request`:
- The ``limit`` parameter is missing or malformed, or ``start_nonce``
- is malformed.
+ The ``limit`` parameter is missing, malformed, given without a
+ value, or not positive; or ``start_nonce`` is malformed or given
+ without a value.
:http:statuscode:`402 Payment required`:
The account has expired and requires payment.
:http:statuscode:`404 Not found`:
@@ -252,6 +309,10 @@ treated as all-zeros in the signed data.
data: string;
}
+ ``data`` is the encrypted block payload as it was uploaded, and
+ hashes to ``block_hash``. ``prev_nonce`` and ``next_nonce`` are
+ absent for the first and last block of the chain respectively.
+
.. http:post:: /backups/${ACCOUNT_KEY}/blocks/${NONCE}
Upload a new block and append (or insert) it into the account's block
@@ -260,8 +321,9 @@ treated as all-zeros in the signed data.
if it differs, the client should use ``PUT`` instead.
The request must include an ``If-None-Match`` header containing the
- quoted hex-encoded SHA-512 hash of the encrypted block data. This
- hash is used by the server to detect duplicates.
+ quoted base32-encoded SHA-512 hash of the encrypted block data. This
+ hash is used by the server to detect duplicates, and the server
+ rejects the upload if the ``data`` in the body does not hash to it.
**Request**
@@ -294,7 +356,8 @@ treated as all-zeros in the signed data.
``upload_sig``
EdDSA signature over the block nonce, ``prev_nonce``,
``next_nonce``, old data hash (for updates, all-zeros for appends),
- and new data hash, signed with the account's private key
+ new data hash and the hash over ``object_refs``, signed with the
+ account's private key
(``TALER_SIGNATURE_SYNC_BLOCK_UPLOAD``).
``prev_nonce``
@@ -306,13 +369,17 @@ treated as all-zeros in the signed data.
Must be omitted for the last block.
``data``
- The encrypted block contents (binary, base64-encoded).
+ The encrypted block contents (binary, base32-encoded).
``object_refs``
Optional object whose keys are blob UIDs and whose values are
16-bit signed integer reference-count deltas. Any objects
referenced here must have been uploaded *beforehand* via
- ``POST /backups/${ACCOUNT_KEY}/objects/${UID}``.
+ ``POST /backups/${ACCOUNT_KEY}/objects/${UID}``, and each UID may
+ appear at most once. The adjustments are applied in the same
+ transaction as the block operation: if any of them names an object
+ the account does not have, or would take a reference count below
+ zero, the entire request is rejected and nothing is modified.
**Response**
@@ -328,26 +395,35 @@ treated as all-zeros in the signed data.
:http:statuscode:`403 Forbidden`:
The signature is invalid or does not match the request.
:http:statuscode:`409 Conflict`:
- The write is outdated (the block data has been modified by another
- device since the caller last fetched it).
- :http:statuscode:`413 Content too large`:
+ The request does not fit the state the server holds, and retrying
+ it unchanged will not help. Either the write is outdated (the
+ block chain has been modified by another device since the caller
+ last fetched it), the nonce is already in use, or ``object_refs``
+ names an object the account does not have or would take a
+ reference count below zero. Nothing was modified.
+ :http:statuscode:`413 Request entity too large`:
The upload exceeds the server's configured upload limit.
:http:statuscode:`500 Internal server error`:
- A database error occurred, or the backup is in an inconsistent
- state (e.g. a referenced block is missing from the chain).
+ A database error occurred.
.. http:put:: /backups/${ACCOUNT_KEY}/blocks/${NONCE}
Replace an existing block's content in-place. Semantics are identical
to ``POST`` on the same endpoint, with one addition: the
- ``If-Match`` header must contain the quoted hex-encoded SHA-512 hash
- of the old block data that is being replaced. The server rejects
- the request with ``409 Conflict`` if the old hash does not match.
+ ``If-Match`` header must contain the quoted base32-encoded SHA-512
+ hash of the old block data that is being replaced. The server rejects
+ the request with ``409 Conflict`` if the old hash, ``prev_nonce`` or
+ ``next_nonce`` do not match the stored block.
The ``upload_sig`` must also cover the old data hash (from
``If-Match``) in addition to the new data hash (from
``If-None-Match``).
+ .. note::
+
+ ``PUT`` stands in for ``PATCH``, which the update operation would
+ otherwise use, until the HTTP server library supports it.
+
**Response**
Same status codes as ``POST``, plus:
@@ -358,7 +434,7 @@ treated as all-zeros in the signed data.
.. http:delete:: /backups/${ACCOUNT_KEY}/blocks/${NONCE}
Delete an existing block from the block chain. The request must
- include an ``If-Match`` header containing the quoted hex-encoded
+ include an ``If-Match`` header containing the quoted base32-encoded
SHA-512 hash of the block data to delete, which the server uses to
detect concurrent modifications.
@@ -377,8 +453,8 @@ treated as all-zeros in the signed data.
``delete_sig``
EdDSA signature over the block nonce, ``prev_nonce``,
- ``next_nonce``, and block hash (from ``If-Match``), signed with
- the account's private key
+ ``next_nonce``, block hash (from ``If-Match``) and the hash over
+ ``object_refs``, signed with the account's private key
(``TALER_SIGNATURE_SYNC_BLOCK_DELETE``).
``prev_nonce``
@@ -393,7 +469,10 @@ treated as all-zeros in the signed data.
Optional object whose keys are blob UIDs and whose values are
16-bit signed integer reference-count deltas (typically negative,
to decrement the refcount of objects that were referenced by the
- deleted block).
+ deleted block). The same rules as for block uploads apply: each
+ UID may appear at most once, and the whole request is rejected if
+ an adjustment names an unknown object or would take a reference
+ count below zero.
**Response**
@@ -408,12 +487,12 @@ treated as all-zeros in the signed data.
:http:statuscode:`404 Not found`:
The specified block does not exist (or was already deleted).
:http:statuscode:`409 Conflict`:
- The ``If-Match`` hash, ``prev_nonce``, or ``next_nonce`` do not
- match the stored block (concurrent modification detected).
+ The ``If-Match`` hash, ``prev_nonce`` or ``next_nonce`` do not
+ match the stored block (concurrent modification detected), or
+ ``object_refs`` names an object the account does not have or would
+ take a reference count below zero. Nothing was modified.
:http:statuscode:`500 Internal server error`:
- A database error occurred, or the backup is in an inconsistent
- state (e.g. a referenced neighbouring block is missing from the
- chain, or a refcount would underflow).
+ A database error occurred.
Hash-indexed object store
~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -427,10 +506,10 @@ zero, which will increase with every referencing block that is uploaded to the
block store. Any blobs with a reference count of zero will be deleted from the
server after a preconfigured expiration period.
-In order to prevent wallets from uploading duplicate blobs, the sync server
-will compare the hash of the encrypted blob provided by the wallet against the
-object store before allowing the upload to proceed, rejecting it in case the
-blob already exists.
+Uploads are keyed by UID and are idempotent: re-uploading a UID that the
+account already holds is accepted and changes nothing, so a wallet that
+is unsure whether a blob is already present can simply upload it again.
+The stored contents of an existing UID are never replaced.
Blob format
+++++++++++
@@ -458,24 +537,77 @@ encrypted blob using SHA-512 and truncated to 32 bytes.
Object store API
++++++++++++++++
+Objects are scoped to the account: a UID is only ever visible to the
+account that uploaded it.
+
.. http:get:: /backups/${ACCOUNT_KEY}/objects/${UID}
Retrieve an existing blob by its UID.
**Response**
- :http:statuscode:`501 Not implemented`:
- This operation is not yet implemented.
+ :http:statuscode:`200 OK`:
+ The body is an `ObjectEntry` object.
+ :http:statuscode:`400 Bad request`:
+ The ``$UID`` is malformed.
+ :http:statuscode:`404 Not found`:
+ The account has no object under that UID. This is also the answer
+ for an account that does not exist.
+ :http:statuscode:`500 Internal server error`:
+ A database error occurred.
+
+ .. ts:def:: ObjectEntry
+
+ interface ObjectEntry {
+ uid: string;
+ data: string;
+ }
.. http:post:: /backups/${ACCOUNT_KEY}/objects/${UID}
Upload an encrypted blob and store it in the hash-indexed object
store. The ``$UID`` is the object's unique identifier.
+ The object is stored with a reference count of zero; it only becomes
+ referenced once a block naming it in ``object_refs`` is uploaded.
+ Until then it is subject to expiry, so blobs should be uploaded
+ shortly before the block that references them.
+
+ **Request**
+
+ The request body is a JSON object:
+
+ .. code-block:: typescript
+
+ interface UploadObjectRequest {
+ object_sig: string;
+ data: string;
+ }
+
+ ``object_sig``
+ EdDSA signature over the ``$UID`` and the hash of ``data``, signed
+ with the account's private key
+ (``TALER_SIGNATURE_SYNC_OBJECT_UPLOAD``).
+
+ ``data``
+ The encrypted blob contents (binary, base32-encoded).
+
**Response**
- :http:statuscode:`501 Not implemented`:
- This operation is not yet implemented.
+ :http:statuscode:`204 No content`:
+ The object was stored. This is also the answer when the account
+ already holds an object under that UID, in which case the stored
+ contents are left as they are.
+ :http:statuscode:`400 Bad request`:
+ The ``$UID`` or the request body is malformed.
+ :http:statuscode:`402 Payment required`:
+ The account has expired and requires payment.
+ :http:statuscode:`403 Forbidden`:
+ The signature is invalid or does not match the request.
+ :http:statuscode:`413 Request entity too large`:
+ The upload exceeds the server's configured upload limit.
+ :http:statuscode:`500 Internal server error`:
+ A database error occurred.
.. TODO: synchronization primitive
@@ -1518,8 +1650,9 @@ Definition of done
* [ ] Design wallet-core API.
* [ ] Wallet-core implementation.
* [x] Design sync API (+ auth).
-* [ ] Server-side implementation (partial: block GET/POST/PUT/DELETE, /config and
- payments done; object store still stubs returning 501).
+* [ ] Server-side implementation (partial: block GET/POST/PUT/DELETE, object
+ store GET/POST with reference counting, /config and payments done;
+ reconciliation mechanism still missing).
* [ ] UI/UX for backup and sync.
Alternatives
