From 1be9da930bd1d532109490a15fb97c27c52ce3ce Mon Sep 17 00:00:00 2001 From: Antoine A <> Date: Thu, 16 Dec 2021 13:38:01 +0100 Subject: Cleanup --- Cargo.lock | 9 ++++---- btc-wire/src/bin/btc-wire-cli.rs | 4 ++-- btc-wire/src/bin/test.rs | 26 +++++++++-------------- btc-wire/src/main.rs | 45 ++++++++++++++++++++++------------------ btc-wire/src/segwit.rs | 2 +- research.md | 14 ++++++------- script/test_gateway.sh | 1 + taler-api/src/api_common.rs | 4 ++-- taler-config/src/lib.rs | 31 ++++++++++++++++++++------- test.conf | 11 +++++++--- uri-pack/src/lib.rs | 8 +++---- wire-gateway/src/json.rs | 2 +- wire-gateway/src/main.rs | 12 +++++------ 13 files changed, 93 insertions(+), 76 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3ec0525..912d200 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1568,11 +1568,10 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" [[package]] name = "tokio" -version = "1.14.0" +version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70e992e41e0d2fb9f755b37446f20900f64446ef54874f40a60c78f021ac6144" +checksum = "fbbf1c778ec206785635ce8ad57fe52b3009ae9e0c9f574a728f3049d3e55838" dependencies = [ - "autocfg", "bytes", "libc", "memchr", @@ -1585,9 +1584,9 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "1.6.0" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9efc1aba077437943f7515666aa2b882dfabfbfdf89c819ea75a8d6e9eaba5e" +checksum = "b557f72f448c511a979e2564e55d74e6c4432fc96ff4f6241bc6bded342643b7" dependencies = [ "proc-macro2", "quote", diff --git a/btc-wire/src/bin/btc-wire-cli.rs b/btc-wire/src/bin/btc-wire-cli.rs index 5e735c0..89dc5a1 100644 --- a/btc-wire/src/bin/btc-wire-cli.rs +++ b/btc-wire/src/bin/btc-wire-cli.rs @@ -77,7 +77,7 @@ struct App { impl App { pub fn start(data_dir: Option) -> Self { - let data_dir = data_dir.unwrap_or(default_data_dir()); + let data_dir = data_dir.unwrap_or_else(default_data_dir); let network = dirty_guess_network(&data_dir); let client = common_rpc(&data_dir, network).expect("Failed to connect to bitcoin core server"); @@ -133,7 +133,7 @@ fn main() { Cmd::Reset(_) => { let path = args .datadir - .unwrap_or(default_data_dir()) + .unwrap_or_else(default_data_dir) .join(Network::RegTest.dir()); if path.exists() { std::fs::remove_dir_all(path).unwrap(); diff --git a/btc-wire/src/bin/test.rs b/btc-wire/src/bin/test.rs index 202adac..658c750 100644 --- a/btc-wire/src/bin/test.rs +++ b/btc-wire/src/bin/test.rs @@ -52,16 +52,16 @@ pub fn main() { { println!("Generate tests wallets"); // Create wallets - rpc.create_wallet(&WIRE, None, None, None, None).ok(); - rpc.create_wallet(&CLIENT, None, None, None, None).ok(); - rpc.create_wallet(&RESERVE, None, None, None, None).ok(); + rpc.create_wallet(WIRE, None, None, None, None).ok(); + rpc.create_wallet(CLIENT, None, None, None, None).ok(); + rpc.create_wallet(RESERVE, None, None, None, None).ok(); } // Load wallets - rpc.load_wallet(&WIRE).ok(); - rpc.load_wallet(&CLIENT).ok(); - rpc.load_wallet(&RESERVE).ok(); + rpc.load_wallet(WIRE).ok(); + rpc.load_wallet(CLIENT).ok(); + rpc.load_wallet(RESERVE).ok(); } // Client initialization @@ -285,15 +285,9 @@ pub fn main() { assert!(match wire_rpc.bounce(&send_id, bounce_fee) { Ok(_) => false, Err(err) => match err { - BounceErr::RPC(err) => match err { - bitcoincore_rpc::Error::JsonRpc(err) => match err { - bitcoincore_rpc::jsonrpc::Error::Rpc(RpcError { code, .. }) => { - code == RpcErrorCode::RpcWalletInsufficientFunds as i32 - } - _ => false, - }, - _ => false, - }, + BounceErr::RPC(bitcoincore_rpc::Error::JsonRpc( + bitcoincore_rpc::jsonrpc::Error::Rpc(RpcError { code, .. }), + )) => code == RpcErrorCode::RpcWalletInsufficientFunds as i32, _ => false, }, }); @@ -411,7 +405,7 @@ impl TestRunner { } } - fn test(&mut self, name: &str, test: impl FnOnce() -> ()) { + fn test(&mut self, name: &str, test: impl FnOnce()) { println!("{}", name.cyan()); let result = std::panic::catch_unwind(AssertUnwindSafe(test)); diff --git a/btc-wire/src/main.rs b/btc-wire/src/main.rs index 825a777..3c3ac57 100644 --- a/btc-wire/src/main.rs +++ b/btc-wire/src/main.rs @@ -1,12 +1,10 @@ use bitcoincore_rpc::{ - bitcoin::{hashes::Hash, Address, Amount as BtcAmount, Block, BlockHash, SignedAmount, Txid}, + bitcoin::{hashes::Hash, Address, Amount as BtcAmount, BlockHash, SignedAmount, Txid}, json::GetTransactionResultDetailCategory as Category, Client as RPC, RpcApi, }; use btc_wire::{ - rpc_utils::{ - common_rpc, default_data_dir, dirty_guess_network, sender_address, wallet_rpc, WIRE, - }, + rpc_utils::{common_rpc, default_data_dir, sender_address, wallet_rpc, Network}, segwit::DecodeSegWitErr, ClientExtended, GetOpReturnErr, GetSegwitErr, }; @@ -58,7 +56,7 @@ impl TryFrom for Status { } fn btc_payto_url(addr: &Address) -> Url { - Url::from_str(&format!("payto://bitcoin/{}", addr.to_string())).unwrap() + Url::from_str(&format!("payto://bitcoin/{}", addr)).unwrap() } fn btc_payto_addr(url: &Url) -> Result { @@ -66,7 +64,7 @@ fn btc_payto_addr(url: &Url) -> Result { return Err("".to_string()); } let str = url.path().trim_start_matches('/'); - return Ok(Address::from_str(str).map_err(|_| "".to_string())?); + return Address::from_str(str).map_err(|_| "".to_string()); } fn btc_amount_to_taler_amount(amount: &SignedAmount) -> Amount { @@ -247,10 +245,10 @@ fn sender(rpc: RPC, mut db: AutoReloadDb, _config: &Config) { // TODO check if transactions are abandoned loop { - let mut db = db.client(); + let db = db.client(); let result: Result<(), Box> = (|| { // Send waiting transactions - for (id, status) in list_waiting(&mut db)? { + for (id, status) in list_waiting(db)? { // Set status to MANUAL to detect database error preventing atomicity let nb = db.execute( "UPDATE tx_out SET status=$1 WHERE id=$2 AND status=$3", @@ -262,7 +260,7 @@ fn sender(rpc: RPC, mut db: AutoReloadDb, _config: &Config) { perform_send(db, &rpc, id)?; } // Try to recover transactions stuck in manual - let mut manuals = list_manual(&mut db)?; + let mut manuals = list_manual(db)?; if !manuals.is_empty() { let last_hash = last_hash(db)?; let txs = rpc.list_since_block(last_hash.as_ref(), None, None, None)?; @@ -370,7 +368,7 @@ fn watcher(rpc: RPC, mut db: AutoReloadDb, config: &Config) { OsRng.fill_bytes(&mut request_uid); let nb = tx.execute( "INSERT INTO tx_out (_date, amount, wtid, debit_acc, credit_acc, exchange_url, status, txid, request_uid) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) ON CONFLICT (wtid) DO NOTHING", - &[&date, &amount.to_string(), &wtid.as_ref(), &btc_payto_url(&debit_addr).to_string(), &btc_payto_url(&credit_addr).to_string(), &config.base_url.to_string(), &(Status::OnChain as i16), &id.as_ref(), &request_uid.as_ref() + &[&date, &amount.to_string(), &wtid.as_ref(), &btc_payto_url(&debit_addr).as_ref(), &btc_payto_url(credit_addr).as_ref(), &config.base_url.as_ref(), &(Status::OnChain as i16), &id.as_ref(), &request_uid.as_ref() ], )?; if nb > 0 { @@ -392,7 +390,7 @@ fn watcher(rpc: RPC, mut db: AutoReloadDb, config: &Config) { let date = SystemTime::UNIX_EPOCH + Duration::from_secs(time); let amount = btc_amount_to_taler_amount(&full.tx.amount); let nb = db.execute("INSERT INTO tx_in (_date, amount, reserve_pub, debit_acc, credit_acc) VALUES ($1, $2, $3, $4, $5) ON CONFLICT (reserve_pub) DO NOTHING ", &[ - &date, &amount.to_string(), &reserve_pub.as_ref(), &btc_payto_url(&debit_addr).to_string(), &btc_payto_url(&credit_addr).to_string() + &date, &amount.to_string(), &reserve_pub.as_ref(), &btc_payto_url(&debit_addr).as_ref(), &btc_payto_url(credit_addr).as_ref() ])?; if nb > 0 { info!("{} << {} {}", &debit_addr, &credit_addr, &amount); @@ -454,21 +452,28 @@ fn main() { // Guess network by trying to connect to a JSON RPC server let data_dir = std::env::args() - .skip(1) - .next() + .nth(1) .map(|str| PathBuf::from_str(&str).unwrap()) - .unwrap_or(default_data_dir()); + .unwrap_or_else(default_data_dir); let config = taler_config::Config::from_path("test.conf"); let config: &'static Config = Box::leak(Box::new(config)); - let network = dirty_guess_network(&data_dir); + let network = match config.btc_chain.as_str() { + "main" => Network::MainNet, + "test" => Network::TestNet, + "regtest" => Network::RegTest, + chain => { + error!("Unsupported chain {}", chain); + std::process::exit(1); + } + }; let rpc = common_rpc(&data_dir, network).unwrap(); - rpc.load_wallet(&WIRE).ok(); - let rpc_watcher = wallet_rpc(&data_dir, network, "wire"); - let rpc_sender = wallet_rpc(&data_dir, network, "wire"); + rpc.load_wallet(&config.btc_wallet).ok(); + let rpc_watcher = wallet_rpc(&data_dir, network, &config.btc_wallet); + let rpc_sender = wallet_rpc(&data_dir, network, &config.btc_wallet); let db_watcher = AutoReloadDb::new(&config.db_url, Duration::from_secs(5)); let db_sender = AutoReloadDb::new(&config.db_url, Duration::from_secs(5)); - let join = std::thread::spawn(move || sender(rpc_sender, db_sender, &config)); - watcher(rpc_watcher, db_watcher, &config); + let join = std::thread::spawn(move || sender(rpc_sender, db_sender, config)); + watcher(rpc_watcher, db_watcher, config); join.join().unwrap(); } diff --git a/btc-wire/src/segwit.rs b/btc-wire/src/segwit.rs index c9d9352..fc3b8be 100644 --- a/btc-wire/src/segwit.rs +++ b/btc-wire/src/segwit.rs @@ -113,7 +113,7 @@ pub fn rand_addresses(hrp: &str, key: &[u8; 32]) -> Vec { .take(2) .collect(); - let mut addresses = encode_segwit_key(hrp, &key).to_vec(); + let mut addresses = encode_segwit_key(hrp, key).to_vec(); addresses.append(&mut rng_address); fastrand::shuffle(&mut addresses); addresses diff --git a/research.md b/research.md index 7454fb4..6add386 100644 --- a/research.md +++ b/research.md @@ -9,15 +9,13 @@ Can we use smart contract to read metadata ? Rust client library OpenEthereum - JSON RCP API -## wire-gateway security - -- Maximum body size -- Maximum body decompresses size - ## TODO - Listen/Notify to replace pooling -- Add and test refund -- Automatic MANUAL fix +- Add and test bound +- check if transactions are abandoned +- suicide after n operations - detect small fork -> warning -- fork -> ERR panic \ No newline at end of file +- fork -> ERR panic + +add biggest soft fork to confirmation threshold \ No newline at end of file diff --git a/script/test_gateway.sh b/script/test_gateway.sh index 7ff2deb..a825fbe 100644 --- a/script/test_gateway.sh +++ b/script/test_gateway.sh @@ -20,6 +20,7 @@ function cleanup() { trap cleanup EXIT source "${BASH_SOURCE%/*}/setup.sh" +ADDRESS=mpTJZxWPerz1Gife6mQSdHT8mMuJK6FP85 echo "---- Setup -----" echo "Load config file" diff --git a/taler-api/src/api_common.rs b/taler-api/src/api_common.rs index bd4553a..778e884 100644 --- a/taler-api/src/api_common.rs +++ b/taler-api/src/api_common.rs @@ -118,7 +118,7 @@ impl<'de> Deserialize<'de> for SafeUint64 { where D: Deserializer<'de>, { - Ok(SafeUint64::try_from(u64::deserialize(deserializer)?).map_err(D::Error::custom)?) + SafeUint64::try_from(u64::deserialize(deserializer)?).map_err(D::Error::custom) } } @@ -297,7 +297,7 @@ impl FromStr for Base32 { } pub fn crockford_base32_encode(bytes: &[u8]) -> String { - base32::encode(base32::Alphabet::Crockford, &bytes) + base32::encode(base32::Alphabet::Crockford, bytes) } impl Display for Base32 { diff --git a/taler-config/src/lib.rs b/taler-config/src/lib.rs index fb7af71..6273cd2 100644 --- a/taler-config/src/lib.rs +++ b/taler-config/src/lib.rs @@ -7,21 +7,36 @@ pub struct Config { pub db_url: String, pub port: u16, pub payto: Url, - pub address: String, pub confirmation: u8, + pub btc_wallet: String, + pub btc_chain: String, } impl Config { pub fn from_path(path: impl AsRef) -> Self { let conf = ini::Ini::load_from_file(path).unwrap(); - let conf = conf.section(Some("main")).unwrap(); + let ex_conf = conf.section(Some("exchange")).unwrap(); + let self_conf = conf.section(Some("depolymerizer-bitcoin")).unwrap(); Self { - base_url: Url::parse(&conf.get("BASE_URL").unwrap()).unwrap(), - db_url: conf.get("DB_URL").unwrap().to_string(), - port: conf.get("PORT").unwrap().parse().unwrap(), - payto: Url::parse(&conf.get("PAYTO").unwrap()).unwrap(), - address: conf.get("ADDRESS").unwrap().to_string(), - confirmation: conf.get("CONFIRMATION").unwrap().parse().unwrap(), + base_url: Url::parse(ex_conf.get("BASE_URL").expect("Missing config BASE_URL")) + .expect("BASE_URL is not a valid url"), + db_url: self_conf + .get("DB_URL") + .expect("Missing config DB_URL") + .to_string(), + port: self_conf + .get("PORT") + .unwrap_or("8080") + .parse() + .expect("Config PORT is not a number"), + payto: Url::parse(self_conf.get("PAYTO").unwrap()).unwrap(), + confirmation: self_conf + .get("CONFIRMATION") + .unwrap_or("1") + .parse() + .expect("Config CONFIRMATION is not a number"), + btc_wallet: self_conf.get("BTC_WALLET").unwrap_or("wire").to_string(), + btc_chain: self_conf.get("BTC_CHAIN").unwrap_or("regtest").to_string(), } } } diff --git a/test.conf b/test.conf index 617c30e..7c503cc 100644 --- a/test.conf +++ b/test.conf @@ -1,7 +1,12 @@ -[main] +[exchange] BASE_URL = http://test.com + +[depolymerizer-bitcoin] DB_URL = postgres://localhost/postgres?user=postgres&password=password PORT = 8060 +UNIXPATH = TODO PAYTO = payto://bitcoin/bcrt1qgkgxkjj27g3f7s87mcvjjsghay7gh34cx39prj -ADDRESS = mpTJZxWPerz1Gife6mQSdHT8mMuJK6FP85 -CONFIRMATION = 1 \ No newline at end of file +CONFIRMATION = 1 +BTC_WALLET = wire +BTC_DATA_DIR = ~/.bitcoin +BTC_CHAIN = regtest \ No newline at end of file diff --git a/uri-pack/src/lib.rs b/uri-pack/src/lib.rs index 3affccb..750075f 100644 --- a/uri-pack/src/lib.rs +++ b/uri-pack/src/lib.rs @@ -74,13 +74,13 @@ pub fn pack_uri(uri: &str) -> Result, EncodeErr> { // Amount of bits we can write in buffer let writable = (8 - buff_bits).min(nb_bits); // Remove non writable bits - let rmv_right = nb >> nb_bits - writable; - let rmv_left = rmv_right << 8 - writable; + let rmv_right = nb >> (nb_bits - writable); + let rmv_left = rmv_right << (8 - writable); // Align remaining bits with buff blank bits let align = rmv_left >> buff_bits; // Write bits in buffer - buff = buff | align; + buff |= align; buff_bits += writable; nb_bits -= writable; @@ -132,7 +132,7 @@ pub fn unpack_uri(bytes: &[u8]) -> Result { // Amount of bits we can read from buff let readable = buff_bits.min(nb_bits); // Remove non writable bits - let rmv_left = buff << 8 - buff_bits; + let rmv_left = buff << (8 - buff_bits); // Align remaining bits with nb blank bits let align = rmv_left >> (8 - readable); // Read bits from buff diff --git a/wire-gateway/src/json.rs b/wire-gateway/src/json.rs index 0aebac9..0393e5d 100644 --- a/wire-gateway/src/json.rs +++ b/wire-gateway/src/json.rs @@ -1,7 +1,7 @@ use hyper::{body::HttpBody, header, http::request::Parts, Body, Response, StatusCode}; use miniz_oxide::inflate::TINFLStatus; -const MAX_ALLOWED_RESPONSE_SIZE: u64 = 1 * 1024 * 1024; // 1MB +const MAX_ALLOWED_RESPONSE_SIZE: u64 = 1024 * 1024; // 1MB #[derive(Debug, thiserror::Error)] pub enum ParseBodyError { diff --git a/wire-gateway/src/main.rs b/wire-gateway/src/main.rs index 209a671..6e81ef2 100644 --- a/wire-gateway/src/main.rs +++ b/wire-gateway/src/main.rs @@ -171,8 +171,8 @@ async fn router( ) -> Result, ServerError> { let response = match parts.uri.path() { "/transfer" => { - assert_method(&parts, Method::POST)?; - let request: TransferRequest = parse_body(&parts, body).await.catch_code( + assert_method(parts, Method::POST)?; + let request: TransferRequest = parse_body(parts, body).await.catch_code( StatusCode::BAD_REQUEST, ErrorCode::GENERIC_PARAMETER_MALFORMED, )?; @@ -209,7 +209,7 @@ async fn router( }; if prev == request { // Idempotence - return Ok(encode_body( + return encode_body( parts, StatusCode::OK, &TransferResponse { @@ -221,7 +221,7 @@ async fn router( }, ) .await - .unexpected()?); + .unexpected(); } else { return Err(ServerError::status(StatusCode::CONFLICT)); } @@ -229,7 +229,7 @@ async fn router( let timestamp = Timestamp::now(); let row = db.query_one("INSERT INTO tx_out (_date, amount, wtid, debit_acc, credit_acc, exchange_url, status, request_uid) VALUES (now(), $1, $2, $3, $4, $5, $6, $7) RETURNING id", &[ - &request.amount.to_string(), &request.wtid.as_ref(), &state.config.payto.to_string(), &request.credit_account.to_string(), &request.exchange_base_url.to_string(), &0i16, &request.request_uid.as_ref() + &request.amount.to_string(), &request.wtid.as_ref(), &state.config.payto.as_ref(), &request.credit_account.as_ref(), &request.exchange_base_url.as_ref(), &0i16, &request.request_uid.as_ref() ]).await?; encode_body( parts, @@ -347,7 +347,7 @@ async fn router( ErrorCode::GENERIC_DB_FETCH_FAILED, )?; let row = db.query_one("INSERT INTO tx_in (_date, amount, reserve_pub, debit_acc, credit_acc) VALUES (now(), $1, $2, $3, $4) RETURNING id", &[ - &request.amount.to_string(), &request.reserve_pub.as_ref(), &request.debit_account.to_string(), &"payto://bitcoin/bcrt1qgkgxkjj27g3f7s87mcvjjsghay7gh34cx39prj" + &request.amount.to_string(), &request.reserve_pub.as_ref(), &request.debit_account.as_ref(), &"payto://bitcoin/bcrt1qgkgxkjj27g3f7s87mcvjjsghay7gh34cx39prj" ]).await.catch_code( StatusCode::BAD_GATEWAY, ErrorCode::GENERIC_DB_FETCH_FAILED, -- cgit v1.2.3