[WIP] Store completed transactions in files instead of DB (mimblewimb…

…le#2148)

Store completed transactions in files instead of DB

src/bin/cmd/wallet_tests.rs

@@ -408,7 +408,7 @@ mod wallet_tests {
 			Ok(())
 		})?;
 
-		// Try using the self-send method
+		// Try using the self-send method, splitting up outputs for the fun of it
 		let arg_vec = vec![
 			"grin",
 			"wallet",
@@ -423,6 +423,8 @@ mod wallet_tests {
 			"mining",
 			"-g",
 			"Self love",
+			"-o",
+			"75",
 			"-s",
 			"smallest",
 			"10",

wallet/src/command.rs

@@ -409,7 +409,7 @@ pub fn repost(
 ) -> Result<(), Error> {
 	controller::owner_single_use(wallet.clone(), |api| {
 		let (_, txs) = api.retrieve_txs(true, Some(args.id), None)?;
-		let stored_tx = txs[0].get_stored_tx();
+		let stored_tx = api.get_stored_tx(&txs[0])?;
 		if stored_tx.is_none() {
 			error!(
 				"Transaction with id {} does not have transaction data. Not reposting.",

wallet/src/controller.rs

@@ -231,7 +231,10 @@ where
 		if let Some(id_string) = params.get("id") {
 			match id_string[0].parse() {
 				Ok(id) => match api.retrieve_txs(true, Some(id), None) {
-					Ok((_, txs)) => Ok((txs[0].confirmed, txs[0].get_stored_tx())),
+					Ok((_, txs)) => {
+						let stored_tx = api.get_stored_tx(&txs[0])?;
+						Ok((txs[0].confirmed, stored_tx))
+					}
 					Err(e) => {
 						error!("retrieve_stored_tx: failed with error: {}", e);
 						Err(e)

wallet/src/display.rs

@@ -178,7 +178,7 @@ pub fn txs(
 			)
 		};
 		let tx_data = match t.tx_hex {
-			Some(_) => format!("Exists"),
+			Some(t) => format!("{}", t),
 			None => "None".to_owned(),
 		};
 		if dark_background_color_scheme {

wallet/src/libwallet/api.rs

@@ -612,7 +612,13 @@ where
 		num_change_outputs: usize,
 		selection_strategy_is_use_all: bool,
 		message: Option<String>,
-	) -> Result<(Slate, impl FnOnce(&mut W, &str) -> Result<(), Error>), Error> {
+	) -> Result<
+		(
+			Slate,
+			impl FnOnce(&mut W, &Transaction) -> Result<(), Error>,
+		),
+		Error,
+	> {
 		let mut w = self.wallet.lock();
 		w.open_with_credentials()?;
 		let parent_key_id = match src_acct_name {
@@ -653,12 +659,11 @@ where
 	pub fn tx_lock_outputs(
 		&mut self,
 		slate: &Slate,
-		lock_fn: impl FnOnce(&mut W, &str) -> Result<(), Error>,
+		lock_fn: impl FnOnce(&mut W, &Transaction) -> Result<(), Error>,
 	) -> Result<(), Error> {
 		let mut w = self.wallet.lock();
 		w.open_with_credentials()?;
-		let tx_hex = util::to_hex(ser::ser_vec(&slate.tx).unwrap());
-		lock_fn(&mut *w, &tx_hex)?;
+		lock_fn(&mut *w, &slate.tx)?;
 		Ok(())
 	}
 
@@ -668,11 +673,10 @@ where
 	/// propagation.
 	pub fn finalize_tx(&mut self, slate: &mut Slate) -> Result<(), Error> {
 		let mut w = self.wallet.lock();
-		let parent_key_id = w.parent_key_id();
 		w.open_with_credentials()?;
 		let context = w.get_private_context(slate.id.as_bytes())?;
 		tx::complete_tx(&mut *w, slate, &context)?;
-		tx::update_tx_hex(&mut *w, &parent_key_id, slate)?;
+		tx::update_stored_tx(&mut *w, slate)?;
 		{
 			let mut batch = w.batch()?;
 			batch.delete_private_context(slate.id.as_bytes())?;
@@ -706,6 +710,12 @@ where
 		Ok(())
 	}
 
+	/// Retrieves a stored transaction from a TxLogEntry
+	pub fn get_stored_tx(&self, entry: &TxLogEntry) -> Result<Option<Transaction>, Error> {
+		let w = self.wallet.lock();
+		w.get_stored_tx(entry)
+	}
+
 	/// Posts a transaction to the chain
 	pub fn post_tx(&self, tx: &Transaction, fluff: bool) -> Result<(), Error> {
 		let tx_hex = util::to_hex(ser::ser_vec(tx).unwrap());

wallet/src/libwallet/internal/selection.rs

@@ -14,6 +14,7 @@
 
 //! Selection of inputs for building transactions
 
+use crate::core::core::Transaction;
 use crate::core::libtx::{build, slate::Slate, tx_fee};
 use crate::keychain::{Identifier, Keychain};
 use crate::libwallet::error::{Error, ErrorKind};
@@ -40,7 +41,7 @@ pub fn build_send_tx_slate<T: ?Sized, C, K>(
 	(
 		Slate,
 		Context,
-		impl FnOnce(&mut T, &str) -> Result<(), Error>,
+		impl FnOnce(&mut T, &Transaction) -> Result<(), Error>,
 	),
 	Error,
 >
@@ -94,42 +95,47 @@ where
 
 	// Return a closure to acquire wallet lock and lock the coins being spent
 	// so we avoid accidental double spend attempt.
-	let update_sender_wallet_fn = move |wallet: &mut T, tx_hex: &str| {
-		let mut batch = wallet.batch()?;
-		let log_id = batch.next_tx_log_id(&parent_key_id)?;
-		let mut t = TxLogEntry::new(parent_key_id.clone(), TxLogEntryType::TxSent, log_id);
-		t.tx_slate_id = Some(slate_id);
-		t.fee = Some(fee);
-		t.tx_hex = Some(tx_hex.to_owned());
-		let mut amount_debited = 0;
-		t.num_inputs = lock_inputs.len();
-		for id in lock_inputs {
-			let mut coin = batch.get(&id).unwrap();
-			coin.tx_log_entry = Some(log_id);
-			amount_debited = amount_debited + coin.value;
-			batch.lock_output(&mut coin)?;
-		}
+	let update_sender_wallet_fn = move |wallet: &mut T, tx: &Transaction| {
+		let tx_entry = {
+			let mut batch = wallet.batch()?;
+			let log_id = batch.next_tx_log_id(&parent_key_id)?;
+			let mut t = TxLogEntry::new(parent_key_id.clone(), TxLogEntryType::TxSent, log_id);
+			t.tx_slate_id = Some(slate_id);
+			let filename = format!("{}.grintx", slate_id);
+			t.tx_hex = Some(filename);
+			t.fee = Some(fee);
+			let mut amount_debited = 0;
+			t.num_inputs = lock_inputs.len();
+			for id in lock_inputs {
+				let mut coin = batch.get(&id).unwrap();
+				coin.tx_log_entry = Some(log_id);
+				amount_debited = amount_debited + coin.value;
+				batch.lock_output(&mut coin)?;
+			}
 
-		t.amount_debited = amount_debited;
-
-		// write the output representing our change
-		for (change_amount, id) in &change_amounts_derivations {
-			t.num_outputs += 1;
-			t.amount_credited += change_amount;
-			batch.save(OutputData {
-				root_key_id: parent_key_id.clone(),
-				key_id: id.clone(),
-				n_child: id.to_path().last_path_index(),
-				value: change_amount.clone(),
-				status: OutputStatus::Unconfirmed,
-				height: current_height,
-				lock_height: 0,
-				is_coinbase: false,
-				tx_log_entry: Some(log_id),
-			})?;
-		}
-		batch.save_tx_log_entry(t, &parent_key_id)?;
-		batch.commit()?;
+			t.amount_debited = amount_debited;
+
+			// write the output representing our change
+			for (change_amount, id) in &change_amounts_derivations {
+				t.num_outputs += 1;
+				t.amount_credited += change_amount;
+				batch.save(OutputData {
+					root_key_id: parent_key_id.clone(),
+					key_id: id.clone(),
+					n_child: id.to_path().last_path_index(),
+					value: change_amount.clone(),
+					status: OutputStatus::Unconfirmed,
+					height: current_height,
+					lock_height: 0,
+					is_coinbase: false,
+					tx_log_entry: Some(log_id),
+				})?;
+			}
+			batch.save_tx_log_entry(t.clone(), &parent_key_id)?;
+			batch.commit()?;
+			t
+		};
+		wallet.store_tx(&format!("{}", tx_entry.tx_slate_id.unwrap()), tx)?;
 		Ok(())
 	};
 

wallet/src/libwallet/internal/tx.rs

@@ -14,11 +14,10 @@
 
 //! Transaction building functions
 
-use crate::util;
 use uuid::Uuid;
 
+use crate::core::core::Transaction;
 use crate::core::libtx::slate::Slate;
-use crate::core::ser;
 use crate::keychain::{Identifier, Keychain};
 use crate::libwallet::internal::{selection, updater};
 use crate::libwallet::types::{Context, NodeClient, TxLogEntryType, WalletBackend};
@@ -74,7 +73,7 @@ pub fn create_send_tx<T: ?Sized, C, K>(
 	(
 		Slate,
 		Context,
-		impl FnOnce(&mut T, &str) -> Result<(), Error>,
+		impl FnOnce(&mut T, &Transaction) -> Result<(), Error>,
 	),
 	Error,
 >
@@ -200,19 +199,13 @@ where
 	Ok((tx.confirmed, tx.tx_hex))
 }
 
-/// Update the stored hex transaction (this update needs to happen when the TX is finalised)
-pub fn update_tx_hex<T: ?Sized, C, K>(
-	wallet: &mut T,
-	_parent_key_id: &Identifier,
-	slate: &Slate,
-) -> Result<(), Error>
+/// Update the stored transaction (this update needs to happen when the TX is finalised)
+pub fn update_stored_tx<T: ?Sized, C, K>(wallet: &mut T, slate: &Slate) -> Result<(), Error>
 where
 	T: WalletBackend<C, K>,
 	C: NodeClient,
 	K: Keychain,
 {
-	let tx_hex = util::to_hex(ser::ser_vec(&slate.tx).unwrap());
-	// This will ignore the parent key, so no need to specify account on the
 	// finalize command
 	let tx_vec = updater::retrieve_txs(wallet, None, Some(slate.id), None)?;
 	let mut tx = None;
@@ -223,14 +216,11 @@ where
 			break;
 		}
 	}
-	let mut tx = match tx {
+	let tx = match tx {
 		Some(t) => t,
 		None => return Err(ErrorKind::TransactionDoesntExist(slate.id.to_string()))?,
 	};
-	tx.tx_hex = Some(tx_hex);
-	let batch = wallet.batch()?;
-	batch.save_tx_log_entry(tx.clone(), &tx.parent_key_id)?;
-	batch.commit()?;
+	wallet.store_tx(&format!("{}", tx.tx_slate_id.unwrap()), &slate.tx)?;
 	Ok(())
 }
 

wallet/src/libwallet/types.rs

@@ -21,7 +21,6 @@ use crate::core::libtx::aggsig;
 use crate::core::ser;
 use crate::keychain::{Identifier, Keychain};
 use crate::libwallet::error::{Error, ErrorKind};
-use crate::util;
 use crate::util::secp::key::{PublicKey, SecretKey};
 use crate::util::secp::{self, pedersen, Secp256k1};
 use chrono::prelude::*;
@@ -102,6 +101,12 @@ where
 	/// Gets an account path for a given label
 	fn get_acct_path(&self, label: String) -> Result<Option<AcctPathMapping>, Error>;
 
+	/// Stores a transaction
+	fn store_tx(&self, uuid: &str, tx: &Transaction) -> Result<(), Error>;
+
+	/// Retrieves a stored transaction from a TxLogEntry
+	fn get_stored_tx(&self, entry: &TxLogEntry) -> Result<Option<Transaction>, Error>;
+
 	/// Create a new write batch to update or remove output data
 	fn batch<'a>(&'a mut self) -> Result<Box<dyn WalletOutputBatch<K> + 'a>, Error>;
 
@@ -156,7 +161,7 @@ where
 	fn tx_log_iter(&self) -> Box<dyn Iterator<Item = TxLogEntry>>;
 
 	/// save a tx log entry
-	fn save_tx_log_entry(&self, t: TxLogEntry, parent_id: &Identifier) -> Result<(), Error>;
+	fn save_tx_log_entry(&mut self, t: TxLogEntry, parent_id: &Identifier) -> Result<(), Error>;
 
 	/// save an account label -> path mapping
 	fn save_acct_path(&mut self, mapping: AcctPathMapping) -> Result<(), Error>;
@@ -595,6 +600,7 @@ pub struct TxLogEntry {
 	pub amount_debited: u64,
 	/// Fee
 	pub fee: Option<u64>,
+	// TODO: rename this to 'stored_tx_file' or something for mainnet
 	/// The transaction json itself, stored for reference or resending
 	pub tx_hex: Option<String>,
 }
@@ -636,17 +642,6 @@ impl TxLogEntry {
 	pub fn update_confirmation_ts(&mut self) {
 		self.confirmation_ts = Some(Utc::now());
 	}
-
-	/// Retrieve the stored transaction, if any
-	pub fn get_stored_tx(&self) -> Option<Transaction> {
-		match self.tx_hex.as_ref() {
-			None => None,
-			Some(t) => {
-				let tx_bin = util::from_hex(t.clone()).unwrap();
-				Some(ser::deserialize::<Transaction>(&mut &tx_bin[..]).unwrap())
-			}
-		}
-	}
 }
 
 /// Map of named accounts to BIP32 paths