Skip to content

Commit

Permalink
Halt melt on exception (#635)
Browse files Browse the repository at this point in the history 
* halt melt for payment status exception

* refactor

* less log

* add fakewallet test throwing exceptions
  • Loading branch information
@callebtc
callebtc authored Oct 6, 2024
1 parent c5ccf65 commit 307d2d7
Show file tree
Hide file tree
Showing 4 changed files with 146 additions and 100 deletions.
2 changes: 2 additions & 0 deletions cashu/core/settings.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -140,7 +140,9 @@ class FakeWalletSettings(MintSettings):
fakewallet_delay_incoming_payment: Optional[float] = Field(default=3.0)
fakewallet_stochastic_invoice: bool = Field(default=False)
fakewallet_payment_state: Optional[str] = Field(default="SETTLED")
fakewallet_payment_state_exception: Optional[bool] = Field(default=False)
fakewallet_pay_invoice_state: Optional[str] = Field(default="SETTLED")
fakewallet_pay_invoice_state_exception: Optional[bool] = Field(default=False)


class MintInformation(CashuSettings):
Expand Down
5 changes: 5 additions & 0 deletions cashu/lightning/fake.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -147,6 +147,9 @@ async def create_invoice(
)

async def pay_invoice(self, quote: MeltQuote, fee_limit: int) -> PaymentResponse:
if settings.fakewallet_pay_invoice_state_exception:
raise Exception("FakeWallet pay_invoice exception")

invoice = decode(quote.request)

if settings.fakewallet_delay_outgoing_payment:
Expand Down Expand Up @@ -189,6 +192,8 @@ async def get_invoice_status(self, checking_id: str) -> PaymentStatus:
)

async def get_payment_status(self, checking_id: str) -> PaymentStatus:
if settings.fakewallet_payment_state_exception:
raise Exception("FakeWallet get_payment_status exception")
if settings.fakewallet_payment_state:
return PaymentStatus(
result=PaymentResult[settings.fakewallet_payment_state]
Expand Down
203 changes: 103 additions & 100 deletions cashu/mint/ledger.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,11 +68,11 @@

class Ledger(LedgerVerification, LedgerSpendingConditions, LedgerTasks, LedgerFeatures):
backends: Mapping[Method, Mapping[Unit, LightningBackend]] = {}
locks: Dict[str, asyncio.Lock] = {} # holds multiprocessing locks
keysets: Dict[str, MintKeyset] = {}
events = LedgerEventManager()
db_read: DbReadHelper
invoice_listener_tasks: List[asyncio.Task] = []
disable_melt: bool = False

def __init__(
self,
Expand Down Expand Up @@ -130,8 +130,7 @@ async def _startup_ledger(self):
logger.error(
"The backend for"
f" {self.backends[method][unit].__class__.__name__} isn't"
f" working properly: '{status.error_message}'",
RuntimeWarning,
f" working properly: '{status.error_message}'"
)
exit(1)
logger.info(f"Backend balance: {status.balance} {unit.name}")
Expand Down Expand Up @@ -880,6 +879,10 @@ async def melt(
Returns:
Tuple[str, List[BlindedMessage]]: Proof of payment and signed outputs for returning overpaid fees to wallet.
"""
# make sure we're allowed to melt
if self.disable_melt:
raise NotAllowedError("Melt is disabled. Please contact the operator.")

# get melt quote and check if it was already paid
melt_quote = await self.get_melt_quote(quote_id=quote)
if not melt_quote.unpaid:
Expand Down Expand Up @@ -913,16 +916,14 @@ async def melt(
raise TransactionError(
f"not enough fee reserve provided for melt. Provided fee reserve: {fee_reserve_provided}, needed: {melt_quote.fee_reserve}"
)

# verify that the amount of the proofs is not larger than the maximum allowed
if settings.mint_max_peg_out and total_provided > settings.mint_max_peg_out:
raise NotAllowedError(
f"Maximum melt amount is {settings.mint_max_peg_out} sat."
)

# verify inputs and their spending conditions
# note, we do not verify outputs here, as they are only used for returning overpaid fees
# we should have used _verify_outputs here already (see above)
# We must have called _verify_outputs here already! (see above)
await self.verify_inputs_and_outputs(proofs=proofs)

# set proofs to pending to avoid race conditions
Expand All @@ -931,114 +932,116 @@ async def melt(
)
previous_state = melt_quote.state
melt_quote = await self.db_write._set_melt_quote_pending(melt_quote)
try:
# if the melt corresponds to an internal mint, mark both as paid
melt_quote = await self.melt_mint_settle_internally(melt_quote, proofs)
# quote not paid yet (not internal), pay it with the backend
if not melt_quote.paid:
logger.debug(f"Lightning: pay invoice {melt_quote.request}")
try:
payment = await self.backends[method][unit].pay_invoice(
melt_quote, melt_quote.fee_reserve * 1000

# if the melt corresponds to an internal mint, mark both as paid
melt_quote = await self.melt_mint_settle_internally(melt_quote, proofs)
# quote not paid yet (not internal), pay it with the backend
if not melt_quote.paid:
logger.debug(f"Lightning: pay invoice {melt_quote.request}")
try:
payment = await self.backends[method][unit].pay_invoice(
melt_quote, melt_quote.fee_reserve * 1000
)
logger.debug(
f"Melt – Result: {payment.result.name}: preimage: {payment.preimage},"
f" fee: {payment.fee.str() if payment.fee is not None else 'None'}"
)
if (
payment.checking_id
and payment.checking_id != melt_quote.checking_id
):
logger.warning(
f"pay_invoice returned different checking_id: {payment.checking_id} than melt quote: {melt_quote.checking_id}. Will use it for potentially checking payment status later."
)
melt_quote.checking_id = payment.checking_id
await self.crud.update_melt_quote(quote=melt_quote, db=self.db)
except Exception as e:
logger.error(f"Exception during pay_invoice: {e}")
payment = PaymentResponse(
result=PaymentResult.UNKNOWN,
error_message=str(e),
)

match payment.result:
case PaymentResult.FAILED | PaymentResult.UNKNOWN:
# explicitly check payment status for failed or unknown payment states
checking_id = payment.checking_id or melt_quote.checking_id
logger.debug(
f"Melt – Result: {payment.result}: preimage: {payment.preimage},"
f" fee: {payment.fee.str() if payment.fee is not None else 'None'}"
f"Payment state is {payment.result.name}.{' Error: ' + payment.error_message + '.' if payment.error_message else ''} Checking status for {checking_id}."
)
if (
payment.checking_id
and payment.checking_id != melt_quote.checking_id
):
logger.warning(
f"pay_invoice returned different checking_id: {payment.checking_id} than melt quote: {melt_quote.checking_id}. Will use it for potentially checking payment status later."
try:
status = await self.backends[method][unit].get_payment_status(
checking_id
)
melt_quote.checking_id = payment.checking_id
await self.crud.update_melt_quote(quote=melt_quote, db=self.db)

except Exception as e:
logger.error(f"Exception during pay_invoice: {e}")
payment = PaymentResponse(
result=PaymentResult.UNKNOWN,
error_message=str(e),
)

match payment.result:
case PaymentResult.FAILED | PaymentResult.UNKNOWN:
# explicitly check payment status for failed or unknown payment states
checking_id = payment.checking_id or melt_quote.checking_id
logger.debug(
f"Payment state is {payment.result}. Checking status for {checking_id}"
except Exception as e:
# Something went wrong. We might have lost connection to the backend. Keep transaction pending and return.
logger.error(
f"Lightning backend error: could not check payment status. Proofs for melt quote {melt_quote.quote} are stuck as PENDING. Disabling melt. Fix your Lightning backend and restart the mint.\nError: {e}"
)
try:
status = await self.backends[method][
unit
].get_payment_status(checking_id)
except Exception as e:
# Something went wrong, better to keep the proofs in pending state
logger.error(
f"Lightning backend error: could not check payment status. Proofs for melt quote {melt_quote.quote} are stuck as PENDING. Error: {e}"
)
return PostMeltQuoteResponse.from_melt_quote(melt_quote)
self.disable_melt = True
return PostMeltQuoteResponse.from_melt_quote(melt_quote)

match status.result:
case PaymentResult.FAILED | PaymentResult.UNKNOWN:
# NOTE: We only throw a payment error if the payment AND a subsequent status check failed
raise LightningError(
f"Lightning payment failed: {payment.error_message}. Error: {status.error_message}"
)
case _:
match status.result:
case PaymentResult.FAILED | PaymentResult.UNKNOWN:
# Everything as expected. Payment AND a status check both agree on a failure. We roll back the transaction.
await self.db_write._unset_proofs_pending(proofs)
await self.db_write._unset_melt_quote_pending(
quote=melt_quote, state=previous_state
)
if status.error_message:
logger.error(
f"Payment state is {status.result} and payment was {payment.result}. Proofs for melt quote {melt_quote.quote} are stuck as PENDING."
f"Status check error: {status.error_message}"
)
return PostMeltQuoteResponse.from_melt_quote(melt_quote)

case PaymentResult.SETTLED:
# payment successful
if payment.fee:
melt_quote.fee_paid = payment.fee.to(
to_unit=unit, round="up"
).amount
if payment.preimage:
melt_quote.payment_preimage = payment.preimage
# set quote as paid
melt_quote.state = MeltQuoteState.paid
melt_quote.paid_time = int(time.time())
# NOTE: This is the only return point for a successful payment

case PaymentResult.PENDING | _:
logger.debug(
f"Lightning payment is pending: {payment.checking_id}"
)
return PostMeltQuoteResponse.from_melt_quote(melt_quote)
raise LightningError(
f"Lightning payment failed{': ' + payment.error_message if payment.error_message else ''}."
)
case _:
# Something went wrong with our implementation or the backend. Status check returned different result than payment. Keep transaction pending and return.
logger.error(
f"Payment state is {status.result.name} and payment was {payment.result}. Proofs for melt quote {melt_quote.quote} are stuck as PENDING. Disabling melt. Fix your Lightning backend and restart the mint."
)
self.disable_melt = True
return PostMeltQuoteResponse.from_melt_quote(melt_quote)

# melt successful, invalidate proofs
await self._invalidate_proofs(proofs=proofs, quote_id=melt_quote.quote)
await self.db_write._unset_proofs_pending(proofs)
case PaymentResult.SETTLED:
# payment successful
if payment.fee:
melt_quote.fee_paid = payment.fee.to(
to_unit=unit, round="up"
).amount
if payment.preimage:
melt_quote.payment_preimage = payment.preimage
# set quote as paid
melt_quote.state = MeltQuoteState.paid
melt_quote.paid_time = int(time.time())
# NOTE: This is the only branch for a successful payment

case PaymentResult.PENDING | _:
logger.debug(
f"Lightning payment is {payment.result.name}: {payment.checking_id}"
)
return PostMeltQuoteResponse.from_melt_quote(melt_quote)

# prepare change to compensate wallet for overpaid fees
return_promises: List[BlindedSignature] = []
if outputs:
return_promises = await self._generate_change_promises(
fee_provided=fee_reserve_provided,
fee_paid=melt_quote.fee_paid,
outputs=outputs,
keyset=self.keysets[outputs[0].id],
)
# melt was successful (either internal or via backend), invalidate proofs
await self._invalidate_proofs(proofs=proofs, quote_id=melt_quote.quote)
await self.db_write._unset_proofs_pending(proofs)

melt_quote.change = return_promises
# prepare change to compensate wallet for overpaid fees
return_promises: List[BlindedSignature] = []
if outputs:
return_promises = await self._generate_change_promises(
fee_provided=fee_reserve_provided,
fee_paid=melt_quote.fee_paid,
outputs=outputs,
keyset=self.keysets[outputs[0].id],
)

await self.crud.update_melt_quote(quote=melt_quote, db=self.db)
await self.events.submit(melt_quote)
melt_quote.change = return_promises

return PostMeltQuoteResponse.from_melt_quote(melt_quote)
await self.crud.update_melt_quote(quote=melt_quote, db=self.db)
await self.events.submit(melt_quote)

except Exception as e:
logger.trace(f"Payment has failed: {e}")
await self.db_write._unset_proofs_pending(proofs)
await self.db_write._unset_melt_quote_pending(
quote=melt_quote, state=previous_state
)
raise e
return PostMeltQuoteResponse.from_melt_quote(melt_quote)

async def swap(
self,
Expand Down
36 changes: 36 additions & 0 deletions tests/test_mint_melt.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -295,3 +295,39 @@ async def test_melt_lightning_pay_invoice_failed_pending(
# expect that proofs are pending
states = await ledger.db_read.get_proofs_states([p.Y for p in wallet.proofs])
assert all([s.pending for s in states])


@pytest.mark.asyncio
@pytest.mark.skipif(is_regtest, reason="only fake wallet")
async def test_melt_lightning_pay_invoice_exception_exception(
ledger: Ledger, wallet: Wallet
):
"""Simulates the case where pay_invoice and get_payment_status raise an exception (due to network issues for example)."""
invoice = await wallet.request_mint(64)
await ledger.get_mint_quote(invoice.id) # fakewallet: set the quote to paid
await wallet.mint(64, id=invoice.id)
# invoice_64_sat = "lnbcrt640n1pn0r3tfpp5e30xac756gvd26cn3tgsh8ug6ct555zrvl7vsnma5cwp4g7auq5qdqqcqzzsxqyz5vqsp5xfhtzg0y3mekv6nsdnj43c346smh036t4f8gcfa2zwpxzwcryqvs9qxpqysgqw5juev8y3zxpdu0mvdrced5c6a852f9x7uh57g6fgjgcg5muqzd5474d7xgh770frazel67eejfwelnyr507q46hxqehala880rhlqspw07ta0"
invoice_62_sat = "lnbcrt620n1pn0r3vepp5zljn7g09fsyeahl4rnhuy0xax2puhua5r3gspt7ttlfrley6valqdqqcqzzsxqyz5vqsp577h763sel3q06tfnfe75kvwn5pxn344sd5vnays65f9wfgx4fpzq9qxpqysgqg3re9afz9rwwalytec04pdhf9mvh3e2k4r877tw7dr4g0fvzf9sny5nlfggdy6nduy2dytn06w50ls34qfldgsj37x0ymxam0a687mspp0ytr8"
quote_id = (
await ledger.melt_quote(
PostMeltQuoteRequest(unit="sat", request=invoice_62_sat)
)
).quote
# quote = await ledger.get_melt_quote(quote_id)
settings.fakewallet_payment_state_exception = True
settings.fakewallet_pay_invoice_state_exception = True

# we expect a pending melt quote because something has gone wrong (for example has lost connection to backend)
resp = await ledger.melt(proofs=wallet.proofs, quote=quote_id)
assert resp.state == MeltQuoteState.pending.value

# the mint should be locked now and not allow any other melts until it is restarted
quote_id = (
await ledger.melt_quote(
PostMeltQuoteRequest(unit="sat", request=invoice_62_sat)
)
).quote
await assert_err(
ledger.melt(proofs=wallet.proofs, quote=quote_id),
"Melt is disabled. Please contact the operator.",
)

0 comments on commit 307d2d7

Please sign in to comment.