From 2d3fec3513ca8af39b0be74278d92e3dab7225fc Mon Sep 17 00:00:00 2001 From: ziggie Date: Sat, 30 Sep 2023 12:31:35 +0200 Subject: [PATCH] add new command to clean height hint cache. It was observed that the height hint cache is poisoned leading to unresolved contracts in lnd. This command is a temporary fix for node runners until the real reason for this behaviour is found. --- README.md | 1 + cmd/chantools/dropheighthintcache.go | 211 +++++++++++++++++++++++++++ cmd/chantools/root.go | 1 + 3 files changed, 213 insertions(+) create mode 100644 cmd/chantools/dropheighthintcache.go diff --git a/README.md b/README.md index 2b3e5e2..f755b07 100644 --- a/README.md +++ b/README.md @@ -417,6 +417,7 @@ Available Commands: dropgraphzombies Remove all channels identified as zombies from the graph to force a re-sync of the graph dumpbackup Dump the content of a channel.backup file dumpchannels Dump all channel information from an lnd channel database + dropheighthintcache Remove all height hint cache data from the channel DB. fakechanbackup Fake a channel backup file to attempt fund recovery filterbackup Filter an lnd channel.backup file and remove certain channels fixoldbackup Fixes an old channel.backup file that is affected by the lnd issue #3881 (unable to derive shachain root key) diff --git a/cmd/chantools/dropheighthintcache.go b/cmd/chantools/dropheighthintcache.go new file mode 100644 index 0000000..2ba333a --- /dev/null +++ b/cmd/chantools/dropheighthintcache.go @@ -0,0 +1,211 @@ +package main + +import ( + "bytes" + "fmt" + "strconv" + "strings" + + "github.com/btcsuite/btcd/chaincfg/chainhash" + "github.com/btcsuite/btcd/txscript" + "github.com/btcsuite/btcd/wire" + "github.com/lightninglabs/chantools/btc" + "github.com/lightninglabs/chantools/lnd" + "github.com/lightningnetwork/lnd/chainntnfs" + "github.com/lightningnetwork/lnd/channeldb" + "github.com/lightningnetwork/lnd/kvdb" + "github.com/spf13/cobra" +) + +var spendHintBucket = []byte("spend-hints") + +type dropHeightHintCacheCommand struct { + APIURL string + ChannelDB string + ChanPoint string + + cmd *cobra.Command +} + +func newDropHeightHintCacheCommand() *cobra.Command { + cc := &dropHeightHintCacheCommand{} + cc.cmd = &cobra.Command{ + Use: "dropheighthintcache", + Short: "Remove all height hints used for spend notifications", + Long: `Removes either all spent height hint entries for +channels remaining in the __waiting_force_close__ state or for an explicit +outpoint which leads to an internal rescan resolving all contracts already due.`, + Example: `chantools dropheighthintcache \ + --channeldb ~/.lnd/data/graph/mainnet/channel.db \ + -chan_point bd278162f98...ecbab00764c8a1:0`, + RunE: cc.Execute, + } + cc.cmd.Flags().StringVar( + &cc.ChannelDB, "channeldb", "", "lnd channel.db file to dump "+ + "channels from", + ) + cc.cmd.Flags().StringVar( + &cc.ChanPoint, "chan_point", "", "outpoint for which the "+ + "height should be removed ", + ) + cc.cmd.Flags().StringVar( + &cc.APIURL, "apiurl", defaultAPIURL, "API URL to use (must "+ + "be esplora compatible)", + ) + return cc.cmd +} + +func (c *dropHeightHintCacheCommand) Execute(_ *cobra.Command, _ []string) error { + if c.ChannelDB == "" { + return fmt.Errorf("channel DB is required") + } + + db, err := lnd.OpenDB(c.ChannelDB, false) + if err != nil { + return fmt.Errorf("error opening rescue DB: %w", err) + } + defer func() { _ = db.Close() }() + + if c.ChanPoint != "" { + return dropHeightHintOutpoint(db, c.ChanPoint, c.APIURL) + } + + // In case no channel point is selected we will only remove the spent + // hint for channels which are borked and in the state + // __waiting_close__ (fundingTx not yet confirmed). + err = dropHeightHintFundingTx(db) + if err != nil { + return err + } + + return nil +} + +// dropHeightHintFundingTx queries the underlying channel.db for channels which +// are in the __waiting_close_channels__ bucket. This means the channel is +// already borked but the funding tx has still not been spent. We observed in +// some cases that the relevant height hint cache was poisoned leading to an +// unrecognized closed channel. Deleting the underlying height hint should +// tigger a rescan form an earlier blockheight and therefore finding the +// confirmed fundingTx. +func dropHeightHintFundingTx(db *channeldb.DB) error { + // We only fetch the waiting force close channels. + channels, err := db.ChannelStateDB().FetchWaitingCloseChannels() + if err != nil { + return err + } + + spendRequests := make([]*chainntnfs.SpendRequest, 0, len(channels)) + + for _, channel := range channels { + spendRequests = append(spendRequests, &chainntnfs.SpendRequest{ + OutPoint: channel.FundingOutpoint, + // We index the SpendRequest entry in the db by the + // outpoint value (for the channel close observer at + // least). + PkScript: txscript.PkScript{}, + }) + } + + // We resolve all the waiting force close channels which might have + // a poisoned height hint cache. + return kvdb.Batch(db.Backend, func(tx kvdb.RwTx) error { + spendHints := tx.ReadWriteBucket(spendHintBucket) + if spendHints == nil { + return chainntnfs.ErrCorruptedHeightHintCache + } + + for _, request := range spendRequests { + var outpoint bytes.Buffer + err := channeldb.WriteElement( + &outpoint, request.OutPoint, + ) + if err != nil { + return err + } + + spendKey := outpoint.Bytes() + if err := spendHints.Delete(spendKey); err != nil { + log.Debugf("outpoint not found in the height "+ + "hint cache: "+ + "%v", request.OutPoint.String()) + + return err + } + log.Infof("deleted height hint for outpoint: "+ + "%v \n", request.OutPoint.String()) + } + + return nil + }) +} + +// dropHeightHintOutpoint deletes the height hint cache for a specific outpoint. +// Sometimes a channel is stuck in a pending state because the spend of a +// channel contract was not recognized. In other words the height hint cache +// for this outpoint was poisoned and we need to delete its value so we trigger +// a clean rescan from the initial height of the channel contract. +func dropHeightHintOutpoint(db *channeldb.DB, chanPoint, apiURL string) error { + api := &btc.ExplorerAPI{BaseURL: apiURL} + // Check that the outpoint is really spent + addr, err := api.Address(chanPoint) + if err != nil { + return err + } + spends, err := api.Spends(addr) + if err != nil || len(spends) == 0 { + return fmt.Errorf("outpoint is not spend yet") + } + outPoint, err := parseChanPoint(chanPoint) + if err != nil { + return err + } + + return kvdb.Update(db.Backend, func(tx kvdb.RwTx) error { + spendHints := tx.ReadWriteBucket(spendHintBucket) + if spendHints == nil { + return chainntnfs.ErrCorruptedHeightHintCache + } + + var outPointBytes bytes.Buffer + err := channeldb.WriteElement( + &outPointBytes, outPoint, + ) + if err != nil { + return err + } + + spendKey := outPointBytes.Bytes() + if err := spendHints.Delete(spendKey); err != nil { + log.Debugf("outpoint not found in the height "+ + "hint cache: "+ + "%v", outPoint.String()) + + return err + } + log.Infof("deleted height hint for outpoint: "+ + "%v \n", outPoint.String()) + + return nil + }, func() {}) +} + +func parseChanPoint(s string) (*wire.OutPoint, error) { + split := strings.Split(s, ":") + if len(split) != 2 || len(split[0]) == 0 || len(split[1]) == 0 { + return nil, fmt.Errorf("invalid channel point") + } + + index, err := strconv.ParseInt(split[1], 10, 64) + if err != nil { + return nil, fmt.Errorf("unable to decode output index: %w", err) + } + + txid, err := chainhash.NewHashFromStr(split[0]) + if err != nil { + return nil, fmt.Errorf("unable to parse hex string: %w", err) + } + + return &wire.OutPoint{Hash: *txid, + Index: uint32(index)}, nil +} diff --git a/cmd/chantools/root.go b/cmd/chantools/root.go index 880b460..7a3ba7b 100644 --- a/cmd/chantools/root.go +++ b/cmd/chantools/root.go @@ -104,6 +104,7 @@ func main() { newDoubleSpendInputsCommand(), newDropChannelGraphCommand(), newDropGraphZombiesCommand(), + newDropHeightHintCacheCommand(), newDumpBackupCommand(), newDumpChannelsCommand(), newDocCommand(),