diff --git a/go.sum b/go.sum index 1ef2b48..2d18adf 100644 --- a/go.sum +++ b/go.sum @@ -1,10 +1,14 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +git.schwanenlied.me/yawning/bsaes.git v0.0.0-20180720073208-c0276d75487e h1:F2x1bq7RaNCIuqYpswggh1+c1JmwdnkHNC9wy1KDip0= git.schwanenlied.me/yawning/bsaes.git v0.0.0-20180720073208-c0276d75487e/go.mod h1:BWqTsj8PgcPriQJGl7el20J/7TuT1d/hSyFDXMEpoEo= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/NebulousLabs/fastrand v0.0.0-20181203155948-6fb6489aac4e h1:n+DcnTNkQnHlwpsrHoQtkrJIO7CBx029fw6oR4vIob4= github.com/NebulousLabs/fastrand v0.0.0-20181203155948-6fb6489aac4e/go.mod h1:Bdzq+51GR4/0DIhaICZEOm+OHvXGwwB2trKZ8B4Y6eQ= +github.com/NebulousLabs/go-upnp v0.0.0-20180202185039-29b680b06c82 h1:MG93+PZYs9PyEsj/n5/haQu2gK0h4tUtSy9ejtMwWa0= github.com/NebulousLabs/go-upnp v0.0.0-20180202185039-29b680b06c82/go.mod h1:GbuBk21JqF+driLX3XtJYNZjGa45YDoa9IqCTzNSfEc= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/Yawning/aez v0.0.0-20180114000226-4dad034d9db2 h1:2be4ykKKov3M1yISM2E8gnGXZ/N2SsPawfnGiXxaYEU= github.com/Yawning/aez v0.0.0-20180114000226-4dad034d9db2/go.mod h1:9pIqrY6SXNL8vjRQE5Hd/OL5GyK/9MrGUWs87z/eFfk= github.com/aead/chacha20 v0.0.0-20180709150244-8b13a72661da h1:KjTM2ks9d14ZYCvmHS9iAKVt9AyzRSqNU1qabPih5BY= github.com/aead/chacha20 v0.0.0-20180709150244-8b13a72661da/go.mod h1:eHEWzANqSiWQsof+nXEI9bUVUyV6F53Fp89EuCh2EAA= @@ -131,7 +135,9 @@ github.com/grpc-ecosystem/grpc-gateway v1.14.3 h1:OCJlWkOUoTnl0neNGlf4fUm3TmbEtg github.com/grpc-ecosystem/grpc-gateway v1.14.3/go.mod h1:6CwZWGDSPRJidgKAtJVvND6soZe6fT7iteq8wDPdhb0= github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/jackpal/gateway v1.0.5 h1:qzXWUJfuMdlLMtt0a3Dgt+xkWQiA5itDEITVJtuSwMc= github.com/jackpal/gateway v1.0.5/go.mod h1:lTpwd4ACLXmpyiCTRtfiNyVnUmqT9RivzCDQetPfnjA= +github.com/jackpal/go-nat-pmp v0.0.0-20170405195558-28a68d0c24ad h1:heFfj7z0pGsNCekUlsFhO2jstxO4b5iQ665LjwM5mDc= github.com/jackpal/go-nat-pmp v0.0.0-20170405195558-28a68d0c24ad/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= github.com/jedib0t/go-pretty v4.3.0+incompatible/go.mod h1:XemHduiw8R651AF9Pt4FwCTKeG3oo7hrHJAoznj9nag= github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= @@ -253,6 +259,7 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5 github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 h1:LnC5Kc/wtumK+WB441p7ynQJzVuNRJiqddSIE3IlSEQ= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/tv42/zbase32 v0.0.0-20160707012821-501572607d02 h1:tcJ6OjwOMvExLlzrAVZute09ocAGa7KqOON60++Gz4E= github.com/tv42/zbase32 v0.0.0-20160707012821-501572607d02/go.mod h1:tHlrkM198S068ZqfrO6S8HsoJq2bF3ETfTL+kt4tInY= github.com/urfave/cli v1.18.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.20.0 h1:fDqGv3UG/4jbVl/QkFwEdddtEDjh/5Ov6X+0B/3bPaw= diff --git a/labels/labels.go b/labels/labels.go index d75376d..3129249 100644 --- a/labels/labels.go +++ b/labels/labels.go @@ -2,6 +2,7 @@ package labels import ( "errors" + "fmt" "strings" ) @@ -12,6 +13,10 @@ const ( // Reserved is used as a prefix to separate labels that are created by // loopd from those created by users. Reserved = "[reserved]" + + // autoOut is the label used for loop out swaps that are automatically + // dispatched. + autoOut = "autoloop-out" ) var ( @@ -23,6 +28,12 @@ var ( ErrReservedPrefix = errors.New("label contains reserved prefix") ) +// AutoOutLabel returns a label with the reserved prefix that identifies +// automatically dispatched loop outs. +func AutoOutLabel() string { + return fmt.Sprintf("%v: %v", Reserved, autoOut) +} + // Validate checks that a label is of appropriate length and is not in our list // of reserved labels. func Validate(label string) error { diff --git a/liquidity/liquidity.go b/liquidity/liquidity.go index af4475c..4b428e4 100644 --- a/liquidity/liquidity.go +++ b/liquidity/liquidity.go @@ -36,6 +36,7 @@ import ( "context" "errors" "fmt" + "sort" "strings" "sync" "time" @@ -43,7 +44,9 @@ import ( "github.com/btcsuite/btcutil" "github.com/lightninglabs/lndclient" "github.com/lightninglabs/loop" + "github.com/lightninglabs/loop/labels" "github.com/lightninglabs/loop/loopdb" + "github.com/lightningnetwork/lnd" "github.com/lightningnetwork/lnd/clock" "github.com/lightningnetwork/lnd/lnwallet/chainfee" "github.com/lightningnetwork/lnd/lnwire" @@ -88,9 +91,21 @@ const ( ) var ( + // defaultBudget is the default autoloop budget we set. This budget will + // only be used for automatically dispatched swaps if autoloop is + // explicitly enabled, so we are happy to set a non-zero value here. The + // amount chosen simply uses the current defaults to provide budget for + // a single swap. We don't have a swap amount to calculate our maximum + // routing fee, so we use 0.16 BTC for now. + defaultBudget = defaultMaximumMinerFee + + ppmToSat(lnd.MaxBtcFundingAmount, defaultSwapFeePPM) + + ppmToSat(defaultMaximumPrepay, defaultPrepayRoutingFeePPM) + + ppmToSat(lnd.MaxBtcFundingAmount, defaultRoutingFeePPM) + // defaultParameters contains the default parameters that we start our // liquidity manger with. defaultParameters = Parameters{ + AutoFeeBudget: defaultBudget, ChannelRules: make(map[lnwire.ShortChannelID]*ThresholdRule), FailureBackOff: defaultFailureBackoff, SweepFeeRateLimit: defaultSweepFeeRateLimit, @@ -125,6 +140,9 @@ var ( // ErrZeroPrepay is returned if a zero maximum prepay is set. ErrZeroPrepay = errors.New("maximum prepay must be non-zero") + + // ErrNegativeBudget is returned if a negative swap budget is set. + ErrNegativeBudget = errors.New("swap budget must be >= 0") ) // Config contains the external functionality required to run the @@ -159,6 +177,16 @@ type Config struct { // Parameters is a set of parameters provided by the user which guide // how we assess liquidity. type Parameters struct { + // AutoFeeBudget is the total amount we allow to be spent on + // automatically dispatched swaps. Once this budget has been used, we + // will stop dispatching swaps until the budget is increased or the + // start date is moved. + AutoFeeBudget btcutil.Amount + + // AutoFeeStartDate is the date from which we will include automatically + // dispatched swaps in our current budget, inclusive. + AutoFeeStartDate time.Time + // FailureBackOff is the amount of time that we require passes after a // channel has been part of a failed loop out swap before we suggest // using it again. @@ -219,12 +247,13 @@ func (p Parameters) String() string { return fmt.Sprintf("channel rules: %v, failure backoff: %v, sweep "+ "fee rate limit: %v, sweep conf target: %v, maximum prepay: "+ "%v, maximum miner fee: %v, maximum swap fee ppm: %v, maximum "+ - "routing fee ppm: %v, maximum prepay routing fee ppm: %v", + "routing fee ppm: %v, maximum prepay routing fee ppm: %v, "+ + "auto budget: %v, budget start: %v", strings.Join(channelRules, ","), p.FailureBackOff, p.SweepFeeRateLimit, p.SweepConfTarget, p.MaximumPrepay, p.MaximumMinerFee, p.MaximumSwapFeePPM, p.MaximumRoutingFeePPM, p.MaximumPrepayRoutingFeePPM, - ) + p.AutoFeeBudget, p.AutoFeeStartDate) } // validate checks whether a set of parameters is valid. It takes the minimum @@ -275,6 +304,10 @@ func (p Parameters) validate(minConfs int32) error { return ErrZeroMinerFee } + if p.AutoFeeBudget < 0 { + return ErrNegativeBudget + } + return nil } @@ -356,6 +389,16 @@ func (m *Manager) SuggestSwaps(ctx context.Context) ( return nil, nil } + // If our start date is in the future, we interpret this as meaning that + // we should start using our budget at this date. This means that we + // have no budget for the present, so we just return. + if m.params.AutoFeeStartDate.After(m.cfg.Clock.Now()) { + log.Debugf("autoloop fee budget start time: %v is in "+ + "the future", m.params.AutoFeeStartDate) + + return nil, nil + } + // Before we get any swap suggestions, we check what the current fee // estimate is to sweep within our target number of confirmations. If // This fee exceeds the fee limit we have set, we will not suggest any @@ -396,6 +439,23 @@ func (m *Manager) SuggestSwaps(ctx context.Context) ( return nil, err } + // Get a summary of our existing swaps so that we can check our autoloop + // budget. + summary, err := m.checkExistingAutoLoops(ctx, loopOut) + if err != nil { + return nil, err + } + + if summary.totalFees() >= m.params.AutoFeeBudget { + log.Debugf("autoloop fee budget: %v exhausted, %v spent on "+ + "completed swaps, %v reserved for ongoing swaps "+ + "(upper limit)", + m.params.AutoFeeBudget, summary.spentFees, + summary.pendingFees) + + return nil, nil + } + eligible, err := m.getEligibleChannels(ctx, loopOut, loopIn) if err != nil { return nil, err @@ -449,7 +509,45 @@ func (m *Manager) SuggestSwaps(ctx context.Context) ( suggestions = append(suggestions, outRequest) } - return suggestions, nil + // If we have no suggestions after we have applied all of our limits, + // just return. + if len(suggestions) == 0 { + return nil, nil + } + + // Sort suggestions by amount in descending order. + sort.SliceStable(suggestions, func(i, j int) bool { + return suggestions[i].Amount > suggestions[j].Amount + }) + + // Run through our suggested swaps in descending order of amount and + // return all of the swaps which will fit within our remaining budget. + var ( + available = m.params.AutoFeeBudget - summary.totalFees() + inBudget []loop.OutRequest + ) + + for _, swap := range suggestions { + fees := worstCaseOutFees( + swap.MaxPrepayRoutingFee, swap.MaxSwapRoutingFee, + swap.MaxSwapFee, swap.MaxMinerFee, swap.MaxPrepayAmount, + ) + + // If the maximum fee we expect our swap to use is less than the + // amount we have available, we add it to our set of swaps that + // fall within the budget and decrement our available amount. + if fees <= available { + available -= fees + inBudget = append(inBudget, swap) + } + + // If we're out of budget, exit early. + if available == 0 { + break + } + } + + return inBudget, nil } // makeLoopOutRequest creates a loop out request from a suggestion. Since we @@ -485,6 +583,87 @@ func (m *Manager) makeLoopOutRequest(suggestion *LoopOutRecommendation, } } +// worstCaseOutFees calculates the largest possible fees for a loop out swap, +// comparing the fees for a successful swap to the cost when the client pays +// the prepay because they failed to sweep the on chain htlc. This is unlikely, +// because we expect clients to be online to sweep, but we want to account for +// every outcome so we include it. +func worstCaseOutFees(prepayRouting, swapRouting, swapFee, minerFee, + prepayAmount btcutil.Amount) btcutil.Amount { + + var ( + successFees = prepayRouting + minerFee + swapFee + swapRouting + noShowFees = prepayRouting + prepayAmount + ) + + if noShowFees > successFees { + return noShowFees + } + + return successFees +} + +// existingAutoLoopSummary provides a summary of the existing autoloops which +// were dispatched during our current budget period. +type existingAutoLoopSummary struct { + // spentFees is the amount we have spent on completed swaps. + spentFees btcutil.Amount + + // pendingFees is the worst-case amount of fees we could spend on in + // flight autoloops. + pendingFees btcutil.Amount +} + +// totalFees returns the total amount of fees that automatically dispatched +// swaps may consume. +func (e *existingAutoLoopSummary) totalFees() btcutil.Amount { + return e.spentFees + e.pendingFees +} + +// checkExistingAutoLoops calculates the total amount that has been spent by +// automatically dispatched swaps that have completed, and the worst-case fee +// total for our set of ongoing, automatically dispatched swaps. +func (m *Manager) checkExistingAutoLoops(ctx context.Context, + loopOuts []*loopdb.LoopOut) (*existingAutoLoopSummary, error) { + + var summary existingAutoLoopSummary + + for _, out := range loopOuts { + if out.Contract.Label != labels.AutoOutLabel() { + continue + } + + // If we have a pending swap, we are uncertain of the fees that + // it will end up paying. We use the worst-case estimate based + // on the maximum values we set for each fee category. This will + // likely over-estimate our fees (because we probably won't + // spend our maximum miner amount). If a swap is not pending, + // it has succeeded or failed so we just record our actual fees + // for the swap provided that the swap completed after our + // budget start date. + if out.State().State.Type() == loopdb.StateTypePending { + prepay, err := m.cfg.Lnd.Client.DecodePaymentRequest( + ctx, out.Contract.PrepayInvoice, + ) + if err != nil { + return nil, err + } + + summary.pendingFees += worstCaseOutFees( + out.Contract.MaxPrepayRoutingFee, + out.Contract.MaxSwapRoutingFee, + out.Contract.MaxSwapFee, + out.Contract.MaxMinerFee, + mSatToSatoshis(prepay.Value), + ) + } else if !out.LastUpdateTime().Before(m.params.AutoFeeStartDate) { + summary.spentFees += out.State().Cost.Total() + } + } + + return &summary, nil +} + // getEligibleChannels takes lists of our existing loop out and in swaps, and // gets a list of channels that are not currently being utilized for a swap. // If an unrestricted swap is ongoing, we return an empty set of channels @@ -653,3 +832,7 @@ func satPerKwToSatPerVByte(satPerKw chainfee.SatPerKWeight) int64 { func ppmToSat(amount btcutil.Amount, ppm int) btcutil.Amount { return btcutil.Amount(uint64(amount) * uint64(ppm) / FeeBase) } + +func mSatToSatoshis(amount lnwire.MilliSatoshi) btcutil.Amount { + return btcutil.Amount(amount / 1000) +} diff --git a/liquidity/liquidity_test.go b/liquidity/liquidity_test.go index d341d5c..39c0210 100644 --- a/liquidity/liquidity_test.go +++ b/liquidity/liquidity_test.go @@ -8,6 +8,7 @@ import ( "github.com/btcsuite/btcutil" "github.com/lightninglabs/lndclient" "github.com/lightninglabs/loop" + "github.com/lightninglabs/loop/labels" "github.com/lightninglabs/loop/loopdb" "github.com/lightninglabs/loop/test" "github.com/lightningnetwork/lnd/clock" @@ -18,7 +19,8 @@ import ( ) var ( - testTime = time.Date(2020, 02, 13, 0, 0, 0, 0, time.UTC) + testTime = time.Date(2020, 02, 13, 0, 0, 0, 0, time.UTC) + testBudgetStart = testTime.Add(time.Hour * -1) chanID1 = lnwire.NewShortChanIDFromInt(1) chanID2 = lnwire.NewShortChanIDFromInt(2) @@ -89,6 +91,17 @@ var ( }, ), } + + // autoOutContract is a contract for an existing loop out that was + // automatically dispatched. This swap is within our test budget period, + // and restricted to a channel that we do not use in our tests. + autoOutContract = &loopdb.LoopOutContract{ + SwapContract: loopdb.SwapContract{ + Label: labels.AutoOutLabel(), + InitiationTime: testBudgetStart, + }, + OutgoingChanSet: loopdb.ChannelSet{999}, + } ) // newTestConfig creates a default test config. @@ -538,6 +551,162 @@ func TestFeeLimits(t *testing.T) { } } +// TestFeeBudget tests limiting of swap suggestions to a fee budget, with and +// without existing swaps. This test uses example channels and rules which need +// a 7500 sat loop out. With our default parameters, and our test quote with +// a prepay of 500, our total fees are (rounded due to int multiplication): +// swap fee: 1 (as set in test quote) +// route fee: 7500 * 0.005 = 37 +// prepay route: 500 * 0.005 = 2 sat +// max miner: set by default params +// Since our routing fees are calculated as a portion of our swap/prepay +// amounts, we use our max miner fee to shift swap cost to values above/below +// our budget, fixing our other fees at 114 sat for simplicity. +func TestFeeBudget(t *testing.T) { + tests := []struct { + name string + + // budget is our autoloop budget. + budget btcutil.Amount + + // maxMinerFee is the maximum miner fee we will pay for swaps. + maxMinerFee btcutil.Amount + + // existingSwaps represents our existing swaps, mapping their + // last update time to their total cost. + existingSwaps map[time.Time]btcutil.Amount + + // expectedSwaps is the set of swaps we expect to be suggested. + expectedSwaps []loop.OutRequest + }{ + { + // Two swaps will cost (78+5000)*2, set exactly 10156 + // budget. + name: "budget for 2 swaps, no existing", + budget: 10156, + maxMinerFee: 5000, + expectedSwaps: []loop.OutRequest{ + chan1Rec, chan2Rec, + }, + }, + { + // Two swaps will cost (78+5000)*2, set 10155 so we can + // only afford one swap. + name: "budget for 1 swaps, no existing", + budget: 10155, + maxMinerFee: 5000, + expectedSwaps: []loop.OutRequest{ + chan1Rec, + }, + }, + { + // Set an existing swap which would limit us to a single + // swap if it were in our period. + name: "existing swaps, before budget period", + budget: 10156, + maxMinerFee: 5000, + existingSwaps: map[time.Time]btcutil.Amount{ + testBudgetStart.Add(time.Hour * -1): 200, + }, + expectedSwaps: []loop.OutRequest{ + chan1Rec, chan2Rec, + }, + }, + { + // Add an existing swap in our budget period such that + // we only have budget left for one more swap. + name: "existing swaps, in budget period", + budget: 10156, + maxMinerFee: 5000, + existingSwaps: map[time.Time]btcutil.Amount{ + testBudgetStart.Add(time.Hour): 500, + }, + expectedSwaps: []loop.OutRequest{ + chan1Rec, + }, + }, + { + name: "existing swaps, budget used", + budget: 500, + maxMinerFee: 1000, + existingSwaps: map[time.Time]btcutil.Amount{ + testBudgetStart.Add(time.Hour): 500, + }, + expectedSwaps: nil, + }, + } + + for _, testCase := range tests { + testCase := testCase + + t.Run(testCase.name, func(t *testing.T) { + cfg, lnd := newTestConfig() + + // Create a swap set of existing swaps with our set of + // existing swap timestamps. + swaps := make( + []*loopdb.LoopOut, 0, + len(testCase.existingSwaps), + ) + + // Add an event with the timestamp and budget set by + // our test case. + for ts, amt := range testCase.existingSwaps { + event := &loopdb.LoopEvent{ + SwapStateData: loopdb.SwapStateData{ + Cost: loopdb.SwapCost{ + Server: amt, + }, + State: loopdb.StateSuccess, + }, + Time: ts, + } + + swaps = append(swaps, &loopdb.LoopOut{ + Loop: loopdb.Loop{ + Events: []*loopdb.LoopEvent{ + event, + }, + }, + Contract: autoOutContract, + }) + } + + cfg.ListLoopOut = func() ([]*loopdb.LoopOut, error) { + return swaps, nil + } + + // Set two channels that need swaps. + lnd.Channels = []lndclient.ChannelInfo{ + channel1, + channel2, + } + + params := defaultParameters + params.ChannelRules = map[lnwire.ShortChannelID]*ThresholdRule{ + chanID1: chanRule, + chanID2: chanRule, + } + params.AutoFeeStartDate = testBudgetStart + params.AutoFeeBudget = testCase.budget + params.MaximumMinerFee = testCase.maxMinerFee + + // Set our custom max miner fee on each expected swap, + // rather than having to create multiple vars for + // different rates. + for i := range testCase.expectedSwaps { + testCase.expectedSwaps[i].MaxMinerFee = + testCase.maxMinerFee + } + + testSuggestSwaps( + t, newSuggestSwapsSetup(cfg, lnd, params), + testCase.expectedSwaps, + ) + }) + } +} + // testSuggestSwapsSetup contains the elements that are used to create a // suggest swaps test. type testSuggestSwapsSetup struct { diff --git a/loopdb/swapstate.go b/loopdb/swapstate.go index c2cddb2..6ace7ce 100644 --- a/loopdb/swapstate.go +++ b/loopdb/swapstate.go @@ -151,6 +151,11 @@ type SwapCost struct { Offchain btcutil.Amount } +// Total returns the total costs represented by swap costs. +func (s SwapCost) Total() btcutil.Amount { + return s.Server + s.Onchain + s.Offchain +} + // SwapStateData is all persistent data to describe the current swap state. type SwapStateData struct { // SwapState is the state the swap is in. diff --git a/test/lightning_client_mock.go b/test/lightning_client_mock.go index daec488..2fe8902 100644 --- a/test/lightning_client_mock.go +++ b/test/lightning_client_mock.go @@ -42,6 +42,13 @@ func (h *mockLightningClient) PayInvoice(ctx context.Context, invoice string, return done } +// DecodePaymentRequest returns a non-nil payment request. +func (h *mockLightningClient) DecodePaymentRequest(_ context.Context, + _ string) (*lndclient.PaymentRequest, error) { + + return &lndclient.PaymentRequest{}, nil +} + func (h *mockLightningClient) WaitForFinished() { h.wg.Wait() }