From f38bc4ca3f0c02fd7f056d159455890e087712af Mon Sep 17 00:00:00 2001 From: Simon Roberts Date: Sun, 26 Sep 2021 18:41:03 +1000 Subject: [PATCH 1/4] Continue from https://github.com/miguelmota/cointop/pull/165 --- cointop/chart.go | 112 ++++++++++++++++++++++++++++----------- pkg/timedata/timedata.go | 51 ++++++++++++++++++ 2 files changed, 133 insertions(+), 30 deletions(-) create mode 100644 pkg/timedata/timedata.go diff --git a/cointop/chart.go b/cointop/chart.go index f0094c7..9c64a2a 100644 --- a/cointop/chart.go +++ b/cointop/chart.go @@ -2,17 +2,25 @@ package cointop import ( "fmt" + "math" "sort" "strings" "sync" "time" "github.com/miguelmota/cointop/pkg/chartplot" + "github.com/miguelmota/cointop/pkg/timedata" "github.com/miguelmota/cointop/pkg/timeutil" "github.com/miguelmota/cointop/pkg/ui" log "github.com/sirupsen/logrus" ) +// Time-series data for a Coin used when building a Portfolio view for chart +type PriceData struct { + coin *Coin + data [][]float64 +} + // ChartView is structure for chart view type ChartView = ui.View @@ -121,7 +129,7 @@ func (ct *Cointop) ChartPoints(symbol string, name string) error { start := nowseconds - int64(rangeseconds.Seconds()) end := nowseconds - var data []float64 + var cacheData [][]float64 keyname := symbol if keyname == "" { @@ -132,21 +140,18 @@ func (ct *Cointop) ChartPoints(symbol string, name string) error { cached, found := ct.cache.Get(cachekey) if found { // cache hit - data, _ = cached.([]float64) + cacheData, _ = cached.([][]float64) log.Debug("ChartPoints() soft cache hit") } - if len(data) == 0 { + if len(cacheData) == 0 { if symbol == "" { convert := ct.State.currencyConversion graphData, err := ct.api.GetGlobalMarketGraphData(convert, start, end) if err != nil { return nil } - for i := range graphData.MarketCapByAvailableSupply { - price := graphData.MarketCapByAvailableSupply[i][1] - data = append(data, price) - } + cacheData = graphData.MarketCapByAvailableSupply } else { convert := ct.State.currencyConversion graphData, err := ct.api.GetCoinGraphData(convert, symbol, name, start, end) @@ -157,20 +162,37 @@ func (ct *Cointop) ChartPoints(symbol string, name string) error { sort.Slice(sorted[:], func(i, j int) bool { return sorted[i][0] < sorted[j][0] }) - for i := range sorted { - price := sorted[i][1] - data = append(data, price) - } + cacheData = sorted } - ct.cache.Set(cachekey, data, 10*time.Second) + ct.cache.Set(cachekey, cacheData, 10*time.Second) if ct.filecache != nil { go func() { - ct.filecache.Set(cachekey, data, 24*time.Hour) + ct.filecache.Set(cachekey, cacheData, 24*time.Hour) }() } } + // Resample cachedata + maxPoints := len(cacheData) + if maxPoints > 2*maxX { + maxPoints = 2 * maxX + } + timeQuantum := timedata.CalculateTimeQuantum(cacheData) + newStart := time.Unix(start, 0).Add(timeQuantum) + newEnd := time.Unix(end, 0).Add(-timeQuantum) + timeData := timedata.ResampleTimeSeriesData(cacheData, float64(newStart.UnixMilli()), float64(newEnd.UnixMilli()), maxPoints) + + // Extract just the values from the data + var data []float64 + for i := range timeData { + value := timeData[i][1] + if math.IsNaN(value) { + value = 0.0 + } + data = append(data, value) + } + chart.SetData(data) ct.State.chartPoints = chart.GetChartPoints(maxX) @@ -203,7 +225,7 @@ func (ct *Cointop) PortfolioChart() error { start := nowseconds - int64(rangeseconds.Seconds()) end := nowseconds - var data []float64 + var allCacheData []PriceData portfolio := ct.GetPortfolioSlice() chartname := ct.SelectedCoinName() for _, p := range portfolio { @@ -218,46 +240,76 @@ func (ct *Cointop) PortfolioChart() error { continue } - var graphData []float64 + var cacheData [][]float64 // [][time,value] cachekey := strings.ToLower(fmt.Sprintf("%s_%s_%s", p.Symbol, convert, strings.Replace(selectedChartRange, " ", "", -1))) cached, found := ct.cache.Get(cachekey) if found { // cache hit - graphData, _ = cached.([]float64) + cacheData, _ = cached.([][]float64) log.Debug("PortfolioChart() soft cache hit") } else { if ct.filecache != nil { - ct.filecache.Get(cachekey, &graphData) + ct.filecache.Get(cachekey, &cacheData) } - if len(graphData) == 0 { + if len(cacheData) == 0 { time.Sleep(2 * time.Second) apiGraphData, err := ct.api.GetCoinGraphData(convert, p.Symbol, p.Name, start, end) if err != nil { return err } - sorted := apiGraphData.Price - sort.Slice(sorted[:], func(i, j int) bool { - return sorted[i][0] < sorted[j][0] + + cacheData = apiGraphData.Price + sort.Slice(cacheData[:], func(i, j int) bool { + return cacheData[i][0] < cacheData[j][0] }) - for i := range sorted { - price := sorted[i][1] - graphData = append(graphData, price) - } } - ct.cache.Set(cachekey, graphData, 10*time.Second) + ct.cache.Set(cachekey, cacheData, 10*time.Second) if ct.filecache != nil { go func() { - ct.filecache.Set(cachekey, graphData, 24*time.Hour) + ct.filecache.Set(cachekey, cacheData, 24*time.Hour) }() } } - for i := range graphData { - price := graphData[i] - sum := p.Holdings * price + allCacheData = append(allCacheData, PriceData{p, cacheData}) + } + + // Calculate how many data points to provide to the chart. Limit maxPoints to 2*maxX + maxPoints := 0 + for _, cacheData := range allCacheData { + if len(cacheData.data) > maxPoints { + maxPoints = len(cacheData.data) + } + } + if maxPoints > 2*maxX { + maxPoints = 2 * maxX + } + + // Use the gap between price samples to adjust start/end in by one interval + var timeQuantum time.Duration + for _, cacheData := range allCacheData { + timeQuantum = timedata.CalculateTimeQuantum(cacheData.data) + if timeQuantum != 0 { + break // use the first one + } + } + newStart := time.Unix(start, 0).Add(timeQuantum) + newEnd := time.Unix(end, 0).Add(-timeQuantum) + + // Resample and sum data + var data []float64 + for _, cacheData := range allCacheData { + coinData := timedata.ResampleTimeSeriesData(cacheData.data, float64(newStart.UnixMilli()), float64(newEnd.UnixMilli()), maxPoints) + // sum (excluding NaN) + for i := range coinData { + price := coinData[i][1] + if math.IsNaN(price) { + price = 0.0 + } + sum := cacheData.coin.Holdings * price if i < len(data) { data[i] += sum } else { diff --git a/pkg/timedata/timedata.go b/pkg/timedata/timedata.go new file mode 100644 index 0000000..c605fdc --- /dev/null +++ b/pkg/timedata/timedata.go @@ -0,0 +1,51 @@ +package timedata + +import ( + "math" + "sort" + "time" + + log "github.com/sirupsen/logrus" +) + +// Resample the [timestamp,value] data given to numsteps between start-end (returns numSteps+1 points). +// If the data does not extend past start/end then there will likely be NaN in the output data. +func ResampleTimeSeriesData(data [][]float64, start float64, end float64, numSteps int) [][]float64 { + var newData [][]float64 + l := len(data) + step := (end - start) / float64(numSteps) + for pos := start; pos <= end; pos += step { + idx := sort.Search(l, func(i int) bool { return data[i][0] >= pos }) + var val float64 + if idx == 0 { + val = math.NaN() // off the left + } else if idx == l { + val = math.NaN() // off the right + } else { + // between two points - linear interpolation + left := data[idx-1] + right := data[idx] + dvdt := (right[1] - left[1]) / (right[0] - left[0]) + val = left[1] + (pos-left[0])*dvdt + } + newData = append(newData, []float64{pos, val}) + } + return newData +} + +// Assuming that the [timestamp,value] data provided is roughly evenly spaced, calculate that interval. +func CalculateTimeQuantum(data [][]float64) time.Duration { + if len(data) > 1 { + minTime := time.UnixMilli(int64(data[0][0])) + maxTime := time.UnixMilli(int64(data[len(data)-1][0])) + return time.Duration(int64(maxTime.Sub(minTime)) / int64(len(data)-1)) + } + return 0 +} + +// Print out all the [timestamp,value] data provided +func DebugLogPriceData(data [][]float64) { + for i := range data { + log.Debugf("%s %.2f", time.Unix(int64(data[i][0]/1000), 0), data[i][1]) + } +} From 49ac2fbc0f1410c88b3cfa89f92758a32e296a73 Mon Sep 17 00:00:00 2001 From: Simon Roberts Date: Sun, 26 Sep 2021 18:53:25 +1000 Subject: [PATCH 2/4] Ask the chart how many data points it needs --- cointop/chart.go | 19 ++----------------- pkg/chartplot/chartplot.go | 12 ++++++++++-- 2 files changed, 12 insertions(+), 19 deletions(-) diff --git a/cointop/chart.go b/cointop/chart.go index 19fd5c8..87ee20f 100644 --- a/cointop/chart.go +++ b/cointop/chart.go @@ -173,14 +173,10 @@ func (ct *Cointop) ChartPoints(symbol string, name string) error { } // Resample cachedata - maxPoints := len(cacheData) - if maxPoints > 2*maxX { - maxPoints = 2 * maxX - } timeQuantum := timedata.CalculateTimeQuantum(cacheData) newStart := time.Unix(start, 0).Add(timeQuantum) newEnd := time.Unix(end, 0).Add(-timeQuantum) - timeData := timedata.ResampleTimeSeriesData(cacheData, float64(newStart.UnixMilli()), float64(newEnd.UnixMilli()), maxPoints) + timeData := timedata.ResampleTimeSeriesData(cacheData, float64(newStart.UnixMilli()), float64(newEnd.UnixMilli()), chart.GetChartDataSize(maxX)) // Extract just the values from the data var data []float64 @@ -276,17 +272,6 @@ func (ct *Cointop) PortfolioChart() error { allCacheData = append(allCacheData, PriceData{p, cacheData}) } - // Calculate how many data points to provide to the chart. Limit maxPoints to 2*maxX - maxPoints := 0 - for _, cacheData := range allCacheData { - if len(cacheData.data) > maxPoints { - maxPoints = len(cacheData.data) - } - } - if maxPoints > 2*maxX { - maxPoints = 2 * maxX - } - // Use the gap between price samples to adjust start/end in by one interval var timeQuantum time.Duration for _, cacheData := range allCacheData { @@ -301,7 +286,7 @@ func (ct *Cointop) PortfolioChart() error { // Resample and sum data var data []float64 for _, cacheData := range allCacheData { - coinData := timedata.ResampleTimeSeriesData(cacheData.data, float64(newStart.UnixMilli()), float64(newEnd.UnixMilli()), maxPoints) + coinData := timedata.ResampleTimeSeriesData(cacheData.data, float64(newStart.UnixMilli()), float64(newEnd.UnixMilli()), chart.GetChartDataSize(maxX)) // sum (excluding NaN) for i := range coinData { price := coinData[i][1] diff --git a/pkg/chartplot/chartplot.go b/pkg/chartplot/chartplot.go index c9101b3..cc7e8b6 100644 --- a/pkg/chartplot/chartplot.go +++ b/pkg/chartplot/chartplot.go @@ -56,10 +56,18 @@ func (c *ChartPlot) SetData(data []float64) { c.t.Data = data } +func (c *ChartPlot) GetChartDataSize(width int) int { + axisYWidth := 30 + return (width * 2) - axisYWidth +} + // GetChartPoints ... func (c *ChartPlot) GetChartPoints(width int) [][]rune { - axisYWidth := 30 - c.t.Data = interpolateData(c.t.Data, (width*2)-axisYWidth) + targetWidth := c.GetChartDataSize(width) + if len(c.t.Data) != targetWidth { + // Don't resample data if it's already the right size + c.t.Data = interpolateData(c.t.Data, targetWidth) + } termui.Body = termui.NewGrid() termui.Body.Width = width termui.Body.AddRows( From 370b9f3a56e1d26aeead56e048875e91a70f7454 Mon Sep 17 00:00:00 2001 From: Simon Roberts Date: Sun, 26 Sep 2021 19:27:27 +1000 Subject: [PATCH 3/4] Fix comments --- cointop/chart.go | 2 +- pkg/chartplot/chartplot.go | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/cointop/chart.go b/cointop/chart.go index 87ee20f..a60ff8f 100644 --- a/cointop/chart.go +++ b/cointop/chart.go @@ -14,7 +14,7 @@ import ( log "github.com/sirupsen/logrus" ) -// Time-series data for a Coin used when building a Portfolio view for chart +// PriceData is the time-series data for a Coin used when building a Portfolio view for chart type PriceData struct { coin *Coin data [][]float64 diff --git a/pkg/chartplot/chartplot.go b/pkg/chartplot/chartplot.go index cc7e8b6..a0b7c78 100644 --- a/pkg/chartplot/chartplot.go +++ b/pkg/chartplot/chartplot.go @@ -56,6 +56,7 @@ func (c *ChartPlot) SetData(data []float64) { c.t.Data = data } +// GetChartDataSize ... func (c *ChartPlot) GetChartDataSize(width int) int { axisYWidth := 30 return (width * 2) - axisYWidth From e8fcd4a7a4b662995f5504001a0ac1a3ed23b3b1 Mon Sep 17 00:00:00 2001 From: Simon Roberts Date: Sun, 26 Sep 2021 19:50:26 +1000 Subject: [PATCH 4/4] Remove interpolateData() --- pkg/chartplot/chartplot.go | 29 +---------------------------- 1 file changed, 1 insertion(+), 28 deletions(-) diff --git a/pkg/chartplot/chartplot.go b/pkg/chartplot/chartplot.go index a0b7c78..24fb2fb 100644 --- a/pkg/chartplot/chartplot.go +++ b/pkg/chartplot/chartplot.go @@ -1,8 +1,6 @@ package chartplot import ( - "math" - "github.com/miguelmota/cointop/pkg/termui" ) @@ -53,6 +51,7 @@ func (c *ChartPlot) SetBorder(enabled bool) { func (c *ChartPlot) SetData(data []float64) { // NOTE: edit `termui.LineChart.shortenFloatVal(float64)` to not // use exponential notation. + // NOTE: data should be the correct width for rendering - see GetChartDataSize() c.t.Data = data } @@ -64,11 +63,6 @@ func (c *ChartPlot) GetChartDataSize(width int) int { // GetChartPoints ... func (c *ChartPlot) GetChartPoints(width int) [][]rune { - targetWidth := c.GetChartDataSize(width) - if len(c.t.Data) != targetWidth { - // Don't resample data if it's already the right size - c.t.Data = interpolateData(c.t.Data, targetWidth) - } termui.Body = termui.NewGrid() termui.Body.Width = width termui.Body.AddRows( @@ -95,24 +89,3 @@ func (c *ChartPlot) GetChartPoints(width int) [][]rune { return points } - -func interpolateData(data []float64, width int) []float64 { - var res []float64 - if len(data) == 0 { - return res - } - stepFactor := float64(len(data)-1) / float64(width-1) - res = append(res, data[0]) - for i := 1; i < width-1; i++ { - step := float64(i) * stepFactor - before := math.Floor(step) - after := math.Ceil(step) - atPoint := step - before - pointBefore := data[int(before)] - pointAfter := data[int(after)] - interpolated := pointBefore + (pointAfter-pointBefore)*atPoint - res = append(res, interpolated) - } - res = append(res, data[len(data)-1]) - return res -}