From 5f2c2b396cf718bc1d9088436da0c66913930209 Mon Sep 17 00:00:00 2001 From: Cloudzp Date: Mon, 3 Jun 2024 20:29:26 +0800 Subject: [PATCH 1/2] add dsp mae debug --- pkg/prediction/dsp/test_data/roc_input.csv | 0 pkg/server/handler/prediction/prediction.go | 63 ++++++++++++++++++++- 2 files changed, 62 insertions(+), 1 deletion(-) create mode 100644 pkg/prediction/dsp/test_data/roc_input.csv diff --git a/pkg/prediction/dsp/test_data/roc_input.csv b/pkg/prediction/dsp/test_data/roc_input.csv new file mode 100644 index 000000000..e69de29bb diff --git a/pkg/server/handler/prediction/prediction.go b/pkg/server/handler/prediction/prediction.go index 1d5fef5cd..7f8b6ec43 100644 --- a/pkg/server/handler/prediction/prediction.go +++ b/pkg/server/handler/prediction/prediction.go @@ -3,7 +3,9 @@ package prediction import ( "context" "fmt" + "math" "net/http" + "time" "github.com/gin-gonic/gin" "github.com/go-echarts/go-echarts/v2/charts" @@ -27,6 +29,12 @@ import ( "github.com/gocrane/crane/pkg/utils/target" ) +const ( + day = time.Hour * 24 + threeDays = time.Hour * 24 * 3 + week = time.Hour * 24 * 7 +) + type DebugHandler struct { craneClient *craneclientset.Clientset predictorManager predictormgr.Manager @@ -88,10 +96,28 @@ func (dh *DebugHandler) Display(c *gin.Context) { return } + dspMAEDay := 0.0 + daySampleLength := int(estimate.SampleRate * day.Seconds()) + if len(estimate.Samples) >= daySampleLength { + dspMAEDay = mae(test.Samples[:daySampleLength-1], estimate.Samples[:daySampleLength-1]) + } + dspMAEThreeDays := 0.0 + threeDaySampleLength := int(estimate.SampleRate * threeDays.Seconds()) + if len(estimate.Samples) >= threeDaySampleLength { + dspMAEThreeDays = mae(test.Samples[:threeDaySampleLength-1], estimate.Samples[:threeDaySampleLength-1]) + } + dspMAEWeekly := 0.0 + eightDaySampleLength := int(estimate.SampleRate * week.Seconds()) + if len(estimate.Samples) >= eightDaySampleLength { + dspMAEWeekly = mae(test.Samples[:eightDaySampleLength-1], estimate.Samples[:eightDaySampleLength-1]) + } + klog.Infof("dspMAE1d: %f,dspMAE3d: %f, dspMAE7d: %f", dspMAEDay, dspMAEThreeDays, dspMAEWeekly) page := components.NewPage() page.AddCharts(plot(history, "history", "green", charts.WithTitleOpts(opts.Title{Title: "history"}))) page.AddCharts(plots([]*dsp.Signal{test, estimate}, []string{"actual", "forecasted"}, charts.WithTitleOpts(opts.Title{Title: "actual/forecasted"}))) + + page.AddCharts(bar(dspMAEDay, dspMAEThreeDays, dspMAEWeekly)) err = page.Render(c.Writer) if err != nil { klog.ErrorS(err, "Failed to display debug time series") @@ -105,6 +131,41 @@ func (dh *DebugHandler) Display(c *gin.Context) { return } +func mae(testData, estimateData []float64) float64 { + mae := 0.0 + for i := 0; i < len(estimateData); i++ { + mae += math.Abs(testData[i] - estimateData[i]) + } + return mae / float64(len(estimateData)) +} + +func bar(dspMAE1d, dspMAE3d, dspMAE7d float64) *charts.Bar { + bar := charts.NewBar() + bar.SetGlobalOptions( + charts.WithInitializationOpts(opts.Initialization{Width: "3000px", Theme: types.ThemeRoma}), + charts.WithTitleOpts(opts.Title{ + Title: "MAE(Mean Absolute Error)", + }), charts.WithTooltipOpts(opts.Tooltip{ + Show: true, + Trigger: "axis", + TriggerOn: "mousemove", + }), + ) + bar.SetXAxis([]string{"1d", "3d", "7d"}). + AddSeries("mae", []opts.BarData{ + { + Value: dspMAE1d, + }, + { + Value: dspMAE3d, + }, + { + Value: dspMAE7d, + }, + }) + return bar +} + func plot(s *dsp.Signal, name string, color string, o ...charts.GlobalOpts) *charts.Line { x := make([]string, 0) y := make([]opts.LineData, 0) @@ -155,7 +216,7 @@ func plots(signals []*dsp.Signal, names []string, o ...charts.GlobalOpts) *chart line := charts.NewLine() line.SetGlobalOptions( - charts.WithInitializationOpts(opts.Initialization{Width: "3000px", Theme: types.ThemeShine}), + charts.WithInitializationOpts(opts.Initialization{Width: "3000px", Theme: types.ThemeRoma}), charts.WithLegendOpts( opts.Legend{ Show: true, From d71331f58783c27582b797ecbea91dab2d8736cd Mon Sep 17 00:00:00 2001 From: Cloudzp Date: Tue, 4 Jun 2024 19:06:17 +0800 Subject: [PATCH 2/2] add percentile debug and mae statistics --- pkg/prediction/percentile/debug.go | 74 +++++++++++++++++++ pkg/server/handler/prediction/prediction.go | 79 +++++++++++++++++++-- 2 files changed, 149 insertions(+), 4 deletions(-) create mode 100644 pkg/prediction/percentile/debug.go diff --git a/pkg/prediction/percentile/debug.go b/pkg/prediction/percentile/debug.go new file mode 100644 index 000000000..9e72ca808 --- /dev/null +++ b/pkg/prediction/percentile/debug.go @@ -0,0 +1,74 @@ +package percentile + +import ( + "fmt" + "time" + + "k8s.io/klog/v2" + + "github.com/gocrane/crane/pkg/common" + "github.com/gocrane/crane/pkg/metricnaming" + "github.com/gocrane/crane/pkg/prediction" + "github.com/gocrane/crane/pkg/prediction/config" +) + +func Debug(predictor prediction.Interface, namer metricnaming.MetricNamer, config *config.Config) ([]*common.TimeSeries, []*common.TimeSeries, error) { + cfg, err := makeInternalConfig(config.Percentile, config.InitMode) + if err != nil { + return nil, nil, err + } + + p := predictor.(*percentilePrediction) + historyTimeSeriesList, err := queryHistoryTimeSeries(p, namer, cfg) + if err != nil { + return nil, nil, err + } + + queryExpr := namer.BuildUniqueKey() + klog.V(4).Infof("process analyzing metric namer: %v, config: %+v", namer.BuildUniqueKey(), *cfg) + + signals := map[string]*aggregateSignal{} + if cfg.aggregated { + signal := newAggregateSignal(cfg) + for _, ts := range historyTimeSeriesList { + for _, s := range ts.Samples { + t := time.Unix(s.Timestamp, 0) + signal.addSample(t, s.Value) + } + } + signals[keyAll] = signal + } else { + for _, ts := range historyTimeSeriesList { + if len(ts.Samples) < 1 { + continue + } + key := prediction.AggregateSignalKey(ts.Labels) + signal := newAggregateSignal(cfg) + for _, s := range ts.Samples { + t := time.Unix(s.Timestamp, 0) + signal.addSample(t, s.Value) + } + signal.labels = ts.Labels + signals[key] = signal + } + } + + return historyTimeSeriesList, p.getPredictedValuesFromSignals(queryExpr, signals, cfg), nil + +} + +func queryHistoryTimeSeries(predictor *percentilePrediction, namer metricnaming.MetricNamer, config *internalConfig) ([]*common.TimeSeries, error) { + p := predictor.GetHistoryProvider() + if p == nil { + return nil, fmt.Errorf("history provider not provisioned") + } + + end := time.Now().Truncate(time.Minute) + start := end.Add(-config.historyLength) + historyTimeSeries, err := predictor.GetHistoryProvider().QueryTimeSeries(namer, start, end, config.sampleInterval) + if err != nil { + klog.ErrorS(err, "Failed to query history time series.") + return nil, err + } + return historyTimeSeries, nil +} diff --git a/pkg/server/handler/prediction/prediction.go b/pkg/server/handler/prediction/prediction.go index 7f8b6ec43..ff8cb7028 100644 --- a/pkg/server/handler/prediction/prediction.go +++ b/pkg/server/handler/prediction/prediction.go @@ -20,9 +20,9 @@ import ( craneclientset "github.com/gocrane/api/pkg/generated/clientset/versioned" "github.com/gocrane/api/prediction/v1alpha1" - "github.com/gocrane/crane/pkg/controller/timeseriesprediction" "github.com/gocrane/crane/pkg/prediction/dsp" + "github.com/gocrane/crane/pkg/prediction/percentile" predictormgr "github.com/gocrane/crane/pkg/predictor" "github.com/gocrane/crane/pkg/server/config" "github.com/gocrane/crane/pkg/server/ginwrapper" @@ -107,9 +107,9 @@ func (dh *DebugHandler) Display(c *gin.Context) { dspMAEThreeDays = mae(test.Samples[:threeDaySampleLength-1], estimate.Samples[:threeDaySampleLength-1]) } dspMAEWeekly := 0.0 - eightDaySampleLength := int(estimate.SampleRate * week.Seconds()) - if len(estimate.Samples) >= eightDaySampleLength { - dspMAEWeekly = mae(test.Samples[:eightDaySampleLength-1], estimate.Samples[:eightDaySampleLength-1]) + weeklyDaySampleLength := int(estimate.SampleRate * week.Seconds()) + if len(estimate.Samples) >= weeklyDaySampleLength { + dspMAEWeekly = mae(test.Samples[:weeklyDaySampleLength-1], estimate.Samples[:weeklyDaySampleLength-1]) } klog.Infof("dspMAE1d: %f,dspMAE3d: %f, dspMAE7d: %f", dspMAEDay, dspMAEThreeDays, dspMAEWeekly) page := components.NewPage() @@ -123,6 +123,67 @@ func (dh *DebugHandler) Display(c *gin.Context) { klog.ErrorS(err, "Failed to display debug time series") } + return + } else if tsp.Spec.PredictionMetrics[0].Algorithm.AlgorithmType == v1alpha1.AlgorithmTypePercentile && tsp.Spec.PredictionMetrics[0].Algorithm.Percentile != nil { + mc, err := timeseriesprediction.NewMetricContext(dh.selectorFetcher, tsp, dh.predictorManager) + if err != nil { + ginwrapper.WriteResponse(c, err, nil) + return + } + + internalConf := mc.ConvertApiMetric2InternalConfig(&tsp.Spec.PredictionMetrics[0]) + internalConf.Percentile.HistoryLength = "7d" + namer := mc.GetMetricNamer(&tsp.Spec.PredictionMetrics[0]) + predictor := dh.predictorManager.GetPredictor(v1alpha1.AlgorithmTypePercentile) + history, estimate, err := percentile.Debug(predictor, namer, internalConf) + if err != nil { + ginwrapper.WriteResponse(c, err, nil) + return + } + if len(history) <= 0 || len(estimate) <= 0 { + c.Writer.WriteHeader(http.StatusBadRequest) + return + } + + samples := history[0].Samples + estimateSamples := estimate[0].Samples + historySignal := &dsp.Signal{SampleRate: 1 / 60.0} + estimateSignal := &dsp.Signal{SampleRate: 1 / 60.0} + for _, v := range samples { + historySignal.Samples = append(historySignal.Samples, v.Value) + estimateSignal.Samples = append(estimateSignal.Samples, estimateSamples[0].Value) + } + + predictMAEDay := 0.0 + daySampleLength := int(estimateSignal.SampleRate * day.Seconds()) + if len(estimateSignal.Samples) >= daySampleLength { + m := maxValue(historySignal.Samples[:daySampleLength-1]) + predictMAEDay = mae([]float64{m}, []float64{estimateSamples[0].Value}) + } + predictMAEThreeDays := 0.0 + threeDaySampleLength := int(estimateSignal.SampleRate * threeDays.Seconds()) + if len(estimateSignal.Samples) >= threeDaySampleLength { + m := maxValue(historySignal.Samples[:threeDaySampleLength-1]) + predictMAEThreeDays = mae([]float64{m}, []float64{estimateSamples[0].Value}) + } + predictMAEWeekly := 0.0 + weeklySampleLength := int(estimateSignal.SampleRate * week.Seconds()) + if len(estimateSignal.Samples) >= weeklySampleLength { + m := maxValue(historySignal.Samples[:weeklySampleLength-1]) + predictMAEWeekly = mae([]float64{m}, []float64{estimateSamples[0].Value}) + } + + page := components.NewPage() + page.AddCharts(plots([]*dsp.Signal{historySignal, estimateSignal}, []string{"actual", "recommended"}, + charts.WithTitleOpts(opts.Title{Title: "actual/recommended"}))) + + page.AddCharts(bar(predictMAEDay, predictMAEThreeDays, predictMAEWeekly)) + err = page.Render(c.Writer) + if err != nil { + klog.ErrorS(err, "Failed to display debug time series") + } + + c.Writer.WriteHeader(http.StatusBadRequest) return } } @@ -131,6 +192,16 @@ func (dh *DebugHandler) Display(c *gin.Context) { return } +func maxValue(testData []float64) float64 { + m := 0.0 + for i := 0; i < len(testData); i++ { + if testData[i] > m { + m = testData[i] + } + } + return m +} + func mae(testData, estimateData []float64) float64 { mae := 0.0 for i := 0; i < len(estimateData); i++ {