Merge pull request #1 from pyroscope-io/feat/baseline-profile

feat: baseline profile

example/go.mod

@@ -2,6 +2,8 @@ module github.com/pyroscope-io/otelpyroscope/example
 
 go 1.16
 
+replace github.com/pyroscope-io/otelpyroscope => ../
+
 require (
 	github.com/pyroscope-io/otelpyroscope v0.1.0
 	go.opentelemetry.io/otel v1.4.1

example/go.sum

@@ -1,5 +1,6 @@
 github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/go-logr/logr v1.2.2 h1:ahHml/yUpnlb96Rp8HCvtYVPY8ZYpxq3g7UYchIYwbs=
 github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
 github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
@@ -10,7 +11,10 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/pyroscope-io/otelpyroscope v0.1.0 h1:LDRYGrFhQUZT7rbJ1PCMOL0TGjNk/SNJbh3Ri7l8OFg=
 github.com/pyroscope-io/otelpyroscope v0.1.0/go.mod h1:Pjgu/PeVua81wS40W4sJ7GaS2mjGUZ7bhUAA1X+8lZ8=
+github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE=
+github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 go.opentelemetry.io/otel v1.4.1 h1:QbINgGDDcoQUoMJa2mMaWno49lja9sHwp6aoa2n3a4g=
@@ -21,6 +25,7 @@ go.opentelemetry.io/otel/sdk v1.4.1 h1:J7EaW71E0v87qflB4cDolaqq3AcujGrtyIPGQoZOB
 go.opentelemetry.io/otel/sdk v1.4.1/go.mod h1:NBwHDgDIBYjwK2WNu1OPgsIc2IJzmBXNnvIJxJc8BpE=
 go.opentelemetry.io/otel/trace v1.4.1 h1:O+16qcdTrT7zxv2J6GejTPFinSwA++cYerC5iSiF8EQ=
 go.opentelemetry.io/otel/trace v1.4.1/go.mod h1:iYEVbroFCNut9QkwEczV9vMRPHNKSSwYZjulEtsmhFc=
+golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7 h1:iGu644GcxtEcrInvDsQRCwJjtCIOlT2V7IRt6ah2Whw=
 golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

example/main.go

@@ -50,8 +50,12 @@ func initTracer() *trace.TracerProvider {
 		trace.WithBatcher(exporter),
 	)
 	otel.SetTracerProvider(otelpyroscope.NewTracerProvider(tp,
-		otelpyroscope.WithDefaultProfileURLBuilder("http://localhost:4040", "example-app"),
+		otelpyroscope.WithAppName("example-app"),
+		otelpyroscope.WithPyroscopeURL("http://localhost:4040"),
 		otelpyroscope.WithRootSpanOnly(true),
+		otelpyroscope.WithAddSpanName(true),
+		otelpyroscope.WithProfileURL(true),
+		otelpyroscope.WithProfileBaselineURL(true),
 	))
 	otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator(
 		propagation.TraceContext{},

otelpyroscope.go

@@ -2,94 +2,186 @@ package otelpyroscope
 
 import (
 	"context"
+	"net/url"
 	"runtime/pprof"
+	"strconv"
+	"strings"
+	"time"
 
 	"go.opentelemetry.io/otel/attribute"
 	"go.opentelemetry.io/otel/trace"
 )
 
-const profileIDLabelName = "profile_id"
+const (
+	profileIDLabelName = "profile_id"
+	spanNameLabelName  = "span_name"
+)
 
 var (
-	profileIDSpanAttributeKey  = attribute.Key("pyroscope.profile.id")
-	profileURLSpanAttributeKey = attribute.Key("pyroscope.profile.url")
+	profileIDSpanAttributeKey          = attribute.Key("pyroscope.profile.id")
+	profileURLSpanAttributeKey         = attribute.Key("pyroscope.profile.url")
+	profileBaselineURLSpanAttributeKey = attribute.Key("pyroscope.profile.baseline.url")
 )
 
-// tracerProvider satisfies open telemetry TracerProvider interface.
-type tracerProvider struct {
-	tp trace.TracerProvider
+type Config struct {
+	AppName                   string
+	PyroscopeURL              string
+	IncludeProfileURL         bool
+	IncludeProfileBaselineURL bool
+	ProfileBaselineLabels     map[string]string
 
-	rootOnly bool
-	buildURL func(string) string
+	RootOnly    bool
+	AddSpanName bool
 }
 
-// NewTracerProvider creates a new tracer provider that annotates pprof
-// profiles with span ID tag. This allows to establish a relationship
-// between pprof profiles and reported tracing spans.
-func NewTracerProvider(tp trace.TracerProvider, options ...Option) trace.TracerProvider {
-	p := tracerProvider{
-		tp:       tp,
-		rootOnly: true,
-	}
-	for _, o := range options {
-		o(&p)
+type Option func(*tracerProvider)
+
+// WithAppName specifies the profiled application name.
+// It should match the name specified in pyroscope configuration.
+// Required, if profile URL or profile baseline URL is enabled.
+func WithAppName(app string) Option {
+	return func(tp *tracerProvider) {
+		tp.config.AppName = app
 	}
-	return &p
 }
 
-type Option func(*tracerProvider)
-
 // WithRootSpanOnly indicates that only the root span is to be profiled.
 // The profile includes samples captured during child span execution
 // but the spans won't have their own profiles and won't be annotated
 // with pyroscope.profile attributes.
-func WithRootSpanOnly(rootOnly bool) Option {
+// The option is enabled by default.
+func WithRootSpanOnly(x bool) Option {
+	return func(tp *tracerProvider) {
+		tp.config.RootOnly = x
+	}
+}
+
+// WithAddSpanName specifies whether the current span name should be added
+// to the profile labels. N.B if the name is dynamic, or too many values
+// are supposed, this may significantly deteriorate performance.
+// By default, span name is not added to profile labels.
+func WithAddSpanName(x bool) Option {
+	return func(tp *tracerProvider) {
+		tp.config.AddSpanName = x
+	}
+}
+
+// WithPyroscopeURL provides a base URL for the profile and baseline URLs.
+// Required, if profile URL or profile baseline URL is enabled.
+func WithPyroscopeURL(addr string) Option {
+	return func(tp *tracerProvider) {
+		tp.config.PyroscopeURL = addr
+	}
+}
+
+// WithProfileURL specifies whether to add the pyroscope.profile.url
+// attribute with the URL to the span profile.
+func WithProfileURL(x bool) Option {
+	return func(tp *tracerProvider) {
+		tp.config.IncludeProfileURL = x
+	}
+}
+
+// WithProfileBaselineURL specifies whether to add the
+// pyroscope.profile.baseline.url attribute with the URL
+// to the baseline profile. See WithProfileBaselineLabels.
+func WithProfileBaselineURL(x bool) Option {
 	return func(tp *tracerProvider) {
-		tp.rootOnly = rootOnly
+		tp.config.IncludeProfileBaselineURL = x
 	}
 }
 
-// WithProfileURLBuilder specifies how profile URL is to be built. Optional.
+// WithProfileBaselineLabels provides a map of extra labels to be added to the
+// baseline query alongside with pprof labels set in runtime. Typically,
+// it should match the labels specified in the Pyroscope profiler config.
+// Note that the map must not be modified.
+func WithProfileBaselineLabels(x map[string]string) Option {
+	return func(tp *tracerProvider) {
+		tp.config.ProfileBaselineLabels = x
+	}
+}
+
+// WithProfileURLBuilder specifies how profile URL is to be built.
+// DEPRECATED: use WithProfileURL
 func WithProfileURLBuilder(b func(profileID string) string) Option {
 	return func(tp *tracerProvider) {
+		tp.config.IncludeProfileURL = true
 		tp.buildURL = b
 	}
 }
 
+// WithDefaultProfileURLBuilder specifies the default profile URL builder.
+// DEPRECATED: use WithProfileURL
 func WithDefaultProfileURLBuilder(addr string, app string) Option {
 	return func(tp *tracerProvider) {
+		tp.config.IncludeProfileURL = true
 		tp.buildURL = DefaultProfileURLBuilder(addr, app)
 	}
 }
 
-func DefaultProfileURLBuilder(addr string, app string) func(string) string {
+// tracerProvider satisfies open telemetry TracerProvider interface.
+type tracerProvider struct {
+	tp trace.TracerProvider
+
+	config   Config
+	buildURL func(string) string
+}
+
+// NewTracerProvider creates a new tracer provider that annotates pprof
+// profiles with span ID tag. This allows to establish a relationship
+// between pprof profiles and reported tracing spans.
+func NewTracerProvider(tp trace.TracerProvider, options ...Option) trace.TracerProvider {
+	p := tracerProvider{
+		tp:     tp,
+		config: Config{RootOnly: true},
+	}
+	for _, o := range options {
+		o(&p)
+	}
+	if p.config.IncludeProfileURL && p.buildURL == nil {
+		p.buildURL = DefaultProfileURLBuilder(p.config.PyroscopeURL, p.config.AppName)
+	}
+	return &p
+}
+
+func DefaultProfileURLBuilder(addr, app string) func(string) string {
 	return func(id string) string {
-		return addr + "?query=" + app + ".cpu%7Bprofile_id%3D%22" + id + "%22%7D"
+		q := make(url.Values, 1)
+		q.Set("query", app+`.cpu{`+profileIDLabelName+`="`+id+`"}`)
+		return addr + "?" + q.Encode()
 	}
 }
 
-func (w tracerProvider) Tracer(name string, opts ...trace.TracerOption) trace.Tracer {
+func (w *tracerProvider) Tracer(name string, opts ...trace.TracerOption) trace.Tracer {
 	return &profileTracer{p: w, tr: w.tp.Tracer(name, opts...)}
 }
 
 type profileTracer struct {
-	p  tracerProvider
+	p  *tracerProvider
 	tr trace.Tracer
 }
 
 func (w profileTracer) Start(ctx context.Context, spanName string, opts ...trace.SpanStartOption) (context.Context, trace.Span) {
-	if w.p.rootOnly && !isRootSpan(trace.SpanContextFromContext(ctx)) {
+	if w.p.config.RootOnly && !isRootSpan(trace.SpanContextFromContext(ctx)) {
 		return w.tr.Start(ctx, spanName, opts...)
 	}
 	ctx, span := w.tr.Start(ctx, spanName, opts...)
 	s := spanWrapper{
-		profileID: trace.SpanContextFromContext(ctx).SpanID().String(),
 		Span:      span,
+		profileID: trace.SpanContextFromContext(ctx).SpanID().String(),
+		startTime: time.Now(),
 		ctx:       ctx,
 		p:         w.p,
 	}
-	ctx = pprof.WithLabels(ctx, pprof.Labels(profileIDLabelName, s.profileID))
+
+	labels := []string{profileIDLabelName, s.profileID}
+	if w.p.config.AddSpanName && spanName != "" {
+		labels = append(labels, spanNameLabelName, spanName)
+	}
+
+	ctx = pprof.WithLabels(ctx, pprof.Labels(labels...))
 	pprof.SetGoroutineLabels(ctx)
+	s.pprofCtx = ctx
 	return ctx, &s
 }
 
@@ -101,10 +193,15 @@ func isRootSpan(s trace.SpanContext) bool {
 
 type spanWrapper struct {
 	trace.Span
-	ctx context.Context
 
+	// Span context.
+	ctx context.Context
+	// Current pprof context with labels.
+	pprofCtx  context.Context
 	profileID string
-	p         tracerProvider
+	startTime time.Time
+
+	p *tracerProvider
 }
 
 func (s spanWrapper) End(options ...trace.SpanEndOption) {
@@ -114,9 +211,59 @@ func (s spanWrapper) End(options ...trace.SpanEndOption) {
 	// scope that is associated with a tracing span.
 	s.SetAttributes(profileIDSpanAttributeKey.String(s.profileID))
 	// Optionally specify the profile URL.
-	if s.p.buildURL != nil {
+	if s.p.config.IncludeProfileURL {
 		s.SetAttributes(profileURLSpanAttributeKey.String(s.p.buildURL(s.profileID)))
 	}
+	if s.p.config.IncludeProfileBaselineURL {
+		s.SetAttributes(profileBaselineURLSpanAttributeKey.String(s.buildProfileBaselineURL()))
+	}
 	s.Span.End(options...)
 	pprof.SetGoroutineLabels(s.ctx)
 }
+
+func (s spanWrapper) buildProfileBaselineURL() string {
+	var b strings.Builder
+	pprof.ForLabels(s.pprofCtx, func(key, value string) bool {
+		if key == profileIDLabelName {
+			return true
+		}
+		if s.p.config.ProfileBaselineLabels != nil {
+			if _, ok := s.p.config.ProfileBaselineLabels[key]; ok {
+				return true
+			}
+		}
+		writeLabel(&b, key, value)
+		return true
+	})
+	for key, value := range s.p.config.ProfileBaselineLabels {
+		if value != "" {
+			writeLabel(&b, key, value)
+		}
+	}
+
+	q := make(url.Values, 9)
+	from := strconv.FormatInt(s.startTime.Unix(), 10)
+	until := strconv.FormatInt(time.Now().Unix(), 10)
+	baselineQuery := s.p.config.AppName + `.cpu{` + b.String() + `}`
+
+	q.Set("query", baselineQuery)
+	q.Set("from", from)
+	q.Set("until", until)
+
+	q.Set("rightQuery", s.p.config.AppName+`.cpu{`+profileIDLabelName+`="`+s.profileID+`"}`)
+	q.Set("rightFrom", from)
+	q.Set("rightUntil", until)
+
+	q.Set("leftQuery", baselineQuery)
+	q.Set("leftFrom", from)
+	q.Set("leftUntil", until)
+
+	return s.p.config.PyroscopeURL + "/comparison?" + q.Encode()
+}
+
+func writeLabel(b *strings.Builder, k, v string) {
+	if b.Len() > 0 {
+		b.WriteByte(',')
+	}
+	b.WriteString(k + `="` + v + `"`)
+}