Skip to content

Commit

Permalink
chore(workflow): adding workflow name and injection time (#100) (#101)
Browse files Browse the repository at this point in the history 
Signed-off-by: shubhamchaudhary <[email protected]>

Co-authored-by: Shubham Chaudhary <[email protected]>
  • Loading branch information
@ispeakc0de
Karthik Satchitanand and ispeakc0de authored Jun 15, 2021
1 parent e16de8a commit 5ab0af6
Show file tree
Hide file tree
Showing 5 changed files with 121 additions and 69 deletions.
29 changes: 24 additions & 5 deletions controller/collect-data.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,9 @@ func (resultDetails *ChaosResultDetails) getExperimentMetricsFromResult(chaosRes
setEndTime(events).
setChaosInjectTime(events).
setChaosEngineName(chaosResult.Spec.EngineName).
setChaosEngineLabel(engine.Labels[EngineLabelKey]).
setChaosEngineContext(engine.Labels[EngineContext]).
setChaosInjectLabel().
setWorkflowName(engine.Labels[WorkFlowName]).
setAppLabel(engine.Spec.Appinfo.Applabel).
setAppNs(engine.Spec.Appinfo.Appns).
setAppKind(engine.Spec.Appinfo.AppKind).
Expand Down Expand Up @@ -151,9 +153,26 @@ func (resultDetails *ChaosResultDetails) setAppKind(appKind string) *ChaosResult
return resultDetails
}

// setChaosEngineLabel sets the chaosEngine label inside resultDetails struct
func (resultDetails *ChaosResultDetails) setChaosEngineLabel(engineLabel string) *ChaosResultDetails {
resultDetails.ChaosEngineLabel = engineLabel
// setChaosEngineContext sets the chaosEngine context inside resultDetails struct
func (resultDetails *ChaosResultDetails) setChaosEngineContext(engineLabel string) *ChaosResultDetails {
resultDetails.ChaosEngineContext = engineLabel
return resultDetails
}

// setWorkflowName sets the workflow name inside resultDetails struct
func (resultDetails *ChaosResultDetails) setWorkflowName(workflowName string) *ChaosResultDetails {
resultDetails.WorkflowName = workflowName
return resultDetails
}

// setChaosInjectLabel sets the chaos inject label inside resultDetails struct
func (resultDetails *ChaosResultDetails) setChaosInjectLabel() *ChaosResultDetails {
injectTime := ""
if resultDetails.InjectionTime != 0 {
injectTime = strconv.Itoa(int(resultDetails.InjectionTime))
}
resultDetails.ChaosInjectLabel = injectTime

return resultDetails
}

Expand Down Expand Up @@ -190,7 +209,7 @@ func (resultDetails *ChaosResultDetails) setChaosInjectTime(events corev1.EventL
chaosInjectTime = maximum(chaosInjectTime, event.LastTimestamp.Unix())
}
}
resultDetails.InjectionTime = float64(chaosInjectTime)
resultDetails.InjectionTime = chaosInjectTime
return resultDetails
}

Expand Down
84 changes: 57 additions & 27 deletions controller/handle-result-deletion.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,9 @@ func (gaugeMetrics *GaugeMetrics) unsetDeletedChaosResults(oldChaosResults, newC
setAppNs(value.AppNs).
setAppKind(value.AppKind).
setChaosEngineName(oldResult.Spec.EngineName).
setChaosEngineLabel(value.Label)
setChaosEngineContext(value.ChaosEngineContext).
setWorkflowName(value.WorkFlowName)
resultDetails.ChaosInjectLabel = value.ChaosInjectLabel

gaugeMetrics.unsetResultChaosMetrics(resultDetails)
}
Expand All @@ -42,38 +44,52 @@ func (gaugeMetrics *GaugeMetrics) unsetDeletedChaosResults(oldChaosResults, newC
}
}

// unsetVerdictMetrics unset the metrics when chaosresult verdict changes
// unsetOutdatedMetrics unset the metrics when chaosresult verdict changes
// if same chaosresult is continuously repeated more than scrape interval then it sets the metrics value to 0
func (gaugeMetrics *GaugeMetrics) unsetVerdictMetrics(resultDetails ChaosResultDetails) float64 {
func (gaugeMetrics *GaugeMetrics) unsetOutdatedMetrics(resultDetails ChaosResultDetails) float64 {
scrapeTime, _ := strconv.Atoi(getEnv("TSDB_SCRAPE_INTERVAL", "10"))
result, ok := matchVerdict[string(resultDetails.UID)]
if ok {
reset := false

switch ok {
case true:
switch {
// if verdict is different then delete the older metrics having outdated verdict
if result.Verdict != resultDetails.Verdict {
gaugeMetrics.ResultVerdict.DeleteLabelValues(resultDetails.Namespace, resultDetails.Name, resultDetails.ChaosEngineName, resultDetails.ChaosEngineLabel, result.Verdict,
fmt.Sprintf("%f", result.ProbeSuccessPercentage), resultDetails.AppLabel, resultDetails.AppNs, resultDetails.AppKind)

// update the values inside matchVerdict
matchVerdict[string(resultDetails.UID)] = result.setCount(1).
setVerdict(resultDetails.Verdict).
setProbeSuccesPercentage(resultDetails.ProbeSuccesPercentage)
return float64(1)
} else {
result.Count++
matchVerdict[string(resultDetails.UID)] = result
case result.Verdict != resultDetails.Verdict:
gaugeMetrics.ResultVerdict.DeleteLabelValues(resultDetails.Namespace, resultDetails.Name, resultDetails.ChaosEngineName,
resultDetails.ChaosEngineContext, result.Verdict, fmt.Sprintf("%f", result.ProbeSuccessPercentage), resultDetails.AppLabel,
resultDetails.AppNs, resultDetails.AppKind, resultDetails.WorkflowName, result.ChaosInjectLabel)
result.Count = 1
// if chaos injectionTime is different then delete the older metrics having outdated data
case result.ChaosInjectLabel != resultDetails.ChaosInjectLabel:
gaugeMetrics.ResultVerdict.DeleteLabelValues(resultDetails.Namespace, resultDetails.Name, resultDetails.ChaosEngineName,
resultDetails.ChaosEngineContext, result.Verdict, fmt.Sprintf("%f", result.ProbeSuccessPercentage), resultDetails.AppLabel,
resultDetails.AppNs, resultDetails.AppKind, resultDetails.WorkflowName, result.ChaosInjectLabel)
gaugeMetrics.ResultAwaitedExperiments.DeleteLabelValues(resultDetails.Namespace, resultDetails.Name, resultDetails.ChaosEngineName,
resultDetails.ChaosEngineContext, resultDetails.WorkflowName, result.ChaosInjectLabel)
result.Count = 1
default:
// if time passed scrape time then reset the value to 0
if result.Count >= scrapeTime {
return float64(0)
reset = true
} else {
result.Count++
}
return float64(1)
}
default:
result = initialiseResultData().
setCount(1)
}

// update the values inside matchVerdict
matchVerdict[string(resultDetails.UID)] = initialiseResultData().
setCount(1).
setVerdict(resultDetails.Verdict).
setProbeSuccesPercentage(resultDetails.ProbeSuccesPercentage)
return float64(0)
matchVerdict[string(resultDetails.UID)] = result.setVerdict(resultDetails.Verdict).
setProbeSuccesPercentage(resultDetails.ProbeSuccesPercentage).
setChaosInjectLabel(resultDetails.ChaosInjectLabel)

if reset {
return float64(0)
}
return float64(1)
}

// getEnv derived the ENVs and sets the default value if env contains empty value
Expand All @@ -89,7 +105,9 @@ func getEnv(key, defaultValue string) string {
//can be used while handling chaosresult deletion
func (resultDetails *ChaosResultDetails) setResultData() {
resultData := initialiseResultData().
setLabel(resultDetails.ChaosEngineLabel).
setContext(resultDetails.ChaosEngineContext).
setWorkflowName(resultDetails.WorkflowName).
setChaosInjectLabel(resultDetails.ChaosInjectLabel).
setAppKind(resultDetails.AppKind).
setNs(resultDetails.AppNs).
setAppLabel(resultDetails.AppLabel).
Expand All @@ -109,9 +127,21 @@ func initialiseResultData() *ResultData {
return &ResultData{}
}

// setLabel sets the label inside resultData struct
func (resultData *ResultData) setLabel(label string) *ResultData {
resultData.Label = label
// setContext sets the engine context inside resultData struct
func (resultData *ResultData) setContext(context string) *ResultData {
resultData.ChaosEngineContext = context
return resultData
}

// setWorkflowName sets the workflow name inside resultData struct
func (resultData *ResultData) setWorkflowName(workflowName string) *ResultData {
resultData.WorkFlowName = workflowName
return resultData
}

// setChaosInjectLabel sets the chaos injection inside resultData struct
func (resultData *ResultData) setChaosInjectLabel(injectTime string) *ResultData {
resultData.ChaosInjectLabel = injectTime
return resultData
}

Expand Down
53 changes: 27 additions & 26 deletions controller/scrape.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,11 +21,11 @@ import (
"os"
"strings"

"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/cloudwatch"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"

"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/cloudwatch"
"github.com/litmuschaos/chaos-exporter/pkg/clients"
"github.com/litmuschaos/chaos-exporter/pkg/log"
litmuschaosv1alpha1 "github.com/litmuschaos/chaos-operator/pkg/apis/litmuschaos/v1alpha1"
Expand Down Expand Up @@ -112,7 +112,7 @@ func (gaugeMetrics *GaugeMetrics) GetLitmusChaosMetrics(clients clients.ClientSe
namespacedScopeMetrics.ExperimentRunCount += resultDetails.AwaitedExperiments + resultDetails.PassedExperiments + resultDetails.FailedExperiments
// setting chaosresult metrics for the given chaosresult

verdictValue := gaugeMetrics.unsetVerdictMetrics(resultDetails)
verdictValue := gaugeMetrics.unsetOutdatedMetrics(resultDetails)
gaugeMetrics.setResultChaosMetrics(resultDetails, verdictValue)
// setting chaosresult aws metrics for the given chaosresult, which can be used for cloudwatch
if awsConfig.Namespace != "" && awsConfig.ClusterName != "" && awsConfig.Service != "" {
Expand Down Expand Up @@ -158,36 +158,37 @@ func (gaugeMetrics *GaugeMetrics) setNamespacedChaosMetrics(namespacedScopeMetri

// setResultChaosMetrics sets metrics for the given chaosresult details
func (gaugeMetrics *GaugeMetrics) setResultChaosMetrics(resultDetails ChaosResultDetails, verdictValue float64) {
gaugeMetrics.ResultAwaitedExperiments.WithLabelValues(resultDetails.Namespace, resultDetails.Name, resultDetails.ChaosEngineName, resultDetails.ChaosEngineLabel).Set(resultDetails.AwaitedExperiments)
gaugeMetrics.ResultPassedExperiments.WithLabelValues(resultDetails.Namespace, resultDetails.Name, resultDetails.ChaosEngineName, resultDetails.ChaosEngineLabel).Set(resultDetails.PassedExperiments)
gaugeMetrics.ResultFailedExperiments.WithLabelValues(resultDetails.Namespace, resultDetails.Name, resultDetails.ChaosEngineName, resultDetails.ChaosEngineLabel).Set(resultDetails.FailedExperiments)
gaugeMetrics.ResultProbeSuccessPercentage.WithLabelValues(resultDetails.Namespace, resultDetails.Name, resultDetails.ChaosEngineName, resultDetails.ChaosEngineLabel).Set(resultDetails.ProbeSuccesPercentage)
gaugeMetrics.ResultAwaitedExperiments.WithLabelValues(resultDetails.Namespace, resultDetails.Name, resultDetails.ChaosEngineName, resultDetails.ChaosEngineContext, resultDetails.WorkflowName,
resultDetails.ChaosInjectLabel).Set(resultDetails.AwaitedExperiments)
gaugeMetrics.ResultPassedExperiments.WithLabelValues(resultDetails.Namespace, resultDetails.Name, resultDetails.ChaosEngineName, resultDetails.ChaosEngineContext).Set(resultDetails.PassedExperiments)
gaugeMetrics.ResultFailedExperiments.WithLabelValues(resultDetails.Namespace, resultDetails.Name, resultDetails.ChaosEngineName, resultDetails.ChaosEngineContext).Set(resultDetails.FailedExperiments)
gaugeMetrics.ResultProbeSuccessPercentage.WithLabelValues(resultDetails.Namespace, resultDetails.Name, resultDetails.ChaosEngineName, resultDetails.ChaosEngineContext).Set(resultDetails.ProbeSuccesPercentage)
switch strings.ToLower(resultDetails.Verdict) {
case "awaited":
gaugeMetrics.ResultVerdict.WithLabelValues(resultDetails.Namespace, resultDetails.Name, resultDetails.ChaosEngineName, resultDetails.ChaosEngineLabel, resultDetails.Verdict, fmt.Sprintf("%f", resultDetails.ProbeSuccesPercentage),
resultDetails.AppLabel, resultDetails.AppNs, resultDetails.AppKind).Set(float64(0))
gaugeMetrics.ResultVerdict.WithLabelValues(resultDetails.Namespace, resultDetails.Name, resultDetails.ChaosEngineName, resultDetails.ChaosEngineContext, resultDetails.Verdict, fmt.Sprintf("%f", resultDetails.ProbeSuccesPercentage),
resultDetails.AppLabel, resultDetails.AppNs, resultDetails.AppKind, resultDetails.WorkflowName, resultDetails.ChaosInjectLabel).Set(float64(0))
default:
gaugeMetrics.ResultVerdict.WithLabelValues(resultDetails.Namespace, resultDetails.Name, resultDetails.ChaosEngineName, resultDetails.ChaosEngineLabel, resultDetails.Verdict, fmt.Sprintf("%f", resultDetails.ProbeSuccesPercentage),
resultDetails.AppLabel, resultDetails.AppNs, resultDetails.AppKind).Set(verdictValue)
gaugeMetrics.ResultVerdict.WithLabelValues(resultDetails.Namespace, resultDetails.Name, resultDetails.ChaosEngineName, resultDetails.ChaosEngineContext, resultDetails.Verdict, fmt.Sprintf("%f", resultDetails.ProbeSuccesPercentage),
resultDetails.AppLabel, resultDetails.AppNs, resultDetails.AppKind, resultDetails.WorkflowName, resultDetails.ChaosInjectLabel).Set(verdictValue)
}
gaugeMetrics.ExperimentStartTime.WithLabelValues(resultDetails.Namespace, resultDetails.Name, resultDetails.ChaosEngineName, resultDetails.ChaosEngineLabel).Set(resultDetails.StartTime)
gaugeMetrics.ExperimentEndTime.WithLabelValues(resultDetails.Namespace, resultDetails.Name, resultDetails.ChaosEngineName, resultDetails.ChaosEngineLabel).Set(resultDetails.EndTime)
gaugeMetrics.ExperimentChaosInjectedTime.WithLabelValues(resultDetails.Namespace, resultDetails.Name, resultDetails.ChaosEngineName, resultDetails.ChaosEngineLabel).Set(resultDetails.InjectionTime)
gaugeMetrics.ExperimentTotalDuration.WithLabelValues(resultDetails.Namespace, resultDetails.Name, resultDetails.ChaosEngineName, resultDetails.ChaosEngineLabel).Set(resultDetails.TotalDuration)
gaugeMetrics.ExperimentStartTime.WithLabelValues(resultDetails.Namespace, resultDetails.Name, resultDetails.ChaosEngineName, resultDetails.ChaosEngineContext).Set(resultDetails.StartTime)
gaugeMetrics.ExperimentEndTime.WithLabelValues(resultDetails.Namespace, resultDetails.Name, resultDetails.ChaosEngineName, resultDetails.ChaosEngineContext).Set(resultDetails.EndTime)
gaugeMetrics.ExperimentChaosInjectedTime.WithLabelValues(resultDetails.Namespace, resultDetails.Name, resultDetails.ChaosEngineName, resultDetails.ChaosEngineContext).Set(float64(resultDetails.InjectionTime))
gaugeMetrics.ExperimentTotalDuration.WithLabelValues(resultDetails.Namespace, resultDetails.Name, resultDetails.ChaosEngineName, resultDetails.ChaosEngineContext).Set(resultDetails.TotalDuration)
}

// unsetResultChaosMetrics unset metrics for the given chaosresult details
func (gaugeMetrics *GaugeMetrics) unsetResultChaosMetrics(resultDetails *ChaosResultDetails) {
gaugeMetrics.ResultAwaitedExperiments.DeleteLabelValues(resultDetails.Namespace, resultDetails.Name, resultDetails.ChaosEngineName, resultDetails.ChaosEngineLabel)
gaugeMetrics.ResultPassedExperiments.DeleteLabelValues(resultDetails.Namespace, resultDetails.Name, resultDetails.ChaosEngineName, resultDetails.ChaosEngineLabel)
gaugeMetrics.ResultFailedExperiments.DeleteLabelValues(resultDetails.Namespace, resultDetails.Name, resultDetails.ChaosEngineName, resultDetails.ChaosEngineLabel)
gaugeMetrics.ResultProbeSuccessPercentage.DeleteLabelValues(resultDetails.Namespace, resultDetails.Name, resultDetails.ChaosEngineName, resultDetails.ChaosEngineLabel)
gaugeMetrics.ResultVerdict.DeleteLabelValues(resultDetails.Namespace, resultDetails.Name, resultDetails.ChaosEngineName, resultDetails.ChaosEngineLabel, resultDetails.Verdict,
fmt.Sprintf("%f", resultDetails.ProbeSuccesPercentage), resultDetails.AppLabel, resultDetails.AppNs, resultDetails.AppKind)
gaugeMetrics.ExperimentStartTime.DeleteLabelValues(resultDetails.Namespace, resultDetails.Name, resultDetails.ChaosEngineName, resultDetails.ChaosEngineLabel)
gaugeMetrics.ExperimentEndTime.DeleteLabelValues(resultDetails.Namespace, resultDetails.Name, resultDetails.ChaosEngineName, resultDetails.ChaosEngineLabel)
gaugeMetrics.ExperimentChaosInjectedTime.DeleteLabelValues(resultDetails.Namespace, resultDetails.Name, resultDetails.ChaosEngineName, resultDetails.ChaosEngineLabel)
gaugeMetrics.ExperimentTotalDuration.DeleteLabelValues(resultDetails.Namespace, resultDetails.Name, resultDetails.ChaosEngineName, resultDetails.ChaosEngineLabel)
gaugeMetrics.ResultAwaitedExperiments.DeleteLabelValues(resultDetails.Namespace, resultDetails.Name, resultDetails.ChaosEngineName, resultDetails.ChaosEngineContext, resultDetails.WorkflowName, resultDetails.ChaosInjectLabel)
gaugeMetrics.ResultPassedExperiments.DeleteLabelValues(resultDetails.Namespace, resultDetails.Name, resultDetails.ChaosEngineName, resultDetails.ChaosEngineContext)
gaugeMetrics.ResultFailedExperiments.DeleteLabelValues(resultDetails.Namespace, resultDetails.Name, resultDetails.ChaosEngineName, resultDetails.ChaosEngineContext)
gaugeMetrics.ResultProbeSuccessPercentage.DeleteLabelValues(resultDetails.Namespace, resultDetails.Name, resultDetails.ChaosEngineName, resultDetails.ChaosEngineContext)
gaugeMetrics.ResultVerdict.DeleteLabelValues(resultDetails.Namespace, resultDetails.Name, resultDetails.ChaosEngineName, resultDetails.ChaosEngineContext, resultDetails.Verdict,
fmt.Sprintf("%f", resultDetails.ProbeSuccesPercentage), resultDetails.AppLabel, resultDetails.AppNs, resultDetails.AppKind, resultDetails.WorkflowName, resultDetails.ChaosInjectLabel)
gaugeMetrics.ExperimentStartTime.DeleteLabelValues(resultDetails.Namespace, resultDetails.Name, resultDetails.ChaosEngineName, resultDetails.ChaosEngineContext)
gaugeMetrics.ExperimentEndTime.DeleteLabelValues(resultDetails.Namespace, resultDetails.Name, resultDetails.ChaosEngineName, resultDetails.ChaosEngineContext)
gaugeMetrics.ExperimentChaosInjectedTime.DeleteLabelValues(resultDetails.Namespace, resultDetails.Name, resultDetails.ChaosEngineName, resultDetails.ChaosEngineContext)
gaugeMetrics.ExperimentTotalDuration.DeleteLabelValues(resultDetails.Namespace, resultDetails.Name, resultDetails.ChaosEngineName, resultDetails.ChaosEngineContext)
}

// setAwsResultChaosMetrics sets aws metrics for the given chaosresult
Expand All @@ -201,7 +202,7 @@ func (awsConfig *AWSConfig) setAwsResultChaosMetrics(resultDetails ChaosResultDe
awsConfig.putAwsMetricData(sess, "chaosresult_probe_success_percentage", "Count", resultDetails.ProbeSuccesPercentage)
awsConfig.putAwsMetricData(sess, "chaosresult_start_time", "Count", resultDetails.StartTime)
awsConfig.putAwsMetricData(sess, "chaosresult_end_time", "Count", resultDetails.EndTime)
awsConfig.putAwsMetricData(sess, "chaosresult_inject_time", "Count", resultDetails.InjectionTime)
awsConfig.putAwsMetricData(sess, "chaosresult_inject_time", "Count", float64(resultDetails.InjectionTime))
awsConfig.putAwsMetricData(sess, "chaosresult_total_duration", "Count", resultDetails.TotalDuration)
}

Expand Down
21 changes: 13 additions & 8 deletions controller/types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,15 +23,18 @@ import (

//EngineLabelKey is key for ChaosEngineLabel
var (
EngineLabelKey = "context"
resultStore = map[string][]ResultData{}
matchVerdict = map[string]*ResultData{}
EngineContext = "context"
WorkFlowName = "workflow_name"
resultStore = map[string][]ResultData{}
matchVerdict = map[string]*ResultData{}
)

// ResultData contains attributes to store metrics parameters
// which can be used while handaling chaosresult deletion
type ResultData struct {
Label string
ChaosEngineContext string
WorkFlowName string
ChaosInjectLabel string
AppKind string
AppNs string
AppLabel string
Expand All @@ -54,11 +57,13 @@ type ChaosResultDetails struct {
ProbeSuccesPercentage float64
StartTime float64
EndTime float64
InjectionTime float64
InjectionTime int64
TotalDuration float64
ChaosEngineName string
ChaosEngineLabel string
ChaosEngineContext string
Verdict string
WorkflowName string
ChaosInjectLabel string
}

// NamespacedScopeMetrics contains metrics for the chaos namespace
Expand Down Expand Up @@ -103,7 +108,7 @@ func (gaugeMetrics *GaugeMetrics) InitializeGaugeMetrics() *GaugeMetrics {
Name: "awaited_experiments",
Help: "Total number of awaited experiments",
},
[]string{"chaosresult_namespace", "chaosresult_name", "chaosengine_name", "chaosengine_context"},
[]string{"chaosresult_namespace", "chaosresult_name", "chaosengine_name", "chaosengine_context", "workflow_name", "chaos_injection_time"},
)

gaugeMetrics.ResultProbeSuccessPercentage = prometheus.NewGaugeVec(prometheus.GaugeOpts{
Expand All @@ -122,7 +127,7 @@ func (gaugeMetrics *GaugeMetrics) InitializeGaugeMetrics() *GaugeMetrics {
Help: "Verdict of the experiments",
},
[]string{"chaosresult_namespace", "chaosresult_name", "chaosengine_name", "chaosengine_context", "chaosresult_verdict",
"probe_success_percentage", "app_label", "app_namespace", "app_kind"},
"probe_success_percentage", "app_label", "app_namespace", "app_kind", "workflow_name", "chaos_injection_time"},
)

gaugeMetrics.ExperimentStartTime = prometheus.NewGaugeVec(prometheus.GaugeOpts{
Expand Down
Loading

0 comments on commit 5ab0af6

Please sign in to comment.