[YUNIKORN-2908] Remove associated metrics when queue is removed (#987)

Closes: #987

Signed-off-by: Craig Condit <[email protected]>

pkg/metrics/init.go

@@ -84,6 +84,15 @@ func GetQueueMetrics(name string) *QueueMetrics {
 	return queueMetrics
 }
 
+func RemoveQueueMetrics(name string) {
+	m.lock.Lock()
+	defer m.lock.Unlock()
+	if metrics, ok := m.queues[name]; ok {
+		metrics.UnregisterMetrics()
+		delete(m.queues, name)
+	}
+}
+
 func GetEventMetrics() *EventMetrics {
 	return m.event
 }

pkg/metrics/queue.go

@@ -132,6 +132,21 @@ func InitQueueMetrics(name string) *QueueMetrics {
 	return q
 }
 
+func (m *QueueMetrics) UnregisterMetrics() {
+	var queueMetricsList = []prometheus.Collector{
+		m.appMetricsLabel,
+		m.appMetricsSubsystem,
+		m.containerMetrics,
+		m.resourceMetricsLabel,
+		m.resourceMetricsSubsystem,
+	}
+
+	// Unregister the metrics
+	for _, metric := range queueMetricsList {
+		prometheus.Unregister(metric)
+	}
+}
+
 func (m *QueueMetrics) incQueueApplications(state string) {
 	m.appMetricsLabel.WithLabelValues(state).Inc()
 	m.appMetricsSubsystem.WithLabelValues(state).Inc()

pkg/metrics/queue_test.go

@@ -246,6 +246,23 @@ func TestQueuePreemptingResourceMetrics(t *testing.T) {
 	verifyResourceMetrics(t, "preempting", "cpu")
 }
 
+func TestRemoveQueueMetrics(t *testing.T) {
+	testQueueName := "root.test"
+	qm = GetQueueMetrics(testQueueName)
+	defer unregisterQueueMetrics()
+
+	qm.SetQueueAllocatedResourceMetrics("cpu", 1)
+	assert.Assert(t, m.queues[testQueueName] != nil)
+	RemoveQueueMetrics(testQueueName)
+	assert.Assert(t, m.queues[testQueueName] == nil)
+	metrics, err := prometheus.DefaultGatherer.Gather()
+	assert.NilError(t, err)
+	for _, metric := range metrics {
+		assert.Assert(t, metric.GetName() != "yunikorn_queue_resource",
+			"Queue metrics are not cleaned up")
+	}
+}
+
 func getQueueMetrics() *QueueMetrics {
 	return InitQueueMetrics("root.test")
 }

pkg/scheduler/objects/queue.go

@@ -986,6 +986,7 @@ func (sq *Queue) RemoveQueue() bool {
 		return false
 	}
 	log.Log(log.SchedQueue).Info("removing queue", zap.String("queue", sq.QueuePath))
+	sq.removeMetrics()
 	// root is always managed and is the only queue with a nil parent: no need to guard
 	sq.parent.removeChildQueue(sq.Name)
 	sq.queueEvents.SendRemoveQueueEvent(sq.QueuePath, sq.isManaged)
@@ -1696,6 +1697,10 @@ func (sq *Queue) updatePreemptingResourceMetrics() {
 	}
 }
 
+func (sq *Queue) removeMetrics() {
+	metrics.RemoveQueueMetrics(sq.QueuePath)
+}
+
 func (sq *Queue) String() string {
 	sq.RLock()
 	defer sq.RUnlock()