Merge pull request #205 from chaudhryfaisal/master

adding option to configure StoreExpireTickInterval and Fixed bug where `Store.Expire()` was only invoked once

cmd/mtail/main.go

@@ -53,8 +53,9 @@ var (
 	emitProgLabel        = flag.Bool("emit_prog_label", true, "Emit the 'prog' label in variable exports.")
 
 	// Ops flags
-	pollInterval    = flag.Duration("poll_interval", 0, "Set the interval to poll all log files for data; must be positive, or zero to disable polling.  With polling mode, only the files found at mtail startup will be polled.")
-	disableFsnotify = flag.Bool("disable_fsnotify", false, "When enabled no fsnotify watcher is created, and mtail falls back to polling mode only.  Only the files known at program startup will be polled.")
+	pollInterval            = flag.Duration("poll_interval", 0, "Set the interval to poll all log files for data; must be positive, or zero to disable polling.  With polling mode, only the files found at mtail startup will be polled.")
+	disableFsnotify         = flag.Bool("disable_fsnotify", false, "When enabled no fsnotify watcher is created, and mtail falls back to polling mode only.  Only the files known at program startup will be polled.")
+	storeExpireTickInterval = flag.Duration("store_expire_tick_interval", time.Hour, "interval to delete expired metrics from store")
 
 	// Debugging flags
 	blockProfileRate     = flag.Int("block_profile_rate", 0, "Nanoseconds of block time before goroutine blocking events reported. 0 turns off.  See https://golang.org/pkg/runtime/#SetBlockProfileRate")
@@ -121,6 +122,7 @@ func main() {
 		mtail.BindAddress(*address, *port),
 		mtail.BuildInfo(buildInfo()),
 		mtail.OverrideLocation(loc),
+		mtail.StoreExpireTickInterval(*storeExpireTickInterval),
 	}
 	if *oneShot {
 		opts = append(opts, mtail.OneShot)

internal/metrics/store.go

@@ -102,6 +102,7 @@ func (s *Store) MarshalJSON() (b []byte, err error) {
 // Expire iterates through the Store looking for metrics that have been marked
 // for expiry, and removing them if their expiration time has passed.
 func (s *Store) Expire() error {
+	glog.Info("Running Store.Expire()")
 	s.Lock()
 	defer s.Unlock()
 	now := time.Now()
@@ -124,14 +125,21 @@ func (s *Store) Expire() error {
 }
 
 // StartExpiryLoop runs a permanent goroutine to expire metrics every hour.
-func (s *Store) StartExpiryLoop() {
+func (s *Store) StartExpiryLoop(duration time.Duration) {
 	go func() {
-		ticker := time.NewTicker(time.Hour)
-		select {
-		case <-ticker.C:
-			err := s.Expire()
-			if err != nil {
-				glog.Info(err)
+if duration <= 0 {
+glog.Infof("Metric store expiration disabled")
+return
+}
+glog.Infof("Configuring StartExpiryLoop with %s", duration.String())
+		ticker := time.NewTicker(duration)
+		for {
+			select {
+			case <-ticker.C:
+				err := s.Expire()
+				if err != nil {
+					glog.Info(err)
+				}
 			}
 		}
 	}()

internal/mtail/mtail.go

@@ -58,9 +58,10 @@ type Server struct {
 	dumpAstTypes bool // if set, mtail prints the program syntax tree after type checking
 	dumpBytecode bool // if set, mtail prints the program bytecode after code generation
 
-	syslogUseCurrentYear bool // if set, use the current year for timestamps that have no year information
-	omitMetricSource     bool // if set, do not link the source program to a metric
-	omitProgLabel        bool // if set, do not put the program name in the metric labels
+	syslogUseCurrentYear    bool          // if set, use the current year for timestamps that have no year information
+	omitMetricSource        bool          // if set, do not link the source program to a metric
+	omitProgLabel           bool          // if set, do not put the program name in the metric labels
+	storeExpireTickInterval time.Duration // interval to expire/delete metrics from store
 }
 
 // StartTailing adds each log path pattern to the tailer.
@@ -344,7 +345,7 @@ func (m *Server) Run() error {
 			return err
 		}
 	} else {
-		m.store.StartExpiryLoop()
+		m.store.StartExpiryLoop(m.storeExpireTickInterval)
 		if err := m.Serve(); err != nil {
 			return err
 		}

internal/mtail/options.go

@@ -50,6 +50,14 @@ func OverrideLocation(loc *time.Location) func(*Server) error {
 	}
 }
 
+// StoreExpireTickInterval sets the interval to run ticker to delete expired metrics from store.
+func StoreExpireTickInterval(interval time.Duration) func(*Server) error {
+	return func(m *Server) error {
+		m.storeExpireTickInterval = interval
+		return nil
+	}
+}
+
 // OneShot sets one-shot mode in the MtailServer.
 func OneShot(m *Server) error {
 	m.oneShot = true