Don't try to take 2 snapshots at a time

Fix #943. Close #1081

CHANGELOG.md

@@ -7,6 +7,8 @@
 
 ### Bugfixes
 
+- [Issue #943](https://github.com/influxdb/influxdb/issues/943). Don't
+  take two snapshots at the same time
 - [Issue #1085](https://github.com/influxdb/influxdb/issues/1085). Set
   the connection string of the local raft node
 - [Issue #996](https://github.com/influxdb/influxdb/issues/996). Fill should

_vendor/raft/server.go

@@ -1203,6 +1203,7 @@ func (s *server) TakeSnapshot() error {
 
 	state, err := s.stateMachine.Save()
 	if err != nil {
+		s.pendingSnapshot = nil
 		return err
 	}
 
@@ -1237,6 +1238,7 @@ func (s *server) saveSnapshot() error {
 
 	// Write snapshot to disk.
 	if err := s.pendingSnapshot.save(); err != nil {
+		s.pendingSnapshot = nil
 		return err
 	}
 

_vendor/raft/snapshot_test.go

@@ -1,6 +1,7 @@
 package raft
 
 import (
+	"errors"
 	"fmt"
 	"io/ioutil"
 	"os"
@@ -121,6 +122,18 @@ func TestSnapshotRequest(t *testing.T) {
 	})
 }
 
+func TestTakeSnapshotWhenStateMachineSaveFails(t *testing.T) {
+	runServerWithMockStateMachine(Leader, func(s Server, m *mock.Mock) {
+		m.On("Save").Return([]byte("foo"), errors.New("test error message"))
+
+		s.Do(&testCommand1{})
+		err := s.TakeSnapshot()
+		assert.Equal(t, err.Error(), "test error message")
+		assert.Nil(t, s.(*server).pendingSnapshot)
+		s.Stop()
+	})
+}
+
 func runServerWithMockStateMachine(state string, fn func(s Server, m *mock.Mock)) {
 	var m mockStateMachine
 	s := newTestServer("1", &testTransporter{})

coordinator/raft_server.go

@@ -330,6 +330,8 @@ const (
 )
 
 func (s *RaftServer) ForceLogCompaction() error {
+	s.mutex.Lock()
+	defer s.mutex.Unlock()
 	err := s.raftServer.TakeSnapshot()
 	if err != nil {
 		log.Error("Cannot take snapshot: %s", err)