Add gossip integration test for renewing expired cert

-Test renew cert before expiration (passes)
-Test renew cert after expiration (previously failed)
-Also moves ExpireCertificate to nwo so that it can be used across integration tests

Signed-off-by: David Enyeart <[email protected]>

integration/gossip/gossip_test.go

@@ -285,8 +285,97 @@ var _ = Describe("Gossip State Transfer and Membership", func() {
 			assertPeerMembershipUpdate(network, peer1Org1, []*nwo.Peer{peer0Org2, peer1Org2}, nwprocs, expectedMsgFromExpirationCallback)
 		})
 	})
+
+	It("updates membership for a peer with a renewed certificate", func() {
+
+		network.Bootstrap()
+		orderer := network.Orderer("orderer")
+		nwprocs.ordererRunner = network.OrdererRunner(orderer)
+		nwprocs.ordererProcess = ifrit.Invoke(nwprocs.ordererRunner)
+		Eventually(nwprocs.ordererProcess.Ready(), network.EventuallyTimeout).Should(BeClosed())
+
+		peer0Org1 := network.Peer("Org1", "peer0")
+		peer0Org2 := network.Peer("Org2", "peer0")
+
+		By("bringing up a peer in each organization")
+		startPeers(nwprocs, false, peer0Org1, peer0Org2)
+
+		channelparticipation.JoinOrdererAppChannel(network, "testchannel", orderer, nwprocs.ordererRunner)
+
+		By("joining peers to channel")
+		network.JoinChannel(channelName, orderer, peer0Org1, peer0Org2)
+
+		By("verifying membership of both peers")
+		bothPeers := []*nwo.Peer{peer0Org1, peer0Org2}
+		network.VerifyMembership(bothPeers, channelName)
+
+		fmt.Println("===MEMBERSHIP VERIFIED===")
+		time.Sleep(5 * time.Second)
+
+		By("stopping, renewing peer0Org2 certificate before expiration, and restarting")
+		stopPeers(nwprocs, peer0Org2)
+		renewPeerCertificate(network, peer0Org2, time.Now().Add(time.Minute))
+
+		fmt.Println("===STOPPED AND RENEWED peer0Org2, RESTARTING, CHECK FOR PKI-ID REPLACEMENT===")
+		time.Sleep(5 * time.Second)
+
+		startPeers(nwprocs, false, peer0Org2)
+
+		By("ensuring that peer0Org1 replaces peer0Org2 PKI-ID")
+		peer0Org1Runner := nwprocs.peerRunners[peer0Org1.ID()]
+		Eventually(peer0Org1Runner.Err(), network.EventuallyTimeout).Should(gbytes.Say("changed its PKI-ID from"))
+
+		fmt.Println("===PKI-ID REPLACED, WAIT FOR MEMBERSHIP===")
+		time.Sleep(5 * time.Second)
+
+		By("verifying membership after cert renewed")
+		//bothPeers = []*nwo.Peer{peer0Org1, peer0Org2}
+		network.VerifyMembership(bothPeers, channelName)
+
+		fmt.Println("===MEMBERSHIP VERIFIED WITH RENEWED CERT, WAIT FOR CERT TO EXPIRE AND THEN RENEW AGAIN ===")
+		time.Sleep(5 * time.Second)
+
+		By("waiting for cert to expire within a minute")
+		Eventually(peer0Org1Runner.Err(), network.EventuallyTimeout).Should(gbytes.Say("gossipping peer identity expired"))
+
+		By("stopping, renewing peer0Org2 certificate again after its expiration, restarting")
+		stopPeers(nwprocs, peer0Org2)
+		renewPeerCertificate(network, peer0Org2, time.Now().Add(time.Hour))
+
+		fmt.Println("===STOPPED AND RENEWED peer0Org2 AGAIN AFTER EXPIRATION, RESTARTING, CHECK FOR PKI-ID REPLACEMENT AGAIN===")
+		time.Sleep(5 * time.Second)
+
+		startPeers(nwprocs, false, peer0Org2)
+
+		By("ensuring that peer0Org1 replaces peer0Org2 PKI-ID after it expired")
+		Eventually(peer0Org1Runner.Err(), network.EventuallyTimeout).Should(gbytes.Say("changed its PKI-ID from"))
+
+	})
 })
 
+// renewPeerCertificate renews the certificate with a given expirationTime and re-writes it to the peer's signcert directory
+func renewPeerCertificate(network *nwo.Network, peer *nwo.Peer, expirationTime time.Time) {
+
+	peerDomain := network.Organization(peer.Organization).Domain
+
+	peerCAKeyPath := filepath.Join(network.RootDir, "crypto", "peerOrganizations", peerDomain, "ca", "priv_sk")
+	peerCAKey, err := os.ReadFile(peerCAKeyPath)
+	Expect(err).NotTo(HaveOccurred())
+
+	peerCACertPath := filepath.Join(network.RootDir, "crypto", "peerOrganizations", peerDomain, "ca", fmt.Sprintf("ca.%s-cert.pem", peerDomain))
+	peerCACert, err := os.ReadFile(peerCACertPath)
+	Expect(err).NotTo(HaveOccurred())
+
+	peerCertPath := filepath.Join(network.PeerLocalMSPDir(peer), "signcerts", fmt.Sprintf("peer0.%s-cert.pem", peerDomain))
+	peerCert, err := os.ReadFile(peerCertPath)
+	Expect(err).NotTo(HaveOccurred())
+
+	renewedCert, _ := nwo.ExpireCertificate(peerCert, peerCACert, peerCAKey, expirationTime)
+	err = os.WriteFile(peerCertPath, renewedCert, 0o600)
+	fmt.Println(peerCertPath)
+	Expect(err).NotTo(HaveOccurred())
+}
+
 func runTransactions(n *nwo.Network, orderer *nwo.Orderer, peer *nwo.Peer, chaincodeName string, channelID string) {
 	for i := 0; i < 5; i++ {
 		sess, err := n.PeerUserSession(peer, "User1", commands.ChaincodeInvoke{

integration/nwo/network.go

@@ -1211,6 +1211,7 @@ func (n *Network) ConfigTxGen(command Command) (*gexec.Session, error) {
 func (n *Network) Discover(command Command) (*gexec.Session, error) {
 	cmd := NewCommand(n.Components.Discover(), command)
 	cmd.Args = append(cmd.Args, "--peerTLSCA", n.CACertsBundlePath())
+	cmd.Env = []string{"FABRIC_LOGGING_SPEC=info:grpc=warn"} // suppress chatty grpc info messages
 	return n.StartSession(cmd, command.SessionName())
 }
 

integration/nwo/signingid.go

@@ -15,9 +15,11 @@ import (
 	"encoding/pem"
 	"fmt"
 	"os"
+	"time"
 
 	"github.com/hyperledger/fabric-lib-go/bccsp/utils"
 	"github.com/hyperledger/fabric-protos-go-apiv2/msp"
+	. "github.com/onsi/gomega"
 	"google.golang.org/protobuf/proto"
 )
 
@@ -74,3 +76,40 @@ func (s *SigningIdentity) Sign(msg []byte) ([]byte, error) {
 		return nil, fmt.Errorf("unexpected key type: %T", key)
 	}
 }
+
+// ExpireCertificate re-creates and re-signs a certificate with a new expirationTime
+func ExpireCertificate(certPEM, caCertPEM, caKeyPEM []byte, expirationTime time.Time) (expiredcertPEM []byte, earlyMadeCACertPEM []byte) {
+	keyAsDER, _ := pem.Decode(caKeyPEM)
+	caKeyWithoutType, err := x509.ParsePKCS8PrivateKey(keyAsDER.Bytes)
+	Expect(err).NotTo(HaveOccurred())
+	caKey := caKeyWithoutType.(*ecdsa.PrivateKey)
+
+	caCertAsDER, _ := pem.Decode(caCertPEM)
+	caCert, err := x509.ParseCertificate(caCertAsDER.Bytes)
+	Expect(err).NotTo(HaveOccurred())
+
+	certAsDER, _ := pem.Decode(certPEM)
+	cert, err := x509.ParseCertificate(certAsDER.Bytes)
+	Expect(err).NotTo(HaveOccurred())
+
+	cert.Raw = nil
+	caCert.Raw = nil
+	// The certificate was made 1 minute ago (1 hour doesn't work since cert will be before original CA cert NotBefore time)
+	cert.NotBefore = time.Now().Add((-1) * time.Minute)
+	// As well as the CA certificate
+	caCert.NotBefore = time.Now().Add((-1) * time.Minute)
+	// The certificate expires now
+	cert.NotAfter = expirationTime
+
+	// The CA signs the certificate
+	certBytes, err := x509.CreateCertificate(rand.Reader, cert, caCert, cert.PublicKey, caKey)
+	Expect(err).NotTo(HaveOccurred())
+
+	// The CA signs its own certificate
+	caCertBytes, err := x509.CreateCertificate(rand.Reader, caCert, caCert, caCert.PublicKey, caKey)
+	Expect(err).NotTo(HaveOccurred())
+
+	expiredcertPEM = pem.EncodeToMemory(&pem.Block{Bytes: certBytes, Type: "CERTIFICATE"})
+	earlyMadeCACertPEM = pem.EncodeToMemory(&pem.Block{Bytes: caCertBytes, Type: "CERTIFICATE"})
+	return
+}

integration/raft/cft_test.go

@@ -8,10 +8,6 @@ package raft
 
 import (
 	"context"
-	"crypto/ecdsa"
-	"crypto/rand"
-	"crypto/x509"
-	"encoding/pem"
 	"fmt"
 	"os"
 	"path"
@@ -647,7 +643,7 @@ var _ = Describe("EndToEnd Crash Fault Tolerance", func() {
 
 			By("Expiring orderer TLS certificates")
 			for filePath, certPEM := range serverTLSCerts {
-				expiredCert, earlyMadeCACert := expireCertificate(certPEM, ordererTLSCACert, ordererTLSCAKey, time.Now())
+				expiredCert, earlyMadeCACert := nwo.ExpireCertificate(certPEM, ordererTLSCACert, ordererTLSCAKey, time.Now())
 				err = os.WriteFile(filePath, expiredCert, 0o600)
 				Expect(err).NotTo(HaveOccurred())
 
@@ -974,7 +970,7 @@ var _ = Describe("EndToEnd Crash Fault Tolerance", func() {
 			originalAdminCert, err := os.ReadFile(adminCertPath)
 			Expect(err).NotTo(HaveOccurred())
 
-			expiredAdminCert, earlyCACert := expireCertificate(originalAdminCert, ordererCACert, ordererCAKey, time.Now())
+			expiredAdminCert, earlyCACert := nwo.ExpireCertificate(originalAdminCert, ordererCACert, ordererCAKey, time.Now())
 			err = os.WriteFile(adminCertPath, expiredAdminCert, 0o600)
 			Expect(err).NotTo(HaveOccurred())
 
@@ -1161,48 +1157,12 @@ func renewOrdererCertificates(network *nwo.Network, orderers ...*nwo.Orderer) {
 	}
 
 	for filePath, certPEM := range serverTLSCerts {
-		renewedCert, _ := expireCertificate(certPEM, ordererTLSCACert, ordererTLSCAKey, time.Now().Add(time.Hour))
+		renewedCert, _ := nwo.ExpireCertificate(certPEM, ordererTLSCACert, ordererTLSCAKey, time.Now().Add(time.Hour))
 		err = os.WriteFile(filePath, renewedCert, 0o600)
 		Expect(err).NotTo(HaveOccurred())
 	}
 }
 
-func expireCertificate(certPEM, caCertPEM, caKeyPEM []byte, expirationTime time.Time) (expiredcertPEM []byte, earlyMadeCACertPEM []byte) {
-	keyAsDER, _ := pem.Decode(caKeyPEM)
-	caKeyWithoutType, err := x509.ParsePKCS8PrivateKey(keyAsDER.Bytes)
-	Expect(err).NotTo(HaveOccurred())
-	caKey := caKeyWithoutType.(*ecdsa.PrivateKey)
-
-	caCertAsDER, _ := pem.Decode(caCertPEM)
-	caCert, err := x509.ParseCertificate(caCertAsDER.Bytes)
-	Expect(err).NotTo(HaveOccurred())
-
-	certAsDER, _ := pem.Decode(certPEM)
-	cert, err := x509.ParseCertificate(certAsDER.Bytes)
-	Expect(err).NotTo(HaveOccurred())
-
-	cert.Raw = nil
-	caCert.Raw = nil
-	// The certificate was made 1 hour ago
-	cert.NotBefore = time.Now().Add((-1) * time.Hour)
-	// As well as the CA certificate
-	caCert.NotBefore = time.Now().Add((-1) * time.Hour)
-	// The certificate expires now
-	cert.NotAfter = expirationTime
-
-	// The CA signs the certificate
-	certBytes, err := x509.CreateCertificate(rand.Reader, cert, caCert, cert.PublicKey, caKey)
-	Expect(err).NotTo(HaveOccurred())
-
-	// The CA signs its own certificate
-	caCertBytes, err := x509.CreateCertificate(rand.Reader, caCert, caCert, caCert.PublicKey, caKey)
-	Expect(err).NotTo(HaveOccurred())
-
-	expiredcertPEM = pem.EncodeToMemory(&pem.Block{Bytes: certBytes, Type: "CERTIFICATE"})
-	earlyMadeCACertPEM = pem.EncodeToMemory(&pem.Block{Bytes: caCertBytes, Type: "CERTIFICATE"})
-	return
-}
-
 func createConfigTx(txData []byte, channelName string, network *nwo.Network, orderer *nwo.Orderer, peer *nwo.Peer) *common.Envelope {
 	ctxEnv, err := protoutil.UnmarshalEnvelope(txData)
 	Expect(err).NotTo(HaveOccurred())