Use nats for pubsub instead of waku

xmtp · Aug 31, 2023 · 2d5ccc6 · 2d5ccc6
1 parent deb9ada
commit 2d5ccc6
Show file tree

Hide file tree

Showing 49 changed files with 675 additions and 5,584 deletions.
diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml
@@ -22,5 +22,4 @@ jobs:
       - name: golangci-lint
         uses: golangci/golangci-lint-action@v3
         with:
-          version: v1.45.2
           args: --timeout=5m --config dev/.golangci.yaml
diff --git a/cmd/xmtpd-e2e/main.go b/cmd/xmtpd-e2e/main.go
@@ -4,16 +4,14 @@ import (
 	"context"
 	"os"
 	"strconv"
-	"strings"
 
 	"github.com/xmtp/xmtp-node-go/pkg/e2e"
+	"github.com/xmtp/xmtp-node-go/pkg/logging"
 	"go.uber.org/zap"
 )
 
 const (
 	localNetworkEnv = "local"
-	localNodesURL   = "http://localhost:8000"
-	remoteNodesURL  = "http://nodes.xmtp.com"
 )
 
 var (
@@ -27,26 +25,20 @@ var (
 )
 
 func main() {
-	ctx := context.Background()
 	log, err := zap.NewProduction()
 	if err != nil {
 		panic(err)
 	}
+	ctx := logging.With(context.Background(), log)
 
 	networkEnv := envVar("XMTPD_E2E_ENV", localNetworkEnv)
-	nodesURL := envVar("XMTPD_E2E_NODES_URL", localNodesURL)
-	if networkEnv != localNetworkEnv {
-		nodesURL = remoteNodesURL
-	}
 
 	apiURL := envVar("XMTPD_E2E_API_URL", remoteAPIURLByEnv[networkEnv])
 
 	runner := e2e.NewRunner(ctx, log, &e2e.Config{
 		Continuous:              envVarBool("E2E_CONTINUOUS"),
 		ContinuousExitOnError:   envVarBool("E2E_CONTINUOUS_EXIT_ON_ERROR"),
 		NetworkEnv:              networkEnv,
-		BootstrapAddrs:          envVarStrings("XMTPD_E2E_BOOTSTRAP_ADDRS"),
-		NodesURL:                nodesURL,
 		APIURL:                  apiURL,
 		DelayBetweenRunsSeconds: envVarInt("XMTPD_E2E_DELAY", 5),
 		GitCommit:               GitCommit,
@@ -66,19 +58,6 @@ func envVar(name, defaultVal string) string {
 	return val
 }
 
-func envVarStrings(name string) []string {
-	val := os.Getenv(name)
-	vals := strings.Split(val, ",")
-	retVals := make([]string, 0, len(vals))
-	for _, v := range vals {
-		if v == "" {
-			continue
-		}
-		retVals = append(retVals, v)
-	}
-	return retVals
-}
-
 func envVarBool(name string) bool {
 	valStr := os.Getenv(name)
 	return valStr != ""

diff --git a/cmd/xmtpd/main.go b/cmd/xmtpd/main.go
@@ -10,10 +10,7 @@ import (
 	"sync"
 	"syscall"
 
-	golog "github.com/ipfs/go-log"
 	"github.com/jessevdk/go-flags"
-	"github.com/pkg/errors"
-	"github.com/status-im/go-waku/waku/v2/utils"
 	"github.com/xmtp/xmtp-node-go/pkg/server"
 	"github.com/xmtp/xmtp-node-go/pkg/tracing"
 	"go.uber.org/zap"
@@ -68,24 +65,11 @@ func main() {
 		}()
 	}
 
-	cleanup, err := initWakuLogging(options)
-	if err != nil {
-		log.Fatal("initializing waku logger", zap.Error(err))
-	}
-	defer cleanup()
-
 	if options.Version {
 		fmt.Printf("Version: %s", Commit)
 		return
 	}
 
-	if options.GenerateKey {
-		if err := server.WritePrivateKeyToFile(options.KeyFile, options.Overwrite); err != nil {
-			log.Fatal("writing private key file", zap.Error(err))
-		}
-		return
-	}
-
 	if options.CreateMessageMigration != "" && options.Store.DbConnectionString != "" {
 		if err := server.CreateMessageMigration(options.CreateMessageMigration, options.Store.DbConnectionString, options.WaitForDB, options.Store.ReadTimeout, options.Store.WriteTimeout, options.Store.MaxOpenConns); err != nil {
 			log.Fatal("creating message db migration", zap.Error(err))
@@ -102,7 +86,7 @@ func main() {
 
 	if options.Tracing.Enable {
 		log.Info("starting tracer")
-		tracing.Start(Commit, utils.Logger())
+		tracing.Start(Commit, log)
 		defer func() {
 			log.Info("stopping tracer")
 			tracing.Stop()
@@ -171,28 +155,6 @@ func fatal(msg string, args ...any) {
 	log.Fatalf(msg, args...)
 }
 
-func initWakuLogging(options server.Options) (func(), error) {
-	err := utils.SetLogLevel(options.LogLevel)
-	if err != nil {
-		return nil, errors.Wrap(err, "parsing log level")
-	}
-	utils.InitLogger(options.LogEncoding)
-
-	lvl, err := golog.LevelFromString(options.LogLevel)
-	if err != nil {
-		return nil, errors.Wrap(err, "parsing log level")
-	}
-	golog.SetAllLoggers(lvl)
-
-	// Note that libp2p reads the encoding from GOLOG_LOG_FMT env var.
-	if options.LogEncoding == "json" && os.Getenv("GOLOG_LOG_FMT") == "" {
-		utils.Logger().Warn("Set GOLOG_LOG_FMT=json to use json for libp2p logs")
-	}
-
-	cleanup := func() { _ = utils.Logger().Sync() }
-	return cleanup, nil
-}
-
 func buildLogger(options server.Options) (*zap.Logger, error) {
 	atom := zap.NewAtomicLevel()
 	level := zapcore.InfoLevel

diff --git a/dev/docker/Dockerfile b/dev/docker/Dockerfile
@@ -26,9 +26,6 @@ LABEL commit=$GIT_COMMIT
 # color, nocolor, json
 ENV GOLOG_LOG_FMT=nocolor
 
-# go-waku default port
-EXPOSE 9000
-
 COPY --from=builder /app/bin/xmtpd /usr/bin/
 
 ENTRYPOINT ["/usr/bin/xmtpd"]

diff --git a/dev/docker/docker-compose.yml b/dev/docker/docker-compose.yml
@@ -17,3 +17,30 @@ services:
       - 9090:9090
     volumes:
       - ./prometheus.yml:/etc/prometheus/prometheus.yml
+  nats-1:
+    image: nats
+    ports:
+      - 14222:4222
+      - 18222:8222
+    command: "--cluster_name=NATS --cluster=nats://0.0.0.0:6222 --http_port=8222 --routes=nats://nats-1:6222,nats://nats-2:6222,nats://nats-3:6222"
+  nats-1-metrics:
+    image: natsio/prometheus-nats-exporter:latest
+    command: "--varz http://nats-1:8222"
+    ports:
+      - 17777:7777
+  nats-2:
+    image: nats
+    command: "--cluster_name=NATS --cluster=nats://0.0.0.0:6222 --http_port=8222 --routes=nats://nats-1:6222,nats://nats-2:6222,nats://nats-3:6222"
+    ports:
+      - 24222:4222
+  nats-2-metrics:
+    image: natsio/prometheus-nats-exporter:latest
+    command: "--varz http://nats-2:8222"
+  nats-3:
+    image: nats
+    command: "--cluster_name=NATS --cluster=nats://0.0.0.0:6222 --http_port=8222 --routes=nats://nats-1:6222,nats://nats-2:6222,nats://nats-3:6222"
+    ports:
+      - 34222:4222
+  nats-3-metrics:
+    image: natsio/prometheus-nats-exporter:latest
+    command: "--varz http://nats-3:8222"
diff --git a/dev/docker/prometheus.yml b/dev/docker/prometheus.yml
@@ -5,5 +5,7 @@ scrape_configs:
   - job_name: prometheus
     static_configs:
       - targets:
-          - 'host.docker.internal:8008'
-          - 'host.docker.internal:8009'
+          - host.docker.internal:8009
+          - nats-1-metrics:7777
+          - nats-2-metrics:7777
+          - nats-3-metrics:7777
diff --git a/dev/docker/up b/dev/docker/up
@@ -3,4 +3,4 @@ set -e
 . dev/docker/env
 
 docker_compose build
-docker_compose up -d
+docker_compose up -d --remove-orphans
diff --git a/dev/e2e/docker/docker-compose.yml b/dev/e2e/docker/docker-compose.yml
@@ -1,10 +1,4 @@
 services:
-  nodes:
-    image: nginx
-    ports:
-      - 8000:80
-    volumes:
-      - ./nodes:/usr/share/nginx/html
   api:
     image: nginx
     ports:
@@ -15,6 +9,15 @@ services:
     image: postgres:13
     environment:
       POSTGRES_PASSWORD: xmtp
+  nats-1:
+    image: nats
+    command: "--cluster_name=NATS --cluster=nats://0.0.0.0:6222 --routes=nats://nats-1:6222,nats://nats-2:6222,nats://nats-3:6222"
+  nats-2:
+    image: nats
+    command: "--cluster_name=NATS --cluster=nats://0.0.0.0:6222 --routes=nats://nats-1:6222,nats://nats-2:6222,nats://nats-3:6222"
+  nats-3:
+    image: nats
+    command: "--cluster_name=NATS --cluster=nats://0.0.0.0:6222 --routes=nats://nats-1:6222,nats://nats-2:6222,nats://nats-3:6222"
   node1:
     build:
       context: ../../../
@@ -26,28 +29,19 @@ services:
       - 18008:8008
     restart: always
     command:
-    - --store
+    - --nats.url=nats://nats-0:4222,nats://nats-1:4222,nats://nats-2:4222
     - --metrics
     - --metrics-address=0.0.0.0
-    - --ws
-    - --ws-port=6001
-    - --port=6000
-    - --lightpush
-    - --filter
     - --api.authn.enable
     - --api.authn.allowlists
     - --log-encoding=json
     - --wait-for-db=10s
-    - --static-node=/dns4/node2/tcp/6001/ws/p2p/16Uiu2HAmPexvM9XDgoac3i3V4PHGHpk11ZoYpNjG5TsuGfeQy79R
-    - --static-node=/dns4/node3/tcp/6001/ws/p2p/16Uiu2HAmRFvBjrt91Xcyi9QVz9mH7G2D1wrDifa3Z2C8azGr3edr
-    - --static-node=/dns4/node4/tcp/6001/ws/p2p/16Uiu2HAmD8P5wpT3nfQauUDiMj9eUpiWU2KrR1ZfVGu7qLjGZVua
     environment:
       ENV: dev
       GOLOG_LOG_FMT: json
       MESSAGE_DB_CONNECTION_STRING: postgres://postgres:xmtp@db:5432/postgres?sslmode=disable
       MESSAGE_DB_READER_CONNECTION_STRING: postgres://postgres:xmtp@db:5432/postgres?sslmode=disable
       AUTHZ_DB_CONNECTION_STRING: postgres://postgres:xmtp@db:5432/postgres?sslmode=disable
-      GOWAKU-NODEKEY: 8a30dcb604b0b53627a5adc054dbf434b446628d4bd1eccc681d223f0550ce67
   node2:
     build:
       context: ../../../
@@ -58,28 +52,19 @@ services:
       - 25555:5555
     restart: always
     command:
-    - --store
+    - --nats.url=nats://nats-1:4222,nats://nats-2:4222,nats://nats-3:4222
     - --metrics
     - --metrics-address=0.0.0.0
-    - --ws
-    - --ws-port=6001
-    - --port=6000
-    - --lightpush
-    - --filter
     - --api.authn.enable
     - --api.authn.allowlists
     - --log-encoding=json
     - --wait-for-db=10s
-    - --static-node=/dns4/node1/tcp/6001/ws/p2p/16Uiu2HAmNCxLZCkXNbpVPBpSSnHj9iq4HZQj7fxRzw2kj1kKSHHA
-    - --static-node=/dns4/node3/tcp/6001/ws/p2p/16Uiu2HAmRFvBjrt91Xcyi9QVz9mH7G2D1wrDifa3Z2C8azGr3edr
-    - --static-node=/dns4/node4/tcp/6001/ws/p2p/16Uiu2HAmD8P5wpT3nfQauUDiMj9eUpiWU2KrR1ZfVGu7qLjGZVua
     environment:
       ENV: dev
       GOLOG_LOG_FMT: json
       MESSAGE_DB_CONNECTION_STRING: postgres://postgres:xmtp@db:5432/postgres?sslmode=disable
       MESSAGE_DB_READER_CONNECTION_STRING: postgres://postgres:xmtp@db:5432/postgres?sslmode=disable
       AUTHZ_DB_CONNECTION_STRING: postgres://postgres:xmtp@db:5432/postgres?sslmode=disable
-      GOWAKU-NODEKEY: 5757057f3daac3fa80e43a06d24edf15fcaba11eb16ede5fd53b78ba68ef1436
   node3:
     build:
       context: ../../../
@@ -90,28 +75,19 @@ services:
       - 35555:5555
     restart: always
     command:
-    - --store
+    - --nats.url=nats://nats-1:4222,nats://nats-2:4222,nats://nats-3:4222
     - --metrics
     - --metrics-address=0.0.0.0
-    - --ws
-    - --ws-port=6001
-    - --port=6000
-    - --lightpush
-    - --filter
     - --api.authn.enable
     - --api.authn.allowlists
     - --log-encoding=json
     - --wait-for-db=10s
-    - --static-node=/dns4/node1/tcp/6001/ws/p2p/16Uiu2HAmNCxLZCkXNbpVPBpSSnHj9iq4HZQj7fxRzw2kj1kKSHHA
-    - --static-node=/dns4/node2/tcp/6001/ws/p2p/16Uiu2HAmPexvM9XDgoac3i3V4PHGHpk11ZoYpNjG5TsuGfeQy79R
-    - --static-node=/dns4/node4/tcp/6001/ws/p2p/16Uiu2HAmD8P5wpT3nfQauUDiMj9eUpiWU2KrR1ZfVGu7qLjGZVua
     environment:
       ENV: dev
       GOLOG_LOG_FMT: json
       MESSAGE_DB_CONNECTION_STRING: postgres://postgres:xmtp@db:5432/postgres?sslmode=disable
       MESSAGE_DB_READER_CONNECTION_STRING: postgres://postgres:xmtp@db:5432/postgres?sslmode=disable
       AUTHZ_DB_CONNECTION_STRING: postgres://postgres:xmtp@db:5432/postgres?sslmode=disable
-      GOWAKU-NODEKEY: f643b771ad9e1bc9519d4dc754370c4ff32dd7a9f76f6026b5a6448289212fd7
   node4:
     build:
       context: ../../../
@@ -122,25 +98,16 @@ services:
       - 45555:5555
     restart: always
     command:
-    - --store
+    - --nats.url=nats://nats-1:4222,nats://nats-2:4222,nats://nats-3:4222
     - --metrics
     - --metrics-address=0.0.0.0
-    - --ws
-    - --ws-port=6001
-    - --port=6000
-    - --lightpush
-    - --filter
     - --api.authn.enable
     - --api.authn.allowlists
     - --log-encoding=json
     - --wait-for-db=10s
-    - --static-node=/dns4/node1/tcp/6001/ws/p2p/16Uiu2HAmNCxLZCkXNbpVPBpSSnHj9iq4HZQj7fxRzw2kj1kKSHHA
-    - --static-node=/dns4/node2/tcp/6001/ws/p2p/16Uiu2HAmPexvM9XDgoac3i3V4PHGHpk11ZoYpNjG5TsuGfeQy79R
-    - --static-node=/dns4/node3/tcp/6001/ws/p2p/16Uiu2HAmRFvBjrt91Xcyi9QVz9mH7G2D1wrDifa3Z2C8azGr3edr
     environment:
       ENV: dev
       GOLOG_LOG_FMT: json
       MESSAGE_DB_CONNECTION_STRING: postgres://postgres:xmtp@db:5432/postgres?sslmode=disable
       MESSAGE_DB_READER_CONNECTION_STRING: postgres://postgres:xmtp@db:5432/postgres?sslmode=disable
       AUTHZ_DB_CONNECTION_STRING: postgres://postgres:xmtp@db:5432/postgres?sslmode=disable
-      GOWAKU-NODEKEY: 3ec2707fdd0584ad441fabcfbe928966378f797f929b3293ac8dc6a37206f3da
diff --git a/dev/e2e/docker/up b/dev/e2e/docker/up
@@ -5,4 +5,4 @@ set -e
 export GO_VERSION="$(go list -f "{{.GoVersion}}" -m)"
 
 docker_compose build
-docker_compose up -d
+docker_compose up -d --remove-orphans
diff --git a/dev/run b/dev/run
@@ -3,13 +3,12 @@ set -e
 
 MESSAGE_DB_DSN="postgres://postgres:xmtp@localhost:15432/postgres?sslmode=disable"
 AUTHZ_DB_DSN="postgres://postgres:xmtp@localhost:15432/postgres?sslmode=disable"
-NODE_KEY="8a30dcb604b0b53627a5adc054dbf434b446628d4bd1eccc681d223f0550ce67"
 
 go run cmd/xmtpd/main.go \
+    --nats.url=nats://localhost:14222,nats://localhost:24222,nats://localhost:34222 \
     --message-db-connection-string "${MESSAGE_DB_DSN}" \
     --message-db-reader-connection-string "${MESSAGE_DB_DSN}" \
     --authz-db-connection-string "${AUTHZ_DB_DSN}" \
-    --nodekey "${NODE_KEY}" \
     --metrics \
     --metrics-period 5s \
     --go-profiling \

diff --git a/dev/start b/dev/start
@@ -4,11 +4,6 @@ set -e
 PORT="${PORT:-9002}"
 
 dev/run \
-    --store \
-    --filter \
-    --lightpush \
-    --ws \
-    --ws-port="${PORT}" \
     --api.authn.enable \
     --api.authn.allowlists \
     "$@"