Add Optimisation (#51)

firehose/Dockerfile

@@ -9,21 +9,23 @@ RUN cargo chef prepare --recipe-path recipe.json
 
 # Stage 3: Extract builder
 FROM chef AS extract_builder
-RUN apt update && apt install -y protobuf-compiler
 COPY --from=planner /app/recipe.json /app/
-RUN cargo chef cook --release --recipe-path /app/recipe.json
+RUN apt-get update && apt-get install -y protobuf-compiler \
+    && cargo chef cook --release --recipe-path /app/recipe.json
 COPY ./firehose-extract/ /app/
 RUN cargo build --release
 
 # Stage 4: Go build
 FROM golang:1.22-alpine as go_build
-COPY ./firehose-core/ /app/firehose-core/
-COPY ./substreams/ /app/substreams/
 WORKDIR /app/firehose-core
+COPY ./firehose-core/go.mod ./firehose-core/go.sum ./
+COPY ./substreams/go.mod ./substreams/go.sum ./
 RUN go mod download
+COPY ./firehose-core/ /app/firehose-core/
+COPY ./substreams/ /app/substreams/
 ARG VERSION="dev"
-RUN apk --no-cache add git
-RUN go build -v -ldflags "-X main.version=${VERSION}" -o /app/firehose-core/firecore ./cmd/firecore
+RUN apk --no-cache add git \
+    && go build -v -ldflags "-X main.version=${VERSION}" -o /app/firehose-core/firecore ./cmd/firecore
 
 # Stage 5: Final stage
 FROM debian
@@ -36,6 +38,7 @@ RUN chmod +x /app/start.sh
 ENV FIREHOSE_PORT_1=10015
 ENV FIREHOSE_PORT_2=10016
 ENV FIREHOSE_PORT_3=10017
+
 # Expose ports
 EXPOSE $FIREHOSE_PORT_1
 EXPOSE $FIREHOSE_PORT_2

firehose/firehose-core/devel/standard/bootstrap.sh

@@ -8,7 +8,7 @@ STORAGE_DIR="/data/storage_dir"
 
 CHAIN_ID="${CHAIN_ID:-$1}"
 
-COMMON_LIVE_BLOCKS_ADDR="$(hostname -I | awk '{print $1}')"
+COMMON_LIVE_BLOCKS_ADDR="$(ifconfig lo0 | awk '$1 == "inet" {print $2}')"
 
 if [ -z "$CHAIN_ID" ]; then
 	echo "Usage: $0 CHAIN_ID"
@@ -18,7 +18,7 @@ fi
 HEIGHT_FILE="$STORAGE_DIR/last_height.txt"
 
 if [[ -f "$HEIGHT_FILE" ]]; then
-  LAST_HEIGHT=$(($(cat "$HEIGHT_FILE") - 1))
+  LAST_HEIGHT=$(($(cat "$HEIGHT_FILE")))
 else
 	LAST_HEIGHT=0
 fi
@@ -39,14 +39,12 @@ start:
      reader-node-path: "$FIREHOSE_EXTRACT_BIN"
      reader-node-arguments: $CHAIN_ID $LAST_HEIGHT
 
-     common-live-blocks-addr: "$COMMON_LIVE_BLOCKS_ADDR:10019"
-     reader-node-grpc-listen-addr: "$COMMON_LIVE_BLOCKS_ADDR:10019"
-#     merger-grpc-listen-addr: "$COMMON_LIVE_BLOCKS_ADDR:10019"
-#     relayer-grpc-listen-addr: :10019
-#
-#     merger-time-between-store-lookups: 5s
-#     merger-time-between-store-pruning: 10s
-#     merger-delete-threads: 10
+     common-live-blocks-addr: :10019
+     reader-node-grpc-listen-addr: :10019
+
+     merger-time-between-store-lookups: 5s
+     merger-time-between-store-pruning: 10s
+     merger-delete-threads: 10
 END
 
 cd "$STORAGE_DIR"

firehose/firehose-core/devel/standard_local/bootstrap.sh

@@ -9,10 +9,12 @@ if [ -z "$CHAIN_ID" ]; then
 	return 1
 fi
 
+COMMON_LIVE_BLOCKS_ADDR="$(ifconfig lo0 | awk '$1 == "inet" {print $2}')"
+
 HEIGHT_FILE="last_height.txt"
 
 if [[ -f "$HEIGHT_FILE" ]]; then
-  LAST_HEIGHT=$(($(cat "$HEIGHT_FILE") - 1))
+  LAST_HEIGHT=$(($(cat "$HEIGHT_FILE")))
 else
 	LAST_HEIGHT=0
 fi
@@ -32,8 +34,15 @@ start:
      reader-node-path: "../../../firehose-extract/target/debug/firehose-extract"
      reader-node-arguments: $CHAIN_ID $LAST_HEIGHT
 
-     common-live-blocks-addr: localhost:10024
-     reader-node-grpc-listen-addr: localhost:10024
+#     common-live-blocks-addr: :10010
+#     reader-node-grpc-listen-addr: :10014
+#     relayer-source: localhost:10024
+#     relayer-grpc-listen-addr: :15011
+#     relayer-source: :10024
+
+
+#     common-live-blocks-addr: "$COMMON_LIVE_BLOCKS_ADDR:10024"
+#     reader-node-grpc-listen-addr: "$COMMON_LIVE_BLOCKS_ADDR:10024"
 
 END
 

firehose/firehose-core/pb/sf/fuel/type/v1/type.go

@@ -31,5 +31,5 @@ func (b *Block) GetFirehoseBlockVersion() int32 {
 }
 
 func (b *Block) GetFirehoseBlockLIBNum() uint64 {
-	return b.GetFirehoseBlockNumber()
+	return b.GetFirehoseBlockNumber() - 1
 }

firehose/firehose-extract/src/main.rs

@@ -35,7 +35,7 @@ async fn main() -> anyhow::Result<()> {
     loop {
         let page_req = PaginationRequest::<String> {
             cursor,
-            results: 32, // 42
+            results: 26,
             direction: PageDirection::Forward,
         };
         let query = FullBlocksQuery::build(page_req.into());