andy/go refactor (#3)

Temporal and Go refactor working

.env.development.example

@@ -1,8 +1,8 @@
-S3_ENDPOINT_URL=https://minio-us-west-1b.andykmccoy.com
-S3_REGION='us-west-1b'
-BUCKET='testing-butcket'
-ACCESS_KEY_ID=machine
-SECRET_ACCESS_KEY=WJYiCoZAgWpZsRCn6y9JpWibxBo3b6rtjiBi
+S3_ENDPOINT_URL=https://minio.example.com
+S3_REGION='us-west-1'
+BUCKET=REPLACE_ME
+AWS_ACCESS_KEY_ID=REPLACE_ME
+AWS_SECRET_ACCESS_KEY=REPLACE_ME
 # TEMPORAL_URL=localhost:7233
 # OUTPUT_DIR='/mnt/output'
 # PBF_PATH='/mnt/input/latest.osm.pbf'

.gitignore

@@ -1,6 +1,16 @@
 .DS_Store
-volumes
-node_modules
-main
-lib
 .env.development.local
+*.dll
+*.dylib
+*.exe
+*.exe~
+*.out
+*.so
+*.test
+bin
+go.work
+go.work.sum
+lib
+main
+node_modules
+volumes

Dockerfile

@@ -1,20 +1,19 @@
-FROM node:current-slim as build
-RUN mkdir /app
-WORKDIR /app
-COPY package.json package-lock.json tsconfig.json /app
-RUN npm install
-COPY ./src ./src
-RUN npm run build
+FROM golang:1.22 as build
+WORKDIR /usr/src/app
+COPY go.mod go.sum Makefile ./
+RUN make install
+
+COPY workflow ./workflow
+RUN make build
 
-FROM node:current-slim
+FROM debian:12-slim
 RUN apt-get update && \
-    apt-get install -y osmium-tool dumb-init && \
+    apt-get upgrade -y && \
+    apt-get install -y ca-certificates osmium-tool dumb-init && \
     mkdir /app
 VOLUME ["/mnt/input", "/mnt/output"]
 WORKDIR /app
-COPY package.json /app
-RUN npm install --production
-
-COPY --from=build /app/lib ./lib
-COPY ./polygons ./polygons
+COPY --from=build /usr/src/app/bin/osm-extractor-workflow .
+COPY polygons ./polygons
+COPY config.json .
 ENTRYPOINT ["/usr/bin/dumb-init", "--"]

Makefile

@@ -0,0 +1,9 @@
+install:
+	go mod download && go mod verify
+build:
+	go build -o ./bin/osm-extractor-workflow ./workflow/cmd
+build_docker: 
+	docker build . -t osm-extractor:local
+run:
+	go run ./workflow/cmd
+all: install build

config.json

@@ -0,0 +1,40 @@
+{
+    "extracts": [
+        {
+            "output": "san-francisco-bay-area.osm.pbf",
+            "directory": "north-america/us/california/san-francisco-bay-area",
+            "description": "San Francisco Bay Area nine counties.",
+            "polygon": {
+                "file_name": "polygons/san-francisco-bay-area-convex.geojson",
+                "file_type": "geojson"
+            }
+        },
+        {
+            "output": "harrisburg-area.osm.pbf",
+            "directory": "north-america/us/pennsylvania/harrisburg-area",
+            "description": "Harrisburg Capital Area",
+            "polygon": {
+                "file_name": "polygons/harrisburg-area.geojson",
+                "file_type": "geojson"
+            }
+        },
+        {
+            "output": "us-virgin-island-saint-croix.osm.pbf",
+            "directory": "north-america/us/us-virgin-island/saint-croix",
+            "description": "US Virgin Island Saint Croix",
+            "polygon": {
+                "file_name": "polygons/us-virgin-island-saint-croix.geojson",
+                "file_type": "geojson"
+            }
+        },
+        {
+            "output": "salt-lake-metro.osm.pbf",
+            "directory": "north-america/us/utah/salt-lake-metro",
+            "description": "Salt Lake City Metro Area",
+            "polygon": {
+                "file_name": "polygons/salt-lake-metro.geojson",
+                "file_type": "geojson"
+            }
+        }
+    ]
+}

docker-compose.yaml

@@ -1,80 +1,19 @@
 services:
-  osm-extractor-workflow-start:
-    container_name: osm-extractor-workflow-start
-    working_dir: /home/node/app
-    env_file:
-      - path: .env.development.local
-        required: true
-    environment:
-      TEMPORAL_URL: host.docker.internal
-    extra_hosts:
-      - "host.docker.internal:host-gateway"
-    image: osm-extractor
-    labels:
-      - 'app=osm-extractor-workflow-start'
-    command: ['node', '/app/lib/start-schedule.js']
   osm-extractor-worker:
     container_name: osm-extractor-worker
-    working_dir: /home/node/app
     env_file:
       - path: .env.development.local
         required: true
     environment:
-      TEMPORAL_URL: host.docker.internal
+      TEMPORAL_URL: host.docker.internal:7233
     extra_hosts:
       - "host.docker.internal:host-gateway"
-    image: osm-extractor
+    image: osm-extractor:local
+    build:
+      context: .
     labels:
       - 'app=osm-extractor-worker'
     volumes:
       - ./volumes/input:/mnt/input 
       - ./volumes/output:/mnt/output
-    command: ['node', '/app/lib/worker.js']
-    depends_on:
-      osm-extractor-workflow-start:
-        condition: service_completed_successfully
-  osm-extractor-workflow-delete:
-    profiles: ['delete']
-    container_name: osm-extractor-workflow-delete
-    working_dir: /home/node/app
-    env_file:
-      - path: .env.development.local
-        required: true
-    environment:
-      TEMPORAL_URL: host.docker.internal
-    extra_hosts:
-      - "host.docker.internal:host-gateway"
-    image: osm-extractor
-    labels:
-      - 'app=osm-extractor-workflow-delete'
-    command: ['node', '/app/lib/delete-schedule.js']
-  osm-extractor-workflow-unpause:
-    profiles: ['unpause']
-    container_name: osm-extractor-workflow-unpause
-    working_dir: /home/node/app
-    env_file:
-      - path: .env.development.local
-        required: true
-    environment:
-      TEMPORAL_URL: host.docker.internal
-    extra_hosts:
-      - "host.docker.internal:host-gateway"
-    image: osm-extractor
-    labels:
-      - 'app=osm-extractor-workflow-unpause'
-    command: ['node', '/app/lib/unpause-schedule.js']
-  osm-extractor-workflow-pause:
-    profiles: ['pause']
-    container_name: osm-extractor-workflow-pause
-    working_dir: /home/node/app
-    env_file:
-      - path: .env.development.local
-        required: true
-    environment:
-      TEMPORAL_URL: host.docker.internal
-    extra_hosts:
-      - "host.docker.internal:host-gateway"
-    image: osm-extractor
-    labels:
-      - 'app=osm-extractor-workflow-pause'
-    command: ['node', '/app/lib/pause-schedule.js']
+    command: ['/app/osm-extractor-workflow', 'worker']

go.mod

@@ -0,0 +1,53 @@
+module github.com/bikehopper/osm-extractor
+
+go 1.22.0
+
+require (
+	github.com/aws/aws-sdk-go-v2 v1.27.2
+	github.com/aws/aws-sdk-go-v2/config v1.27.18
+	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.24
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.55.1
+	github.com/google/uuid v1.6.0
+	go.temporal.io/sdk v1.27.0
+)
+
+require (
+	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2 // indirect
+	github.com/aws/aws-sdk-go-v2/credentials v1.17.18 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.5 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.9 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.9 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.9 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.11 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.11 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.9 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.20.11 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.24.5 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.28.12 // indirect
+	github.com/aws/smithy-go v1.20.2 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang/mock v1.6.0 // indirect
+	github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 // indirect
+	github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 // indirect
+	github.com/jmespath/go-jmespath v0.4.0 // indirect
+	github.com/pborman/uuid v1.2.1 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/robfig/cron v1.2.0 // indirect
+	github.com/stretchr/objx v0.5.2 // indirect
+	github.com/stretchr/testify v1.9.0 // indirect
+	go.temporal.io/api v1.34.0 // indirect
+	golang.org/x/exp v0.0.0-20231127185646-65229373498e // indirect
+	golang.org/x/net v0.25.0 // indirect
+	golang.org/x/sys v0.20.0 // indirect
+	golang.org/x/text v0.15.0 // indirect
+	golang.org/x/time v0.3.0 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20240521202816-d264139d666e // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20240521202816-d264139d666e // indirect
+	google.golang.org/grpc v1.64.0 // indirect
+	google.golang.org/protobuf v1.34.1 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+)