Merge branch 'master' into master

.dockerignore

@@ -1,4 +1,3 @@
 build/
 *.egg-info/
 venv/
-.git/

.env

@@ -0,0 +1,3 @@
+# This file is for use with docker compose so that mounting Neo4j volumes doesn't fail with perms errs
+GID=10001
+UID=10001

.github/workflows/publish-to-ghcr.yml

@@ -44,7 +44,7 @@ jobs:
       - name: Build and push
         uses: docker/build-push-action@v3
         with:
-          file: dist.Dockerfile
+          file: Dockerfile
           context: .
           push: true # push the image to ghcr
           tags: ${{ steps.meta.outputs.tags }}

.github/workflows/test_suite.yml

@@ -90,7 +90,7 @@ jobs:
       - name: make test_integration
         run: make test_integration
 
-  build-dist-docker-image:
+  build-docker-image:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
@@ -106,7 +106,7 @@ jobs:
       - name: Build
         uses: docker/build-push-action@v3
         with:
-          file: dist.Dockerfile
+          file: Dockerfile
           push: false # only build the image, don't push it anywhere
           context: .
           tags: ${{ steps.meta.outputs.tags }}

.gitignore

@@ -16,3 +16,4 @@ build/
 .compose
 generated
 dist/
+.local

Dockerfile

@@ -1,33 +1,20 @@
-FROM ubuntu:focal
+# This is a thin distribution of the cartography software.
+FROM python:3.10-slim
 
-WORKDIR /srv/cartography
+# the UID and GID to run cartography as
+# (https://github.com/hexops/dockerfile#do-not-use-a-uid-below-10000).
+ARG uid=10001
+ARG gid=10001
 
-ENV PATH=/venv/bin:$PATH
-RUN apt-get update && \
-    DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends python3.10-dev python3-pip python3-setuptools openssl libssl-dev gcc pkg-config libffi-dev libxml2-dev libxmlsec1-dev curl make git && \
-    apt-get clean && \
-    rm -rf /var/lib/apt/lists/*
+WORKDIR /var/cartography
+ENV HOME=/var/cartography
 
-# Installs pip supported by python3.10
-RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && python3.10 get-pip.py
+RUN pip install cartography
 
-# Create cartography user so that we can give it ownership of the directory later for unit&integ tests
-RUN groupadd cartography && \
-    useradd -s /bin/bash -d /home/cartography -m -g cartography cartography
+USER ${uid}:${gid}
 
-# Installs python dependencies
-COPY setup.py test-requirements.txt ./
-RUN pip install -e . && \
-    pip install -r test-requirements.txt && \
-    # Grant write access to the directory for unit and integration test coverage files
-    chmod -R a+w /srv/cartography
+# verify that the binary at least runs
+RUN cartography -h
 
-# Install cartography, setting the owner so that tests work
-COPY --chown=cartography:cartography . /srv/cartography
-
-USER cartography
-
-# Sets the directory as safe due to a mismatch in the user that cloned the repo
-# and the user that is going to run the unit&integ tests.
-RUN git config --global --add safe.directory /srv/cartography
-RUN /usr/bin/git config --local user.name "cartography"
+ENTRYPOINT ["cartography"]
+CMD ["-h"]

dev.Dockerfile

@@ -0,0 +1,28 @@
+# Builds cartography container for development by performing a Python editable install of the current source code.
+FROM python:3.10-slim
+
+# the UID and GID to run cartography as
+# (https://github.com/hexops/dockerfile#do-not-use-a-uid-below-10000).
+ARG uid=10001
+ARG gid=10001
+
+RUN apt-get update && \
+    DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends make git && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+# Assumption: current working directory is the cartography source tree from github.
+COPY . /var/cartography
+WORKDIR /var/cartography
+ENV HOME=/var/cartography
+
+RUN pip install -U -e . && \
+    pip install -r test-requirements.txt && \
+    # Grant write access to the directory for unit and integration test coverage files
+    chmod -R a+w /var/cartography && \
+    # Sets the directory as safe due to a mismatch in the user that cloned the repo
+    # and the user that is going to run the unit&integ tests. This lets pre-commit work.
+    git config --global --add safe.directory /var/cartography && \
+    git config --local user.name "cartography"
+
+USER ${uid}:${gid}

docker-compose.yml

@@ -1,7 +1,11 @@
-version: "3.7"
+# This docker-compose is intended to help you quick-start or develop cartography.
+# It is also a good starting point for your own customizations.
+# If you want to modify this and contribute your change upstream, please file a GitHub issue first. It's hard to make
+# this so that it will support as many users as possible, so we've tried to make this with a minimum set of
+# functionality that you can extend on your own.
 services:
   neo4j:
-    image: neo4j:4.4.5-community
+    image: neo4j:4.4-community
     restart: unless-stopped
     ports:
       - 7474:7474
@@ -12,6 +16,7 @@ services:
       - ./.compose/neo4j/import:/import
       - ./.compose/neo4j/logs:/logs
       - ./.compose/neo4j/plugins:/plugins
+    user: "${UID}:${GID}"
     environment:
       # Raise memory limits:
       - NEO4J_dbms_memory_pagecache_size=1G
@@ -33,19 +38,39 @@ services:
         interval: 10s
         timeout: 10s
         retries: 10
+
+# Runs the standard cartography image available at ghcr.io.
   cartography:
-    # As seen in docs, we build with `cd /path/to/cartography && docker build -t lyft/cartography .`
-    # and then run with `docker-compose up -d`.
-    image: lyft/cartography
+    image: ghcr.io/lyft/cartography:latest
     # EXAMPLE: Our ENTRYPOINT is cartography, running specific command to sync AWS
     # command: ["-v", "--neo4j-uri=bolt://neo4j:7687", "--aws-sync-all-profiles"]
-    user: cartography
     init: true
     restart: on-failure
     depends_on:
       - neo4j
     volumes:
-      - ~/.aws:/cartography/.aws/
+    # Provide AWS creds to the container
+      - ~/.aws:/var/cartography/.aws/
+    environment:
+      # Point to the neo4j service defined in this docker-compose file.
+      - NEO4J_URL=bolt://cartography-neo4j-1:7687
+
+# Intended to run local automated tests, custom sync scripts, and local changes.
+  cartography-dev:
+    # See dev instructions: we assume that you have built this with
+    # `docker build -t lyft/cartography-dev . -f dev.Dockerfile`.
+    # Do not push this image remotely!
+    image: lyft/cartography-dev
+    init: true
+    restart: on-failure
+    depends_on:
+      - neo4j
+    volumes:
+      # Provide AWS creds to the container
+      - ~/.aws:/var/cartography/.aws/
+      # For pre-commit to work
+      - .:/var/cartography
+      - ./.cache/pre-commit:/var/cartography/.cache/pre-commit
     environment:
       # Point to the neo4j service defined in this docker-compose file.
       - NEO4J_URL=bolt://cartography-neo4j-1:7687