feat: Use uv package manager

fix celery
codecov · Nov 21, 2024 · 0bd1c5f · 0bd1c5f
1 parent 4a2bb9b
commit 0bd1c5f
Show file tree

Hide file tree

Showing 8 changed files with 2,695 additions and 597 deletions.
diff --git a/Makefile b/Makefile
@@ -7,7 +7,7 @@ branch = $(shell git branch | grep \* | cut -f2 -d' ')
 epoch := $(shell date +"%s")
 AR_REPO ?= codecov/api
 DOCKERHUB_REPO ?= codecov/self-hosted-api
-REQUIREMENTS_TAG := requirements-v1-$(shell sha1sum requirements.txt | cut -d ' ' -f 1)-$(shell sha1sum docker/Dockerfile.requirements | cut -d ' ' -f 1)
+REQUIREMENTS_TAG := requirements-v1-$(shell sha1sum uv.lock | cut -d ' ' -f 1)-$(shell sha1sum docker/Dockerfile.requirements | cut -d ' ' -f 1)
 VERSION := release-${sha}
 CODECOV_UPLOAD_TOKEN ?= "notset"
 CODECOV_STATIC_TOKEN ?= "notset"
@@ -21,7 +21,7 @@ API_DOMAIN ?= api
 PROXY_NETWORK ?= api_default
 
 # Codecov CLI version to use
-CODECOV_CLI_VERSION := 0.5.1
+CODECOV_CLI_VERSION := 9.0.4
 
 build:
 	make build.requirements
@@ -31,13 +31,13 @@ check-for-migration-conflicts:
 	python manage.py check_for_migration_conflicts
 
 test:
-	COVERAGE_CORE=sysmon python -m pytest --cov=./ --junitxml=junit.xml -o junit_family=legacy
+	COVERAGE_CORE=sysmon pytest --cov=./ --junitxml=junit.xml -o junit_family=legacy
 
 test.unit:
-	COVERAGE_CORE=sysmon python -m pytest --cov=./ -m "not integration" --cov-report=xml:unit.coverage.xml --junitxml=unit.junit.xml -o junit_family=legacy
+	COVERAGE_CORE=sysmon pytest --cov=./ -m "not integration" --cov-report=xml:unit.coverage.xml --junitxml=unit.junit.xml -o junit_family=legacy
 
 test.integration:
-	COVERAGE_CORE=sysmon python -m pytest --cov=./ -m "integration" --cov-report=xml:integration.coverage.xml --junitxml=integration.junit.xml -o junit_family=legacy
+	COVERAGE_CORE=sysmon pytest --cov=./ -m "integration" --cov-report=xml:integration.coverage.xml --junitxml=integration.junit.xml -o junit_family=legacy
 
 lint:
 	make lint.install
@@ -179,17 +179,17 @@ push.self-hosted-rolling:
 	docker push ${DOCKERHUB_REPO}:rolling
 
 shell:
-	docker-compose exec api bash
+	docker compose exec api bash
 
 test_env.up:
 	env | grep GITHUB > .testenv; true
-	TIMESERIES_ENABLED=${TIMESERIES_ENABLED} docker-compose up -d
+	TIMESERIES_ENABLED=${TIMESERIES_ENABLED} docker compose up -d
 
 test_env.prepare:
-	docker-compose exec api make test_env.container_prepare
+	docker compose exec api make test_env.container_prepare
 
 test_env.check_db:
-	docker-compose exec api make test_env.container_check_db
+	docker compose exec api make test_env.container_check_db
 	make test_env.check-for-migration-conflicts
 
 test_env.install_cli:
@@ -205,18 +205,18 @@ test_env.container_check_db:
 	while ! nc -vz timescale 5432; do sleep 1; echo "waiting for timescale"; done
 
 test_env.run_unit:
-	docker-compose exec api make test.unit
+	docker compose exec api make test.unit
 
 test_env.run_integration:
-	#docker-compose exec api make test.integration
+	#docker compose exec api make test.integration
 	echo "Skipping. No Tests"
 
 test_env.check-for-migration-conflicts:
-	docker-compose exec api python manage.py check_for_migration_conflicts
+	docker compose exec api python manage.py check_for_migration_conflicts
 
 test_env.upload:
-	docker-compose exec api make test_env.container_upload CODECOV_UPLOAD_TOKEN=${CODECOV_UPLOAD_TOKEN} CODECOV_URL=${CODECOV_URL}
-	docker-compose exec api make test_env.container_upload_test_results CODECOV_UPLOAD_TOKEN=${CODECOV_UPLOAD_TOKEN} CODECOV_URL=${CODECOV_URL}
+	docker compose exec api make test_env.container_upload CODECOV_UPLOAD_TOKEN=${CODECOV_UPLOAD_TOKEN} CODECOV_URL=${CODECOV_URL}
+	docker compose exec api make test_env.container_upload_test_results CODECOV_UPLOAD_TOKEN=${CODECOV_UPLOAD_TOKEN} CODECOV_URL=${CODECOV_URL}
 
 test_env.container_upload:
 	codecovcli -u ${CODECOV_URL} upload-process --flag unit-latest-uploader --flag unit  \
@@ -229,13 +229,13 @@ test_env.container_upload_test_results:
 	--files-search-exclude-folder=api/internal/tests/unit/views/cassetes/** || true
 
 test_env.static_analysis:
-	docker-compose exec api make test_env.container_static_analysis CODECOV_STATIC_TOKEN=${CODECOV_STATIC_TOKEN}
+	docker compose exec api make test_env.container_static_analysis CODECOV_STATIC_TOKEN=${CODECOV_STATIC_TOKEN}
 
 test_env.label_analysis:
-	docker-compose exec api make test_env.container_label_analysis CODECOV_STATIC_TOKEN=${CODECOV_STATIC_TOKEN}
+	docker compose exec api make test_env.container_label_analysis CODECOV_STATIC_TOKEN=${CODECOV_STATIC_TOKEN}
 
 test_env.ats:
-	docker-compose exec api make test_env.container_ats CODECOV_UPLOAD_TOKEN=${CODECOV_UPLOAD_TOKEN}
+	docker compose exec api make test_env.container_ats CODECOV_UPLOAD_TOKEN=${CODECOV_UPLOAD_TOKEN}
 
 test_env.container_static_analysis:
 	codecovcli -u ${CODECOV_URL} static-analysis --token=${CODECOV_STATIC_TOKEN}

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -1,5 +1,3 @@
-version: "3"
-
 services:
   api:
     image: ${API_DOCKER_REPO}:${API_DOCKER_VERSION}
@@ -17,11 +15,11 @@ services:
       - SETUP__TIMESERIES__ENABLED=${TIMESERIES_ENABLED-true}
       # Improves pytest-cov performance in python 3.12
       # https://github.com/nedbat/coveragepy/issues/1665#issuecomment-1937075835
-      - COVERAGE_CORE=sysmon 
+      - COVERAGE_CORE=sysmon
     env_file:
       - .testenv
   postgres:
-    image: postgres:14.7-alpine
+    image: postgres:14-alpine
     environment:
       - POSTGRES_USER=postgres
       - POSTGRES_HOST_AUTH_METHOD=trust
@@ -32,7 +30,7 @@ services:
         tmpfs:
           size: 2048M
   timescale:
-    image: timescale/timescaledb-ha:pg14-latest
+    image: timescale/timescaledb:latest-pg14
     environment:
       - POSTGRES_USER=postgres
       - POSTGRES_HOST_AUTH_METHOD=trust
@@ -44,4 +42,3 @@ services:
           size: 2048M
   redis:
     image: redis:6-alpine
-
diff --git a/docker/Dockerfile b/docker/Dockerfile
@@ -6,9 +6,8 @@ ARG BERGLAS_VERSION=2.0.6
 FROM us-docker.pkg.dev/berglas/berglas/berglas:$BERGLAS_VERSION as berglas
 
 FROM $REQUIREMENTS_IMAGE as app
-COPY . /app
 WORKDIR /app
-RUN pip install setuptools==71.1.0
+ADD . /app
 RUN python manage.py collectstatic --no-input
 
 FROM app as local
@@ -20,7 +19,6 @@ COPY --chmod=755 --from=berglas /bin/berglas /usr/local/bin/berglas
 
 FROM app as self-hosted
 
-RUN pip uninstall -y typing
 # set settings module
 ENV DJANGO_SETTINGS_MODULE="codecov.settings_enterprise"
 # Remove the settings dev and enterprise files.
@@ -57,6 +55,7 @@ FROM self-hosted as self-hosted-runtime
 USER root
 ARG EXTERNAL_DEPS_FOLDER=./external_deps
 RUN mkdir $EXTERNAL_DEPS_FOLDER
+# TODO - replace the pip install here?
 RUN pip install --target $EXTERNAL_DEPS_FOLDER psycopg2-binary tlslite-ng
 RUN chown codecov:application $EXTERNAL_DEPS_FOLDER
 USER codecov

diff --git a/docker/Dockerfile.requirements b/docker/Dockerfile.requirements
@@ -1,34 +1,62 @@
 # syntax=docker/dockerfile:1.4
-ARG PYTHON_IMAGE=python:3.12-slim-bookworm
+ARG PYTHON_IMAGE=ghcr.io/astral-sh/uv:python3.12-bookworm-slim
 
 # BUILD STAGE - Download dependencies from GitHub that require SSH access
 FROM $PYTHON_IMAGE as build
 
 RUN apt-get update
 RUN apt-get install -y \
     build-essential \
+    git \
     libffi-dev \
     libpq-dev \
     curl
 
-COPY requirements.txt /
-WORKDIR /pip-packages/
-RUN pip wheel -r /requirements.txt
-RUN rm -rf /pip-packages/src
+ENV UV_LINK_MODE=copy \
+    UV_COMPILE_BYTECODE=1 \
+    UV_PYTHON_DOWNLOADS=never \
+    UV_PYTHON=python \
+    UV_PROJECT_ENVIRONMENT=/worker
+
+# Then, add the rest of the project source code and install it
+# Installing separately from its dependencies allows optimal layer caching
+RUN --mount=type=cache,target=/root/.cache/uv \
+    --mount=type=bind,source=uv.lock,target=uv.lock \
+    --mount=type=bind,source=pyproject.toml,target=pyproject.toml \
+    uv sync --frozen
+
+RUN --mount=type=cache,target=/root/.cache/uv \
+    --mount=type=bind,source=uv.lock,target=uv.lock \
+    --mount=type=bind,source=pyproject.toml,target=pyproject.toml \
+    uv export --no-hashes --frozen --format requirements-txt > requirements.txt
+
+RUN grep -v '^-e ' requirements.txt > requirements.remote.txt
+
+# build all remote wheels
+RUN pip wheel -w wheels --find-links wheels -r requirements.remote.txt
+
+# build all local packages to wheels
+RUN --mount=type=cache,target=/root/.cache/uv \
+    --mount=type=bind,source=uv.lock,target=uv.lock \
+    --mount=type=bind,source=pyproject.toml,target=pyproject.toml \
+    uv build --all-packages --wheel -o wheels
 
 # RUNTIME STAGE - Copy packages from build stage and install runtime dependencies
 FROM $PYTHON_IMAGE
 
 # Our postgres driver psycopg2 requires libpq-dev as a runtime dependency
 RUN apt-get update
 RUN apt-get install -y \
+    git \
     libpq-dev \
     make \
     curl \
     libexpat1 \
     && pip install --upgrade pip
 
-WORKDIR /pip-packages/
-COPY --from=build /pip-packages/ /pip-packages/
+COPY --from=build /wheels/ /wheels/
 
-RUN pip install --no-deps --no-index --find-links=/pip-packages/ /pip-packages/*
+RUN --mount=type=cache,target=/root/.cache/uv \
+    --mount=type=bind,source=uv.lock,target=uv.lock \
+    --mount=type=bind,source=pyproject.toml,target=pyproject.toml \
+    uv pip install --no-deps --no-index --find-links=wheels wheels/* --system
diff --git a/pyproject.toml b/pyproject.toml
@@ -0,0 +1,85 @@
+[project]
+name = "api"
+version = "0.1.0"
+description = "Codecov's API layer"
+readme = "README.md"
+requires-python = ">=3.12"
+dependencies = [
+    "aiodataloader",
+    "ariadne==0.23",
+    "ariadne_django==0.3.0",
+    "codecovopentelem",
+    "celery>=5.3.6",
+    "cerberus",
+    "ddtrace",
+    "Django>=4.2.16",
+    "django-cors-headers",
+    "django-csp",
+    "django-dynamic-fixture",
+    "django-filter",
+    "django-model-utils",
+    "django-postgres-extra>=2.0.8",
+    "django-prometheus",
+    "djangorestframework==3.15.2",
+    "drf-spectacular",
+    "drf-spectacular-sidecar",
+    "elastic-apm",
+    "google-cloud-pubsub",
+    "gunicorn>=22.0.0",
+    "idna>=3.7",
+    "minio",
+    "opentelemetry-instrumentation-django>=0.45b0",
+    "opentelemetry-sdk>=1.24.0",
+    "opentracing",
+    "polars==1.12.0",
+    "psycopg2",
+    "PyJWT",
+    "pydantic",
+    "python-dateutil",
+    "python-json-logger",
+    "python-redis-lock",
+    "pytz",
+    "redis",
+    "regex",
+    "requests",
+    "sentry-sdk>=2.13.0",
+    "setproctitle",
+    "shared",
+    "simplejson",
+    "starlette==0.40.0",
+    "stripe>=9.6.0",
+    "test-results-parser",
+    "timestring",
+    "urllib3>=1.26.19",
+    "vcrpy",
+    "whitenoise",
+    "django-autocomplete-light",
+    "django-better-admin-arrayfield",
+    "certifi>=2024.07.04",
+]
+
+[build-system]
+requires = ["setuptools"]
+build-backend = "setuptools.build_meta"
+
+[tool.setuptools]
+py-modules = []
+
+[tool.uv]
+dev-dependencies = [
+    "factory-boy",
+    "fakeredis",
+    "freezegun",
+    "pre-commit",
+    "pytest>=7.2.0",
+    "pytest-cov",
+    "pytest-asyncio",
+    "pytest-django",
+    "pytest-mock",
+    "vcrpy",
+]
+
+
+[tool.uv.sources]
+codecovopentelem = { git = "https://github.com/codecov/opentelem-python", rev = "df1e241927d4794fc0e24f96431f0fb730edac21" }
+shared = { git = "https://github.com/codecov/shared", rev = "9c31870de026a8a3f026f0753b1991e8c832fadb" }
diff --git a/requirements.in b/requirements.in