From 9d60e9fc8d210cb67e02b198dca8be876d98e74c Mon Sep 17 00:00:00 2001 From: David Asabina Date: Sat, 9 Mar 2024 12:56:11 -0400 Subject: [PATCH 1/8] chore: Parameterise platform to get system-compat cloud-sql-proxy On macOS, I don't need the Linux build for cloud-sql-proxy but rather the darwin build. Parameterizing the platform allow this part of the Makefile to work more generally on developer systems. --- Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 89377ca0e..d35236ded 100644 --- a/Makefile +++ b/Makefile @@ -6,6 +6,7 @@ DOCKERPORT := 8080 LOCALPORT := 6200 DBPROXYPORT := 6201 BROWSE_DOCKER_RUN := docker run --cpus 2 --rm -p ${LOCALPORT}:${DOCKERPORT} -e PORT=${DOCKERPORT} -v ${HOME}/arxiv/arxiv-browse/tests:/tests --name ${NAME} --env-file "${PWD}/tests/docker.env" --security-opt="no-new-privileges=true" +PLATFORM := $(shell uname -s | tr '[:upper:]' '[:lower:]') default: venv @@ -17,7 +18,7 @@ venv: .prerequisit . venv/bin/activate && poetry install /usr/local/bin/cloud-sql-proxy: - curl -o ./cloud-sql-proxy https://storage.googleapis.com/cloud-sql-connectors/cloud-sql-proxy/v2.8.2/cloud-sql-proxy.linux.amd64 + curl -o ./cloud-sql-proxy https://storage.googleapis.com/cloud-sql-connectors/cloud-sql-proxy/v2.8.2/cloud-sql-proxy.${PLATFORM}.amd64 sudo install -m 755 cloud-sql-proxy /usr/local/bin rm -f ./cloud-sql-proxy From b990802ceee533451d38828ef1778f5957fc53f0 Mon Sep 17 00:00:00 2001 From: David Asabina Date: Mon, 11 Mar 2024 20:02:26 -0400 Subject: [PATCH 2/8] chore: Parameterise bin path --- Makefile | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index d35236ded..d15ca26d2 100644 --- a/Makefile +++ b/Makefile @@ -6,6 +6,7 @@ DOCKERPORT := 8080 LOCALPORT := 6200 DBPROXYPORT := 6201 BROWSE_DOCKER_RUN := docker run --cpus 2 --rm -p ${LOCALPORT}:${DOCKERPORT} -e PORT=${DOCKERPORT} -v ${HOME}/arxiv/arxiv-browse/tests:/tests --name ${NAME} --env-file "${PWD}/tests/docker.env" --security-opt="no-new-privileges=true" +BIN_PATH ?= /usr/local/bin PLATFORM := $(shell uname -s | tr '[:upper:]' '[:lower:]') default: venv @@ -17,12 +18,13 @@ venv: .prerequisit . venv/bin/activate && pip install poetry==1.3.2 . venv/bin/activate && poetry install -/usr/local/bin/cloud-sql-proxy: +${BIN_PATH}/cloud-sql-proxy: curl -o ./cloud-sql-proxy https://storage.googleapis.com/cloud-sql-connectors/cloud-sql-proxy/v2.8.2/cloud-sql-proxy.${PLATFORM}.amd64 - sudo install -m 755 cloud-sql-proxy /usr/local/bin + mkdir -p ${BIN_PATH} + sudo install -m 755 cloud-sql-proxy ${BIN_PATH} rm -f ./cloud-sql-proxy -.prerequisit: /usr/local/bin/cloud-sql-proxy +.prerequisit: ${BIN_PATH}/cloud-sql-proxy sudo apt install -y libmysqlclient-dev touch .prerequisit @@ -30,10 +32,10 @@ run: venv . venv/bin/activate && python main.py proxy: - /usr/local/bin/cloud-sql-proxy --address 0.0.0.0 --port 1234 arxiv-production:us-east4:arxiv-production-rep4 > /dev/null 2>&1 & + ${BIN_PATH}/cloud-sql-proxy --address 0.0.0.0 --port 1234 arxiv-production:us-east4:arxiv-production-rep4 > /dev/null 2>&1 & dev-proxy: - /usr/local/bin/cloud-sql-proxy --address 0.0.0.0 --port ${DBPROXYPORT} arxiv-development:us-east4:arxiv-db-dev > /dev/null 2>&1 & + ${BIN_PATH}/cloud-sql-proxy --address 0.0.0.0 --port ${DBPROXYPORT} arxiv-development:us-east4:arxiv-db-dev > /dev/null 2>&1 & tests/.env: ~/.arxiv/browse.env cp $< $@ From a6b1463617f4decc6ad46e87092a0322fdb84fdf Mon Sep 17 00:00:00 2001 From: David Asabina Date: Mon, 11 Mar 2024 20:06:11 -0400 Subject: [PATCH 3/8] chore: Track devenv gitignore and lockfile --- .gitignore | 11 ++++ devenv.lock | 156 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 167 insertions(+) create mode 100644 devenv.lock diff --git a/.gitignore b/.gitignore index aff0eaa9a..efb942e31 100644 --- a/.gitignore +++ b/.gitignore @@ -194,3 +194,14 @@ TAG.txt .htaccess /.prerequisit + +# Devenv +.devenv* +devenv.local.nix + +# direnv +.direnv + +# pre-commit +.pre-commit-config.yaml + diff --git a/devenv.lock b/devenv.lock new file mode 100644 index 000000000..53e9b4bc5 --- /dev/null +++ b/devenv.lock @@ -0,0 +1,156 @@ +{ + "nodes": { + "devenv": { + "locked": { + "dir": "src/modules", + "lastModified": 1710144971, + "narHash": "sha256-CjTOdoBvT/4AQncTL20SDHyJNgsXZjtGbz62yDIUYnM=", + "owner": "cachix", + "repo": "devenv", + "rev": "6c0bad0045f1e1802f769f7890f6a59504825f4d", + "type": "github" + }, + "original": { + "dir": "src/modules", + "owner": "cachix", + "repo": "devenv", + "type": "github" + } + }, + "flake-compat": { + "flake": false, + "locked": { + "lastModified": 1696426674, + "narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=", + "owner": "edolstra", + "repo": "flake-compat", + "rev": "0f9255e01c2351cc7d116c072cb317785dd33b33", + "type": "github" + }, + "original": { + "owner": "edolstra", + "repo": "flake-compat", + "type": "github" + } + }, + "flake-utils": { + "inputs": { + "systems": "systems" + }, + "locked": { + "lastModified": 1701680307, + "narHash": "sha256-kAuep2h5ajznlPMD9rnQyffWG8EM/C73lejGofXvdM8=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "4022d587cbbfd70fe950c1e2083a02621806a725", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "gitignore": { + "inputs": { + "nixpkgs": [ + "pre-commit-hooks", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1703887061, + "narHash": "sha256-gGPa9qWNc6eCXT/+Z5/zMkyYOuRZqeFZBDbopNZQkuY=", + "owner": "hercules-ci", + "repo": "gitignore.nix", + "rev": "43e1aa1308018f37118e34d3a9cb4f5e75dc11d5", + "type": "github" + }, + "original": { + "owner": "hercules-ci", + "repo": "gitignore.nix", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1710159071, + "narHash": "sha256-CT0WKgcmlcWZPZL/sSSICN/Vbm4Of0ZDgxc0GFf6sYU=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "0fbcc4b2e8571f4af39be41752581ea09dd9ab06", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixpkgs-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs-stable": { + "locked": { + "lastModified": 1704874635, + "narHash": "sha256-YWuCrtsty5vVZvu+7BchAxmcYzTMfolSPP5io8+WYCg=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "3dc440faeee9e889fe2d1b4d25ad0f430d449356", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-23.11", + "repo": "nixpkgs", + "type": "github" + } + }, + "pre-commit-hooks": { + "inputs": { + "flake-compat": "flake-compat", + "flake-utils": "flake-utils", + "gitignore": "gitignore", + "nixpkgs": [ + "nixpkgs" + ], + "nixpkgs-stable": "nixpkgs-stable" + }, + "locked": { + "lastModified": 1708018599, + "narHash": "sha256-M+Ng6+SePmA8g06CmUZWi1AjG2tFBX9WCXElBHEKnyM=", + "owner": "cachix", + "repo": "pre-commit-hooks.nix", + "rev": "5df5a70ad7575f6601d91f0efec95dd9bc619431", + "type": "github" + }, + "original": { + "owner": "cachix", + "repo": "pre-commit-hooks.nix", + "type": "github" + } + }, + "root": { + "inputs": { + "devenv": "devenv", + "nixpkgs": "nixpkgs", + "pre-commit-hooks": "pre-commit-hooks" + } + }, + "systems": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} From b346a94655a1c775e7dab5a45dabd3e422d4b87b Mon Sep 17 00:00:00 2001 From: David Asabina Date: Mon, 11 Mar 2024 20:13:37 -0400 Subject: [PATCH 4/8] chore: Track devenv.nix and yaml --- devenv.nix | 17 +++++++++++++++++ devenv.yaml | 3 +++ 2 files changed, 20 insertions(+) create mode 100644 devenv.nix create mode 100644 devenv.yaml diff --git a/devenv.nix b/devenv.nix new file mode 100644 index 000000000..9f2175065 --- /dev/null +++ b/devenv.nix @@ -0,0 +1,17 @@ +{ pkgs, ... }: + +{ + packages = [ + pkgs.git + pkgs.libmysqlclient + ]; + + languages.python = { + enable = true; + poetry = { + enable = true; + activate.enable = true; + }; + venv.enable = true; + }; +} diff --git a/devenv.yaml b/devenv.yaml new file mode 100644 index 000000000..c7cb5ceda --- /dev/null +++ b/devenv.yaml @@ -0,0 +1,3 @@ +inputs: + nixpkgs: + url: github:NixOS/nixpkgs/nixpkgs-unstable From 4bdf5fef66f2808f10e1a876b9945b32aac4a6d6 Mon Sep 17 00:00:00 2001 From: David Asabina Date: Tue, 12 Mar 2024 11:07:29 -0400 Subject: [PATCH 5/8] chore: Set BIN_PATH and install cloud-sql-proxy in devenv --- devenv.nix | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/devenv.nix b/devenv.nix index 9f2175065..6d3632c64 100644 --- a/devenv.nix +++ b/devenv.nix @@ -1,6 +1,15 @@ { pkgs, ... }: { + env = { + BIN_PATH = "bin"; + }; + + enterShell = '' + # See to it that cloud-sql-proxy is installed in to dev env + make $BIN_PATH/cloud-sql-proxy + ''; + packages = [ pkgs.git pkgs.libmysqlclient From 3902a5da606e5cdf38a1bba8672c9213e1080ec9 Mon Sep 17 00:00:00 2001 From: David Asabina Date: Tue, 12 Mar 2024 11:07:52 -0400 Subject: [PATCH 6/8] chore: Run poetry install in devenv --- devenv.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/devenv.nix b/devenv.nix index 6d3632c64..86a16b63a 100644 --- a/devenv.nix +++ b/devenv.nix @@ -20,6 +20,7 @@ poetry = { enable = true; activate.enable = true; + install.enable = true; }; venv.enable = true; }; From 5a00c0dfd23a6276ca23044eabaa0ae869ac9b3d Mon Sep 17 00:00:00 2001 From: David Asabina Date: Tue, 12 Mar 2024 11:09:04 -0400 Subject: [PATCH 7/8] chore: Gitignore bin --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index efb942e31..fe874ab9d 100644 --- a/.gitignore +++ b/.gitignore @@ -205,3 +205,4 @@ devenv.local.nix # pre-commit .pre-commit-config.yaml +/bin \ No newline at end of file From af9cf63606847d0168c59554c15c01e4c72de5b3 Mon Sep 17 00:00:00 2001 From: David Asabina Date: Tue, 12 Mar 2024 11:09:16 -0400 Subject: [PATCH 8/8] doc: Outline non-Debian setup and devenv setup --- README.md | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/README.md b/README.md index 21be4391f..53e9abb68 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,40 @@ By default, the application will use the directory trees in metadata and PDF files. These paths can be overridden via environment variables (see `browse/config.py`). +### Note for non-Debian users + +In case you are not working on a Debian-based distribution, there are a few things to take note of: +1. You may not want to install into `/usr/local/bin` to avoid polluting your system state and on some OSes, like NixOS, this may be prohibitively difficult. + Use the `BIN_PATH` env var to set a different directory to install binaries, like cloud-sql-proxy, into. +2. You may not have `apt` available as a package manager on your system and may therefore have to install the mysqlclient lib in a manner appropriate for your OS. + Refer to your OS's documentation to install the mysqlclient lib and touch the .prerequisit file in order to allow to satisfy the neccessary prerequisit for `make venv` to continue. + ```bash + touch .prerequisit + ``` + + Note that on macOS, `apt` is the [Java relic predating JDK 7](https://docs.oracle.com/javase//7/docs/technotes/guides/apt/GettingStarted.html#deprecated) which definitely can't be used to install packages. + +```bash +# TODO: manually install mysqlclient-dev +touch .prerequisit +BIN_PATH=bin make venv +``` + +> :warning: Using the `make venv` rule may open you up for some weird state issues on non-Debian platforms that are clang and grpcio related. See https://github.com/grpc/grpc/issues/30723#issuecomment-1231809453 for an example of such an issue and a imperative workaround. For a declarative way to manage your environment, consider the use of the [devenv.sh](#note-for-devenvsh-users) as documented below. + +### Note for devenv.sh users + +> :bulb: devenv.sh is a Nix-based configuration management tool that allows one to declaratively manage development environements. + +In case you don't want to be bothered managing system-level dependencies, feel free to use the [devenv.sh](https://devenv.sh/) config provided in this repo. Refer to https://devenv.sh/getting-started/ to get started with devenv. With devenv set up, the following will automagically happen when entering the project directory (in a terminal): +1. mysqlproxy will be installed +2. `BIN_PATH` will be set to a working-directory local path such that the Makefile rules still work for you +3. cloud-sql-proxy will be installed into your working directory (at the path specified by the `BIN_PATH` env var inside of the devenv.nix file) +4. a venv will be created in the `.venv` directory local to your working directory +5.`poetry install` will be executed + +> :bulb: The use of the provided devenv config renders the use of `make venv` and `make run` moot. Rely on devenv.sh to manage your venv instead and run `python main.py` directly instead of relying on `make run` (since you don't need to spin up/activate your venv anymore). The use of `make proxy` and `make dev-proxy` should still work due to us having specified the correct `BIN_PATH` in the devenv config. + ### Running Browse with .env file First, you'd need to create the '.env' file somewhere. Using tests/.env is suggested.