From 5dd93f6c4705f0eae5b72fa0fdd56ac11bed2ddf Mon Sep 17 00:00:00 2001 From: Zak Lee Date: Tue, 22 Oct 2024 08:18:45 -0700 Subject: [PATCH 1/4] update computer-use-demo workflows (#35) * update computer-use-demo workflows * update github target --- .github/workflows/build.yaml | 6 ++++-- .github/workflows/tests.yaml | 8 +++++++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 25729df3..ff091e56 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -4,11 +4,13 @@ name: build on: pull_request: paths: - - computer-use-demo/** + - .github/** + - computer-use-demo/** push: branches: - main paths: + - .github/** - computer-use-demo/** jobs: build: @@ -86,7 +88,7 @@ jobs: uses: docker/build-push-action@v5 with: platforms: linux/${{ matrix.platform }} - context: . + context: computer-use-demo push: true tags: ${{ env.PUSH_TAGS }} cache-from: type=gha,scope=computer-use-${{ matrix.platform }} diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index c9d1b481..ade2d45a 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -1,9 +1,15 @@ name: tests on: - pull_request: {} + pull_request: + paths: + - .github/** + - computer-use-demo/** push: branches: - main + paths: + - .github/** + - computer-use-demo/** jobs: ruff: runs-on: ubuntu-latest From e2625090f8ad01d32cab6435d6a89658124f2766 Mon Sep 17 00:00:00 2001 From: Zak Lee Date: Tue, 22 Oct 2024 08:25:59 -0700 Subject: [PATCH 2/4] re-add precommit (#37) --- .pre-commit-config.yaml | 24 ++++++++++++++++++++++++ computer-use-demo/README.md | 6 ++++++ pyproject.toml | 4 ++++ 3 files changed, 34 insertions(+) create mode 100644 .pre-commit-config.yaml create mode 100644 pyproject.toml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..16804e3c --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,24 @@ +files: ^computer-use-demo/ +default_stages: [pre-commit, pre-push] +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v2.3.0 + hooks: + - id: check-yaml + - id: end-of-file-fixer + - id: trailing-whitespace + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.6.7 + hooks: + - id: ruff + name: Run `ruff` to autofix lint errors + args: [--fix-only] + - id: ruff + name: Run `ruff` to format code + entry: ruff format --force-exclude + - id: ruff + name: Run `ruff` to lint code + - repo: https://github.com/RobertCraigie/pyright-python + rev: v1.1.384 + hooks: + - id: pyright diff --git a/computer-use-demo/README.md b/computer-use-demo/README.md index c31739ea..8ef5cbc9 100644 --- a/computer-use-demo/README.md +++ b/computer-use-demo/README.md @@ -48,6 +48,7 @@ Once the container is running, open your browser to [http://localhost:8080](http The container stores settings like API key and custom system prompt in `~/.anthropic/`. Mount this directory to persist these settings between container runs. Alternative access points: + - Streamlit interface only: [http://localhost:8501](http://localhost:8501) - Desktop view only: [http://localhost:6080/vnc.html](http://localhost:6080/vnc.html) - Direct VNC connection: `vnc://localhost:5900` (for VNC clients) @@ -94,6 +95,7 @@ docker run \ ``` ### Vertex + You'll need to pass in Google Cloud credentials with appropriate permissions to use Claude on Vertex. ```bash @@ -112,11 +114,13 @@ docker run \ -p 8080:8080 \ -it computer-use-demo ``` + This example shows how to use the Google Cloud Application Default Credentials to authenticate with Vertex. You can also set `GOOGLE_APPLICATION_CREDENTIALS` to use an arbitrary credential file, see the [Google Cloud Authentication documentation](https://cloud.google.com/docs/authentication/application-default-credentials#GAC) for more details. ## Screen size + Environment variables `WIDTH` and `HEIGHT` can be used to set the screen size. For example: ```bash @@ -137,6 +141,7 @@ We do not recommend sending screenshots in resolutions above [XGA/WXGA](https:// Relying on the image resizing behavior in the API will result in lower model accuracy and slower performance than implementing scaling in your tools directly. The `computer` tool implementation in this project demonstrates how to scale both images and coordinates from higher resolutions to the suggested resolutions. ## Development + ```bash ./setup.sh # configure venv, install development dependencies, and install pre-commit hooks docker build . -t computer-use-demo:local # manually build the docker image (optional) @@ -151,4 +156,5 @@ docker run \ -p 8080:8080 \ -it computer-use-demo:local # can also use ghcr.io/anthropics/anthropic-quickstarts:computer-use-demo-latest ``` + The docker run command above mounts the repo inside the docker image, such that you can edit files from the host. Streamlit is already configured with auto reloading. diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..d9ed5fba --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,4 @@ +[tool.pyright] +venvPath = "computer-use-demo" +venv = ".venv" +useLibraryCodeForTypes = false \ No newline at end of file From 6df3c6acc171fab4ba1162debde52e5029f47ef6 Mon Sep 17 00:00:00 2001 From: Andrew Edstrom Date: Tue, 22 Oct 2024 10:21:55 -0600 Subject: [PATCH 3/4] Add note about where to find the demo app for Vertex after it starts up (#38) * Add note about where to find the app for Vertex after it starts up * Move instructions on accessing the app to the end of the quickstart section * dummy change to trigger CI * Add link to section below * Fix AWS profile --- computer-use-demo/README.md | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/computer-use-demo/README.md b/computer-use-demo/README.md index 8ef5cbc9..a7d09e80 100644 --- a/computer-use-demo/README.md +++ b/computer-use-demo/README.md @@ -43,15 +43,7 @@ docker run \ -it ghcr.io/anthropics/anthropic-quickstarts:computer-use-demo-latest ``` -Once the container is running, open your browser to [http://localhost:8080](http://localhost:8080) to access the combined interface that includes both the agent chat and desktop view. - -The container stores settings like API key and custom system prompt in `~/.anthropic/`. Mount this directory to persist these settings between container runs. - -Alternative access points: - -- Streamlit interface only: [http://localhost:8501](http://localhost:8501) -- Desktop view only: [http://localhost:6080/vnc.html](http://localhost:6080/vnc.html) -- Direct VNC connection: `vnc://localhost:5900` (for VNC clients) +Once the container is running, see the [Accessing the demo app](#accessing-the-demo-app) section below for instructions on how to connect to the interface. ### Bedrock @@ -75,6 +67,8 @@ docker run \ -it ghcr.io/anthropics/anthropic-quickstarts:computer-use-demo-latest ``` +Once the container is running, see the [Accessing the demo app](#accessing-the-demo-app) section below for instructions on how to connect to the interface. + #### Option 2: Use an access key and secret ```bash @@ -94,6 +88,8 @@ docker run \ -it ghcr.io/anthropics/anthropic-quickstarts:computer-use-demo-latest ``` +Once the container is running, see the [Accessing the demo app](#accessing-the-demo-app) section below for instructions on how to connect to the interface. + ### Vertex You'll need to pass in Google Cloud credentials with appropriate permissions to use Claude on Vertex. @@ -115,10 +111,24 @@ docker run \ -it computer-use-demo ``` +Once the container is running, see the [Accessing the demo app](#accessing-the-demo-app) section below for instructions on how to connect to the interface. + This example shows how to use the Google Cloud Application Default Credentials to authenticate with Vertex. You can also set `GOOGLE_APPLICATION_CREDENTIALS` to use an arbitrary credential file, see the [Google Cloud Authentication documentation](https://cloud.google.com/docs/authentication/application-default-credentials#GAC) for more details. +### Accessing the demo app + +Once the container is running, open your browser to [http://localhost:8080](http://localhost:8080) to access the combined interface that includes both the agent chat and desktop view. + +The container stores settings like the API key and custom system prompt in `~/.anthropic/`. Mount this directory to persist these settings between container runs. + +Alternative access points: + +- Streamlit interface only: [http://localhost:8501](http://localhost:8501) +- Desktop view only: [http://localhost:6080/vnc.html](http://localhost:6080/vnc.html) +- Direct VNC connection: `vnc://localhost:5900` (for VNC clients) + ## Screen size Environment variables `WIDTH` and `HEIGHT` can be used to set the screen size. For example: From 4ec875e4a8210d422b9865ac582ce0da054a8915 Mon Sep 17 00:00:00 2001 From: Rahim Nathwani Date: Tue, 22 Oct 2024 09:24:09 -0700 Subject: [PATCH 4/4] Fix README.md typo (#40) --- computer-use-demo/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/computer-use-demo/README.md b/computer-use-demo/README.md index a7d09e80..7feb9df7 100644 --- a/computer-use-demo/README.md +++ b/computer-use-demo/README.md @@ -14,7 +14,7 @@ This repository helps you get started with computer use on Claude, with reference implementations of: -* Build files to create a Docker container with all nescessary dependencies +* Build files to create a Docker container with all necessary dependencies * A computer use agent loop using the Anthropic API, Bedrock, or Vertex to access the updated Claude 3.5 Sonnet model * Anthropic-defined computer use tools * A streamlit app for interacting with the agent loop