Release/2.0 acacia

.dockerignore

@@ -0,0 +1,11 @@
+node_modules
+
+.git
+.github
+
+dist
+.env*
+docker-compose.yml
+Dockerfile
+*.md
+claudia.*

.github/workflows/build-docker.yml

@@ -0,0 +1,94 @@
+name: build-docker-image
+
+on:
+  push:
+    branches:
+      - 'main'
+    tags:
+      - 'v*'
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+        # qemu and buildx are for multi-platform images and caching
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      # set up local caching
+      - name: Cache Docker layers
+        uses: actions/cache@v3
+        with:
+          path: /tmp/.buildx-cache
+          key: ${{ runner.os }}-buildx-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-buildx-
+
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v4
+        with:
+          images: |
+            verida/storage-node
+            ghcr.io/verida/storage-node
+          tags: |
+            type=ref,event=branch
+            type=ref,event=pr
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}.{{minor}}
+            type=sha
+
+      # get the correct AW role to push image to ECR
+      - name: Configure AWS credentials from Testnet account
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          role-duration-seconds: 1500
+          role-to-assume: arn:aws:iam::737954963756:role/github-testnet-deploy-role
+          aws-region: us-east-1
+
+      # login to push to DockerHub
+      - name: Login to DockerHub
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      # login to push to GHCR
+      - name: Login to GHCR
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build and push
+        uses: docker/build-push-action@v3
+        with:
+          platforms: linux/amd64 # ,linux/arm64,linux/arm/v7
+          context: .
+          push: ${{ github.event_name != 'pull_request' }}
+          cache-from: type=local,src=/tmp/.buildx-cache
+          cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max
+          tags: ${{ steps.meta.output.tags }}
+          labels: ${{ steps.meta.output.labels }}
+
+      # Temp fix
+      # https://github.com/docker/build-push-action/issues/252
+      # https://github.com/moby/buildkit/issues/1896
+      - name: Move cache
+        run: |
+          rm -rf /tmp/.buildx-cache
+          mv /tmp/.buildx-cache-new /tmp/.buildx-cache

.gitignore

@@ -4,6 +4,7 @@ dist
 .DS_Store
 .vscode
 curl_tests
+dbdata
 
 .env
-.env*
+.env*

.nvmrc

@@ -1 +1 @@
-14.17.1
+14.20.0

CHANGELOG.md

@@ -1,3 +1,17 @@
+2023-01-13 (2.0.0)
+--------------------
+
+- Support blockchain DID registry
+- Support refresh and access tokens
+- Support server side replication, including cehcks and recovery
+- Support device disconnections
+- Support maintaining database of context specific databases to enable monitoring of usage
+- Support database deletion
+- Support create, update, retreive and delete DID documents
+- Support `/lookup/did` endpoint to provide a DID document caching service
+- Remove lambda support
+- Support docker containers
+
 
 2021-09-17 (1.2.0)
 --------------------

Dockerfile

@@ -0,0 +1,33 @@
+FROM node:14.20.0-slim as node
+
+ENV NODE_ENV=production
+EXPOSE 5000
+
+RUN mkdir /storage-node && chown -R node:node /storage-node
+WORKDIR /storage-node
+
+# this ensures we fix simlinks for npx and yarn
+RUN corepack disable && corepack enable
+
+RUN apt-get update \
+    && apt-get -qq install -y --no-install-recommends \
+    git ca-certificates dnsutils \
+    && rm -rf /var/lib/apt/lists/*
+
+# new stage
+FROM node AS source
+
+USER node
+# Copy the current directory
+COPY --chown=node:node . .
+
+# we need devDependencies to build. Setting --production=false ignores NODE_ENV (deliberatly)
+RUN yarn install --production=false --frozen-lockfile
+# we have to build inside the container in case we are on a Mac building a linux image
+RUN yarn build
+
+
+### prod stage
+# Note: use --init option when running the container to have better signal forwarding
+FROM source as prod
+CMD ["node", "--trace-warnings", "./dist/server.js"]

README.md

@@ -10,6 +10,23 @@ Key features:
 - Adding a second layer of security by managing per-database ACL validation rules
 - Providing applications with user's database connection strings
 
+## How Authorization Works
+
+This is the login flow:
+
+1. The Verida Account makes a request to the storage node API to obtain an auth JWT to be signed (`/auth/generateAuthJwt`). This prevents replay attacks.
+2. The Verida Account signs a consent message using their private key. This consent message proves the user wants to unlock a specific application context
+3. The Verida Account submits the signed authorization request (`/auth/authenticate`). Assuming the signed AuthJWT is valid, the storage node returns a refresh token and an access token
+4. The Verida Account can then use the access token to either; 1) make storage node requests (ie: create database) or 2) directly access CouchDB as an authenticated user (using `Bearer` token auth)
+5. When the access token expires, the Verida Account can use the refresh token to request a new access token (`/auth/connect`)
+6. If a refresh token is close to expiry, the Verida Account can use the active refresh token to obtain a new refresh token (`/auth/regenerateRefreshToken`)
+
+When a Verida Account authenticates, it can designate an `authenticate` requst to be linked to a particular device by specifying the `deviceId` in the request.
+
+This allows a specific device to be linked to a refresh token. A call to `/auth/invalidateDeviceId` can be used to invalidate any refresh tokens linked to the specified `deviceId`. This allows the Verida Vault to remotely log out an application that previously logged in.
+
+Note: This only invalidates the refresh token. The access token will remain valid until it expires. It's for this reason that access tokens are configured to have a short expiry (5 minutes by default). CouchDB does not support manually invalidating access tokens, so we have to take this timeout approach to invalidation.
+
 ## Usage
 
 ```bash
@@ -30,7 +47,6 @@ This server is running on the Verida Testnet and is accessible by any applicatio
 
 A `sample.env` is included. Copy this to `.env` and update the configuration:
 
-- `HASH_KEY`: A unique hash key that is used as entropy when generating an alpha numeric username from a DID. Set this to a unique value when first running the server. DO NOT change this key once the server is up and running as you will end up with a mismatch of usernames. If you run multiple servers in front of a cluster of CouchDB instances, all servers must use the same `HASH_KEY`.
 - `DID_SERVER_URL`: URL of a Verida DID Server endpoint.
 - `DB_PROTOCOL`: Protocol to use when connecting to CouchDB (`http` or `https`).
 - `DB_USER`: Username of CouchDB Admin (has access to create users and databases).
@@ -40,12 +56,15 @@ A `sample.env` is included. Copy this to `.env` and update the configuration:
 - `DB_REJECT_UNAUTHORIZED_SSL`: Boolean indicating if unauthorized SSL certificates should be rejected (`true` or `false`). Defaults to `false` for development testing. Must be `true` for production environments otherwise SSL certificates won't be verified.
 - `DB_PUBLIC_USER`: Alphanumeric string for a public database user. These credentials can be requested by anyone and provide access to all databases where the permissions have been set to `public`.
 - `DB_PUBLIC_PASS`: Alphanumeric string for a public database password.
+- `ACCESS_TOKEN_EXPIRY`: Number of seconds before an access token expires. The protocol will use the refresh token to obtain a new access token. CouchDB does not support a way to force the expiry of an issued token, so the access token expiry should always be set to 5 minutes (300)
+- `REFRESH_TOKEN_EXPIRY`: Number of seconds before a refresh token expires. Users will be forced to re-login once this time limit is reached. This should be set to 7 days (604800).
+- `ACCESS_JWT_SIGN_PK`: The access token private key. The base64 version of this must be specified in the CouchDB configuration under `jwt_keys/hmac:_default`
+- `REFRESH_JWT_SIGN_PK`: The refresh token private key
 
 ### Setting up environment variables on Windows
 
 * On a powershell execute the following ( replica of `.env` )
 ```bash
-$env:HASH_KEY="this_is_not_prod_hash_key"
 $env:DID_SERVER_URL="https://dids.testnet.verida.io:5001"
 $env:DID_CACHE_DURATION=3600
 $env:DB_PROTOCOL="http"
@@ -63,27 +82,83 @@ $env:DB_PUBLIC_PASS="784c2n780c9cn0789"
 - CORS must be enabled so that database requests can come from any domain name
 - A valid user must be enforced for security reasons
 
+[Ensure `{chttpd_auth, jwt_authentication_handler}` is added to the list of the active `chttpd/authentication_handlers`](https://docs.couchdb.org/en/stable/api/server/authn.html?highlight=jwt#jwt-authentication)
+
+
+
 ```
-[httpd]
-WWW-Authenticate = Basic realm="administrator"
+[couchdb]
+single_node=true
+
+[chttpd]
+authentication_handlers = {chttpd_auth, jwt_authentication_handler}, {chttpd_auth, cookie_authentication_handler}, {chttpd_auth, default_authentication_handler}
 enable_cors = true
 
 [chttpd_auth]
 require_valid_user = true
 
+[jwt_auth]
+required_claims = exp
+
+[jwt_keys]
+hmac:_default = <base64 secret key>
+
 [cors]
 origins = *
 credentials = true
 methods = GET, PUT, POST, HEAD, DELETE
 headers = accept, authorization, content-type, origin, referer, x-csrf-token
 ```
 
-## Lambda deployment
+The `hmac:_default` key is a base64 encoded representation of the access token JWT private key
+
+## Generating JWT key
+
+Note: A secret key (string) suitable for `jwt_keys` can be base64 encoded with the following:
+
+```
+const secretKey = 'secretKey'
+const encodedKey = Buffer.from(secretKey).toString('base64')
+```
+
+This can be tested via curl:
+
+```
+curl -H "Host: localhost:5984" \
+ -H "accept: application/json, text/plain, */*" \
+ -H "authorization: Bearer <bearer_token>" \
+  "http://localhost:5984/_session"
+```
+
+Where:
+
+- `bearer_token` - A bearer token generated via the `test/jwt` unit test
+- `localhost` - Replace this with the hostname of the server being tested
+
+## Docker
+
+You can spin up storage node API on your machine with Docker:
+```shell
+docker run --init --env-file=.env verida/storage-node:latest
+```
+
+### Deploying a new Docker Image to Docker Hub
+
+Note that this uses the experimental `buildx` command to build both AMD64 (Intel/AMD servers) and ARM64 (Mac) images.
+
+* Login (details in BitWarden)
+```
+docker buildx build --platform linux/amd64,linux/arm64 --push -t verida/storage-node:latest .
+```
+
+
+## Tests
 
-We use [Claudia.js](https://claudiajs.com/) to turn our Express app into an Express-on-Lambda app.
+Run tests with `yarn run tests`
 
-Before doing any Lambda deployments you **MUST** translate your `.env` file (or one for production) to JSON as `.env.prod.json`.
-See the [Claudia Docs for information](https://claudiajs.com/news/2016/11/24/claudia-2.2.0-environment-vars.html).
+Note: The tests in `server.js` require the server to be running locally. The other tests operate fine without the server running.
 
-Verida staff can see the [internal Verida repo]( https://github.com/verida/infrastructure/blob/develop/storage_node.md) for docs on this. 
+Common issues when running tests:
 
+1. `Bad key`: The key in CouchDB configuration for `jwt_keys/hmac:_default` is not a valid Base64 encoded key
+2. `HMAC error`: The key in CouchDB configuration for `jwt_keys/hmac:_default` does not match `ACCESS_JWT_SIGN_PK` in `.env`