diff --git a/stack/README.md b/stack/README.md index 9392720..aaabd3c 100644 --- a/stack/README.md +++ b/stack/README.md @@ -1,12 +1,24 @@ # Docker stack -Run the complete demo stack using the following command: +You can start the stack with a full demo dataset: ```sh -docker compose pull -docker compose up -d --build +docker compose \ + --profile full-demo-dataset \ + up -d --build ``` +You can also add a basic Kubernetes + OCI demo dataset, by running the following command: + +```sh +docker compose \ + --profile basic-metadata \ + --profile k8s-demo-data \ + up -d --build +``` + +You will see in a next section how to build this dataset by yourself, in case you want to try with your own Kubernetes cluster. + Here is where each component is exposed: - Blueprint: http://127.0.0.1:8081/ @@ -15,27 +27,23 @@ Here is where each component is exposed: ## Generate triples -### Graphs - -Here is the list of graphs that are used for this demo: - -- Kubernetes + OCI data: `urn:graph:k8s` +Here is how some of the triples were generated, in case you want to generate your own, based on your own data. ### Kubernetes triples #### Without a Kubernetes cluster -Once you started the Docker Compose stack, you can run the following command: +Once you have started the Docker Compose stack, you can run the following command (this is the one that was used in the first part already): ```sh -docker compose -f docker-compose.data.yaml up add-demo-k8s-data --build +docker compose --profile k8s-demo-data up -d --build ``` This will run a job that will import the triples that are in the `demo-data/k8s.ttl` file. Those triples were generated by starting a Kubernetes cluster using Docker Desktop. -This will help you to populate the store with some useful data without the need of running a Kubernetes cluster, so that you can start playing with them. +This will help you populate the store with some useful data without the need of running a Kubernetes cluster, so that you can start playing with them. To learn more about how those triples were generated, read the next section. @@ -48,7 +56,7 @@ If you want to quickly get a cluster locally, just: - [enable Kubernetes from Docker Desktop](https://docs.docker.com/desktop/kubernetes/) if you are using MacOS or Windows, - [install microk8s](https://microk8s.io/#install-microk8s) + run the following command: `microk8s config > ~/.kube/config`, if you are using Linux. -By default it will use the selected `context` (you get get it by running: `config current-context`) that you have when running [`kubectl`](https://kubernetes.io/docs/tasks/tools/#kubectl). +By default it will use the selected `context` (you get it by running: `config current-context`) that you have when running [`kubectl`](https://kubernetes.io/docs/tasks/tools/#kubectl). For more information, you can read the README of the project here: https://github.com/zazuko/k8s-rdf-exporter. To generate triples, you can run the following command within this directory: @@ -61,9 +69,22 @@ npx @zazuko/k8s-rdf-exporter@latest \ --base-iri-oci=http://127.0.0.1:8080/oci/ > ./volumes/k8s-data/data.ttl ``` -The `demo-data/k8s.ttl` was was generated that way, using a Kubernetes cluster created using Docker Desktop. +The `demo-data/k8s.ttl` was generated that way, using a Kubernetes cluster created using Docker Desktop. Importing generated triples to the Fuseki triplestore (run one of both): -- for the custom file with your cluster (`volumes/k8s-data/data.ttl`): `docker compose -f docker-compose.data.yaml up add-custom-k8s-data --build` -- for the demo file (`demo-data/k8s.ttl`): `docker compose -f docker-compose.data.yaml up add-demo-k8s-data --build` +- for the custom file with your cluster (`volumes/k8s-data/data.ttl`): `docker compose --profile k8s-custom-data up -d --build` +- for the demo file (`demo-data/k8s.ttl`): `docker compose --profile k8s-demo-data up -d --build` + +Running one of the above commands will import the triples into the Fuseki triplestore and removes the previous k8s data, not the other data or any metadata. + +## Reset everything + +To reset everything (it will also remove your custom k8s data if any), you can run the following command: + +```sh +docker compose --profile='*' down +rm -rf ./volumes/ +``` + +This can be useful if you want to start from scratch and try other kinds of data. diff --git a/stack/alpine-curl/Dockerfile b/stack/alpine-curl/Dockerfile deleted file mode 100644 index 4ea6312..0000000 --- a/stack/alpine-curl/Dockerfile +++ /dev/null @@ -1,3 +0,0 @@ -FROM docker.io/library/alpine:latest -WORKDIR /app -RUN apk add --no-cache curl diff --git a/stack/docker-compose.data.yaml b/stack/docker-compose.data.yaml deleted file mode 100644 index 54f769c..0000000 --- a/stack/docker-compose.data.yaml +++ /dev/null @@ -1,20 +0,0 @@ -services: - # Add k8s demo data (for those who don't have a Kubernetes cluster) - add-demo-k8s-data: - build: - context: ./alpine-curl/ - dockerfile: Dockerfile - restart: "no" - command: 'curl -vvv --fail-with-body -u admin:admin http://store:3030/blueprint/data?graph=urn:graph:k8s --data-binary @data.ttl --header "Content-Type: text/turtle"' - volumes: - - ./demo-data/k8s.ttl:/app/data.ttl:ro - - # Add custom k8s data generated locally (will not be in the git history) - add-custom-k8s-data: - build: - context: ./alpine-curl/ - dockerfile: Dockerfile - restart: "no" - command: 'curl -vvv --fail-with-body -u admin:admin http://store:3030/blueprint/data?graph=urn:graph:k8s --data-binary @data.ttl --header "Content-Type: text/turtle"' - volumes: - - ./volumes/k8s-data/data.ttl:/app/data.ttl:ro diff --git a/stack/docker-compose.yaml b/stack/docker-compose.yaml index 8051518..8de6278 100644 --- a/stack/docker-compose.yaml +++ b/stack/docker-compose.yaml @@ -11,6 +11,8 @@ services: - ./volumes/fuseki/:/fuseki/databases/ store-data: + profiles: + - basic-metadata build: context: ./fuseki-data/ dockerfile: Dockerfile @@ -19,6 +21,7 @@ services: - ./fuseki-data:/app/data:ro blueprint: + image: ghcr.io/zazuko/blueprint:latest build: context: .. dockerfile: Dockerfile @@ -40,3 +43,37 @@ services: - SPARQL_ENDPOINT_URL=http://store:3030/blueprint/query - SPARQL_USERNAME=admin - SPARQL_PASSWORD=admin + + # Add k8s demo data (for those who don't have a Kubernetes cluster) + add-demo-k8s-data: + profiles: + - k8s-demo-data + build: + context: ./fuseki-data/ + dockerfile: Dockerfile + restart: "no" + volumes: + - ./demo-data/k8s.ttl:/app/data/k8s.ttl:ro + + # Add custom k8s data generated locally (will not be in the git history) + add-custom-k8s-data: + profiles: + - k8s-custom-data + build: + context: ./fuseki-data/ + dockerfile: Dockerfile + restart: "no" + volumes: + - ./volumes/k8s-data/data.ttl:/app/data/k8s.ttl:ro + + # Add demo dataset (will not be in the git history) + add-demo-dataset: + profiles: + - full-demo-dataset + build: + context: ./fuseki-data/ + dockerfile: Dockerfile + restart: "no" + environment: + - DOWNLOAD_URL=https://download.zazukoians.org/blueprint/demo-data.nt + - DOWNLOAD_NAME=demo-dataset.nt diff --git a/stack/fuseki-data/load-data.sh b/stack/fuseki-data/load-data.sh index 7a37f04..1d9b829 100755 --- a/stack/fuseki-data/load-data.sh +++ b/stack/fuseki-data/load-data.sh @@ -1,5 +1,8 @@ #!/bin/sh +DOWNLOAD_URL="${DOWNLOAD_URL:-}" +DOWNLOAD_NAME="${DOWNLOAD_NAME:-}" + set -eu echo "==========================" @@ -14,11 +17,39 @@ echo "Loading data to triple store…" echo "=============================" echo "" -# loop over all .ttl files -for f in /app/data/*.ttl; do - graph=$(basename -s .ttl "${f}") - echo "Loading '${f}' in graph urn:graph:${graph}…" - curl -s --fail-with-body -u admin:admin "http://store:3030/blueprint/data?graph=urn:graph:${graph}" --data-binary "@${f}" --header "Content-Type: text/turtle" +# Make sure the data directory exists +mkdir -p /app/data + +# If DOWNLOAD_URL is set, download the data +if [ -n "${DOWNLOAD_URL}" ]; then + if [ -z "${DOWNLOAD_NAME}" ]; then + DOWNLOAD_NAME="data$(date -u +'%Y%m%d%H%M%S').ttl" + fi + + echo "Downloading data from '${DOWNLOAD_URL}' as '${DOWNLOAD_NAME}'…" + curl -vvv -L -o "/app/data/${DOWNLOAD_NAME}" "${DOWNLOAD_URL}" + echo "" +fi + +ls -alh /app/data/* + +# Loop over all files in the data directory and load them +for f in /app/data/*; do + filename=$(basename "${f}") + ext="${filename##*.}" + graph="${filename%.*}" + if [ "${ext}" = "ttl" ]; then + echo "Loading '${f}' in graph urn:graph:${graph}…" + curl -s -X PUT --fail-with-body -u admin:admin "http://store:3030/blueprint/data?graph=urn:graph:${graph}" \ + --data-binary "@${f}" --header "Content-Type: text/turtle" + elif [ "${ext}" = "nt" ]; then + echo "Loading '${f}' in graph urn:graph:${graph}…" + curl -s -X PUT --fail-with-body -u admin:admin "http://store:3030/blueprint/data?graph=urn:graph:${graph}" \ + --data-binary "@${f}" --header "Content-Type: application/n-triples" + else + echo "Skipping '${f}'…" + continue + fi echo "" done