Improved runtime of workflow

.github/workflows/fetch-data-and-push.yml

@@ -3,10 +3,10 @@ name: Fetch JSON Data and Push to Artsdata
 on:
   workflow_call:
     inputs:
-      api-url:
+      api-urls:
         required: true
         type: string
-        description: 'The URL of the API to fetch data from'
+        description: 'A comma-separated string of API URLs to fetch data from'
     secrets:
       publisher-uri:
         required: true
@@ -20,8 +20,7 @@ jobs:
       ports:
         - 7333:7333
     outputs:
-      ttl_file_name: ${{ steps.extract-file-name.outputs.ttl_file_name }}
-      artifact_name: ${{ steps.extract-file-name.outputs.artifact_name }}
+      matrix: ${{ steps.process-urls.outputs.matrix }}
 
     steps:
       - name: Install requirements
@@ -30,37 +29,51 @@ jobs:
       - name: Run ontorefine server
         run: /opt/ontorefine/dist/bin/ontorefine &
 
-      - name: Download data from the API
-        run: |
-          curl '${{ inputs.api-url }}' >> entities.json
-
       - name: Download project-configuration-file
         id: step-4
         run: | 
           curl 'https://raw.githubusercontent.com/culturecreates/artsdata-planet-spektrix/main/ontorefine/configuration.json' >> project-config.json
 
-      - name: Extract file name and artifact name from URL
-        id: extract-file-name
+      - name: Process URLs
+        id: process-urls
+        shell: bash
         run: |
-          api_url="${{ inputs.api-url }}"
-          api_name=$(echo "${api_url}" | awk -F/ '{print $(NF-3)}')
-          echo "ttl_file_name=spektrix-${api_name}-entities.ttl" >> $GITHUB_OUTPUT
-          echo "artifact_name=spektrix-${api_name}" >> $GITHUB_OUTPUT
+          mkdir -p outputs
+          matrix="[]"
+          IFS=',' read -ra URLS <<< "${{ inputs.api-urls }}"
+          for api_url in "${URLS[@]}"; do
+            curl "$api_url" >> entities.json
+            api_name=$(echo "$api_url" | awk -F/ '{print $(NF-3)}')
+            ttl_file_name="spektrix-${api_name}-entities.ttl"
+            artifact_name="spektrix-${api_name}"
+
+            json_object="{\"file_name\": \"${ttl_file_name}\", \"artifact\": \"${artifact_name}\"}"
+
+            if [ "$matrix" == "[]" ]; then
+              matrix="[${json_object}]"
+            else
+              matrix=$(echo $matrix | sed "s/]$/,$json_object]/")
+            fi
 
-      - name: Transform json data to RDF
+            /opt/ontorefine/dist/bin/ontorefine-cli \
+            transform entities.json \
+            -u http://localhost:7333 \
+            --configurations project-config.json \
+            -f json >> outputs/$ttl_file_name
+
+          done
+          formatted_matrix=$(echo "$matrix" | sed 's/^\[/{ "include": [/; s/]$/]}/')
+          echo "matrix=$formatted_matrix" >> $GITHUB_OUTPUT
+      
+      - name: Print TTL file names
         run: |
-          /opt/ontorefine/dist/bin/ontorefine-cli \
-          transform entities.json \
-          -u http://localhost:7333 \
-          --configurations project-config.json \
-          -f json >> ${{ steps.extract-file-name.outputs.ttl_file_name }}
+          echo ${{ steps.process-urls.outputs.matrix }}
 
       - name: Upload artifacts
         uses: actions/upload-artifact@v4
         with:
-          name: ${{ steps.extract-file-name.outputs.artifact_name }}
-          path: "./${{ steps.extract-file-name.outputs.ttl_file_name }}"
-          retention-days: 1
+          name: all-ttl-files
+          path: outputs/
 
   commit-and-push:
     runs-on: ubuntu-latest
@@ -73,30 +86,27 @@ jobs:
         id: download-step
         uses: actions/download-artifact@v4
         with:
-          name: ${{needs.fetch-data.outputs.artifact_name}}
-          path: ./${{ needs.fetch-data.outputs.ttl_file_name }}
+          name: all-ttl-files
+          path: outputs/
 
-      - name: Copy files
+      - name: Commit files to GitHub
         run: |
-          mkdir -p outputs
-          cp "${{ steps.download-step.outputs.download-path }}/${{ needs.fetch-data.outputs.ttl_file_name }}" ./outputs/"${{ needs.fetch-data.outputs.ttl_file_name }}"
-
-      - name: Commit file to GitHub
-        run: |
-            git pull
-            git config --global user.email "[email protected]"
-            git config --global user.name "GitHub Actions"
-            git add "outputs/${{ needs.fetch-data.outputs.ttl_file_name }}"
-            git commit -m "Push Entities"
-            git push
+          git pull
+          git config --global user.email "[email protected]"
+          git config --global user.name "GitHub Actions"
+          git add "outputs/"
+          git commit -m "Push Entities"
+          git push
 
   artsdata-push:
     runs-on: ubuntu-latest
-    needs: [commit-and-push, fetch-data]
+    needs: [fetch-data, commit-and-push]
+    strategy:
+      matrix: ${{fromJson(needs.fetch-data.outputs.matrix)}}
     steps:
-    - name: Action setup
-      uses: culturecreates/[email protected]
-      with:
-        artifact: ${{ needs.fetch-data.outputs.artifact_name }}
-        publisher: ${{ secrets.publisher-uri }}
-        downloadUrl: https://raw.githubusercontent.com/culturecreates/artsdata-planet-spektrix/outputs/${{ needs.fetch-data.outputs.ttl_file_name }}
+      - name: Action setup
+        uses: culturecreates/[email protected]
+        with:
+          artifact: ${{ matrix.artifact }}
+          publisher: ${{ secrets.publisher-uri }}
+          downloadUrl: https://raw.githubusercontent.com/culturecreates/artsdata-planet-spektrix/outputs/${{ matrix.file_name }}

.github/workflows/main.yml

@@ -5,14 +5,8 @@ on:
 
 jobs:
   call-fetch-data:
-    strategy:
-        matrix:
-          url:
-            # - 'https://system.spektrix.com/gatewaytheatre/api/v3/events'
-            - 'https://system.spektrix.com/kaymeekcentre/api/v3/events'
-            - 'https://system.spektrix.com/terrabrucemajestic/api/v3/events'
     uses: ./.github/workflows/fetch-data-and-push.yml
     with:
-      api-url: ${{ matrix.url }}
+      api-urls: 'https://system.spektrix.com/kaymeekcentre/api/v3/events,https://system.spektrix.com/terrabrucemajestic/api/v3/events'
     secrets:
-        publisher-uri: ${{secrets.PUBLISHER_URI_GREGORY}}
+      publisher-uri: ${{ secrets.PUBLISHER_URI_GREGORY }}