keystone-enclave · grg-haas · Jun 24, 2024 · Mar 27, 2024 · Jun 13, 2024 · Jun 18, 2024
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -222,7 +222,7 @@ jobs:
   # Combine cache directories to save space
   combine-caches:
     runs-on: ubuntu-latest
-    if: ${{ always() }}
+    if: success() || failure()
     needs: build
     steps:
       - name: Install dependencies
@@ -378,6 +378,6 @@ jobs:
     uses: ./.github/workflows/build-runtime.yml
 
   # System tests, which are run for simulatable and self-hostable platforms
-  test-system:
+  test-system-functionality:
     needs: build
     uses: ./.github/workflows/test-system.yml
diff --git a/.github/workflows/test-system.yml b/.github/workflows/test-system.yml
@@ -3,19 +3,31 @@ on:
   workflow_call:
 
 jobs:
-  test-generic:
-    runs-on: ubuntu-latest
+  test-system:
+    runs-on: ${{ matrix.platform == 'generic' && 'ubuntu-latest' || matrix.platform }}
+    environment: ${{ matrix.platform != 'generic' && 'track' || null }}
     strategy:
+      fail-fast: false
       matrix:
-        platform: [generic]
+        platform: [generic, mpfs, cva6]
         bits: [32, 64]
+        exclude:
+          # mpfs is not 32 bit
+          - platform: mpfs
+            bits: 32
+          # ignore 32-bit cva6 for now
+          - platform: cva6
+            bits: 32
 
     steps:
-    # We don't need submodules here since Keystone is a monorepo!
     - name: Checkout Keystone
       uses: actions/checkout@v4
       with:
-        submodules: 'false'
+        submodules: 'true'
+        sparse-checkout: |
+          .
+          scripts/
+          mkutils/
 
     - name: Restore build directory
       uses: actions/download-artifact@v4
@@ -26,188 +38,63 @@ jobs:
     - name: Decompress build directory
       run: cat build.tar.xz | xz -d -T0 | tar -xf -
 
-    - name: Test Keystone system
-      run: |
-        # Fix permissions on the key
-        chmod 600 build-${{ matrix.platform }}${{ matrix.bits }}/buildroot.build/target/root/.ssh/id-rsa
-
-        # Launch QEMU
-        export KEYSTONE_PLATFORM=${{ matrix.platform }}
-        export KEYSTONE_BITS=${{ matrix.bits }}
-        export QEMU_PORT=$(( RANDOM + 1024 ))
-        export LD_LIBRARY_PATH=build-${{ matrix.platform }}${{ matrix.bits }}/buildroot.build/host/lib
-        screen -L -dmS qemu bash -c "make run 2>&1 | tee run.log"
-
-        # TODO: check for connectivity instead of sleeping
-        sleep 60
-
-        export CALL_LOGFILE=cmd.log
-        echo "" > $CALL_LOGFILE
-
-        KEYSTONE_COMMAND="modprobe keystone-driver" make call
-        KEYSTONE_COMMAND="/usr/share/keystone/examples/tests.ke" make call
-        KEYSTONE_COMMAND="/usr/share/keystone/examples/attestor.ke" make call
-        KEYSTONE_COMMAND="poweroff" make call
-
-    - name: Check expected
-      run: |
-        [[ -z $(diff cmd.log scripts/ci/expected.log) ]]
-
-    - name: Upload run log
-      if: failure()
-      uses: actions/upload-artifact@v4
-      with:
-        name: test-keystone-${{ matrix.platform }}${{ matrix.bits }}-run.log
-        path: run.log
-
-    - name: Upload cmd log
-      if: failure()
-      uses: actions/upload-artifact@v4
-      with:
-        name: test-keystone-${{ matrix.platform }}${{ matrix.bits }}-cmd.log
-        path: cmd.log
-
-  test-mpfs:
-    runs-on: [self-hosted, mpfs]
-    environment: track
-    steps:
-    # We don't need submodules here since Keystone is a monorepo!
-    - name: Checkout Keystone
-      uses: actions/checkout@v4
-      with:
-        submodules: 'false'
-
-    - name: Restore build directory
-      uses: actions/download-artifact@v4
-      with:
-        name: keystone-mpfs64-builddir
-        path: .
-
-    - name: Decompress build directory
-      run: cat build.tar.xz | xz -d -T0 | tar -xf -
-
-    # Test the firmware, first by flashing it
-    - name: Flash HSS
+    - name: Flash and check firmware
       env:
-        POWER_ON_CMD: ${{ vars.POWER_ON_CMD_MPFS }}
-        POWER_OFF_CMD: ${{ vars.POWER_OFF_CMD_MPFS }}
-        SC_INSTALL_DIR: ${{ vars.SC_INSTALL_DIR }}
+        KEYSTONE_PLATFORM: ${{ matrix.platform }}
+        KEYSTONE_BITS: ${{ matrix.bits }}
+        LOGFILE: fw-program.log
       run: |
-        $POWER_ON_CMD
-        export FPGENPROG=$(which fpgenprog)
-        make -C build-mpfs64/buildroot.build/build/hss-v2023.06 program 2>/dev/null >program.log
-        $POWER_OFF_CMD
-
-        # Check if we succeeded
-        [[ ! -z $(cat program.log | grep "mpfsBootmodeProgrammer completed successfully") ]]
+        if [[ -f scripts/ci/plat/${{ matrix.platform }}/flash-firmware.sh ]]; then
+          scripts/ci/plat/${{ matrix.platform }}/flash-firmware.sh
+        fi
 
-    - name: Upload HSS program log
-      if: failure()
+    - name: Upload firmware programming log
+      if: ${{ matrix.platform != 'generic' && failure() }}
       uses: actions/upload-artifact@v4
       with:
-        name: test-keystone-mpfs64-prog-hss.log
-        path: program.log
+        name: test-keystone-${{ matrix.platform }}${{ matrix.bits }}-fw-program.log
+        path: fw-program.log
 
-    # And then verifying that we can actually get to the command line
-    - name: Check HSS ok
+    - name: Flash and check OS
       env:
-        POWER_ON_CMD: ${{ vars.POWER_ON_CMD_MPFS }}
-        POWER_OFF_CMD: ${{ vars.POWER_OFF_CMD_MPFS }}
-        FIND_TTY_CMD: ${{ vars.FIND_TTY_CMD }}
+        KEYSTONE_PLATFORM: ${{ matrix.platform }}
+        KEYSTONE_BITS: ${{ matrix.bits }}
+        LOGFILE: os-program.log
       run: |
-        # Collect serial output
-        TTYDEV=$($FIND_TTY_CMD 0)
-        screen -L -dmS mpfs-tty bash -c "stty raw -echo 115200 < $TTYDEV ; cat $TTYDEV > run-hss.log"
-        $POWER_ON_CMD ; sleep 30 ; $POWER_OFF_CMD
-        screen -XS mpfs-tty quit
-
-        # At least the first hart should have started
-        [[ ! -z $(cat run-hss.log | sed -e 's/\x1b\[[0-9;]*m//g' | grep "u54 State Change:  \[Running\]") ]]
+        if [[ -f scripts/ci/plat/${{ matrix.platform }}/flash-os.sh ]]; then
+          scripts/ci/plat/${{ matrix.platform }}/flash-os.sh
+        fi
 
-    - name: Upload HSS run log
-      if: failure()
+    - name: Upload OS programming log
+      if: ${{ matrix.platform != 'generic' && failure() }}
       uses: actions/upload-artifact@v4
       with:
-        name: test-keystone-mpfs64-run-hss.log
-        path: run-hss.log
-
-    # Now we also need to flash the disk. First, get into usbdmsc
-    - name: Flash OS
-      env:
-        POWER_ON_CMD: ${{ vars.POWER_ON_CMD_MPFS }}
-        POWER_OFF_CMD: ${{ vars.POWER_OFF_CMD_MPFS }}
-        FIND_TTY_CMD: ${{ vars.FIND_TTY_CMD }}
-      run: |
-        # Wait for the board to come up a bit. We'll hammer it with serial
-        # input to ensure that we halt the boot at HSS
-        TTYDEV=$($FIND_TTY_CMD 0)
-        $POWER_ON_CMD
-        NOW=$(date +%s)
-        stty raw -echo 115200 < "$TTYDEV"
-        while [[ $(( $(date +%s) - $NOW )) -lt 10 ]]; do echo 'a' > "$TTYDEV" ; done
-
-        echo "" > "$TTYDEV"
-        echo "usbdmsc" > "$TTYDEV"
-
-        # Wait a bit for the USB to connect then flash
-        sleep 10
-        FOUND_DEVICE=""
-        for d in /dev/sd? ; do
-          if [[ ! -z $(udevadm info --query=all -n "$d" | grep -i polarfire) ]]; then
-            FOUND_DEVICE="yes"
-            dd if=build-mpfs64/buildroot.build/images/sdcard.img of="$d" bs=4M oflag=direct
-            break
-          fi
-        done
-
-        $POWER_OFF_CMD
-        [[ ! -z "$FOUND_DEVICE" ]]
+        name: test-keystone-${{ matrix.platform }}${{ matrix.bits }}-os-program.log
+        path: os-program.log
 
     - name: Test Keystone system
       env:
-        POWER_ON_CMD: ${{ vars.POWER_ON_CMD_MPFS }}
-        POWER_OFF_CMD: ${{ vars.POWER_OFF_CMD_MPFS }}
-        FIND_TTY_CMD: ${{ vars.FIND_TTY_CMD }}
-        KEYSTONE_IP: ${{ vars.BOARD_IP_MPFS }}
+        KEYSTONE_PLATFORM: ${{ matrix.platform }}
+        KEYSTONE_BITS: ${{ matrix.bits }}
+        LOGFILE: run.log
+        CMD_LOGFILE: cmd.log
       run: |
-        # Fix permissions on the key
-        chmod 600 build-mpfs64/buildroot.build/target/root/.ssh/id-rsa
-
-        # Start the board
-        TTYDEV=$($FIND_TTY_CMD 1)
-        export KEYSTONE_PLATFORM=mpfs
-        export KEYSTONE_BITS=64
-        screen -L -dmS mpfs-tty bash -c "stty raw -echo 115200 < $TTYDEV ; cat $TTYDEV > run.log"
-        $POWER_ON_CMD
-
-        # TODO: check for connectivity instead of sleeping
-        sleep 60
-
-        export CALL_LOGFILE=cmd.log
-        echo "" > $CALL_LOGFILE
-
-        KEYSTONE_COMMAND="modprobe keystone-driver" make call
-        KEYSTONE_COMMAND="/usr/share/keystone/examples/tests.ke" make call
-        # Todo: attestation does not yet work in mpfs
-        #KEYSTONE_COMMAND="/usr/share/keystone/examples/attestor.ke" make call
-
-        $POWER_OFF_CMD
-        screen -XS mpfs-tty quit
+        scripts/ci/plat/${{ matrix.platform }}/test.sh
 
     - name: Check expected
       run: |
-        [[ -z $(diff cmd.log scripts/ci/expected-mpfs.log) ]]
+        [[ -z $(diff -wB cmd.log scripts/ci/plat/${{ matrix.platform }}/expected.log) ]]
 
     - name: Upload run log
       if: failure()
       uses: actions/upload-artifact@v4
       with:
-        name: test-keystone-mpfs64-run.log
+        name: test-keystone-${{ matrix.platform }}${{ matrix.bits }}-run.log
         path: run.log
 
     - name: Upload cmd log
       if: failure()
       uses: actions/upload-artifact@v4
       with:
-        name: test-keystone-mpfs64-cmd.log
+        name: test-keystone-${{ matrix.platform }}${{ matrix.bits }}-cmd.log
         path: cmd.log
diff --git a/.gitmodules b/.gitmodules
@@ -10,3 +10,6 @@
 [submodule "overlays/microchip"]
 	path = overlays/microchip
 	url = https://github.com/linux4microchip/buildroot-external-microchip
+[submodule "scripts/ci/utils/relay_ft245r"]
+	path = scripts/ci/utils/relay_ft245r
+	url = https://github.com/vpatron/relay_ft245r
diff --git a/mkutils/plat/cva6/run.mk b/mkutils/plat/cva6/run.mk
@@ -5,16 +5,30 @@
 
 
 PAYLOAD                 = $(BUILDROOT_BUILDDIR)/images/fw_payload.bin
-SDDEVICE_PART1          = $(SD_DEVICE)1
-SDDEVICE_PART2          = $(SD_DEVICE)2
-
-
+KERNEL			= $(BUILDROOT_BUILDDIR)/images/uImage
+SDDEVICE_PART1          = $(shell lsblk $(SD_DEVICE) -no PATH | head -2 | tail -1)
+SDDEVICE_PART2          = $(shell lsblk $(SD_DEVICE) -no PATH | head -3 | tail -1)
 
 flash: $(SD_DEVICE)
 	$(info PAYLOAD INFORMATION)
 	$(info $(PAYLOAD))
 	$(info $(SD_DEVICE))
 	$(info $(SDDEVICE_PART1))
 	$(info $(SDDEVICE_PART2))
-	sgdisk --clear -g --new=1:2048:40M --new=2:512M:0 --typecode=1:3000 --typecode=2:8300 $(SD_DEVICE)
+	sgdisk --clear -g --new=1:2048:4M --new=2:512M:0 --typecode=1:3000 --typecode=2:8300 $(SD_DEVICE)
 	dd if=$(PAYLOAD) of=$(SDDEVICE_PART1) status=progress oflag=sync bs=1M
+	dd if=$(KERNEL) of=$(SDDEVICE_PART2) status=progress oflag=sync bs=1M
+
+CALL_LOGFILE ?= $(shell mktemp)
+call:
+	$(call log,info,Calling command on the CVA6 board)
+	ssh -i $(BUILDROOT_BUILDDIR)/target/root/.ssh/id-rsa \
+                -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
+                 root@$(KEYSTONE_IP) $(KEYSTONE_COMMAND) 2>&1 | \
+                 grep -v "Warning: Permanently added" | tee -a $(CALL_LOGFILE)
+
+debug-connect:
+	$(call log,info,Connecting to OpenOCD)
+	$(BUILDROOT_BUILDDIR)/host/bin/riscv64-buildroot-linux-gnu-gdb \
+                -iex "set KEYSTONE=$(KEYSTONE)" \
+                -x $(KEYSTONE)/scripts/gdb/cva6.cfg
diff --git a/mkutils/plat/mpfs/run.mk b/mkutils/plat/mpfs/run.mk
@@ -20,7 +20,7 @@ run:
 
 CALL_LOGFILE ?= $(shell mktemp)
 call:
-	$(call log,info,Calling command in QEMU)
+	$(call log,info,Calling command on the MPFS board)
 	ssh -i $(BUILDROOT_BUILDDIR)/target/root/.ssh/id-rsa \
 		-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
 		 root@$(KEYSTONE_IP) $(KEYSTONE_COMMAND) 2>&1 | \