Implement support for MPFS (#412)

keystone-enclave · Jan 25, 2024 · 767e246 · 767e246
1 parent 874b3d8
commit 767e246
Show file tree

Hide file tree

Showing 83 changed files with 1,591 additions and 11,728 deletions.
diff --git a/.github/workflows/build-runtime.yml b/.github/workflows/build-runtime.yml
@@ -7,17 +7,20 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        platform: [generic, hifive_unmatched, cva6]
+        platform: [generic, hifive_unmatched, cva6, mpfs]
         bits: [32, 64]
         exclude:
           # unmatched is not 32 bit
           - platform: hifive_unmatched
             bits: 32
+          # mpfs is not 32 bit
+          - platform: mpfs
+            bits: 32
 
     steps:
     # We don't need submodules here since Keystone is a monorepo!
     - name: Checkout Keystone
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       with:
         submodules: 'false'
 

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -14,12 +14,15 @@ jobs:
     # platform that we support here
     strategy:
       matrix:
-        platform: [generic, hifive_unmatched, cva6]
+        platform: [generic, hifive_unmatched, cva6, mpfs]
         bits: [32, 64]
         exclude:
           # unmatched is not 32 bit
           - platform: hifive_unmatched
             bits: 32
+          # mpfs is not 32 bit
+          - platform: mpfs
+            bits: 32
 
     # Output cache keys that were used so we can consolidate them later. Note
     # that this is a matrix job, and job outputs for these are not well supported
@@ -43,7 +46,7 @@ jobs:
     # First, we need to get the version of Keystone we are working on. We
     # will also need submodules here since we are doing full builds
     - name: Checkout Keystone
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       with:
         submodules: 'true'
 
@@ -72,7 +75,7 @@ jobs:
     # since we really only care about keeping the latest cache anyways.
     - name: Restore buildroot packages
       id: restore-buildroot-dl
-      uses: actions/cache/restore@v3
+      uses: actions/cache/restore@v4
       with:
         path: dl.tar
         key: buildroot-dl-${{ steps.cache-keys.outputs.YMDH }}
@@ -84,7 +87,7 @@ jobs:
 
     - name: Restore ccache
       id: restore-ccache
-      uses: actions/cache/restore@v3
+      uses: actions/cache/restore@v4
       with:
         path: ccache.tar.xz
         key: ccache-${{ steps.cache-keys.outputs.YMDH }}
@@ -142,6 +145,10 @@ jobs:
         # Needed by end-to-end tests
         COMPRESSDIRS="$COMPRESSDIRS $BASEDIR/target/root/"
 
+        if [[ "${{ matrix.platform }}" == "mpfs" ]]; then
+          COMPRESSDIRS="$COMPRESSDIRS $BASEDIR/build/hss-v2023.06"
+        fi
+
         tar -cf - $COMPRESSDIRS | xz -9 -T0 > build.tar.xz
 
     - name: Compress cache directories
@@ -155,7 +162,9 @@ jobs:
         fi
 
         if [[ $(du -s buildroot-ccache | awk -F' ' '{ print $1 }') -gt 4 ]]; then
-          tar -cf - buildroot-ccache | xz -9 -T0 > ccache.tar.xz
+          # Ignore character device files, which are used as "whiteouts" in overlayfs
+          find buildroot-ccache -type f -not -type c -print0 | \
+            tar --null -cf - -T - | xz -9 -T0 > ccache.tar.xz
         fi
 
     - name: Upload build directory
@@ -230,14 +239,14 @@ jobs:
           fi
 
       - name: Restore buildroot packages
-        uses: actions/cache/restore@v3
+        uses: actions/cache/restore@v4
         if: ${{ steps.check-caches.outputs.BUILDROOT_DL_UPDATE == 'true' }}
         with:
           path: dl.tar
           key: ${{ needs.build.outputs.buildroot-dl-matched-key }}
 
       - name: Restore ccache
-        uses: actions/cache/restore@v3
+        uses: actions/cache/restore@v4
         if: ${{ steps.check-caches.outputs.BUILDROOT_CCACHE_UPDATE == 'true' }}
         with:
           path: ccache.tar.xz
@@ -286,14 +295,14 @@ jobs:
           fi
 
       - name: Save buildroot download cache
-        uses: actions/cache/save@v3
+        uses: actions/cache/save@v4
         if: ${{ steps.check-caches.outputs.BUILDROOT_DL_UPDATE == 'true' }}
         with:
           path: dl.tar
           key: ${{ needs.build.outputs.buildroot-dl-primary-key }}
 
       - name: Save ccache
-        uses: actions/cache/save@v3
+        uses: actions/cache/save@v4
         if: ${{ steps.check-caches.outputs.BUILDROOT_CCACHE_UPDATE == 'true' }}
         with:
           path: ccache.tar.xz
@@ -309,7 +318,7 @@ jobs:
     steps:
     # We don't need submodules here since Keystone is a monorepo!
     - name: Checkout Keystone
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       with:
         submodules: 'false'
 
@@ -324,7 +333,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Checkout Keystone
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       with:
         submodules: 'true'
 
@@ -343,7 +352,7 @@ jobs:
     needs: build
     uses: ./.github/workflows/build-runtime.yml
 
-  # System tests, which are run for simulatable platforms
+  # System tests, which are run for simulatable and self-hostable platforms
   test-system:
     needs: build
     uses: ./.github/workflows/test-system.yml
diff --git a/.github/workflows/test-system.yml b/.github/workflows/test-system.yml
@@ -3,7 +3,7 @@ on:
   workflow_call:
 
 jobs:
-  test-system:
+  test-generic:
     runs-on: ubuntu-latest
     strategy:
       matrix:
@@ -13,7 +13,7 @@ jobs:
     steps:
     # We don't need submodules here since Keystone is a monorepo!
     - name: Checkout Keystone
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       with:
         submodules: 'false'
 
@@ -34,6 +34,7 @@ jobs:
         # Launch QEMU
         export KEYSTONE_PLATFORM=${{ matrix.platform }}
         export KEYSTONE_BITS=${{ matrix.bits }}
+        export QEMU_PORT=$(( RANDOM + 1024 ))
         export LD_LIBRARY_PATH=build-${{ matrix.platform }}${{ matrix.bits }}/buildroot.build/host/lib
         screen -L -dmS qemu bash -c "make run 2>&1 | tee run.log"
 
@@ -65,3 +66,148 @@ jobs:
       with:
         name: test-keystone-${{ matrix.platform }}${{ matrix.bits }}-cmd.log
         path: cmd.log
+
+  test-mpfs:
+    runs-on: [self-hosted, mpfs]
+    environment: track
+    steps:
+    # We don't need submodules here since Keystone is a monorepo!
+    - name: Checkout Keystone
+      uses: actions/checkout@v4
+      with:
+        submodules: 'false'
+
+    - name: Restore build directory
+      uses: actions/download-artifact@v4
+      with:
+        name: keystone-mpfs64-builddir
+        path: .
+
+    - name: Decompress build directory
+      run: cat build.tar.xz | xz -d -T0 | tar -xf -
+
+    # Test the firmware, first by flashing it
+    - name: Flash HSS
+      env:
+        POWER_ON_CMD: ${{ vars.POWER_ON_CMD_MPFS }}
+        POWER_OFF_CMD: ${{ vars.POWER_OFF_CMD_MPFS }}
+        SC_INSTALL_DIR: ${{ vars.SC_INSTALL_DIR }}
+      run: |
+        $POWER_ON_CMD
+        export FPGENPROG=$(which fpgenprog)
+        make -C build-mpfs64/buildroot.build/build/hss-v2023.06 program 2>/dev/null >program.log
+        $POWER_OFF_CMD
+
+        # Check if we succeeded
+        [[ ! -z $(cat program.log | grep "mpfsBootmodeProgrammer completed successfully") ]]
+
+    - name: Upload HSS program log
+      if: failure()
+      uses: actions/upload-artifact@v4
+      with:
+        name: test-keystone-mpfs64-prog-hss.log
+        path: program.log
+
+    # And then verifying that we can actually get to the command line
+    - name: Check HSS ok
+      env:
+        POWER_ON_CMD: ${{ vars.POWER_ON_CMD_MPFS }}
+        POWER_OFF_CMD: ${{ vars.POWER_OFF_CMD_MPFS }}
+        FIND_TTY_CMD: ${{ vars.FIND_TTY_CMD }}
+      run: |
+        # Collect serial output
+        TTYDEV=$($FIND_TTY_CMD 0)
+        screen -L -dmS mpfs-tty bash -c "stty raw -echo 115200 < $TTYDEV ; cat $TTYDEV > run-hss.log"
+        $POWER_ON_CMD ; sleep 30 ; $POWER_OFF_CMD
+        screen -XS mpfs-tty quit
+
+        # At least the first hart should have started
+        [[ ! -z $(cat run-hss.log | sed -e 's/\x1b\[[0-9;]*m//g' | grep "u54 State Change:  \[Running\]") ]]
+
+    - name: Upload HSS run log
+      if: failure()
+      uses: actions/upload-artifact@v4
+      with:
+        name: test-keystone-mpfs64-run-hss.log
+        path: run-hss.log
+
+    # Now we also need to flash the disk. First, get into usbdmsc
+    - name: Flash OS
+      env:
+        POWER_ON_CMD: ${{ vars.POWER_ON_CMD_MPFS }}
+        POWER_OFF_CMD: ${{ vars.POWER_OFF_CMD_MPFS }}
+        FIND_TTY_CMD: ${{ vars.FIND_TTY_CMD }}
+      run: |
+        # Wait for the board to come up a bit. We'll hammer it with serial
+        # input to ensure that we halt the boot at HSS
+        TTYDEV=$($FIND_TTY_CMD 0)
+        $POWER_ON_CMD
+        NOW=$(date +%s)
+        stty raw -echo 115200 < "$TTYDEV"
+        while [[ $(( $(date +%s) - $NOW )) -lt 10 ]]; do echo 'a' > "$TTYDEV" ; done
+
+        echo "" > "$TTYDEV"
+        echo "usbdmsc" > "$TTYDEV"
+
+        # Wait a bit for the USB to connect then flash
+        sleep 10
+        FOUND_DEVICE=""
+        for d in /dev/sd? ; do
+          if [[ ! -z $(udevadm info --query=all -n "$d" | grep -i polarfire) ]]; then
+            FOUND_DEVICE="yes"
+            dd if=build-mpfs64/buildroot.build/images/sdcard.img of="$d" bs=4M oflag=direct
+            break
+          fi
+        done
+
+        $POWER_OFF_CMD
+        [[ ! -z "$FOUND_DEVICE" ]]
+
+    - name: Test Keystone system
+      env:
+        POWER_ON_CMD: ${{ vars.POWER_ON_CMD_MPFS }}
+        POWER_OFF_CMD: ${{ vars.POWER_OFF_CMD_MPFS }}
+        FIND_TTY_CMD: ${{ vars.FIND_TTY_CMD }}
+        KEYSTONE_IP: ${{ vars.BOARD_IP_MPFS }}
+      run: |
+        # Fix permissions on the key
+        chmod 600 build-mpfs64/buildroot.build/target/root/.ssh/id-rsa
+
+        # Start the board
+        TTYDEV=$($FIND_TTY_CMD 1)
+        export KEYSTONE_PLATFORM=mpfs
+        export KEYSTONE_BITS=64
+        screen -L -dmS mpfs-tty bash -c "stty raw -echo 115200 < $TTYDEV ; cat $TTYDEV > run.log"
+        $POWER_ON_CMD
+
+        # TODO: check for connectivity instead of sleeping
+        sleep 60
+
+        export CALL_LOGFILE=cmd.log
+        echo "" > $CALL_LOGFILE
+
+        KEYSTONE_COMMAND="modprobe keystone-driver" make call
+        KEYSTONE_COMMAND="/usr/share/keystone/examples/tests.ke" make call
+        # Todo: attestation does not yet work in mpfs
+        #KEYSTONE_COMMAND="/usr/share/keystone/examples/attestor.ke" make call
+
+        $POWER_OFF_CMD
+        screen -XS mpfs-tty quit
+
+    - name: Check expected
+      run: |
+        [[ -z $(diff cmd.log scripts/ci/expected-mpfs.log) ]]
+
+    - name: Upload run log
+      if: failure()
+      uses: actions/upload-artifact@v4
+      with:
+        name: test-keystone-mpfs64-run.log
+        path: run.log
+
+    - name: Upload cmd log
+      if: failure()
+      uses: actions/upload-artifact@v4
+      with:
+        name: test-keystone-mpfs64-cmd.log
+        path: cmd.log
diff --git a/.gitmodules b/.gitmodules
@@ -7,3 +7,6 @@
 [submodule "overlays/keystone/board/cva6/cva6-sdk"]
 	path = overlays/keystone/board/cva6/cva6-sdk
 	url = https://github.com/openhwgroup/cva6-sdk
+[submodule "overlays/microchip"]
+	path = overlays/microchip
+	url = https://github.com/linux4microchip/buildroot-external-microchip
diff --git a/Makefile b/Makefile
@@ -24,7 +24,7 @@ export BUILDROOT_OVERLAYDIR     ?= $(BUILDDIR)/overlay
 export BUILDROOT_BUILDDIR       ?= $(BUILDDIR)/buildroot.build
 
 
-# options: generic, cva6, hifive_unmatched
+# options: generic, cva6, hifive_unmatched, mpfs
 export KEYSTONE_PLATFORM        ?= generic
 export KEYSTONE_BITS            ?= 64
 
@@ -33,7 +33,8 @@ include mkutils/log.mk
 
 BUILDROOT_CONFIGFILE    ?= riscv$(KEYSTONE_BITS)_$(KEYSTONE_PLATFORM)_defconfig
 ifeq ($(KEYSTONE_PLATFORM),mpfs)
-	EXTERNALS += microchip
+	EXTERNALS		+= microchip
+	ADDITIONAL_OVERLAYS  	:= \$$(BR2_EXTERNAL_MCHP_PATH)/board/microchip/icicle/rootfs-overlay
 endif
 
 # Highest priority external
@@ -69,7 +70,7 @@ BUILDROOT_CCACHE ?= $(HOME)/.buildroot-ccache
 $(BUILDROOT_BUILDDIR)/.config: $(BUILDROOT_BUILDDIR)
 	$(call log,info,Configuring Buildroot with $(BUILDROOT_CONFIGFILE))
 	$(MAKE) $(BUILDROOT_MAKEFLAGS) $(BUILDROOT_CONFIGFILE)
-	echo "BR2_ROOTFS_OVERLAY=\"$(BUILDROOT_OVERLAYDIR)\"" >> $(BUILDROOT_BUILDDIR)/.config
+	echo "BR2_ROOTFS_OVERLAY=\"$(BUILDROOT_OVERLAYDIR) $(ADDITIONAL_OVERLAYS)\"" >> $(BUILDROOT_BUILDDIR)/.config
 	echo "BR2_CCACHE_DIR=$(BUILDROOT_CCACHE)" >> $(BUILDROOT_BUILDDIR)/.config
 
 # Overlay