Add type to data structure list of blob locations. Test multiarch pushs.

extras/hyperbee-dump.js

@@ -0,0 +1,31 @@
+#!/usr/bin/env node
+
+import path from 'path'
+import hypercore from 'hypercore'
+import Hyperbee from 'hyperbee'
+import { baseDir } from '../lib/utils.js'
+
+const myArgs = process.argv.slice(2)
+
+if (myArgs.length !== 1) {
+  console.error('usage: hyperbee-dump.js <feedid>')
+  process.exit(1)
+}
+
+main(myArgs[0]).catch(e => console.error(`top-level exception: ${e}`))
+
+async function main (feed) {
+  const corePath = path.join(await baseDir(), 'hypercores', feed)
+
+  const core = hypercore(corePath, { createIfMissing: false })
+
+  const bee = new Hyperbee(core, {
+    keyEncoding: 'utf-8',
+    valueEncoding: 'binary'
+  })
+  await bee.ready()
+
+  for await (const { key, value } of bee.createReadStream()) {
+    console.log(`${key} ==> ${value}`)
+  }
+}

lib/distributed-storage.js

@@ -54,7 +54,11 @@ class DistributedStorage {
       }
 
       const { contentType, size } = blobObject
-      const ipfsCid = blobObject.locations[0] // currently, support only ipfs cid and a single location for every blob
+      const ipfsCid = blobObject.locations[0].ref // currently, support only ipfs cid and a single location for every blob
+
+      if (!ipfsCid) {
+        return { stream: null, size: null, contentType: null }
+      }
 
       debug(`retrieving blob stream from IPFS with CID ${ipfsCid}, based on digest ${dockerDigest}. size ${size}, content type: ${contentType}`)
 
@@ -136,7 +140,7 @@ class DistributedStorage {
       const blobObject = {
         size: streamSize,
         contentType,
-        locations: [ipfsCid]
+        locations: [{ type: 'ipfs', ref: ipfsCid }]
       }
       await blobs.put(digest, blobObject)
     } else {

lib/utils.js

@@ -2,11 +2,13 @@ import fs from 'fs/promises'
 import os from 'os'
 import path from 'path'
 
-export async function baseDir () {
+export async function baseDir (makeDir = true) {
   const baseDir = process.env.WHALESONG_DATA ?? path.join(
     os.homedir(),
     '.whalesong'
   )
-  await fs.mkdir(baseDir, { recursive: true })
+  if (makeDir) {
+    await fs.mkdir(baseDir, { recursive: true })
+  }
   return baseDir
 }

tests/push-containerd-pull-containerd.sh

@@ -21,9 +21,33 @@ ctr image pull --plain-http "$WHALESONG_IMAGE"
 OUTPUT=$(ctr run --rm "$WHALESONG_IMAGE" testcontainer python -c 'print("hi from ci")')
 
 if [[ $OUTPUT == "hi from ci" ]]; then
-	echo "Pull/push test successful!"
+	echo "Pull/push test successful for single platform!"
 else
-	echo "Pull/push test failed!"
+	echo "Pull/push test failed for single platform!"
+	exit 1
+fi
+
+# Repeat test but push _all_ platforms, i.e., test creating an index (manifest list) too.
+ctr image rm "$FULL_TEST_IMAGE" "$WHALESONG_IMAGE" || true
+ctr image pull --all-platforms "$FULL_TEST_IMAGE"
+ctr image tag "$FULL_TEST_IMAGE" "$WHALESONG_IMAGE"
+
+# Push all platforms with containerd.
+ctr image push --plain-http "$WHALESONG_IMAGE"
+
+# Clean stored image before pulling back down again
+ctr image rm "$FULL_TEST_IMAGE" "$WHALESONG_IMAGE"
+
+# Pull using containerd
+ctr image pull --plain-http "$WHALESONG_IMAGE"
+
+# Run image using containerd
+OUTPUT=$(ctr run --rm "$WHALESONG_IMAGE" testcontainer python -c 'print("hi from ci")')
+
+if [[ $OUTPUT == "hi from ci" ]]; then
+	echo "Pull/push test successful for multiple platforms!"
+else
+	echo "Pull/push test failed for multiple platforms!"
 	exit 1
 fi
 

tests/push-containerd-pull-docker.sh

@@ -18,14 +18,36 @@ docker pull "$WHALESONG_IMAGE"
 OUTPUT=$(docker run --rm "$WHALESONG_IMAGE" python -c 'print("hi from ci")')
 
 if [[ $OUTPUT == "hi from ci" ]]; then
-	echo "Pull/push test successful!"
+	echo "Pull/push test successful for single platform!"
 else
-	echo "Pull/push test failed!"
+	echo "Pull/push test failed for single platform!"
+	exit 1
+fi
+
+# Repeat test but push _all_ platforms, i.e., test creating an index (manifest list) too.
+ctr image rm "$FULL_TEST_IMAGE" "$WHALESONG_IMAGE" || true
+docker rmi "$WHALESONG_IMAGE" || true
+
+ctr image pull --all-platforms "$FULL_TEST_IMAGE"
+ctr image tag "$FULL_TEST_IMAGE" "$WHALESONG_IMAGE"
+
+# Push all platforms with containerd.
+ctr image push --plain-http "$WHALESONG_IMAGE"
+
+# Pull using docker
+docker pull "$WHALESONG_IMAGE"
+
+# Run image using docker
+OUTPUT=$(docker run --rm "$WHALESONG_IMAGE" python -c 'print("hi from ci")')
+
+if [[ $OUTPUT == "hi from ci" ]]; then
+	echo "Pull/push test successful for multiple platforms!"
+else
+	echo "Pull/push test failed for multiple platforms!"
 	exit 1
 fi
 
 # Optional in CI: cleanup
 set +e
 ctr image rm "$FULL_TEST_IMAGE" "$WHALESONG_IMAGE" || true
 docker rmi "$WHALESONG_IMAGE" || true
-