Merge branch 'TurboWarp:develop' into develop

package.json

@@ -35,6 +35,7 @@
   },
   "dependencies": {
     "@turbowarp/json": "^0.1.2",
+    "@turbowarp/nanolog": "^0.1.0",
     "@vernier/godirect": "1.5.0",
     "arraybuffer-loader": "^1.0.6",
     "atob": "2.1.2",
@@ -44,10 +45,9 @@
     "diff-match-patch": "1.0.4",
     "format-message": "6.2.1",
     "htmlparser2": "3.10.0",
-    "immutable": "3.8.1",
+    "immutable": "3.8.2",
     "jszip": "^3.1.5",
-    "minilog": "3.1.0",
-    "scratch-parser": "5.1.1",
+    "scratch-parser": "github:TurboWarp/scratch-parser#master",
     "scratch-sb1-converter": "0.2.7",
     "scratch-translate-extension-languages": "0.0.20191118205314",
     "text-encoding": "0.7.0",

src/compiler/irgen.js

@@ -1414,12 +1414,14 @@ class ScriptTreeGenerator {
 
         const blockInfo = this.getBlockInfo(block.opcode);
         const blockType = (blockInfo && blockInfo.info && blockInfo.info.blockType) || BlockType.COMMAND;
-        const substacks = [];
+        const substacks = {};
         if (blockType === BlockType.CONDITIONAL || blockType === BlockType.LOOP) {
-            const branchCount = blockInfo.info.branchCount;
-            for (let i = 0; i < branchCount; i++) {
-                const inputName = i === 0 ? 'SUBSTACK' : `SUBSTACK${i + 1}`;
-                substacks.push(this.descendSubstack(block, inputName));
+            for (const inputName in block.inputs) {
+                if (!inputName.startsWith('SUBSTACK')) continue;
+                const branchNum = inputName === 'SUBSTACK' ? 1 : +inputName.substring('SUBSTACK'.length);
+                if (!isNaN(branchNum)) {
+                    substacks[branchNum] = this.descendSubstack(block, inputName);
+                }
             }
         }
 

src/compiler/jsgen.js

@@ -773,9 +773,9 @@ class JSGenerator {
                 this.source += `const ${branchVariable} = createBranchInfo(${blockType === BlockType.LOOP});\n`;
                 this.source += `while (${branchVariable}.branch = +(${this.generateCompatibilityLayerCall(node, false, branchVariable)})) {\n`;
                 this.source += `switch (${branchVariable}.branch) {\n`;
-                for (let i = 0; i < node.substacks.length; i++) {
-                    this.source += `case ${i + 1}: {\n`;
-                    this.descendStack(node.substacks[i], new Frame(false));
+                for (const index in node.substacks) {
+                    this.source += `case ${+index}: {\n`;
+                    this.descendStack(node.substacks[index], new Frame(false));
                     this.source += `break;\n`;
                     this.source += `}\n`; // close case
                 }

src/engine/runtime.js

@@ -522,6 +522,22 @@ class Runtime extends EventEmitter {
          * Responsible for managing custom fonts.
          */
         this.fontManager = new FontManager(this);
+
+        /**
+         * Maps extension ID to a JSON-serializable value.
+         * @type {Object.<string, object>}
+         */
+        this.extensionStorage = {};
+
+        /**
+         * Total number of scratch-storage load() requests since the runtime was created or cleared.
+         */
+        this.totalAssetRequests = 0;
+
+        /**
+         * Total number of finished or errored scratch-storage load() requests since the runtime was created or cleared.
+         */
+        this.finishedAssetRequests = 0;
     }
 
     /**
@@ -661,6 +677,14 @@ class Runtime extends EventEmitter {
         return 'AFTER_EXECUTE';
     }
 
+    /**
+     * Event name for reporting asset download progress. Fired with finished, total
+     * @const {string}
+     */
+    static get ASSET_PROGRESS () {
+        return 'ASSET_PROGRESS';
+    }
+
     /**
      * Event name when the project is started (threads may not necessarily be
      * running).
@@ -1313,7 +1337,7 @@ class Runtime extends EventEmitter {
             type: extendedOpcode,
             inputsInline: true,
             category: categoryInfo.name,
-            extensions: blockInfo.extensions ?? [],
+            extensions: [],
             colour: blockInfo.color1 ?? categoryInfo.color1,
             colourSecondary: blockInfo.color2 ?? categoryInfo.color2,
             colourTertiary: blockInfo.color3 ?? categoryInfo.color3
@@ -1335,10 +1359,21 @@ class Runtime extends EventEmitter {
         // the category block icon.
         const iconURI = blockInfo.blockIconURI || categoryInfo.blockIconURI;
 
+        // All extension blocks have from_extension
+        blockJSON.extensions.push('from_extension');
+
+        // Allow easily detecting which blocks use default colors
+        if (
+            blockJSON.colour === defaultExtensionColors[0] &&
+            blockJSON.colourSecondary === defaultExtensionColors[1] &&
+            blockJSON.colourTertiary === defaultExtensionColors[2]
+        ) {
+            blockJSON.extensions.push('default_extension_colors');
+        }
+
         if (iconURI) {
-            if (!blockJSON.extensions.includes('scratch_extension')) {
-                blockJSON.extensions.push('scratch_extension');
-            }
+            // scratch_extension is a misleading name - this is for fixing the icon rendering
+            blockJSON.extensions.push('scratch_extension');
             blockJSON.message0 = '%1 %2';
             const iconJSON = {
                 type: 'field_image',
@@ -1451,6 +1486,14 @@ class Runtime extends EventEmitter {
         const inputs = context.inputList.join('');
         const blockXML = `<block type="${xmlEscape(extendedOpcode)}">${mutation}${inputs}</block>`;
 
+        if (blockInfo.extensions) {
+            for (const extension of blockInfo.extensions) {
+                if (!blockJSON.extensions.includes(extension)) {
+                    blockJSON.extensions.push(extension);
+                }
+            }
+        }
+
         return {
             info: context.blockInfo,
             json: context.blockJSON,
@@ -2252,6 +2295,7 @@ class Runtime extends EventEmitter {
         });
 
         this.targets.map(this.disposeTarget, this);
+        this.extensionStorage = {};
         // tw: explicitly emit a MONITORS_UPDATE instead of relying on implicit behavior of _step()
         const emptyMonitorState = OrderedMap({});
         if (!emptyMonitorState.equals(this._monitorState)) {
@@ -2281,6 +2325,8 @@ class Runtime extends EventEmitter {
         this.getNumberOfCloudVariables = newCloudDataManager.getNumberOfCloudVariables;
         this.addCloudVariable = this._initializeAddCloudVariable(newCloudDataManager);
         this.removeCloudVariable = this._initializeRemoveCloudVariable(newCloudDataManager);
+
+        this.resetProgress();
     }
 
     /**
@@ -3418,6 +3464,39 @@ class Runtime extends EventEmitter {
         this.externalCommunicationMethods[method] = enabled;
         this.updatePrivacy();
     }
+
+    emitAssetProgress () {
+        this.emit(Runtime.ASSET_PROGRESS, this.finishedAssetRequests, this.totalAssetRequests);
+    }
+
+    resetProgress () {
+        this.finishedAssetRequests = 0;
+        this.totalAssetRequests = 0;
+        this.emitAssetProgress();
+    }
+
+    /**
+     * Wrap an asset loading promise with progress support.
+     * @template T
+     * @param {Promise<T>} promise
+     * @returns {Promise<T>}
+     */
+    wrapAssetRequest (promise) {
+        this.totalAssetRequests++;
+        this.emitAssetProgress();
+
+        return promise
+            .then(result => {
+                this.finishedAssetRequests++;
+                this.emitAssetProgress();
+                return result;
+            })
+            .catch(error => {
+                this.finishedAssetRequests++;
+                this.emitAssetProgress();
+                throw error;
+            });
+    }
 }
 
 /**

src/engine/target.js

@@ -70,6 +70,12 @@ class Target extends EventEmitter {
          * @type {Object.<string, *>}
          */
         this._edgeActivatedHatValues = {};
+
+        /**
+         * Maps extension ID to a JSON-serializable value.
+         * @type {Object.<string, object>}
+         */
+        this.extensionStorage = {};
     }
 
     /**

src/extension-support/extension-manager.js

@@ -287,7 +287,7 @@ class ExtensionManager {
                     dispatch.call('runtime', '_refreshExtensionPrimitives', info);
                 })
                 .catch(e => {
-                    log.error(`Failed to refresh built-in extension primitives: ${e}`);
+                    log.error('Failed to refresh built-in extension primitives', e);
                 })
         );
         return Promise.all(allPromises);

src/extension-support/tw-unsandboxed-extension-runner.js

@@ -2,6 +2,7 @@ const ScratchCommon = require('./tw-extension-api-common');
 const createScratchX = require('./tw-scratchx-compatibility-layer');
 const AsyncLimiter = require('../util/async-limiter');
 const createTranslate = require('./tw-l10n');
+const staticFetch = require('../util/tw-static-fetch');
 
 /* eslint-disable require-await */
 
@@ -97,6 +98,12 @@ const setupUnsandboxedExtensionAPI = vm => new Promise(resolve => {
 
     Scratch.fetch = async (url, options) => {
         const actualURL = url instanceof Request ? url.url : url;
+
+        const staticFetchResult = staticFetch(url);
+        if (staticFetchResult) {
+            return staticFetchResult;
+        }
+
         if (!await Scratch.canFetch(actualURL)) {
             throw new Error(`Permission to fetch ${actualURL} rejected.`);
         }

src/extensions/scratch3_music/index.js

@@ -925,6 +925,13 @@ class Scratch3MusicBlocks {
         };
     }
 
+    _isConcurrencyLimited () {
+        return (
+            this.runtime.runtimeOptions.miscLimits &&
+            this._concurrencyCounter > Scratch3MusicBlocks.CONCURRENCY_LIMIT
+        );
+    }
+
     /**
      * Play a drum sound for some number of beats.
      * @param {object} args - the block arguments.
@@ -987,7 +994,7 @@ class Scratch3MusicBlocks {
         if (util.runtime.audioEngine === null) return;
         if (util.target.sprite.soundBank === null) return;
         // If we're playing too many sounds, do not play the drum sound.
-        if (this._concurrencyCounter > Scratch3MusicBlocks.CONCURRENCY_LIMIT) {
+        if (this._isConcurrencyLimited()) {
             return;
         }
 
@@ -1088,7 +1095,7 @@ class Scratch3MusicBlocks {
         if (util.target.sprite.soundBank === null) return;
 
         // If we're playing too many sounds, do not play the note.
-        if (this._concurrencyCounter > Scratch3MusicBlocks.CONCURRENCY_LIMIT) {
+        if (this._isConcurrencyLimited()) {
             return;
         }
 

src/serialization/sb2.js

@@ -500,9 +500,10 @@ const parseScratchAssets = function (object, runtime, topLevel, zip) {
             // the file name of the costume should be the baseLayerID followed by the file ext
             const assetFileName = `${costumeSource.baseLayerID}.${ext}`;
             const textLayerFileName = costumeSource.textLayerID ? `${costumeSource.textLayerID}.png` : null;
-            costumePromises.push(deserializeCostume(costume, runtime, zip, assetFileName, textLayerFileName)
-                .then(() => loadCostume(costume.md5, costume, runtime, 2 /* optVersion */))
-            );
+            costumePromises.push(runtime.wrapAssetRequest(
+                deserializeCostume(costume, runtime, zip, assetFileName, textLayerFileName)
+                    .then(() => loadCostume(costume.md5, costume, runtime, 2 /* optVersion */))
+            ));
         }
     }
     // Sounds from JSON
@@ -535,10 +536,10 @@ const parseScratchAssets = function (object, runtime, topLevel, zip) {
             // the file name of the sound should be the soundID (provided from the project.json)
             // followed by the file ext
             const assetFileName = `${soundSource.soundID}.${ext}`;
-            soundPromises.push(
+            soundPromises.push(runtime.wrapAssetRequest(
                 deserializeSound(sound, runtime, zip, assetFileName)
                     .then(() => loadSound(sound, runtime, soundBank))
-            );
+            ));
         }
     }
 

src/serialization/sb3.js

@@ -634,6 +634,26 @@ const serializeTarget = function (target, extensions) {
     return obj;
 };
 
+/**
+ * @param {Record<string, unknown>} extensionStorage extensionStorage object
+ * @param {Set<string>} extensions extension IDs
+ * @returns {Record<string, unknown>|null}
+ */
+const serializeExtensionStorage = (extensionStorage, extensions) => {
+    const result = {};
+    let isEmpty = true;
+    for (const [key, value] of Object.entries(extensionStorage)) {
+        if (extensions.has(key) && value !== null && typeof value !== 'undefined') {
+            isEmpty = false;
+            result[key] = extensionStorage[key];
+        }
+    }
+    if (isEmpty) {
+        return null;
+    }
+    return result;
+};
+
 const getSimplifiedLayerOrdering = function (targets) {
     const layerOrders = targets.map(t => t.getLayerOrder());
     return MathUtil.reducedSortOrdering(layerOrders);
@@ -712,7 +732,17 @@ const serialize = function (runtime, targetId, {allowOptimization = true} = {})
         });
     }
 
-    const serializedTargets = flattenedOriginalTargets.map(t => serializeTarget(t, extensions));
+    const serializedTargets = flattenedOriginalTargets.map(t => serializeTarget(t, extensions))
+        .map((serialized, index) => {
+            // can't serialize extensionStorage until the list of used extensions is fully known
+            const target = originalTargetsToSerialize[index];
+            const targetExtensionStorage = serializeExtensionStorage(target.extensionStorage, extensions);
+            if (targetExtensionStorage) {
+                serialized.extensionStorage = targetExtensionStorage;
+            }
+            return serialized;
+        });
+
     const fonts = runtime.fontManager.serializeJSON();
 
     if (targetId) {
@@ -731,6 +761,11 @@ const serialize = function (runtime, targetId, {allowOptimization = true} = {})
         return serializedTargets[0];
     }
 
+    const globalExtensionStorage = serializeExtensionStorage(runtime.extensionStorage, extensions);
+    if (globalExtensionStorage) {
+        obj.extensionStorage = globalExtensionStorage;
+    }
+
     obj.targets = serializedTargets;
 
     obj.monitors = serializeMonitors(runtime.getMonitorState(), runtime, extensions);
@@ -1076,8 +1111,8 @@ const parseScratchAssets = function (object, runtime, zip) {
         // we're always loading the 'sb3' representation of the costume
         // any translation that needs to happen will happen in the process
         // of building up the costume object into an sb3 format
-        return deserializeCostume(costume, runtime, zip)
-            .then(() => loadCostume(costumeMd5Ext, costume, runtime));
+        return runtime.wrapAssetRequest(deserializeCostume(costume, runtime, zip)
+            .then(() => loadCostume(costumeMd5Ext, costume, runtime)));
         // Only attempt to load the costume after the deserialization
         // process has been completed
     });
@@ -1101,8 +1136,8 @@ const parseScratchAssets = function (object, runtime, zip) {
         // we're always loading the 'sb3' representation of the costume
         // any translation that needs to happen will happen in the process
         // of building up the costume object into an sb3 format
-        return deserializeSound(sound, runtime, zip)
-            .then(() => loadSound(sound, runtime, assets.soundBank));
+        return runtime.wrapAssetRequest(deserializeSound(sound, runtime, zip)
+            .then(() => loadSound(sound, runtime, assets.soundBank)));
         // Only attempt to load the sound after the deserialization
         // process has been completed.
     });
@@ -1274,6 +1309,9 @@ const parseScratchObject = function (object, runtime, extensions, zip, assets) {
     if (Object.prototype.hasOwnProperty.call(object, 'draggable')) {
         target.draggable = object.draggable;
     }
+    if (Object.prototype.hasOwnProperty.call(object, 'extensionStorage')) {
+        target.extensionStorage = object.extensionStorage;
+    }
     Promise.all(costumePromises).then(costumes => {
         sprite.costumes = costumes;
     });
@@ -1501,6 +1539,9 @@ const deserialize = function (json, runtime, zip, isSingleSprite) {
         .then(targets => replaceUnsafeCharsInVariableIds(targets))
         .then(targets => {
             monitorObjects.map(monitorDesc => deserializeMonitor(monitorDesc, runtime, targets, extensions));
+            if (Object.prototype.hasOwnProperty.call(json, 'extensionStorage')) {
+                runtime.extensionStorage = json.extensionStorage;
+            }
             return targets;
         })
         .then(targets => ({