Sync HTML with an audio track, narrating the presentation

gulpfile.js

@@ -232,6 +232,7 @@ function makeBrowserifyTask(name) {
 
 const playerBrowserifyTask = parallel(
     makeBrowserifyTask("player"),
+    makeBrowserifyTask("narrated"),
     makeBrowserifyTask("presenter")
 );
 
@@ -266,11 +267,13 @@ function makeTemplateTask(target, tpl) {
 
 const electronTemplatesTask = parallel(
     makeTemplateTask("electron", "player"),
+    makeTemplateTask("electron", "narrated"),
     makeTemplateTask("electron", "presenter")
 );
 
 const browserTemplatesTask = parallel(
     makeTemplateTask("browser", "player"),
+    makeTemplateTask("browser", "narrated"),
     makeTemplateTask("browser", "presenter")
 );
 

src/js/Storage.js

@@ -275,9 +275,11 @@ export class Storage {
         const svgName           = this.backend.getName(this.svgFileDescriptor);
         const htmlFileName      = replaceFileExtWith(svgName, ".sozi.html");
         const presenterFileName = replaceFileExtWith(svgName, "-presenter.sozi.html");
+        const narratedFileName  = replaceFileExtWith(svgName, ".narrated.sozi.html");
         // TODO Save only if SVG is more recent than HTML.
         this.createHTMLFile(htmlFileName, location);
         this.createPresenterHTMLFile(presenterFileName, location, htmlFileName);
+        this.createNarratedHTMLFile(narratedFileName, location, htmlFileName);
     }
 
     /** Create the presentation HTML file if it does not exist.
@@ -312,10 +314,30 @@ export class Storage {
     async createPresenterHTMLFile(name, location, htmlFileName) {
         try {
             const fileDescriptor = await this.backend.find(name, location);
-            this.backend.save(fileDescriptor, this.exportPresenterHTML(htmlFileName));
+            await this.backend.save(fileDescriptor, this.exportPresenterHTML(htmlFileName));
         }
         catch (err) {
-            this.backend.create(name, location, "text/html", this.exportPresenterHTML(htmlFileName));
+            await this.backend.create(name, location, "text/html", this.exportPresenterHTML(htmlFileName));
+        }
+    }
+
+    /** Create the narrated HTML file if it does not exist.
+     *
+     * @param {string} name - The name of the HTML file to create.
+     * @param {any} location - The location of the file (backend-dependent).
+     * @param {string} htmlFileName - The name of the presentation HTML file.
+     */
+    async createNarratedHTMLFile(name, location, htmlFileName) {
+        const t = this.presentation.narrativeType;
+        if (!t || t === "none") {
+            return;
+        }
+        try {
+            const fileDescriptor = await this.backend.find(name, location);
+            await this.backend.save(fileDescriptor, this.exportNarratedHTML(htmlFileName));
+        }
+        catch (err) {
+            await this.backend.create(name, location, "text/html", this.exportNarratedHTML(htmlFileName));
         }
     }
 
@@ -403,6 +425,20 @@ export class Storage {
         });
     }
 
+    /** Generate the content of the narrated HTML file.
+     *
+     * The result is derived from the `narrated.html` template.
+     *
+     * @param {string} htmlFileName - The name of the presentation HTML file to play.
+     * @returns {string} - An HTML document content, as text.
+     */
+    exportNarratedHTML(htmlFileName) {
+        return nunjucks.render("narrated.html", {
+            pres: this.presentation,
+            soziHtml: htmlFileName
+        });
+    }
+
     /** Get the path of a file relative to the location of the current SVG file.
      *
      * @param {string} filePath - The path of a file.

src/js/model/Presentation.js

@@ -756,6 +756,25 @@ export class Presentation extends EventEmitter {
          */
         this.updateURLOnFrameChange = true;
 
+        /** The narrative file type or "none" if narration is disabled.
+         *
+         * @default
+         * @type {string} */
+        this.narrativeType = "none";
+
+        /** The narrative file name or relative path.
+         *
+         * @default
+         * @type {string} */
+        this.narrativeFile = "narrative.flac";
+
+        /** The narrative time-to-slide data mapping the time
+         * moments (in seconds) to their corresponding frame number (one-based).
+         *
+         * @default
+         * @type {string} */
+        this.narrativeTimeToSlide = "0";
+
         /** The last export document type.
          *
          * @default
@@ -922,6 +941,9 @@ export class Presentation extends EventEmitter {
             enableMouseRotation       : this.enableMouseRotation,
             enableMouseNavigation     : this.enableMouseNavigation,
             updateURLOnFrameChange    : this.updateURLOnFrameChange,
+            narrativeType             : this.narrativeType,
+            narrativeFile             : this.narrativeFile,
+            narrativeTimeToSlide      : this.narrativeTimeToSlide,
             exportType                : this.exportType,
             exportToPDFPageSize       : this.exportToPDFPageSize,
             exportToPDFPageOrientation: this.exportToPDFPageOrientation,
@@ -978,6 +1000,9 @@ export class Presentation extends EventEmitter {
         copyIfSet(this, storable, "enableMouseRotation");
         copyIfSet(this, storable, "enableMouseNavigation");
         copyIfSet(this, storable, "updateURLOnFrameChange");
+        copyIfSet(this, storable, "narrativeType");
+        copyIfSet(this, storable, "narrativeFile");
+        copyIfSet(this, storable, "narrativeTimeToSlide");
         copyIfSet(this, storable, "exportType");
         copyIfSet(this, storable, "exportToPDFPageSize");
         copyIfSet(this, storable, "exportToPDFPageOrientation");

src/js/narrated.js

@@ -0,0 +1,94 @@
+
+window.addEventListener("load", () => {
+    const narrative = document.getElementById("narrative")
+
+    const timeToSlide = (() => {
+        const timeToSlide = new Map()
+        let prevSlide = 0
+        for (const x of narrative.dataset.timeToSlide.split(",")) {
+            const [time, slide] = x.split(":")
+            prevSlide = parseInt((slide === undefined) ? prevSlide+1 : slide)
+            timeToSlide.set(parseInt(time), prevSlide - 1)
+        }
+        return timeToSlide
+    })()
+    const slideToTimeIntervals = (() => {
+        const slideToTimeIntervals = new Map()
+        const process = (start, end, slide) => {
+            let timeIntervals = slideToTimeIntervals.get(slide)
+            if (timeIntervals === undefined) {
+                timeIntervals = []
+                slideToTimeIntervals.set(slide, timeIntervals)
+            }
+            timeIntervals.push([start, end])
+        }
+        let start, slide
+        for (const [end, nextSlide] of timeToSlide) {
+            if (start === undefined) {
+                start = end
+                slide = nextSlide
+                continue
+            }
+            process(start, end, slide)
+            start = end
+            slide = nextSlide
+        }
+        if (start !== undefined) {
+            process(start, Number.POSITIVE_INFINITY, slide)
+        }
+        return slideToTimeIntervals
+    })()
+
+    let currentSlide = 0
+
+    const updateAudio = (newSlide) => {
+        currentSlide = newSlide
+        const timeIntervals = slideToTimeIntervals.get(newSlide)
+        if (timeIntervals === undefined) {
+            narrative.pause()
+            return
+        }
+        const t = narrative.currentTime
+        let nearestTime
+        for (const [start, end] of timeIntervals) {
+            if (start <= t && t < end) {
+                return
+            }
+            if (nearestTime === undefined || start <= t) {
+                nearestTime = start
+            }
+        }
+        narrative.currentTime = nearestTime
+    }
+    const updateSlide = (target) => {
+        const t = narrative.currentTime
+        let targetSlide = 0
+        for (const [time, slide] of timeToSlide) {
+            if (time <= t) {
+                targetSlide = slide
+            }
+        }
+        if (targetSlide !== currentSlide) {
+            currentSlide = targetSlide
+            target.postMessage({name:"moveToFrame", args: [currentSlide]},"*")
+        }
+    }
+    window.addEventListener("message", (e) => {
+        switch(e.data.name) {
+            case "loaded":
+                const target = e.source
+                target.postMessage({name:"notifyOnFrameChange"},"*")
+                const frame = document.querySelector("iframe")
+                setInterval(() => {
+                    frame.focus()
+                }, 1000)
+                narrative.ontimeupdate = () => {
+                    updateSlide(target)
+                }
+                break
+            case "frameChange":
+                updateAudio(e.data.index)
+                break
+        }
+    })
+}, false)

src/js/view/Properties.js

@@ -77,6 +77,7 @@ export class Properties extends VirtualDOMView {
     render() {
         switch (this.mode) {
             case "preferences": return this.renderPreferences();
+            case "narration":   return this.renderNarrationProperties();
             case "export":      return this.renderExportTool();
             default:            return this.renderPresentationProperties();
         }
@@ -368,6 +369,86 @@ export class Properties extends VirtualDOMView {
         ]);
     }
 
+    /** Render the properties view for a presentation-wide narrative.
+     *
+     * Three properties will be configured:
+     *   1. narrativeType: the audio tag type; this field is a hint to
+     *      guide user in selection of an appropriate format, although
+     *      the browser can automatically detect the correct type by
+     *      itself. If this value is set to "none", narration will be
+     *      disabled. If it is set to flac or ogg, the corresponding
+     *      mimetype will be set for the <audio> type:
+     *        audio/flac or audio/ogg
+     *   2. narrativeFile: the name (or relative path) of the narrative
+     *      file which must be available in side of the HTML file.
+     *   3. narrativeTimeToSlide: the time-to-slide data value
+     *      Check the {narrativeTimeToSlideHelp} for details.
+     *
+     * @returns {VNode} - A virtual DOM tree.
+     */
+    renderNarrationProperties() {
+        const controller = this.controller;
+        const _ = controller.gettext;
+        const disabled = controller.presentation.narrativeType === "none";
+        const toDefaultMode = () => this.toggleMode("default");
+
+        return h("div.properties", [
+            h("div.back", {
+                title: _("Back to presentation properties"),
+                onClick() { toDefaultMode(); }
+            }, h("i.fa.fa-arrow-left", )),
+
+            h("h1", _("Narration")),
+
+            h("label", {for: "field-narrativeType"}, _("Narrative file type")),
+            this.renderSelectField("narrativeType", controller.getPresentationProperty, controller.setPresentationProperty, {
+                none: _("None (disable narration)"),
+                flac: _("Free Lossless Audio Codec (FLAC)"),
+                ogg:  _("Ogg (a container for Vorbis or Opus)"),
+            }),
+
+            h("label.side-by-side", {for: "field-narrativeFile"}, [
+                _("Narrative file name"),
+                this.renderHelp(_("Name or path of the narrative file"),
+                  () => controller.info(
+                    _("Name or path of the narrative file (relative to the HTML file location)"),
+                    true
+                  )
+                )
+            ]),
+            this.renderTextField("narrativeFile", disabled, controller.getPresentationProperty, controller.setPresentationProperty, false),
+
+            h("label.side-by-side", {for: "field-narrativeTimeToSlide"}, [
+                _("Time to frame number mapping"),
+                this.renderHelp(_("Click here to see the syntax for this field"), () => controller.info(this.narrativeTimeToSlideHelp, true))
+            ]),
+            this.renderTextField("narrativeTimeToSlide", disabled, controller.getPresentationProperty, controller.setPresentationProperty, false)
+        ]);
+    }
+
+    /** The HTML of the help message for time-to-slide data.
+     *
+     * @readonly
+     * @type {string}
+     */
+    get narrativeTimeToSlideHelp() {
+        const _ = this.controller.gettext;
+        return [
+            _("Format of the narrative time-to-slide data:"),
+            "<ul><li>" + [
+                _("A string of comma-separated descriptors"),
+                _("Each descriptor specifies the time instant (in seconds) which should trigger a slide transition"),
+                _("Descriptor can specify the target slide number (counting from one); for this purpose the time and slide index must be separated by a colon"),
+            ].join("<li>") + "</ul>",
+            _("Examples:"),
+            "<ul><li>" + [
+                _("\"0\": Switch to the first slide when narrative is at zero second without any other slide transition"),
+                _("\"0,5,7\": Switch to the 1st, 2nd, and 3rd slides at 0, 5, and 7 seconds of narrative"),
+                _("\"0,5,7:1,10:4\": At time 7s, return to the 1st slide and at 10s resume to the 4th slide"),
+            ].join("<li>") + "</ul>",
+        ].join("<br>");
+    }
+
     /** The HTML of the help message for include/exclude lists in export.
      *
      * @readonly

src/js/view/Toolbar.js

@@ -144,6 +144,11 @@ export class Toolbar extends VirtualDOMView {
                     title: _("Reload the SVG document"),
                     onclick() { controller.reload(); }
                 }, h("i.fa.fa-refresh")),
+                h("button", {
+                    title: _("Narrate the presentation"),
+                    className: properties.mode === "narration" ? "active" : undefined,
+                    onclick() { properties.toggleMode("narration"); }
+                }, h("i.fa.fa-microphone")), // alternatives are file-audio-o and volume-up
                 // TODO disable the Export button if the feature is not available
                 h("button", {
                     title: _("Export the presentation"),

src/templates/narrated.html

@@ -0,0 +1,47 @@
+{# This Source Code Form is subject to the terms of the Mozilla Public
+ # License, v. 2.0. If a copy of the MPL was not distributed with this
+ # file, You can obtain one at http://mozilla.org/MPL/2.0/. #}
+
+{% raw %}
+<!doctype html>
+<html>
+    <head>
+        <meta charset="utf-8">
+        <meta http-equiv="X-UA-Compatible" content="IE=edge">
+        <title>{{ pres.title }}</title>
+        <style>
+            html, body {
+                width: 100%;
+                height: 100%;
+                margin: 0;
+            }
+            .presentation-container {
+                position: fixed;
+                top: 0;
+                left: 0;
+                width: 100%;
+                height: 100%;
+            }
+            iframe {
+                border: none;
+            }
+            audio {
+                position: fixed;
+                right: 0;
+                bottom: 0;
+                width: 25%;
+            }
+        </style>
+    </head>
+    <body>
+        <iframe class="presentation-container" src="{{ soziHtml }}">
+            The Sozi presentation should play here.
+        </iframe>
+        <audio id="narrative" controls data-time-to-slide="{{ pres.narrativeTimeToSlide }}">
+             <source src="{{ pres.narrativeFile }}" type="audio/{{ pres.narrativeType }}" preload="auto">
+             Audio HTML5 tag is not supported!
+        </audio>
+{% endraw %}
+        <script>{{'{% raw %}'}}{{ js|safe }}{{'{% endraw %}'}}</script>
+    </body>
+</html>