Adds WARC generation for offline viewing

.gitignore

@@ -1,3 +1,4 @@
+*.warc
 node_modules
 /dist
 /aws.json

README.md

@@ -60,3 +60,13 @@ The file is in this format:
 
 Publishing the site content requires AWS credentials which are not stored in this repository. Attempts to use the `gulp publish` command will fail.
 
+**WARC Generation**
+
+To generate [WARC](https://iipc.github.io/warc-specifications/specifications/warc-format/warc-1.1/) files for all languages:
+
+1. Start the local web server with `gulp`.
+2. In another terminal, run `gulp offline`.
+
+This process will take a few minutes and at the end there will be one WARC file for each language.
+
+These files can be viewed through WARC players like [replayweb.page](https://replayweb.page/).

gulp/index.js

@@ -1,5 +1,7 @@
 
 const { series, watch, src, dest } = require('gulp');
+const log = require('fancy-log');
+const chalk = require('chalk');
 
 /** **************************************************************************************************************** **/
 
@@ -24,6 +26,8 @@ exports.push = pushToProd;
 const cloudfront = require('./cloudfront');
 exports.cloudfront = cloudfront;
 
+const { offlineTask } = require('./offline');
+
 exports.new = require('../build/new-post.js');
 
 function copyProd () {
@@ -32,6 +36,8 @@ function copyProd () {
 
 /** **************************************************************************************************************** **/
 
+exports.offline_quick = offlineTask;
+exports.offline_full = offline_full;
 exports.dev  = series(devBuildTask);
 exports.prod = series(prodBuildTask);
 exports.publish = series(
@@ -47,11 +53,10 @@ exports.testpush = pushToProd.dryrun;
 /** **************************************************************************************************************** **/
 
 function watcher () {
-
   watch([
-    'public/**/*.{md,hbs,html}',
-    'posts/**/*.{md,hbs,html}',
-    'templates/*.{md,hbs,html}',
+    'public/**/*.{md,hbs,html,js,json}',
+    'posts/**/*.{md,hbs,html,js,json}',
+    'templates/*.{md,hbs,html,js,json}',
   ], pagesTask);
 
   watch([
@@ -73,11 +78,41 @@ function watcher () {
   server();
 }
 
-function server () {
+// This thing is buggy as hell and barely works
+async function offline_full(callback) {
+  const { spawn } = require('child_process');
+  log(process.execPath);
+  const srv_proc = spawn(process.execPath, ['./server.js']);
+  srv_proc.stdout.on('data', (data) => {
+    process.stdout.write(data);
+  });
+  srv_proc.stderr.on('data', (data) => {
+    process.stderr.write(data);
+  });
+
+  await new Promise(resolve => setTimeout(resolve, 500));
+
+  // Run the main task
+  offlineTask();
+
+  await new Promise(resolve => setTimeout(resolve, 500));
+
+  // Finish
+  srv_proc.kill('SIGTERM');
+  if (callback !== undefined) {
+    callback();
+  }
+}
+
+function server (callback) {
   var forever = require('forever');
-  var srv = new forever.Monitor('server.js');
+  var srv = new forever.Monitor('server.js', {fork: true});
   srv.start();
   forever.startServer(srv);
+
+  if (callback !== undefined) {
+    callback();
+  }
 }
 
 exports.watch = series(devBuildTask, watcher);