generate missing-template doc

.gitignore

@@ -3,3 +3,4 @@ afwiki-latest-pages-articles.xml
 node_modules
 dump.rdb
 .DS_STORE
+tmp.txt

package.json

@@ -26,7 +26,7 @@
     "prettysize": "1.1.0",
     "sunday-driver": "1.0.1",
     "worker-nodes": "1.6.1",
-    "wtf_wikipedia": "6.2.0",
+    "wtf_wikipedia": "6.2.1",
     "yargs": "12.0.2"
   },
   "devDependencies": {

scripts/missing-templates.js

@@ -0,0 +1,37 @@
+//first, run dumpster with {} piped to a text file
+// npm run watch > ./tmp.txt
+const sundayDriver = require('sunday-driver');
+
+let templates = {}
+
+let obj = {
+  file: './tmp.txt',
+  splitter: '\n',
+  each: (line, resume) => {
+    if (line[0] === ':') {
+      templates[line] = templates[line] || 0
+      templates[line] += 1
+    }
+    resume()
+  }
+}
+
+let p = sundayDriver(obj);
+p.then(() => {
+  let keys = Object.keys(templates)
+  keys = keys.sort((a, b) => {
+    if (templates[a] > templates[b]) {
+      return -1
+    }
+    return 1
+  })
+  let show = keys.slice(0, 1000)
+  show.forEach((key) => {
+    let k = key.replace('\n', '')
+    k = k.replace('::', '')
+    k = k.replace(/^ +/, '')
+    k = k.replace(/ /g, '_')
+    let link = `* [${k}](https://en.wikipedia.org/wiki/Template:${k})`
+    console.log(link + '\t    -   ' + templates[key])
+  })
+})

src/worker/02-parseWiki.js

@@ -16,7 +16,7 @@ const escapeXML = function(str) {
 const parseWiki = function(page, options, worker) {
   try {
     page.wiki = escapeXML(page.wiki || '');
-    let doc = wtf(page.wiki);
+    let doc = wtf(page.wiki, options);
     //dont insert this if it's a redirect
     if (options.skip_redirects === true && doc.isRedirect()) {
       worker.counts.redirects += 1