From e0f6960bdc20141b6aa57d4b756a1c110fbda535 Mon Sep 17 00:00:00 2001
From: lakhoune <a.b.lakhoune@gmail.com>
Date: Thu, 17 Aug 2023 18:17:09 +0200
Subject: [PATCH 1/2] [ENH] Intent coverage Fixes #102

---
 src/model-training.js | 148 +++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 147 insertions(+), 1 deletion(-)

diff --git a/src/model-training.js b/src/model-training.js
index 766ce808..14cd6938 100644
--- a/src/model-training.js
+++ b/src/model-training.js
@@ -12,7 +12,10 @@ const production = "env:development" === "env:production";
  *
  */
 class ModelTraining extends LitElement {
-  static properties = {};
+  static properties = {
+    missingIntents: { type: Set, state: true },
+    intentCoverage: { type: Number, state: true },
+  };
   static styles = css`
     #editor {
       height: 512px;
@@ -118,6 +121,13 @@ class ModelTraining extends LitElement {
             <div
               class="d-flex flex-row mb-3 justify-content-end align-items-center"
             >
+              <button
+                type="button"
+                class="btn btn-lg btn-info"
+                @click="${this.showIntentCoverage}"
+              >
+                Intent Coverage
+              </button>
               <i id="trainingStatus" class="form-text text-muted me-1"></i>
               <button
                 type="button"
@@ -136,18 +146,64 @@ class ModelTraining extends LitElement {
             </div>
           </form>
         </div>
+        <!-- Vertically centered modal -->
+
+        <div class="modal fade" id="intentCoverageModal" tabindex="-1">
+          <div class="modal-dialog modal-dialog-centered modal-dialog-scrollable">
+            <div class="modal-content">
+              <div class="modal-header">
+                <h5 class="modal-title">Intent Coverage: ${Math.round(
+                  this.intentCoverage
+                )}%
+                </h5>
+                <button
+                  type="button"
+                  class="btn-close"
+                  data-bs-dismiss="modal"
+                  aria-label="Close"
+                ></button>
+              </div>
+              <div id="intentCoverageModalBody" class="modal-body">
+                ${
+                  this.intentCoverage === 100
+                    ? html`<h5>
+                        All intents are covered by your training data. Great
+                        job!
+                      </h5>`
+                    : html`
+                        <h5>Missing Intents:</h5>
+
+                        <ul class="list-group list-group-flush">
+                          <!-- loop over missing intents -->
+                          ${this.missingIntents.map((intent) => {
+                            return html` <li class="list-group-item">
+                              ${intent}
+                            </li>`;
+                          })}
+                        </ul>
+                      `
+                }
+              
+            </div>
+          </div>
+        </div>
       </main>
     `;
   }
 
   constructor() {
     super();
+    this.missingIntents = [];
+    this.intentCoverage = 0;
   }
 
   async firstUpdated() {
     this.dataName = this.htmlQuery("#dataName");
     this.loadName = this.htmlQuery("#loadNameInput");
     this.curModels = [];
+    this.modal = new bootstrap.Modal(
+      this.shadowRoot.getElementById("intentCoverageModal")
+    );
 
     const y = await ModelOps.getY(true);
 
@@ -229,6 +285,15 @@ class ModelTraining extends LitElement {
   async submitForm() {
     var _this = this;
     const y = await ModelOps.getY(true);
+    await this.determineIntentCoverage();
+    if (
+      this.intentCoverage < 100 &&
+      !confirm(
+        "Your training data is missing some intents. Do you still wish to submit it? You can check the missing intents by clicking the Intent Coverage button."
+      )
+    ) {
+      return;
+    }
     const trainingUrl = y.getText("sbfManager").toString() + "/trainAndLoad/";
     const rasaUrl = y.getText("rasa").toString();
     $(_this.htmlQuery("#trainingStatus")).text(
@@ -255,6 +320,87 @@ class ModelTraining extends LitElement {
     });
   }
 
+  async extractIntentsFromBotModel() {
+    const y = await ModelOps.getY(true);
+    const model = y.getMap("data").get("model");
+    if (!model) {
+      return new Set();
+    }
+    const incomingMessageNodes = Object.values(model["nodes"]).filter(
+      (node) => node.type === "Incoming Message"
+    );
+    const leadsToEdges = Object.values(model["edges"]).filter(
+      (edge) => edge.type === "leadsTo"
+    );
+    const intents = new Set();
+
+    for (const node of incomingMessageNodes) {
+      for (const attribute of Object.values(node.attributes)) {
+        if (
+          attribute.name === "Intent Keyword" &&
+          attribute.value.value !== ""
+        ) {
+          let intentValue = attribute.value.value;
+          if (intentValue.includes(",")) {
+            for (const int of intentValue.split(",")) {
+              intents.add(int.trim());
+            }
+          } else {
+            intents.add(intentValue);
+          }
+        }
+      }
+    }
+
+    for (const edge of leadsToEdges) {
+      if (edge.label.value.value !== "") {
+        let intentValue = edge.label.value.value;
+        if (intentValue.includes(",")) {
+          for (const int of intentValue.split(",")) {
+            intents.add(int.trim());
+          }
+        } else {
+          intents.add(intentValue);
+        }
+      }
+    }
+
+    return intents;
+  }
+
+  async extractIntentsFromNLUModel() {
+    const intents = new Set();
+    for (const line of this.editor.getText().split("\n")) {
+      if (line.trimStart().includes("intent:")) {
+        let intentValue = line.trimStart().split("intent:")[1].trim();
+        intents.add(intentValue);
+      }
+    }
+    return intents;
+  }
+
+  async determineIntentCoverage() {
+    const botModelIntents = await this.extractIntentsFromBotModel();
+    const nluModelIntents = await this.extractIntentsFromNLUModel();
+    // check if all intents from bot model are covered by nlu model
+    const missingIntents = new Set();
+    for (const intent of botModelIntents) {
+      if (!nluModelIntents.has(intent) && intent !== "default") {
+        missingIntents.add(intent);
+      }
+    }
+
+    this.missingIntents = Array.from(missingIntents).sort();
+    const coverage = (1 - missingIntents.size / botModelIntents.size) * 100;
+    this.intentCoverage = coverage;
+  }
+
+  async showIntentCoverage() {
+    await this.determineIntentCoverage();
+    // open modal
+    this.modal.show();
+  }
+
   async retrieveStatus() {
     const y = await ModelOps.getY(true);
     const trainingStatusUrl =

From ebab0415c43313e36a99cae703c0b13fdffb3d75 Mon Sep 17 00:00:00 2001
From: lakhoune <a.b.lakhoune@gmail.com>
Date: Sat, 19 Aug 2023 18:39:27 +0200
Subject: [PATCH 2/2] whenever a bot is submitted we also store the botmodel as
 backup

---
 src/bot.manager.widget.js | 71 ++++++++++++++++++++++-----------------
 1 file changed, 41 insertions(+), 30 deletions(-)

diff --git a/src/bot.manager.widget.js b/src/bot.manager.widget.js
index a27a35c0..c7e45613 100644
--- a/src/bot.manager.widget.js
+++ b/src/bot.manager.widget.js
@@ -218,6 +218,12 @@ class BotManagerWidget extends LitElement {
     xhr.open("POST", endpoint + "/bots");
     xhr.setRequestHeader("Content-Type", "application/json");
     xhr.send(JSON.stringify(model));
+    let botName;
+    const botNode = Object.values(model["nodes"]).find(
+      (node) => node.type === "Bot"
+    );
+    botName = botNode.label.value.value;
+    this.storeModel(botName);
   }
 
   deleteModel() {
@@ -313,8 +319,13 @@ class BotManagerWidget extends LitElement {
     xhr.send();
   }
 
-  storeModel() {
-    var name = y.getText("storeName").toString();
+  storeModel(name) {
+    let botName = "";
+    if (typeof name === "string") {
+      botName = name;
+    } else {
+      botName = "backup-" + y.getText("storeName").toString();
+    }
     var endpoint = y.getText("sbfManager").toString();
     var model = y.getMap("data").get("model");
     var storeStatus = $("#storeStatus");
@@ -324,35 +335,34 @@ class BotManagerWidget extends LitElement {
     storeStatus.text("Storing...");
     btn.prop("disabled", true);
 
-    if (name && model) {
-     fetch(endpoint + "/models/" + name, {
-       method: "POST",
-       headers: {
-         "Content-Type": "application/json",
-       },
-       body: JSON.stringify(model),
-     })
-       .then((response) => {
-         if (response.ok) {
-           alert("Your bot model has been successfully backed up");
-           this.updateMenu();
-         } else {
-           throw new Error(
-             "Your bot model could not be backed up. Make sure that the SBF endpoint is correct."
-           );
-         }
-       })
-       .catch((error) => {
-         alert(error.message);
-       })
-       .finally(() => {
-         spinner.hide();
-         btn.prop("disabled", false);
-         // cleanStatus("storeStatus");
-       });
-
+    if (botName && model) {
+      fetch(endpoint + "/models/" + botName, {
+        method: "POST",
+        headers: {
+          "Content-Type": "application/json",
+        },
+        body: JSON.stringify(model),
+      })
+        .then((response) => {
+          if (response.ok) {
+            if (!name) alert("Your bot model has been successfully backed up");
+            this.updateMenu();
+          } else {
+            throw new Error(
+              "Your bot model could not be backed up. Make sure that the SBF endpoint is correct."
+            );
+          }
+        })
+        .catch((error) => {
+          console.error(error);
+        })
+        .finally(() => {
+          spinner.hide();
+          btn.prop("disabled", false);
+          // cleanStatus("storeStatus");
+        });
     } else {
-      if (!name) {
+      if (!botName) {
         alert("The model name is invalid.");
       } else {
         alert("The model name is empty.");
@@ -505,6 +515,7 @@ class BotManagerWidget extends LitElement {
             <div id="modelstorer" class="col col-4">
               <label for="store-model" class="form-label">Store model</label>
               <div class="input-group mb-3">
+                 <span class="input-group-text" id="basic-addon3">backup-</span>
                 <div id="storeNameInput"></div>
 
                 <button id="store-model" class="btn btn-outline-primary"  @click="${this.storeModel}">