Merge branch 'main' of https://github.com/biothings/biothings_explorer …

…into dev

prod_revisions.txt

@@ -2,12 +2,12 @@
 > @biothings-explorer/[email protected] get_rev
 > ./scripts/get_rev.sh
 
-# Generated from "npm run get_rev" on  Tue Aug 8 11:57:42 EDT 2023
-https://github.com/biothings/biothings_explorer.git 9ccce71 9ccce715ee19115e05fdbe4e40aac7a8f7802c43
-https://github.com/biothings/api-respone-transform.js.git 2baec27 2baec2713f2e3d0e912955d0fc5ed6234775522f
+# Generated from "npm run get_rev" on  Thu Aug 10 12:08:24 EDT 2023
+https://github.com/biothings/biothings_explorer.git 566af8e 566af8e8b82519b9e56e2af41afd799b21eb4ac0
+https://github.com/biothings/api-respone-transform.js.git ed920f8 ed920f849054af32c77f1d13a0646852a1e2888a
 https://github.com/biothings/call-apis.js.git c6c6421 c6c64216b0ae52ad293c6efae210b7dcda4956e3
 https://github.com/biothings/smartapi-kg.js.git 874e120 874e1201efaa34a9a53434bae1cad2ade489b5c5
 https://github.com/biothings/bte_trapi_query_graph_handler.git 32eb684 32eb684ee67b1908cbe15e64a8d307b8e2ea68e6
 https://github.com/biothings/node-expansion.git 524df56 524df56767495bf397ecacb2f5a299a5b18b8bff
-https://github.com/biothings/biolink-model.js.git abfb7b7 abfb7b754251887677a90890d887a72a27c0cb85
+https://github.com/biothings/biolink-model.js.git c31f19f c31f19f35f6773b544eaad82747f680891aeaf6b
 https://github.com/biothings/biomedical_id_resolver.js.git b7ffbe8 b7ffbe8b9e0e222c6d392232ef64a9abfb877ce3

src/controllers/threading/threadHandler.js

@@ -14,20 +14,29 @@ const { getQueryQueue } = require("../async/asyncquery_queue");
 const Sentry = require("@sentry/node");
 const ErrorHandler = require("../../middlewares/error.js");
 
+const SYNC_MIN_CONCURRENCY = 2;
+const ASYNC_MIN_CONCURRENCY = 3;
+
 // On Prod: 0.25 ratio * 16 cores * 4 instances = 16 threads
-const SYNC_CONCURRENCY_RATIO = 0.25
-/** On Prod: 0.5 ratio * 16 cores * 4 instances = 32 threads
+const SYNC_CONCURRENCY_RATIO = 0.25;
+/** On Prod: 0.25 ratio * 16 cores * 4 instances = 16 threads
  * Async has 3 queues:
  * - general
  * - by api
  * - by team
  *
- * Distribution between those is set manually. General should get priority.
+ * Distribution between those is seen below.
  * */
-const ASYNC_CONCURRENCY_RATIO = 0.5
+const ASYNC_CONCURRENCY_RATIO = 0.25;
+
+let SYNC_CONCURRENCY = Math.ceil(os.cpus().length * SYNC_CONCURRENCY_RATIO);
+if (SYNC_CONCURRENCY < SYNC_MIN_CONCURRENCY) SYNC_CONCURRENCY = SYNC_MIN_CONCURRENCY;
+let ASYNC_CONCURRENCY = Math.ceil(os.cpus().length * ASYNC_CONCURRENCY_RATIO);
+if (ASYNC_CONCURRENCY < ASYNC_MIN_CONCURRENCY) ASYNC_CONCURRENCY = ASYNC_MIN_CONCURRENCY;
 
-const SYNC_CONCURRENCY = Math.ceil(os.cpus().length * SYNC_CONCURRENCY_RATIO)
-const ASYNC_CONCURRENCY = Math.ceil(os.cpus().length * ASYNC_CONCURRENCY_RATIO)
+const ASYNC_MAIN_CONCURRENCY = ASYNC_CONCURRENCY <= 9 ? Math.ceil(ASYNC_CONCURRENCY / 3) : ASYNC_CONCURRENCY - 6;
+const ASYNC_BY_API_CONCURRENCY = ASYNC_CONCURRENCY <= 9 ? Math.floor(ASYNC_CONCURRENCY / 3) : 3;
+const ASYNC_BY_TEAM_CONCURRENCY = ASYNC_CONCURRENCY <= 9 ? Math.floor(ASYNC_CONCURRENCY / 3) : 3;
 
 if (!global.threadpool && !isWorkerThread && !(process.env.USE_THREADING === "false")) {
   const env = {
@@ -186,7 +195,7 @@ async function runTask(req, task, route, res, useBullSync = true) {
       },
     },
     params: req.params,
-    endpoint: req.originalUrl
+    endpoint: req.originalUrl,
   };
   if (queryQueue && useBullSync) {
     const nanoid = customAlphabet("0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ", 10);
@@ -294,7 +303,7 @@ function taskResponse(response, status = undefined) {
 function taskError(error) {
   if (global.parentPort) {
     if (ErrorHandler.shouldHandleError(error)) {
-        Sentry.captureException(error);
+      Sentry.captureException(error);
     }
     global.parentPort.postMessage({ threadId, err: error });
     return undefined;
@@ -316,6 +325,33 @@ if (!global.queryQueue.bte_sync_query_queue && !isWorkerThread) {
   }
 }
 
+if (!global.queryQueue.bte_query_queue && !isWorkerThread) {
+  getQueryQueue("bte_query_queue");
+  if (global.queryQueue.bte_query_queue) {
+    global.queryQueue.bte_query_queue.process(ASYNC_MAIN_CONCURRENCY, async job => {
+      return await runBullTask(job, "asyncquery_v1");
+    });
+  }
+}
+
+if (!global.queryQueue.bte_query_queue_by_api && !isWorkerThread) {
+  getQueryQueue("bte_query_queue_by_api");
+  if (global.queryQueue.bte_query_queue_by_api) {
+    global.queryQueue.bte_query_queue_by_api.process(ASYNC_BY_API_CONCURRENCY, async job => {
+      return await runBullTask(job, "asyncquery_v1_by_api");
+    });
+  }
+}
+
+if (!global.queryQueue.bte_query_queue_by_team && !isWorkerThread) {
+  getQueryQueue("bte_query_queue_by_team");
+  if (global.queryQueue.bte_query_queue_by_team) {
+    global.queryQueue.bte_query_queue_by_team.process(ASYNC_BY_TEAM_CONCURRENCY, async job => {
+      return await runBullTask(job, "asyncquery_v1_by_team");
+    });
+  }
+}
+
 module.exports = {
   runTask,
   runBullTask,

src/routes/v1/asyncquery_status.js

@@ -55,7 +55,7 @@ class VCheckQueryStatus {
         logs = logs.logs.map(log => JSON.parse(log));
         let [status, description] = {
           // convert to TRAPI states
-          completed: ["Completed", "The query has finixhed executing."],
+          completed: ["Completed", "The query has finished executing."],
           failed: ["Failed", job.failedReason],
           delayed: ["Queued", "The query is queued, but has been delayed."],
           active: ["Running", "The query is currently being processed."],

src/routes/v1/asyncquery_v1.js

@@ -16,16 +16,6 @@ const predicatesPath = path.resolve(
   process.env.STATIC_PATH ? `${process.env.STATIC_PATH}/data/predicates.json` : "../../../data/predicates.json",
 );
 
-if (!global.queryQueue.bte_query_queue && isMainThread) {
-  getQueryQueue("bte_query_queue");
-  if (global.queryQueue.bte_query_queue) {
-    global.queryQueue.bte_query_queue.process(22, async job => {
-      return await runBullTask(job, path.parse(__filename).name);
-    });
-    // path.resolve(__dirname, "../../controllers/async/processors/async_v1.js"),
-  }
-}
-
 class V1RouteAsyncQuery {
   setRoutes(app) {
     app

src/routes/v1/asyncquery_v1_by_api.js

@@ -16,15 +16,6 @@ const predicatesPath = path.resolve(
   process.env.STATIC_PATH ? `${process.env.STATIC_PATH}/data/predicates.json` : "../../../data/predicates.json",
 );
 
-if (!global.queryQueue.bte_query_queue_by_api && isMainThread) {
-  getQueryQueue("bte_query_queue_by_api");
-  if (global.queryQueue.bte_query_queue_by_api) {
-    global.queryQueue.bte_query_queue_by_api.process(5, async job => {
-      return await runBullTask(job, path.parse(__filename).name);
-    });
-  }
-}
-
 class V1RouteAsyncQueryByAPI {
   setRoutes(app) {
     app

src/routes/v1/asyncquery_v1_by_team.js

@@ -16,15 +16,6 @@ const predicatesPath = path.resolve(
   process.env.STATIC_PATH ? `${process.env.STATIC_PATH}/data/predicates.json` : "../../../data/predicates.json",
 );
 
-if (!global.queryQueue.bte_query_queue_by_team && isMainThread) {
-  getQueryQueue("bte_query_queue_by_team");
-  if (global.queryQueue.bte_query_queue_by_team) {
-    global.queryQueue.bte_query_queue_by_team.process(5, async job => {
-      return await runBullTask(job, path.parse(__filename).name);
-    });
-  }
-}
-
 class V1RouteAsyncQueryByTeam {
   setRoutes(app) {
     app

src/web-app/src/views/HomeView.vue

@@ -13,7 +13,7 @@ import LinkOut from '../components/LinkOutIcon.vue'
     <div class="text-center flex-grow">
       <h2 class="text-white text-3xl"><Logo class="w-3/4 inline"></Logo> Is Ready!</h2>
       <div class="flex p-3 justify-center items-center">
-        <a class="main-btn-outline" href="https://biothings.io/explorer/" target="_blank" rel="nonopenner">
+        <a class="main-btn-outline" href="https://explorer.biothings.io/" target="_blank" rel="nonopenner">
           Learn more about BTE <LinkOut></LinkOut>
         </a>
       </div>