feat: limit and orchestration pull connections to mongo (#3814)

Co-authored-by: Ihar <[email protected]>

application-events/.env.docker

@@ -1,2 +1,7 @@
 MQ_ADDRESS="message-broker"
 MONGODB_SERVER_URL=mongodb://mongo:27017
+
+# Mongo init
+MIN_POOL_SIZE="1"
+MAX_POOL_SIZE="5"
+MAX_IDLE_TIME_MS="30000"

application-events/.env.example

@@ -1,3 +1,7 @@
 MQ_ADDRESS=localhost
 MONGODB_SERVER_URL=mongodb://mongo:27017
 
+# Mongo init
+MIN_POOL_SIZE="1"
+MAX_POOL_SIZE="5"
+MAX_IDLE_TIME_MS="30000"

application-events/package.json

@@ -20,6 +20,9 @@
         "@azure/core-rest-pipeline": "1.12.1",
         "image-size": "1.0.2"
     },
+    "imports": {
+      "#constants": "./dist/constants/index.js"
+    },
     "dependencies": {
         "@guardian/common": "^2.13.0",
         "@guardian/interfaces": "^2.13.0",

application-events/src/constants/index.ts

@@ -0,0 +1 @@
+export { DEFAULT as DEFAULT_MONGO } from './mongo.js';

application-events/src/constants/mongo.ts

@@ -0,0 +1,5 @@
+export const DEFAULT = {
+    MIN_POOL_SIZE: '1',
+    MAX_POOL_SIZE: '5',
+    MAX_IDLE_TIME_MS: '30000',
+};

application-events/src/singletons/MongodbConnection.ts

@@ -1,6 +1,8 @@
 import { MikroORM } from '@mikro-orm/core';
 import { MongoDriver } from '@mikro-orm/mongodb';
 import { Webhook } from '../entities/Webhook.js';
+import process from 'process';
+import { DEFAULT_MONGO } from '#constants';
 
 export default class MongodbConnection {
 
@@ -14,6 +16,11 @@ export default class MongodbConnection {
       dbName: 'application_events',
       driver: MongoDriver,
       entities: [ Webhook ],
+      driverOptions: {
+        minPoolSize: parseInt(process.env.MIN_POOL_SIZE ?? DEFAULT_MONGO.MIN_POOL_SIZE),
+        maxPoolSize: parseInt(process.env.MAX_POOL_SIZE  ?? DEFAULT_MONGO.MAX_POOL_SIZE),
+        maxIdleTimeMS: parseInt(process.env.MAX_IDLE_TIME_MS  ?? DEFAULT_MONGO.MAX_IDLE_TIME_MS)
+      },
     });
   };
 

application-events/tsconfig.json

@@ -9,7 +9,10 @@
         "esModuleInterop": true,
         "skipLibCheck": true,
         "isolatedModules": true,
-        "moduleResolution": "node"
+        "moduleResolution": "node",
+        "paths": {
+          "#constants": ["constants/index.js"]
+        }
     },
     "include": ["./src/**/*", "./tests/**/*.ts"],
     "exclude": ["node_modules", "**/*.spec.ts"]

application-events/tsconfig.production.json

@@ -9,7 +9,10 @@
     "esModuleInterop": true,
     "skipLibCheck": true,
     "isolatedModules": true,
-    "moduleResolution": "node"
+    "moduleResolution": "node",
+    "paths": {
+      "#constants": ["constants/index.js"]
+    }
   },
   "include": [
     "./src/**/*",

auth-service/configs/.env.auth

@@ -42,3 +42,8 @@ HASHICORP_ENCRIPTION_ALG="sha512"
 
 ACCESS_TOKEN_UPDATE_INTERVAL=60000
 REFRESH_TOKEN_UPDATE_INTERVAL=31536000000
+
+# Mongo init
+MIN_POOL_SIZE="1"
+MAX_POOL_SIZE="5"
+MAX_IDLE_TIME_MS="30000"

auth-service/configs/.env.auth.develop

@@ -41,3 +41,8 @@ AZURE_VAULT_NAME=guardianVault
 
 ACCESS_TOKEN_UPDATE_INTERVAL=60000
 REFRESH_TOKEN_UPDATE_INTERVAL=31536000000
+
+# Mongo init
+MIN_POOL_SIZE="1"
+MAX_POOL_SIZE="5"
+MAX_IDLE_TIME_MS="30000"

auth-service/configs/.env.auth.template

@@ -41,3 +41,8 @@ AZURE_VAULT_NAME=
 
 ACCESS_TOKEN_UPDATE_INTERVAL=60000
 REFRESH_TOKEN_UPDATE_INTERVAL=31536000000
+
+# Mongo init
+MIN_POOL_SIZE="1"
+MAX_POOL_SIZE="5"
+MAX_IDLE_TIME_MS="30000"

auth-service/src/app.ts

@@ -13,6 +13,7 @@ import { MicroserviceOptions, Transport } from '@nestjs/microservices';
 import { MeecoAuthService } from './api/meeco-service.js';
 import { ApplicationEnvironment } from './environment.js';
 import { RoleService } from './api/role-service.js';
+import { DEFAULT_MONGO } from '#constants';
 
 Promise.all([
     Migration({
@@ -25,9 +26,12 @@ Promise.all([
     MikroORM.init<MongoDriver>({
         ...COMMON_CONNECTION_CONFIG,
         driverOptions: {
-            useUnifiedTopology: true
+            useUnifiedTopology: true,
+            minPoolSize: parseInt(process.env.MIN_POOL_SIZE ?? DEFAULT_MONGO.MIN_POOL_SIZE, 10),
+            maxPoolSize: parseInt(process.env.MAX_POOL_SIZE  ?? DEFAULT_MONGO.MAX_POOL_SIZE, 10),
+            maxIdleTimeMS: parseInt(process.env.MAX_IDLE_TIME_MS  ?? DEFAULT_MONGO.MAX_IDLE_TIME_MS, 10)
         },
-        ensureIndexes: true
+        ensureIndexes: true,
     }),
     MessageBrokerChannel.connect('AUTH_SERVICE'),
     NestFactory.createMicroservice<MicroserviceOptions>(AppModule, {

auth-service/src/constants/index.ts

@@ -1 +1,3 @@
 export { REQUIRED_PROPS as USER_REQUIRED_PROPS, REGISTER_REQUIRED_PROPS } from './user.js';
+
+export { DEFAULT as DEFAULT_MONGO } from './mongo.js';

auth-service/src/constants/mongo.ts

@@ -0,0 +1,5 @@
+export const DEFAULT = {
+    MIN_POOL_SIZE: '1',
+    MAX_POOL_SIZE: '5',
+    MAX_IDLE_TIME_MS: '30000',
+};

common/src/helpers/migration.ts

@@ -1,6 +1,12 @@
 import { MikroORM } from '@mikro-orm/core';
 import { MongoDriver } from '@mikro-orm/mongodb';
 import { Migrator } from '@mikro-orm/migrations-mongodb';
+import process from 'process';
+
+const DEFAULT_MIN_POOL_SIZE = '1';
+const DEFAULT_MAX_POOL_SIZE = '5';
+const DEFAULT_MAX_IDLE_TIME_MS = '30000';
+const RADIX = 10;
 
 /**
  * Define migration process
@@ -10,6 +16,11 @@ export async function Migration(initConfig: any, migrations?: string[]) {
     const orm = await MikroORM.init<MongoDriver>({
       ...initConfig,
       extensions: [Migrator],
+      driverOptions: {
+          minPoolSize: parseInt(process.env.MIN_POOL_SIZE ?? DEFAULT_MIN_POOL_SIZE, RADIX),
+          maxPoolSize: parseInt(process.env.MAX_POOL_SIZE ?? DEFAULT_MAX_POOL_SIZE, RADIX),
+          maxIdleTimeMS: parseInt(process.env.MAX_IDLE_TIME_MS ?? DEFAULT_MAX_IDLE_TIME_MS, RADIX),
+      },
     });
 
     const migrator = orm.getMigrator();

common/src/secret-manager/migrations/migrations.ts

@@ -18,6 +18,11 @@ const workerEnvPath = path.join(process.cwd(), '../worker-service/.env')
 const guardianCertsPath = path.join('../guardian-service/tls/vault/client.js')
 const workerCertsPath = path.join('../worker-service/tls/vault/client.js')
 
+const DEFAULT_MIN_POOL_SIZE = '1';
+const DEFAULT_MAX_POOL_SIZE = '5';
+const DEFAULT_MAX_IDLE_TIME_MS = '30000';
+const RADIX = 10;
+
 /**
  * Set common configs for Vault
  */
@@ -127,10 +132,13 @@ async function migrate() {
     entities: [
       'dist/secret-manager/migrations/vault-account.js'
     ],
-    driverOptions: {
-      useUnifiedTopology: true
-    },
-    ensureIndexes: true,
+      driverOptions: {
+        useUnifiedTopology: true,
+          minPoolSize: parseInt(process.env.MIN_POOL_SIZE ?? DEFAULT_MIN_POOL_SIZE, RADIX),
+          maxPoolSize: parseInt(process.env.MAX_POOL_SIZE ?? DEFAULT_MAX_POOL_SIZE, RADIX),
+          maxIdleTimeMS: parseInt(process.env.MAX_IDLE_TIME_MS ?? DEFAULT_MAX_IDLE_TIME_MS, RADIX),
+      },
+      ensureIndexes: true,
   })
 
   DataBaseHelper.orm = db;

configs/.env..guardian.system

@@ -139,3 +139,8 @@ OPENAI_API_KEY=...
 HOST_CACHE='cache'
 PORT_CACHE='6379'
 ENABLE_CACHE='true'
+
+# MONGO_INIT
+MIN_POOL_SIZE="1"
+MAX_POOL_SIZE="5"
+MAX_IDLE_TIME_MS="30000"

configs/.env.develop.guardian.system

@@ -140,3 +140,8 @@ OPENAI_API_KEY=...
 HOST_CACHE='cache'
 PORT_CACHE='6379'
 ENABLE_CACHE='true'
+
+# MONGO_INIT
+MIN_POOL_SIZE="1"
+MAX_POOL_SIZE="5"
+MAX_IDLE_TIME_MS="30000"

configs/.env.template.guardian.system

@@ -154,4 +154,9 @@ REFRESH_TOKEN_UPDATE_INTERVAL=31536000000
 #CACHE
 HOST_CACHE='cache'
 PORT_CACHE='6379'
-ENABLE_CACHE='true'
+ENABLE_CACHE='true'
+
+# MONGO_INIT
+MIN_POOL_SIZE="1"
+MAX_POOL_SIZE="5"
+MAX_IDLE_TIME_MS="30000"

configs/.env_SSV

@@ -36,4 +36,9 @@ WORKER_2_SERVICE_CHANNEL="worker.2"
 #CACHE
 HOST_CACHE='cache'
 PORT_CACHE='6379'
-ENABLE_CACHE='true'
+ENABLE_CACHE='true'
+
+# MONGO_INIT
+MIN_POOL_SIZE="1"
+MAX_POOL_SIZE="5"
+MAX_IDLE_TIME_MS="30000"

logger-service/configs/.env.logger

@@ -10,3 +10,8 @@ MQ_ADDRESS="localhost"
 DB_HOST="localhost"
 MQ_MAX_PAYLOAD="1048576"
 #MQ_MESSAGE_CHUNK=5000000
+
+# Mongo init
+MIN_POOL_SIZE="1"
+MAX_POOL_SIZE="5"
+MAX_IDLE_TIME_MS="30000"

logger-service/configs/.env.logger.develop

@@ -10,3 +10,8 @@ MQ_ADDRESS="localhost"
 DB_HOST="localhost"
 MQ_MAX_PAYLOAD="1048576"
 #MQ_MESSAGE_CHUNK=5000000
+
+# Mongo init
+MIN_POOL_SIZE="1"
+MAX_POOL_SIZE="5"
+MAX_IDLE_TIME_MS="30000"

logger-service/configs/.env.logger.template

@@ -10,3 +10,8 @@ MQ_ADDRESS=""
 DB_HOST=""
 MQ_MAX_PAYLOAD=""
 #MQ_MESSAGE_CHUNK=5000000
+
+# Mongo init
+MIN_POOL_SIZE="1"
+MAX_POOL_SIZE="5"
+MAX_IDLE_TIME_MS="30000"

logger-service/package.json

@@ -43,6 +43,9 @@
   "main": "dist/index.js",
   "name": "logger-service",
   "packageManager": "[email protected]",
+  "imports": {
+    "#constants": "./dist/constants/index.js"
+  },
   "scripts": {
     "build": "tsc",
     "build:prod": "tsc --project tsconfig.production.json",

logger-service/src/app.ts

@@ -6,6 +6,7 @@ import { NestFactory } from '@nestjs/core';
 import { Deserializer, IncomingRequest, MicroserviceOptions, Serializer, Transport } from '@nestjs/microservices';
 import process from 'process';
 import { AppModule } from './app.module.js';
+import { DEFAULT_MONGO } from '#constants';
 
 export class LoggerSerializer implements Serializer {
     serialize(value: any, options?: Record<string, any>): any {
@@ -31,9 +32,12 @@ Promise.all([
     MikroORM.init<MongoDriver>({
         ...COMMON_CONNECTION_CONFIG,
         driverOptions: {
-            useUnifiedTopology: true
+            useUnifiedTopology: true,
+            minPoolSize: parseInt(process.env.MIN_POOL_SIZE ?? DEFAULT_MONGO.MIN_POOL_SIZE, 10),
+            maxPoolSize: parseInt(process.env.MAX_POOL_SIZE  ?? DEFAULT_MONGO.MAX_POOL_SIZE, 10),
+            maxIdleTimeMS: parseInt(process.env.MAX_IDLE_TIME_MS  ?? DEFAULT_MONGO.MAX_IDLE_TIME_MS, 10)
         },
-        ensureIndexes: true
+        ensureIndexes: true,
     }),
     MessageBrokerChannel.connect('LOGGER_SERVICE'),
     NestFactory.createMicroservice<MicroserviceOptions>(AppModule,{

logger-service/src/constants/index.ts

@@ -0,0 +1 @@
+export { DEFAULT as DEFAULT_MONGO } from './mongo.js';

logger-service/src/constants/mongo.ts

@@ -0,0 +1,5 @@
+export const DEFAULT = {
+    MIN_POOL_SIZE: '1',
+    MAX_POOL_SIZE: '5',
+    MAX_IDLE_TIME_MS: '30000',
+};

logger-service/tsconfig.json

@@ -15,7 +15,8 @@
         "baseUrl": "./src",
         "paths": {
             "@entity/*": ["entity/*"],
-            "@api/*": ["api/*"]
+            "@api/*": ["api/*"],
+            "#constants": ["constants/index.js"]
         }
     },
     "include": [

logger-service/tsconfig.production.json

@@ -15,7 +15,8 @@
         "baseUrl": "./src",
         "paths": {
             "@entity/*": ["entity/*"],
-            "@api/*": ["api/*"]
+            "@api/*": ["api/*"],
+            "#constants": ["constants/index.js"]
         }
     },
     "include": [

notification-service/configs/.env.notification

@@ -4,3 +4,8 @@ DB_DATABASE="notification_db"
 MQ_ADDRESS="localhost"
 DB_HOST="localhost"
 MQ_MAX_PAYLOAD="1048576"
+
+# Mongo init
+MIN_POOL_SIZE="1"
+MAX_POOL_SIZE="5"
+MAX_IDLE_TIME_MS="30000"

notification-service/configs/.env.notification.develop

@@ -4,3 +4,8 @@ DB_DATABASE="notification_db"
 MQ_ADDRESS="localhost"
 DB_HOST="localhost"
 MQ_MAX_PAYLOAD="1048576"
+
+# Mongo init
+MIN_POOL_SIZE="1"
+MAX_POOL_SIZE="5"
+MAX_IDLE_TIME_MS="30000"

notification-service/configs/.env.notification.template

@@ -4,3 +4,8 @@ DB_DATABASE="notification_db"
 MQ_ADDRESS="localhost"
 DB_HOST="localhost"
 MQ_MAX_PAYLOAD="1048576"
+
+# Mongo init
+MIN_POOL_SIZE="1"
+MAX_POOL_SIZE="5"
+MAX_IDLE_TIME_MS="30000"

notification-service/package.json

@@ -42,6 +42,9 @@
   "main": "dist/index.js",
   "name": "notification-service",
   "packageManager": "[email protected]",
+  "imports": {
+    "#constants": "./dist/constants/index.js"
+  },
   "scripts": {
     "build": "tsc",
     "build:prod": "tsc --project tsconfig.production.json",

notification-service/src/app.ts

@@ -6,6 +6,7 @@ import { NestFactory } from '@nestjs/core';
 import { MicroserviceOptions, Transport } from '@nestjs/microservices';
 import process from 'process';
 import { AppModule } from './app.module.js';
+import { DEFAULT_MONGO } from '#constants';
 
 Promise.all([
     Migration({
@@ -19,6 +20,9 @@ Promise.all([
         ...COMMON_CONNECTION_CONFIG,
         driverOptions: {
             useUnifiedTopology: true,
+            minPoolSize: parseInt(process.env.MIN_POOL_SIZE ?? DEFAULT_MONGO.MIN_POOL_SIZE, 10),
+            maxPoolSize: parseInt(process.env.MAX_POOL_SIZE  ?? DEFAULT_MONGO.MAX_POOL_SIZE, 10),
+            maxIdleTimeMS: parseInt(process.env.MAX_IDLE_TIME_MS  ?? DEFAULT_MONGO.MAX_IDLE_TIME_MS, 10)
         },
         ensureIndexes: true,
     }),

notification-service/src/constants/index.ts

@@ -0,0 +1 @@
+export { DEFAULT as DEFAULT_MONGO } from './mongo.js';