From c15cbb423956dad4a5731c51ff360600473a4abe Mon Sep 17 00:00:00 2001 From: Justin Brooks Date: Mon, 26 Aug 2024 20:07:05 -0400 Subject: [PATCH] init gke target scanner queue --- ...dispatcher.yaml => apps-event-worker.yaml} | 14 +- .../Dockerfile | 6 +- apps/event-worker/README.md | 1 + .../eslint.config.js | 3 +- .../package.json | 7 +- .../src/config.ts | 2 +- apps/event-worker/src/index.ts | 18 ++ apps/event-worker/src/redis.ts | 7 + apps/event-worker/src/target-scan/gke.ts | 107 ++++++++ apps/event-worker/src/target-scan/google.ts | 134 +++++++++ apps/event-worker/src/target-scan/index.ts | 59 ++++ apps/event-worker/src/target-scan/upsert.ts | 25 ++ apps/event-worker/src/utils.ts | 11 + .../tsconfig.json | 3 - apps/job-dispatcher/README.md | 3 - apps/job-dispatcher/src/index.ts | 11 - docker-compose.yaml | 10 + packages/node-sdk/tsup.config.js | 9 + packages/validators/package.json | 4 + packages/validators/src/events/index.ts | 16 ++ pnpm-lock.yaml | 254 +++++++++++++++++- 21 files changed, 665 insertions(+), 39 deletions(-) rename .github/workflows/{apps-job-dispatcher.yaml => apps-event-worker.yaml} (80%) rename apps/{job-dispatcher => event-worker}/Dockerfile (83%) create mode 100644 apps/event-worker/README.md rename apps/{job-dispatcher => event-worker}/eslint.config.js (59%) rename apps/{job-dispatcher => event-worker}/package.json (83%) rename apps/{job-dispatcher => event-worker}/src/config.ts (70%) create mode 100644 apps/event-worker/src/index.ts create mode 100644 apps/event-worker/src/redis.ts create mode 100644 apps/event-worker/src/target-scan/gke.ts create mode 100644 apps/event-worker/src/target-scan/google.ts create mode 100644 apps/event-worker/src/target-scan/index.ts create mode 100644 apps/event-worker/src/target-scan/upsert.ts create mode 100644 apps/event-worker/src/utils.ts rename apps/{job-dispatcher => event-worker}/tsconfig.json (88%) delete mode 100644 apps/job-dispatcher/README.md delete mode 100644 apps/job-dispatcher/src/index.ts create mode 100644 packages/node-sdk/tsup.config.js create mode 100644 packages/validators/src/events/index.ts diff --git a/.github/workflows/apps-job-dispatcher.yaml b/.github/workflows/apps-event-worker.yaml similarity index 80% rename from .github/workflows/apps-job-dispatcher.yaml rename to .github/workflows/apps-event-worker.yaml index 37927473..a7d1fb12 100644 --- a/.github/workflows/apps-job-dispatcher.yaml +++ b/.github/workflows/apps-event-worker.yaml @@ -4,16 +4,16 @@ on: pull_request: branches: ["*"] paths: - - apps/job-dispatcher/** + - apps/event-worker/** - packages/db/** - - .github/workflows/apps-job-dispatcher.yaml + - .github/workflows/apps-event-worker.yaml - pnpm-lock.yaml push: branches: ["main"] paths: - - apps/job-dispatcher/** + - apps/event-worker/** - packages/db/** - - .github/workflows/apps-job-dispatcher.yaml + - .github/workflows/apps-event-worker.yaml - pnpm-lock.yaml jobs: @@ -41,7 +41,7 @@ jobs: id: meta uses: docker/metadata-action@v4 with: - images: ctrlplane/job-dispatchers + images: ctrlplane/event-worker tags: | type=sha,format=short,prefix= @@ -50,7 +50,7 @@ jobs: if: github.ref != 'refs/heads/main' with: push: false - file: apps/job-dispatchers/Dockerfile + file: apps/event-worker/Dockerfile tags: ${{ steps.meta.outputs.tags }} - name: Build and Push @@ -58,5 +58,5 @@ jobs: if: github.ref == 'refs/heads/main' with: push: true - file: apps/job-dispatchers/Dockerfile + file: apps/event-worker/Dockerfile tags: ${{ steps.meta.outputs.tags }} diff --git a/apps/job-dispatcher/Dockerfile b/apps/event-worker/Dockerfile similarity index 83% rename from apps/job-dispatcher/Dockerfile rename to apps/event-worker/Dockerfile index f492f5e0..2ad6d568 100644 --- a/apps/job-dispatcher/Dockerfile +++ b/apps/event-worker/Dockerfile @@ -25,16 +25,16 @@ COPY tooling/typescript/package.json ./tooling/typescript/package.json COPY packages/db/package.json ./packages/db/package.json COPY packages/job-dispatch/package.json ./packages/job-dispatch/package.json -COPY apps/job-dispatcher/package.json ./apps/job-dispatcher/package.json +COPY apps/event-worker/package.json ./apps/event-worker/package.json RUN pnpm install --frozen-lockfile COPY . . -RUN turbo build --filter=...@ctrlplane/job-dispatcher +RUN turbo build --filter=...@ctrlplane/event-worker RUN addgroup --system --gid 1001 nodejs RUN adduser --system --uid 1001 nodejs USER nodejs -CMD node apps/job-dispatcher/dist/index.js \ No newline at end of file +CMD node apps/event-worker/dist/index.js \ No newline at end of file diff --git a/apps/event-worker/README.md b/apps/event-worker/README.md new file mode 100644 index 00000000..42eb2480 --- /dev/null +++ b/apps/event-worker/README.md @@ -0,0 +1 @@ +# Event Worker diff --git a/apps/job-dispatcher/eslint.config.js b/apps/event-worker/eslint.config.js similarity index 59% rename from apps/job-dispatcher/eslint.config.js rename to apps/event-worker/eslint.config.js index dba3e699..0a6674b9 100644 --- a/apps/job-dispatcher/eslint.config.js +++ b/apps/event-worker/eslint.config.js @@ -1,9 +1,10 @@ -import baseConfig from "@ctrlplane/eslint-config/base"; +import baseConfig, { requireJsSuffix } from "@ctrlplane/eslint-config/base"; /** @type {import('typescript-eslint').Config} */ export default [ { ignores: [".nitro/**", ".output/**"], }, + requireJsSuffix, ...baseConfig, ]; diff --git a/apps/job-dispatcher/package.json b/apps/event-worker/package.json similarity index 83% rename from apps/job-dispatcher/package.json rename to apps/event-worker/package.json index 2c3732a1..d54ebd22 100644 --- a/apps/job-dispatcher/package.json +++ b/apps/event-worker/package.json @@ -1,5 +1,5 @@ { - "name": "@ctrlplane/job-dispatcher", + "name": "@ctrlplane/event-worker", "private": true, "type": "module", "scripts": { @@ -12,12 +12,17 @@ }, "dependencies": { "@ctrlplane/db": "workspace:*", + "@ctrlplane/logger": "workspace:*", + "@ctrlplane/validators": "workspace:*", "@google-cloud/container": "^5.16.0", "@kubernetes/client-node": "^0.21.0", "@t3-oss/env-core": "^0.10.1", + "bullmq": "^5.12.10", "cron": "^3.1.7", "dotenv": "^16.4.5", "google-auth-library": "^9.13.0", + "ioredis": "^5.4.1", + "ms": "^2.1.3", "semver": "^7.6.2", "zod": "catalog:" }, diff --git a/apps/job-dispatcher/src/config.ts b/apps/event-worker/src/config.ts similarity index 70% rename from apps/job-dispatcher/src/config.ts rename to apps/event-worker/src/config.ts index c19091b2..eec809ad 100644 --- a/apps/job-dispatcher/src/config.ts +++ b/apps/event-worker/src/config.ts @@ -5,6 +5,6 @@ import { z } from "zod"; dotenv.config(); export const env = createEnv({ - server: { POSTGRES_URL: z.string().url() }, + server: { POSTGRES_URL: z.string().url(), REDIS_URL: z.string().url() }, runtimeEnv: process.env, }); diff --git a/apps/event-worker/src/index.ts b/apps/event-worker/src/index.ts new file mode 100644 index 00000000..ff3ff819 --- /dev/null +++ b/apps/event-worker/src/index.ts @@ -0,0 +1,18 @@ +import { logger } from "@ctrlplane/logger"; + +import { redis } from "./redis"; +import { createTargetScanWorker } from "./target-scan"; + +const targetScanWorker = createTargetScanWorker(); + +const shutdown = () => { + logger.warn("Exiting..."); + + targetScanWorker.close(); + redis.quit(); + + process.exit(0); +}; + +process.on("SIGTERM", shutdown); +process.on("SIGINT", shutdown); diff --git a/apps/event-worker/src/redis.ts b/apps/event-worker/src/redis.ts new file mode 100644 index 00000000..d09f2b40 --- /dev/null +++ b/apps/event-worker/src/redis.ts @@ -0,0 +1,7 @@ +import IORedis from "ioredis"; + +import { env } from "./config"; + +export const redis = new IORedis(env.REDIS_URL, { + maxRetriesPerRequest: null, +}); diff --git a/apps/event-worker/src/target-scan/gke.ts b/apps/event-worker/src/target-scan/gke.ts new file mode 100644 index 00000000..ee48f297 --- /dev/null +++ b/apps/event-worker/src/target-scan/gke.ts @@ -0,0 +1,107 @@ +import { CoreV1Api } from "@kubernetes/client-node"; +import { Job } from "bullmq"; +import _ from "lodash"; + +import { TargetProviderGoogle, Workspace } from "@ctrlplane/db/schema"; +import { logger } from "@ctrlplane/logger"; + +import { + clusterToTarget, + connectToCluster, + getClusters, + getGoogleClusterClient, +} from "./google.js"; +import { UpsertTarget } from "./upsert.js"; + +const log = logger.child({ label: "target-scan/gke" }); + +export const getGkeTargets = async ( + workspace: Workspace, + config: TargetProviderGoogle, +) => { + const { googleServiceAccountEmail } = workspace; + log.info( + `Scaning ${config.projectIds.join(", ")} using ${googleServiceAccountEmail}`, + { workspaceId: workspace.id, config, googleServiceAccountEmail }, + ); + const googleClusterClient = await getGoogleClusterClient( + googleServiceAccountEmail, + ); + + const clusters = ( + await Promise.allSettled( + config.projectIds.map(async (project) => { + const clusters = await getClusters(googleClusterClient, project); + return { project, clusters }; + }), + ) + ) + .filter((result) => result.status === "fulfilled") + .map((v) => v.value); + + const kubernetesApiTargets: UpsertTarget[] = clusters.flatMap( + ({ project, clusters }) => + clusters.map((cluster) => + clusterToTarget( + workspace.id, + config.targetProviderId, + project, + cluster, + ), + ), + ); + const kubernetesNamespaceTargets = ( + await Promise.all( + clusters.flatMap(({ project, clusters }, idx) => { + return clusters.flatMap(async (cluster) => { + if (cluster.name == null || cluster.location == null) return []; + + const kubeConfig = await connectToCluster( + googleClusterClient, + project, + cluster.name, + cluster.location, + ); + + const k8sApi = kubeConfig.makeApiClient(CoreV1Api); + + try { + const response = await k8sApi.listNamespace(); + const namespaces = response.body.items; + return namespaces + .filter((n) => n.metadata != null) + .map((n) => + _.merge( + clusterToTarget( + workspace.id, + config.targetProviderId, + project, + cluster, + ), + { + name: `${cluster.name ?? cluster.id ?? ""}/${n.metadata!.name}`, + kind: "KubernetesNamespace", + identifier: `${project}/${cluster.name}/${n.metadata!.name}`, + config: { + namespace: n.metadata!.name, + }, + labels: { + ...n.metadata?.labels, + "kubernetes/namespace": n.metadata!.name, + }, + }, + ), + ); + } catch { + console.log( + `Unable to connect to cluster: ${cluster.name}/${cluster.id}`, + ); + return []; + } + }); + }), + ) + ).flat(); + + return [...kubernetesApiTargets, ...kubernetesNamespaceTargets]; +}; diff --git a/apps/event-worker/src/target-scan/google.ts b/apps/event-worker/src/target-scan/google.ts new file mode 100644 index 00000000..fcd40f36 --- /dev/null +++ b/apps/event-worker/src/target-scan/google.ts @@ -0,0 +1,134 @@ +import type { ClusterManagerClient } from "@google-cloud/container"; +import Container from "@google-cloud/container"; +import { google } from "@google-cloud/container/build/protos/protos.js"; +import { KubeConfig } from "@kubernetes/client-node"; +import { GoogleAuth } from "google-auth-library"; +import { SemVer } from "semver"; + +import { omitNullUndefined } from "../utils.js"; + +const sourceCredentials = new GoogleAuth({ + scopes: ["https://www.googleapis.com/auth/cloud-platform"], +}); + +export const getGoogleClusterClient = async ( + targetPrincipal?: string | null, +) => { + return new Container.v1.ClusterManagerClient({ + clientOptions: + targetPrincipal != null + ? { + sourceClient: await sourceCredentials.getClient(), + targetPrincipal, + lifetime: 3600, // Token lifetime in seconds + delegates: [], + targetScopes: ["https://www.googleapis.com/auth/cloud-platform"], + } + : {}, + }); +}; + +export const getClusters = async ( + clusterClient: ClusterManagerClient, + projectId: string, +) => { + const request = { parent: `projects/${projectId}/locations/-` }; + const [response] = await clusterClient.listClusters(request); + const { clusters } = response; + return clusters ?? []; +}; + +export const connectToCluster = async ( + clusterClient: ClusterManagerClient, + project: string, + clusterName: string, + clusterLocation: string, +) => { + const [credentials] = await clusterClient.getCluster({ + name: `projects/${project}/locations/${clusterLocation}/clusters/${clusterName}`, + }); + const kubeConfig = new KubeConfig(); + kubeConfig.loadFromOptions({ + clusters: [ + { + name: clusterName, + server: `https://${credentials.endpoint}`, + caData: credentials.masterAuth!.clusterCaCertificate!, + }, + ], + users: [ + { + name: clusterName, + token: (await sourceCredentials.getAccessToken())!, + }, + ], + contexts: [ + { + name: clusterName, + user: clusterName, + cluster: clusterName, + }, + ], + currentContext: clusterName, + }); + return kubeConfig; +}; + +export const clusterToTarget = ( + workspaceId: string, + providerId: string, + project: string, + cluster: google.container.v1.ICluster, +) => { + const masterVersion = new SemVer(cluster.currentMasterVersion ?? "0"); + const nodeVersion = new SemVer(cluster.currentNodeVersion ?? "0"); + const autoscaling = String( + cluster.autoscaling?.enableNodeAutoprovisioning ?? false, + ); + + const appUrl = `https://console.cloud.google.com/kubernetes/clusters/details/${cluster.location}/${cluster.name}/details?project=${project}`; + return { + workspaceId, + name: cluster.name ?? cluster.id ?? "", + providerId, + identifier: `${project}/${cluster.name}`, + version: "kubernetes/v1", + kind: "KubernetesAPI", + config: { + name: cluster.name, + status: cluster.status, + cluster: { + certificateAuthorityData: cluster.masterAuth?.clusterCaCertificate, + endpoint: `https://${cluster.endpoint}`, + }, + }, + labels: omitNullUndefined({ + "ctrlplane/url": appUrl, + + "google/self-link": cluster.selfLink, + "google/project": project, + "google/location": cluster.location, + "google/autopilot": cluster.autopilot?.enabled, + + "kubernetes/cluster-name": cluster.name, + "kubernetes/cluster-id": cluster.id, + "kubernetes/distribution": "gke", + "kubernetes/status": cluster.status, + "kubernetes/node-count": String(cluster.currentNodeCount ?? "unknown"), + + "kubernetes/master-version": masterVersion.version, + "kubernetes/master-version-major": String(masterVersion.major), + "kubernetes/master-version-minor": String(masterVersion.minor), + "kubernetes/master-version-patch": String(masterVersion.patch), + + "kubernetes/node-version": nodeVersion.version, + "kubernetes/node-version-major": String(nodeVersion.major), + "kubernetes/node-version-minor": String(nodeVersion.minor), + "kubernetes/node-version-patch": String(nodeVersion.patch), + + "kubernetes/autoscaling-enabled": autoscaling, + + ...(cluster.resourceLabels ?? {}), + }), + }; +}; diff --git a/apps/event-worker/src/target-scan/index.ts b/apps/event-worker/src/target-scan/index.ts new file mode 100644 index 00000000..83bdcdc7 --- /dev/null +++ b/apps/event-worker/src/target-scan/index.ts @@ -0,0 +1,59 @@ +import type { TargetScanEvent } from "@ctrlplane/validators/events"; +import { Queue, Worker } from "bullmq"; +import ms from "ms"; + +import { eq, takeFirstOrNull } from "@ctrlplane/db"; +import { db } from "@ctrlplane/db/client"; +import { + targetProvider, + targetProviderGoogle, + workspace, +} from "@ctrlplane/db/schema"; +import { logger } from "@ctrlplane/logger"; +import { Channel } from "@ctrlplane/validators/events"; + +import { redis } from "../redis.js"; +import { getGkeTargets } from "./gke.js"; +import { upsertTargets } from "./upsert.js"; + +const targetScanQueue = new Queue(Channel.TargetScan, { connection: redis }); +const requeue = (data: any, delay: number) => + targetScanQueue.add(Channel.TargetScan, data, { delay }); + +export const createTargetScanWorker = () => + new Worker( + Channel.TargetScan, + async (job) => { + const { targetProviderId } = job.data; + + const tp = await db + .select() + .from(targetProvider) + .where(eq(targetProvider.id, targetProviderId)) + .innerJoin(workspace, eq(targetProvider.workspaceId, workspace.id)) + .leftJoin( + targetProviderGoogle, + eq(targetProvider.id, targetProviderGoogle.targetProviderId), + ) + .then(takeFirstOrNull); + if (tp == null) return; + + logger.info( + `Received scanning request for "${tp.target_provider.name}" (${targetProviderId}).`, + ); + + if (tp.target_provider_google != null) { + logger.info("Found Google config, scanning for GKE targets"); + const gkeTargets = await getGkeTargets( + tp.workspace, + tp.target_provider_google, + ); + + await upsertTargets(db, tp.workspace.id, gkeTargets); + } + + await requeue(job.data, ms("5m")); + // + }, + { connection: redis, concurrency: 10 }, + ); diff --git a/apps/event-worker/src/target-scan/upsert.ts b/apps/event-worker/src/target-scan/upsert.ts new file mode 100644 index 00000000..3cf12964 --- /dev/null +++ b/apps/event-worker/src/target-scan/upsert.ts @@ -0,0 +1,25 @@ +import { buildConflictUpdateColumns, sql, Tx } from "@ctrlplane/db"; +import { Target, target } from "@ctrlplane/db/schema"; + +export type UpsertTarget = Pick< + Target, + | "version" + | "name" + | "kind" + | "config" + | "labels" + | "providerId" + | "identifier" + | "workspaceId" +>; + +export const upsertTargets = (db: Tx, providerId: string, ts: UpsertTarget[]) => + db + .insert(target) + .values(ts) + .onConflictDoUpdate({ + target: [target.identifier, target.workspaceId], + setWhere: sql`target.provider_id = ${providerId}`, + set: buildConflictUpdateColumns(target, ["labels"]), + }) + .returning(); diff --git a/apps/event-worker/src/utils.ts b/apps/event-worker/src/utils.ts new file mode 100644 index 00000000..a85fb555 --- /dev/null +++ b/apps/event-worker/src/utils.ts @@ -0,0 +1,11 @@ +import { Target } from "@ctrlplane/db/schema"; + +export function omitNullUndefined(obj: object) { + return Object.entries(obj).reduce>( + (acc, [key, value]) => { + if (value !== null && value !== undefined) acc[key] = value; + return acc; + }, + {}, + ); +} diff --git a/apps/job-dispatcher/tsconfig.json b/apps/event-worker/tsconfig.json similarity index 88% rename from apps/job-dispatcher/tsconfig.json rename to apps/event-worker/tsconfig.json index ef186d5c..34755271 100644 --- a/apps/job-dispatcher/tsconfig.json +++ b/apps/event-worker/tsconfig.json @@ -6,9 +6,6 @@ "noEmit": false, "tsBuildInfoFile": "node_modules/.cache/tsbuildinfo.json", "baseUrl": ".", - "paths": { - "@/*": ["src/*"] - }, "esModuleInterop": true, "importsNotUsedAsValues": "remove" }, diff --git a/apps/job-dispatcher/README.md b/apps/job-dispatcher/README.md deleted file mode 100644 index cedca48d..00000000 --- a/apps/job-dispatcher/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# Rollout Job - -Job that checks if releases are ready for rollout diff --git a/apps/job-dispatcher/src/index.ts b/apps/job-dispatcher/src/index.ts deleted file mode 100644 index 017ebe29..00000000 --- a/apps/job-dispatcher/src/index.ts +++ /dev/null @@ -1,11 +0,0 @@ -import { CronJob } from "cron"; - -const run = () => { - console.log("Running managed providers"); -}; - -const job = new CronJob("* * * * *", run); - -console.log("Starting managed providers cronjob"); -run(); -job.start(); diff --git a/docker-compose.yaml b/docker-compose.yaml index c395ba70..aa60b576 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -27,6 +27,16 @@ services: volumes: - rabbitmq-data:/var/lib/rabbitmq + redis: + image: redis:latest + container_name: ctrlplane-redis + restart: always + ports: + - "6379:6379" + volumes: + - redis-data:/data + volumes: db-data: rabbitmq-data: + redis-data: diff --git a/packages/node-sdk/tsup.config.js b/packages/node-sdk/tsup.config.js new file mode 100644 index 00000000..b3bc70d5 --- /dev/null +++ b/packages/node-sdk/tsup.config.js @@ -0,0 +1,9 @@ +import { defineConfig } from "tsup"; +export default defineConfig({ + entry: ["src/index.ts"], + format: ["cjs", "esm"], // Build for commonJS and ESmodules + dts: true, // Generate declaration file (.d.ts) + splitting: false, + sourcemap: true, + clean: true, +}); diff --git a/packages/validators/package.json b/packages/validators/package.json index c5926455..030f6832 100644 --- a/packages/validators/package.json +++ b/packages/validators/package.json @@ -11,6 +11,10 @@ "./targets": { "types": "./src/targets/index.ts", "default": "./dist/targets/index.js" + }, + "./events": { + "types": "./src/events/index.ts", + "default": "./dist/events/index.js" } }, "license": "MIT", diff --git a/packages/validators/src/events/index.ts b/packages/validators/src/events/index.ts new file mode 100644 index 00000000..ef8e4c1c --- /dev/null +++ b/packages/validators/src/events/index.ts @@ -0,0 +1,16 @@ +import { z } from "zod"; + +export enum Channel { + JobExecutionSync = "job-execution-sync", + DispatchJob = "dispatch-job", + TargetScan = "target-scan", +} + +export const targetScanEvent = z.object({ targetProviderId: z.string() }); +export type TargetScanEvent = z.infer; + +export const dispatchJobEvent = z.object({ jobConfigId: z.string() }); +export type DispatchJobEvent = z.infer; + +export const jobExecutionSyncEvent = z.object({ jobExecutionId: z.string() }); +export type JobExecutionSyncEvent = z.infer; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index caae83a9..fb6495b3 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -183,6 +183,73 @@ importers: specifier: ^7.8.0 version: 7.16.1(eslint@9.9.0(jiti@1.21.6))(typescript@5.5.4) + apps/event-worker: + dependencies: + '@ctrlplane/db': + specifier: workspace:* + version: link:../../packages/db + '@ctrlplane/logger': + specifier: workspace:* + version: link:../../packages/logger + '@ctrlplane/validators': + specifier: workspace:* + version: link:../../packages/validators + '@google-cloud/container': + specifier: ^5.16.0 + version: 5.16.0 + '@kubernetes/client-node': + specifier: ^0.21.0 + version: 0.21.0(bufferutil@4.0.8)(utf-8-validate@6.0.4) + '@t3-oss/env-core': + specifier: ^0.10.1 + version: 0.10.1(typescript@5.5.4)(zod@3.23.8) + bullmq: + specifier: ^5.12.10 + version: 5.12.10 + cron: + specifier: ^3.1.7 + version: 3.1.7 + dotenv: + specifier: ^16.4.5 + version: 16.4.5 + google-auth-library: + specifier: ^9.13.0 + version: 9.13.0 + ioredis: + specifier: ^5.4.1 + version: 5.4.1 + ms: + specifier: ^2.1.3 + version: 2.1.3 + semver: + specifier: ^7.6.2 + version: 7.6.3 + zod: + specifier: 'catalog:' + version: 3.23.8 + devDependencies: + '@ctrlplane/eslint-config': + specifier: workspace:^ + version: link:../../tooling/eslint + '@ctrlplane/prettier-config': + specifier: workspace:^ + version: link:../../tooling/prettier + '@ctrlplane/tsconfig': + specifier: workspace:* + version: link:../../tooling/typescript + eslint: + specifier: 'catalog:' + version: 9.9.0(jiti@1.21.6) + prettier: + specifier: 'catalog:' + version: 3.3.3 + tsx: + specifier: ^4.11.0 + version: 4.16.2 + typescript: + specifier: ^5.4.5 + version: 5.5.4 + apps/job-policy-checker: dependencies: '@ctrlplane/db': @@ -223,7 +290,7 @@ importers: specifier: ^5.4.5 version: 5.5.4 - apps/provider: + apps/target-provider: dependencies: '@ctrlplane/db': specifier: workspace:* @@ -236,7 +303,7 @@ importers: version: 0.21.0(bufferutil@4.0.8)(utf-8-validate@6.0.4) '@t3-oss/env-core': specifier: ^0.10.1 - version: 0.10.1(typescript@5.5.3)(zod@3.23.8) + version: 0.10.1(typescript@5.5.4)(zod@3.23.8) cron: specifier: ^3.1.7 version: 3.1.7 @@ -273,7 +340,7 @@ importers: version: 4.16.2 typescript: specifier: ^5.4.5 - version: 5.5.3 + version: 5.5.4 apps/webservice: dependencies: @@ -1157,7 +1224,7 @@ importers: version: 1.13.4(eslint@9.9.0(jiti@1.21.6)) eslint-plugin-import: specifier: ^2.29.1 - version: 2.29.1(eslint@9.9.0(jiti@1.21.6)) + version: 2.29.1(@typescript-eslint/parser@7.16.1(eslint@9.9.0(jiti@1.21.6))(typescript@5.5.3))(eslint@9.9.0(jiti@1.21.6)) eslint-plugin-jsx-a11y: specifier: ^6.8.0 version: 6.9.0(eslint@9.9.0(jiti@1.21.6)) @@ -2088,6 +2155,9 @@ packages: '@internationalized/string@3.2.3': resolution: {integrity: sha512-9kpfLoA8HegiWTeCbR2livhdVeKobCnVv8tlJ6M2jF+4tcMqDo94ezwlnrUANBWPgd8U7OXIHCk2Ov2qhk4KXw==} + '@ioredis/commands@1.2.0': + resolution: {integrity: sha512-Sx1pU8EM64o2BrqNpEO1CNLtKQwyhuXuqyfH7oGKCk+1a33d2r5saW8zNwm3j6BTExtjrv2BxTgzzkMwts6vGg==} + '@isaacs/cliui@8.0.2': resolution: {integrity: sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==} engines: {node: '>=12'} @@ -2147,6 +2217,36 @@ packages: react: ^16.8.0 || ^17.0.0 || ^18.0.0 react-dom: ^16.8.0 || ^17.0.0 || ^18.0.0 + '@msgpackr-extract/msgpackr-extract-darwin-arm64@3.0.3': + resolution: {integrity: sha512-QZHtlVgbAdy2zAqNA9Gu1UpIuI8Xvsd1v8ic6B2pZmeFnFcMWiPLfWXh7TVw4eGEZ/C9TH281KwhVoeQUKbyjw==} + cpu: [arm64] + os: [darwin] + + '@msgpackr-extract/msgpackr-extract-darwin-x64@3.0.3': + resolution: {integrity: sha512-mdzd3AVzYKuUmiWOQ8GNhl64/IoFGol569zNRdkLReh6LRLHOXxU4U8eq0JwaD8iFHdVGqSy4IjFL4reoWCDFw==} + cpu: [x64] + os: [darwin] + + '@msgpackr-extract/msgpackr-extract-linux-arm64@3.0.3': + resolution: {integrity: sha512-YxQL+ax0XqBJDZiKimS2XQaf+2wDGVa1enVRGzEvLLVFeqa5kx2bWbtcSXgsxjQB7nRqqIGFIcLteF/sHeVtQg==} + cpu: [arm64] + os: [linux] + + '@msgpackr-extract/msgpackr-extract-linux-arm@3.0.3': + resolution: {integrity: sha512-fg0uy/dG/nZEXfYilKoRe7yALaNmHoYeIoJuJ7KJ+YyU2bvY8vPv27f7UKhGRpY6euFYqEVhxCFZgAUNQBM3nw==} + cpu: [arm] + os: [linux] + + '@msgpackr-extract/msgpackr-extract-linux-x64@3.0.3': + resolution: {integrity: sha512-cvwNfbP07pKUfq1uH+S6KJ7dT9K8WOE4ZiAcsrSes+UY55E/0jLYc+vq+DO7jlmqRb5zAggExKm0H7O/CBaesg==} + cpu: [x64] + os: [linux] + + '@msgpackr-extract/msgpackr-extract-win32-x64@3.0.3': + resolution: {integrity: sha512-x0fWaQtYp4E6sktbsdAqnehxDgEc/VwM7uLsRCYWaiGu0ykYdZPiS8zCWdnjHwyiumousxfBm4SO31eXqwEZhQ==} + cpu: [x64] + os: [win32] + '@napi-rs/simple-git-android-arm-eabi@0.1.16': resolution: {integrity: sha512-dbrCL0Pl5KZG7x7tXdtVsA5CO6At5ohDX3myf5xIYn9kN4jDFxsocl8bNt6Vb/hZQoJd8fI+k5VlJt+rFhbdVw==} engines: {node: '>= 10'} @@ -4557,6 +4657,9 @@ packages: resolution: {integrity: sha512-4T53u4PdgsXqKaIctwF8ifXlRTTmEPJ8iEPWFdGZvcf7sbwYo6FKFEX9eNNAnzFZ7EzJAQ3CJeOtCRA4rDp7Pw==} engines: {node: '>=6.14.2'} + bullmq@5.12.10: + resolution: {integrity: sha512-lXH8Caj+FvYHiBS0QBEpQOq57RcVuEPziBC5cBlWguCVNfn1UMSri22bRrynDKuof8o9XB43ctmYASUpoa0DeQ==} + bundle-require@5.0.0: resolution: {integrity: sha512-GuziW3fSSmopcx4KRymQEJVbZUfqlCqcq7dvs6TYwKRZiegK/2buMxQTPs6MGlNv50wms1699qYO54R8XfRX4w==} engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} @@ -4849,6 +4952,10 @@ packages: create-require@1.1.1: resolution: {integrity: sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==} + cron-parser@4.9.0: + resolution: {integrity: sha512-p0SaNjrHOnQeR8/VnfGbmg9te2kfyYSQ7Sc/j/6DtPL3JQvKxmjO9TSjNFpujqV3vEYYBvNNvXSxzyksBWAx1Q==} + engines: {node: '>=12.0.0'} + cron@3.1.7: resolution: {integrity: sha512-tlBg7ARsAMQLzgwqVxy8AZl/qlTc5nibqYwtNGoCrd+cV+ugI+tvZC1oT/8dFH8W455YrywGykx/KMmAqOr7Jw==} @@ -5161,6 +5268,10 @@ packages: resolution: {integrity: sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg==} engines: {node: '>= 0.8', npm: 1.2.8000 || >= 1.4.16} + detect-libc@2.0.3: + resolution: {integrity: sha512-bwy0MGW55bG41VqxxypOsdSdGqLwXPI/focwgTYCFMbdUiBAxLg9CFzG08sz2aqzknwiX7Hkl0bQENjg8iLByw==} + engines: {node: '>=8'} + detect-node-es@1.1.0: resolution: {integrity: sha512-ypdmJU/TbBby2Dxibuv7ZLW3Bs1QEmM7nHjEANfohJLvE0XVujisn1qPJcZxg+qDucsr+bP6fLD1rPS3AhJ7EQ==} @@ -6168,6 +6279,10 @@ packages: invariant@2.2.4: resolution: {integrity: sha512-phJfQVBuaJM5raOpJjSfkiD6BpbCE4Ns//LaXl6wGYtUBY83nWS6Rf9tXm2e8VaK60JEjYldbPif/A2B1C2gNA==} + ioredis@5.4.1: + resolution: {integrity: sha512-2YZsvl7jopIa1gaePkeMtd9rAcSjOOjPtpcLlOeusyO+XH2SK5ZcT+UCrElPP+WVIInh2TzeI4XW9ENaSLVVHA==} + engines: {node: '>=12.22.0'} + ip-address@9.0.5: resolution: {integrity: sha512-zHtQzGojZXTwZTHQqra+ETKd4Sn3vgi7uBmlPoXVWZqYvuKmtI0l/VZTjqGmJY9x88GGOaZ9+G9ES8hC4T4X8g==} engines: {node: '>= 12'} @@ -6568,9 +6683,15 @@ packages: lodash.castarray@4.4.0: resolution: {integrity: sha512-aVx8ztPv7/2ULbArGJ2Y42bG1mEQ5mGjpdvrbJcJFU3TbYybe+QlLS4pst9zV52ymy2in1KpFPiZnAOATxD4+Q==} + lodash.defaults@4.2.0: + resolution: {integrity: sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ==} + lodash.get@4.4.2: resolution: {integrity: sha512-z+Uw/vLuy6gQe8cfaFWD7p0wVv8fJl3mbzXh33RS+0oW2wvUqiRXiQ69gLWSLpgB5/6sU+r6BlQR0MBILadqTQ==} + lodash.isarguments@3.1.0: + resolution: {integrity: sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg==} + lodash.isplainobject@4.0.6: resolution: {integrity: sha512-oSXzaWypCMHkPC3NvBEaPHf0KsA5mvPrOPgQWDsbg8n7orZ290M0BmC/jgRZ4vcJ6DTAhjrsSYgdsW/F+MFOBA==} @@ -6944,6 +7065,13 @@ packages: ms@2.1.3: resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} + msgpackr-extract@3.0.3: + resolution: {integrity: sha512-P0efT1C9jIdVRefqjzOQ9Xml57zpOXnIuS+csaB4MdZbTdmGDLo8XhzBG1N7aO11gKDDkJvBLULeFTo46wwreA==} + hasBin: true + + msgpackr@1.11.0: + resolution: {integrity: sha512-I8qXuuALqJe5laEBYoFykChhSXLikZmUhccjGsPuSJ/7uPip2TJ7lwdIQwWSAi0jGZDXv4WOP8Qg65QZRuXxXw==} + murmurhash@2.0.1: resolution: {integrity: sha512-5vQEh3y+DG/lMPM0mCGPDnyV8chYg/g7rl6v3Gd8WMF9S429ox3Xk8qrk174kWhG767KQMqqxLD1WnGd77hiew==} @@ -7069,6 +7197,9 @@ packages: no-case@2.3.2: resolution: {integrity: sha512-rmTZ9kz+f3rCvK2TD1Ue/oZlns7OGoIWP4fc3llxxRXlOkHKoWPPWJOfFYpITabSow43QJbRIoHQXtt10VldyQ==} + node-abort-controller@3.1.1: + resolution: {integrity: sha512-AGK2yQKIjRuqnc6VkX2Xj5d+QW8xZ87pa1UK6yA6ouUyuxfHuMP6umE5QK7UmTeOAymo+Zx1Fxiuw9rVx8taHQ==} + node-domexception@1.0.0: resolution: {integrity: sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==} engines: {node: '>=10.5.0'} @@ -7086,6 +7217,10 @@ packages: resolution: {integrity: sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==} engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + node-gyp-build-optional-packages@5.2.2: + resolution: {integrity: sha512-s+w+rBWnpTMwSFbaE0UXsRlg7hU4FjekKU4eyAih5T8nJuNZT1nNsskXpxmeqSK9UzkBl6UgRlnKc8hz8IEqOw==} + hasBin: true + node-gyp-build@4.8.1: resolution: {integrity: sha512-OSs33Z9yWr148JZcbZd5WiAXhh/n9z8TxQcdMhIOlpN9AhWpLfvVFO73+m77bBABQMaY9XSvIa+qk0jlI7Gcaw==} hasBin: true @@ -7785,6 +7920,14 @@ packages: react: ^16.0.0 || ^17.0.0 || ^18.0.0 react-dom: ^16.0.0 || ^17.0.0 || ^18.0.0 + redis-errors@1.2.0: + resolution: {integrity: sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==} + engines: {node: '>=4'} + + redis-parser@3.0.0: + resolution: {integrity: sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A==} + engines: {node: '>=4'} + redis@4.6.15: resolution: {integrity: sha512-2NtuOpMW3tnYzBw6S8mbXSX7RPzvVFCA2wFJq9oErushO2UeBkxObk+uvo7gv7n0rhWeOj/IzrHO8TjcFlRSOg==} @@ -8148,6 +8291,9 @@ packages: stacktrace-js@2.0.2: resolution: {integrity: sha512-Je5vBeY4S1r/RnLydLl0TBTi3F2qdfWmYsGvtfZgEI+SCprPppaIhQf5nGcal4gI4cGpCV/duLcAzT1np6sQqg==} + standard-as-callback@2.1.0: + resolution: {integrity: sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==} + state-local@1.0.7: resolution: {integrity: sha512-HTEHMNieakEnoe33shBYcZ7NX83ACUjCu8c40iOGEZsngj9zRnkqS9j1pqQPXwobB0ZcVTk27REb7COQ0UR59w==} @@ -9623,6 +9769,8 @@ snapshots: dependencies: '@swc/helpers': 0.5.5 + '@ioredis/commands@1.2.0': {} + '@isaacs/cliui@8.0.2': dependencies: string-width: 5.1.2 @@ -9724,6 +9872,24 @@ snapshots: react: 18.3.1 react-dom: 18.3.1(react@18.3.1) + '@msgpackr-extract/msgpackr-extract-darwin-arm64@3.0.3': + optional: true + + '@msgpackr-extract/msgpackr-extract-darwin-x64@3.0.3': + optional: true + + '@msgpackr-extract/msgpackr-extract-linux-arm64@3.0.3': + optional: true + + '@msgpackr-extract/msgpackr-extract-linux-arm@3.0.3': + optional: true + + '@msgpackr-extract/msgpackr-extract-linux-x64@3.0.3': + optional: true + + '@msgpackr-extract/msgpackr-extract-win32-x64@3.0.3': + optional: true + '@napi-rs/simple-git-android-arm-eabi@0.1.16': optional: true @@ -12739,6 +12905,18 @@ snapshots: node-gyp-build: 4.8.1 optional: true + bullmq@5.12.10: + dependencies: + cron-parser: 4.9.0 + ioredis: 5.4.1 + msgpackr: 1.11.0 + node-abort-controller: 3.1.1 + semver: 7.6.3 + tslib: 2.6.3 + uuid: 9.0.1 + transitivePeerDependencies: + - supports-color + bundle-require@5.0.0(esbuild@0.23.1): dependencies: esbuild: 0.23.1 @@ -13030,6 +13208,10 @@ snapshots: create-require@1.1.1: {} + cron-parser@4.9.0: + dependencies: + luxon: 3.4.4 + cron@3.1.7: dependencies: '@types/luxon': 3.4.2 @@ -13371,8 +13553,7 @@ snapshots: delayed-stream@1.0.0: {} - denque@2.1.0: - optional: true + denque@2.1.0: {} depd@2.0.0: {} @@ -13382,6 +13563,9 @@ snapshots: destroy@1.2.0: {} + detect-libc@2.0.3: + optional: true + detect-node-es@1.1.0: {} devlop@1.1.0: @@ -13836,16 +14020,17 @@ snapshots: transitivePeerDependencies: - supports-color - eslint-module-utils@2.8.1(eslint-import-resolver-node@0.3.9)(eslint@9.9.0(jiti@1.21.6)): + eslint-module-utils@2.8.1(@typescript-eslint/parser@7.16.1(eslint@9.9.0(jiti@1.21.6))(typescript@5.5.3))(eslint-import-resolver-node@0.3.9)(eslint@9.9.0(jiti@1.21.6)): dependencies: debug: 3.2.7 optionalDependencies: + '@typescript-eslint/parser': 7.16.1(eslint@9.9.0(jiti@1.21.6))(typescript@5.5.3) eslint: 9.9.0(jiti@1.21.6) eslint-import-resolver-node: 0.3.9 transitivePeerDependencies: - supports-color - eslint-plugin-import@2.29.1(eslint@9.9.0(jiti@1.21.6)): + eslint-plugin-import@2.29.1(@typescript-eslint/parser@7.16.1(eslint@9.9.0(jiti@1.21.6))(typescript@5.5.3))(eslint@9.9.0(jiti@1.21.6)): dependencies: array-includes: 3.1.8 array.prototype.findlastindex: 1.2.5 @@ -13855,7 +14040,7 @@ snapshots: doctrine: 2.1.0 eslint: 9.9.0(jiti@1.21.6) eslint-import-resolver-node: 0.3.9 - eslint-module-utils: 2.8.1(eslint-import-resolver-node@0.3.9)(eslint@9.9.0(jiti@1.21.6)) + eslint-module-utils: 2.8.1(@typescript-eslint/parser@7.16.1(eslint@9.9.0(jiti@1.21.6))(typescript@5.5.3))(eslint-import-resolver-node@0.3.9)(eslint@9.9.0(jiti@1.21.6)) hasown: 2.0.2 is-core-module: 2.13.1 is-glob: 4.0.3 @@ -13865,6 +14050,8 @@ snapshots: object.values: 1.2.0 semver: 6.3.1 tsconfig-paths: 3.15.0 + optionalDependencies: + '@typescript-eslint/parser': 7.16.1(eslint@9.9.0(jiti@1.21.6))(typescript@5.5.3) transitivePeerDependencies: - eslint-import-resolver-typescript - eslint-import-resolver-webpack @@ -14822,6 +15009,20 @@ snapshots: dependencies: loose-envify: 1.4.0 + ioredis@5.4.1: + dependencies: + '@ioredis/commands': 1.2.0 + cluster-key-slot: 1.1.2 + debug: 4.3.5 + denque: 2.1.0 + lodash.defaults: 4.2.0 + lodash.isarguments: 3.1.0 + redis-errors: 1.2.0 + redis-parser: 3.0.0 + standard-as-callback: 2.1.0 + transitivePeerDependencies: + - supports-color + ip-address@9.0.5: dependencies: jsbn: 1.1.0 @@ -15162,8 +15363,12 @@ snapshots: lodash.castarray@4.4.0: {} + lodash.defaults@4.2.0: {} + lodash.get@4.4.2: {} + lodash.isarguments@3.1.0: {} + lodash.isplainobject@4.0.6: {} lodash.merge@4.6.2: {} @@ -15813,6 +16018,22 @@ snapshots: ms@2.1.3: {} + msgpackr-extract@3.0.3: + dependencies: + node-gyp-build-optional-packages: 5.2.2 + optionalDependencies: + '@msgpackr-extract/msgpackr-extract-darwin-arm64': 3.0.3 + '@msgpackr-extract/msgpackr-extract-darwin-x64': 3.0.3 + '@msgpackr-extract/msgpackr-extract-linux-arm': 3.0.3 + '@msgpackr-extract/msgpackr-extract-linux-arm64': 3.0.3 + '@msgpackr-extract/msgpackr-extract-linux-x64': 3.0.3 + '@msgpackr-extract/msgpackr-extract-win32-x64': 3.0.3 + optional: true + + msgpackr@1.11.0: + optionalDependencies: + msgpackr-extract: 3.0.3 + murmurhash@2.0.1: {} mute-stream@0.0.8: {} @@ -15984,6 +16205,8 @@ snapshots: dependencies: lower-case: 1.1.4 + node-abort-controller@3.1.1: {} + node-domexception@1.0.0: {} node-fetch@2.7.0: @@ -15996,6 +16219,11 @@ snapshots: fetch-blob: 3.2.0 formdata-polyfill: 4.0.10 + node-gyp-build-optional-packages@5.2.2: + dependencies: + detect-libc: 2.0.3 + optional: true + node-gyp-build@4.8.1: optional: true @@ -16775,6 +17003,12 @@ snapshots: tiny-invariant: 1.3.3 victory-vendor: 36.9.2 + redis-errors@1.2.0: {} + + redis-parser@3.0.0: + dependencies: + redis-errors: 1.2.0 + redis@4.6.15: dependencies: '@redis/bloom': 1.2.0(@redis/client@1.5.17) @@ -17241,6 +17475,8 @@ snapshots: stack-generator: 2.0.10 stacktrace-gps: 3.1.2 + standard-as-callback@2.1.0: {} + state-local@1.0.7: {} statuses@2.0.1: {}