Merge pull request #38 from jerolimov/create-demo-data-script

Create a TypeScript script to create some demo data

scripts/db-create-demo-data.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+set -e
+
+cd db
+
+if [ ! -d node_modules ]; then
+    yarn install
+fi
+
+export PGHOST=localhost
+export PGPORT=5432
+
+export PGDATABASE=$(oc get -n openshift-pipelines configmap tekton-results-postgres -o 'jsonpath={.data.POSTGRES_DB}')
+export PGUSERNAME=$(oc get -n openshift-pipelines secret tekton-results-postgres -o 'jsonpath={.data.POSTGRES_USER}' | base64 -d)
+export PGPASSWORD=$(oc get -n openshift-pipelines secret tekton-results-postgres -o 'jsonpath={.data.POSTGRES_PASSWORD}' | base64 -d)
+
+yarn run create-demo-data

scripts/db-show-stats.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+set -e
+
+cd db
+
+if [ ! -d node_modules ]; then
+    yarn install
+fi
+
+export PGHOST=localhost
+export PGPORT=5432
+
+export PGDATABASE=$(oc get -n openshift-pipelines configmap tekton-results-postgres -o 'jsonpath={.data.POSTGRES_DB}')
+export PGUSERNAME=$(oc get -n openshift-pipelines secret tekton-results-postgres -o 'jsonpath={.data.POSTGRES_USER}' | base64 -d)
+export PGPASSWORD=$(oc get -n openshift-pipelines secret tekton-results-postgres -o 'jsonpath={.data.POSTGRES_PASSWORD}' | base64 -d)
+
+yarn run show-stats

scripts/db/create-demo-data.ts

@@ -0,0 +1,147 @@
+import fs from 'fs';
+import { load } from 'js-yaml';
+import postgres from 'postgres';
+
+import { toDate, toDateTime, copyDay, nextDay } from './utils/date-utils';
+import {
+  getNumberOfPipelinesPerDay,
+  getNumberOfFailuresPerDay,
+} from './utils/history-utils';
+
+const namespace = 'test';
+
+const pipelineRunStrings = {
+  succeeded: fs.readFileSync('./yamls/PipelineRun-succeeded.yaml', {
+    encoding: 'utf-8',
+  }),
+  failed: fs.readFileSync('./yamls/PipelineRun-failed.yaml', {
+    encoding: 'utf-8',
+  }),
+};
+
+const createPipelineRuns = async (startDay: Date, endDay: Date) => {
+  const sql = postgres(undefined, { idle_timeout: 5 });
+
+  await sql`DELETE FROM results WHERE parent=${namespace} AND id LIKE 'demo-data-%'`;
+  await sql`DELETE FROM records WHERE parent=${namespace} AND id LIKE 'demo-data-%'`;
+
+  let targetNumberOfPipelineRuns = 0;
+  for (
+    let day = copyDay(startDay);
+    day.getTime() < endDay.getTime();
+    day = nextDay(day)
+  ) {
+    targetNumberOfPipelineRuns += getNumberOfPipelinesPerDay(day);
+  }
+  console.log(
+    `Will create ${targetNumberOfPipelineRuns} PipelineRuns between ${toDate(
+      startDay,
+    )} and ${toDate(endDay)}`,
+  );
+
+  let createdNumberOfPipelineRuns = 0;
+  for (
+    let day = copyDay(startDay);
+    day.getTime() < endDay.getTime();
+    day = nextDay(day)
+  ) {
+    const numberOfPipelines = getNumberOfPipelinesPerDay(day);
+    const numberOfFailures = getNumberOfFailuresPerDay(day);
+
+    console.log(
+      `day: ${toDate(
+        day,
+      )} / numberOfPipelines: ${numberOfPipelines} / numberOfFailures: ${numberOfFailures}`,
+    );
+
+    for (let i = 0; i < numberOfPipelines; i++) {
+      // Do all the PipelineRuns between 8am and 8pm
+      const startWorkingTimeInMinutes = 8 * 60;
+      const endWorkingTimeInMinutes = 20 * 60;
+      const durationInMinutes = numberOfPipelines + (i % 5);
+      const failure = i % (numberOfPipelines / numberOfFailures) === 0;
+      // console.log(
+      //   `startWorkingTimeInMinutes: ${startWorkingTimeInMinutes} / endWorkingTimeInMinutes: ${endWorkingTimeInMinutes} / durationInMinutes: ${durationInMinutes}`,
+      // );
+
+      const creationTimestamp = new Date(
+        day.getTime() +
+          (startWorkingTimeInMinutes +
+            (endWorkingTimeInMinutes - startWorkingTimeInMinutes) *
+              (i / numberOfPipelines)) *
+            60 *
+            1000,
+      );
+      const completionTimestamp = new Date(
+        creationTimestamp.getTime() + durationInMinutes * 60 * 1000,
+      );
+      // console.log(
+      //   `creationTimestamp: ${toDateTime(
+      //     creationTimestamp,
+      //   )} / completionTimestamp: ${toDateTime(
+      //     completionTimestamp,
+      //   )} / durationInMinutes: ${durationInMinutes} / failure: ${failure}`,
+      // );
+
+      const creationTimestampAsKey = creationTimestamp
+        .toISOString()
+        .replace(/[^0-9]/g, '');
+      const id = 'demo-data-' + creationTimestampAsKey;
+
+      const pipelineName = 'demo-data';
+      const pipelineRunName = 'demo-data-' + creationTimestampAsKey;
+      const pipelineRunWithVariables = failure
+        ? pipelineRunStrings.failed
+        : pipelineRunStrings.succeeded;
+
+      const pipelineRun = load(
+        pipelineRunWithVariables
+          .replace(/\$NAMESPACE/g, namespace)
+          .replace(/\$PIPELINE_NAME/g, pipelineName)
+          .replace(/\$PIPELINERUN_NAME/g, pipelineRunName)
+          .replace(
+            /\$CREATIONTIMESTAMP/g,
+            '"' + creationTimestamp.toISOString() + '"',
+          )
+          .replace(
+            /\$COMPLETIONTIMESTAMP/g,
+            '"' + completionTimestamp.toISOString() + '"',
+          ),
+      );
+      // console.log(pipelineRun);
+
+      await sql`INSERT INTO results ${sql({
+        parent: namespace,
+        id: id,
+      })}`;
+      await sql`INSERT INTO records ${sql({
+        parent: namespace,
+        id: id,
+        result_id: id,
+        // result_name: '',
+        type: 'tekton.dev/v1beta1.PipelineRun',
+        data: pipelineRun,
+        // name: '',
+        // etag: '',
+        created_time: creationTimestamp,
+        updated_time: completionTimestamp,
+      })}`;
+
+      createdNumberOfPipelineRuns++;
+    }
+  }
+
+  console.log(
+    `Create ${createdNumberOfPipelineRuns} / ${targetNumberOfPipelineRuns} PipelineRuns between ${toDate(
+      startDay,
+    )} and ${toDate(endDay)}`,
+  );
+};
+
+// Currently the UI can show the last 4 weeks only.
+const last4WeeksInMinutes = 4 * 7 * 24 * 60;
+const startDay = new Date(
+  new Date().getTime() - last4WeeksInMinutes * 60 * 1000,
+);
+const endDay = new Date();
+createPipelineRuns(startDay, endDay);

scripts/db/package.json

@@ -0,0 +1,15 @@
+{
+  "private": true,
+  "name": "db",
+  "version": "0.1.0",
+  "main": "index.js",
+  "scripts": {
+    "create-demo-data": "ts-node create-demo-data.ts",
+    "show-stats": "ts-node show-stats.ts"
+  },
+  "dependencies": {
+    "js-yaml": "^4.1.0",
+    "postgres": "^3.4.3",
+    "ts-node": "^10.9.1"
+  }
+}

scripts/db/show-stats.ts

@@ -0,0 +1,55 @@
+import postgres from 'postgres';
+
+const stats = async () => {
+  const sql = postgres(undefined, { idle_timeout: 5 });
+
+  const pipelineRunTypes = [
+    'tekton.dev/v1beta1.PipelineRun',
+    'tekton.dev/v1.PipelineRun',
+  ];
+  const sinceDays = 7;
+  const since = new Date(
+    new Date().getTime() - sinceDays * 24 * 60 * 60 * 1000,
+  );
+  console.log('Number of PipelineRuns since', since);
+
+  const [{ count: all }] = await sql`
+    SELECT count(*)
+    FROM records
+    WHERE type in (${pipelineRunTypes})
+    AND (data->'status'->>'startTime')::TIMESTAMP WITH TIME ZONE > ${since}::TIMESTAMP WITH TIME ZONE
+  `;
+  console.log('- All PipelineRuns:', Number(all));
+
+  const [{ count: fromPipeline }] = await sql`
+    SELECT count(*)
+    FROM records
+    WHERE type in (${pipelineRunTypes})
+    AND (data->'status'->>'startTime')::TIMESTAMP WITH TIME ZONE > ${since}::TIMESTAMP WITH TIME ZONE
+    AND data->'spec'->'pipelineRef'->>'name' IS NOT NULL
+  `;
+  console.log('- Started from a Pipeline:', Number(fromPipeline));
+
+  const [{ count: fromRepository }] = await sql`
+    SELECT count(*)
+    FROM records
+    WHERE type in (${pipelineRunTypes})
+    AND (data->'status'->>'startTime')::TIMESTAMP WITH TIME ZONE > ${since}::TIMESTAMP WITH TIME ZONE
+    AND data->'metadata'->'labels'->>'pipelinesascode.tekton.dev/repository' IS NOT NULL
+  `;
+  console.log('- Started from a Repository:', Number(fromRepository));
+
+  const [{ count: standalonePipelineRun }] = await sql`
+    SELECT count(*)
+    FROM records
+    WHERE type in ('tekton.dev/v1beta1.PipelineRun', 'tekton.dev/v1.PipelineRun')
+    AND (data->'status'->>'startTime')::TIMESTAMP WITH TIME ZONE > ${since}::TIMESTAMP WITH TIME ZONE
+    AND data->'spec'->'pipelineRef'->>'name' IS NULL
+    AND data->'metadata'->'labels'->>'pipelinesascode.tekton.dev/repository' IS NULL
+  `;
+  console.log('- Standalone PipelineRuns:', Number(standalonePipelineRun));
+
+  await sql.end({ timeout: 5 });
+};
+
+stats();

scripts/db/tsconfig.json

@@ -0,0 +1,5 @@
+{
+  "compilerOptions": {
+    "esModuleInterop": true
+  }
+}

scripts/db/utils/date-utils.ts

@@ -0,0 +1,15 @@
+export const toDate = (date: Date) =>
+  date.toLocaleDateString(undefined, {
+    weekday: 'short',
+    day: 'numeric',
+    month: 'short',
+    year: 'numeric',
+  });
+
+export const toDateTime = (date: Date) => date.toISOString();
+
+export const copyDay = (date: Date) =>
+  new Date(date.getFullYear(), date.getMonth(), date.getDate());
+
+export const nextDay = (date: Date) =>
+  new Date(date.getFullYear(), date.getMonth(), date.getDate() + 1);

scripts/db/utils/history-utils.ts

@@ -0,0 +1,31 @@
+// Contains 32 values so that index 0 can be ignored!
+// Values are between 0 and 3 will be used to
+// 1. add some ups and downs in the PipelineRuns per day
+// 2. define the number of failures per day
+const dateOfMonth = [
+  -1, 1, 0, 2, 0, 2, 3, 1, 2, 0, 3, 2, 1, 2, 3, 2, 0, 1, 1, 2, 0, 3, 0, 1, 2, 0,
+  1, 0, 2, 1, 1, 0,
+];
+console.assert(dateOfMonth.length == 32);
+
+export const getNumberOfPipelinesPerDay = (day: Date) => {
+  // increase it over time:
+  // -  1 for Jan 2020
+  // -  4 for Dec 2020
+  // -  8 for Dec 2021
+  // - 12 for Dec 2022
+  // - 16 for Dec 2023
+  const yearAddend = Math.max((day.getFullYear() - 2020) * 4, 0);
+  const monthAddend = Math.floor((day.getMonth() + 1) / 4) + 1;
+  const dayOfMonthAddend = dateOfMonth[day.getDate()];
+  let weekdayFactor = 1;
+  if (day.getDay() === 6) weekdayFactor = 0.2; // Saturday
+  if (day.getDay() === 0) weekdayFactor = 0.1; // Sunday
+  return Math.ceil(
+    (yearAddend + monthAddend + dayOfMonthAddend) * weekdayFactor,
+  );
+};
+
+export const getNumberOfFailuresPerDay = (day: Date) => {
+  return dateOfMonth[day.getDate()];
+};

scripts/db/yamls/PipelineRun-failed.yaml

@@ -0,0 +1,63 @@
+apiVersion: tekton.dev/v1
+kind: PipelineRun
+metadata:
+  namespace: $NAMESPACE
+  name: $PIPELINERUN_NAME
+  creationTimestamp: $CREATIONTIMESTAMP
+  labels:
+    tekton.dev/pipeline: $PIPELINE_NAME
+spec:
+  params:
+    - name: NAME
+      value: world
+  pipelineSpec:
+    tasks:
+      - name: echo-hello
+        taskSpec:
+          metadata: {}
+          spec: null
+          steps:
+            - computeResources: {}
+              image: ubuntu
+              name: echo
+              script: |
+                #!/usr/bin/env bash
+                echo "Hello"
+      - name: echo-name
+        runAfter:
+          - echo-hello
+        taskSpec:
+          metadata: {}
+          spec: null
+          steps:
+            - computeResources: {}
+              image: ubuntu
+              name: echo
+              script: |
+                #!/usr/bin/env bash
+                echo "$(params.NAME)"
+      - name: echo-bye
+        taskSpec:
+          metadata: {}
+          spec: null
+          steps:
+            - computeResources: {}
+              image: ubuntu
+              name: echo
+              script: |
+                #!/usr/bin/env bash
+                sleep 3
+                echo "Goodbye!"
+  taskRunTemplate:
+    serviceAccountName: pipeline
+  timeouts:
+    pipeline: 1h0m0s
+status:
+  startTime: $CREATIONTIMESTAMP
+  completionTime: $COMPLETIONTIMESTAMP
+  conditions:
+    - lastTransitionTime: $COMPLETIONTIMESTAMP
+      message: 'Tasks Completed: 2 (Failed: 1, Cancelled 0), Skipped: 1'
+      reason: Failed
+      status: 'False'
+      type: Succeeded