generated from openshift/console-plugin-template
-
Notifications
You must be signed in to change notification settings - Fork 15
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #38 from jerolimov/create-demo-data-script
Create a TypeScript script to create some demo data
- Loading branch information
Showing
12 changed files
with
559 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,18 @@ | ||
#!/bin/bash | ||
|
||
set -e | ||
|
||
cd db | ||
|
||
if [ ! -d node_modules ]; then | ||
yarn install | ||
fi | ||
|
||
export PGHOST=localhost | ||
export PGPORT=5432 | ||
|
||
export PGDATABASE=$(oc get -n openshift-pipelines configmap tekton-results-postgres -o 'jsonpath={.data.POSTGRES_DB}') | ||
export PGUSERNAME=$(oc get -n openshift-pipelines secret tekton-results-postgres -o 'jsonpath={.data.POSTGRES_USER}' | base64 -d) | ||
export PGPASSWORD=$(oc get -n openshift-pipelines secret tekton-results-postgres -o 'jsonpath={.data.POSTGRES_PASSWORD}' | base64 -d) | ||
|
||
yarn run create-demo-data |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,18 @@ | ||
#!/bin/bash | ||
|
||
set -e | ||
|
||
cd db | ||
|
||
if [ ! -d node_modules ]; then | ||
yarn install | ||
fi | ||
|
||
export PGHOST=localhost | ||
export PGPORT=5432 | ||
|
||
export PGDATABASE=$(oc get -n openshift-pipelines configmap tekton-results-postgres -o 'jsonpath={.data.POSTGRES_DB}') | ||
export PGUSERNAME=$(oc get -n openshift-pipelines secret tekton-results-postgres -o 'jsonpath={.data.POSTGRES_USER}' | base64 -d) | ||
export PGPASSWORD=$(oc get -n openshift-pipelines secret tekton-results-postgres -o 'jsonpath={.data.POSTGRES_PASSWORD}' | base64 -d) | ||
|
||
yarn run show-stats |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,147 @@ | ||
import fs from 'fs'; | ||
import { load } from 'js-yaml'; | ||
import postgres from 'postgres'; | ||
|
||
import { toDate, toDateTime, copyDay, nextDay } from './utils/date-utils'; | ||
import { | ||
getNumberOfPipelinesPerDay, | ||
getNumberOfFailuresPerDay, | ||
} from './utils/history-utils'; | ||
|
||
const namespace = 'test'; | ||
|
||
const pipelineRunStrings = { | ||
succeeded: fs.readFileSync('./yamls/PipelineRun-succeeded.yaml', { | ||
encoding: 'utf-8', | ||
}), | ||
failed: fs.readFileSync('./yamls/PipelineRun-failed.yaml', { | ||
encoding: 'utf-8', | ||
}), | ||
}; | ||
|
||
const createPipelineRuns = async (startDay: Date, endDay: Date) => { | ||
const sql = postgres(undefined, { idle_timeout: 5 }); | ||
|
||
await sql`DELETE FROM results WHERE parent=${namespace} AND id LIKE 'demo-data-%'`; | ||
await sql`DELETE FROM records WHERE parent=${namespace} AND id LIKE 'demo-data-%'`; | ||
|
||
let targetNumberOfPipelineRuns = 0; | ||
for ( | ||
let day = copyDay(startDay); | ||
day.getTime() < endDay.getTime(); | ||
day = nextDay(day) | ||
) { | ||
targetNumberOfPipelineRuns += getNumberOfPipelinesPerDay(day); | ||
} | ||
console.log( | ||
`Will create ${targetNumberOfPipelineRuns} PipelineRuns between ${toDate( | ||
startDay, | ||
)} and ${toDate(endDay)}`, | ||
); | ||
|
||
let createdNumberOfPipelineRuns = 0; | ||
for ( | ||
let day = copyDay(startDay); | ||
day.getTime() < endDay.getTime(); | ||
day = nextDay(day) | ||
) { | ||
const numberOfPipelines = getNumberOfPipelinesPerDay(day); | ||
const numberOfFailures = getNumberOfFailuresPerDay(day); | ||
|
||
console.log( | ||
`day: ${toDate( | ||
day, | ||
)} / numberOfPipelines: ${numberOfPipelines} / numberOfFailures: ${numberOfFailures}`, | ||
); | ||
|
||
for (let i = 0; i < numberOfPipelines; i++) { | ||
// Do all the PipelineRuns between 8am and 8pm | ||
const startWorkingTimeInMinutes = 8 * 60; | ||
const endWorkingTimeInMinutes = 20 * 60; | ||
const durationInMinutes = numberOfPipelines + (i % 5); | ||
const failure = i % (numberOfPipelines / numberOfFailures) === 0; | ||
// console.log( | ||
// `startWorkingTimeInMinutes: ${startWorkingTimeInMinutes} / endWorkingTimeInMinutes: ${endWorkingTimeInMinutes} / durationInMinutes: ${durationInMinutes}`, | ||
// ); | ||
|
||
const creationTimestamp = new Date( | ||
day.getTime() + | ||
(startWorkingTimeInMinutes + | ||
(endWorkingTimeInMinutes - startWorkingTimeInMinutes) * | ||
(i / numberOfPipelines)) * | ||
60 * | ||
1000, | ||
); | ||
const completionTimestamp = new Date( | ||
creationTimestamp.getTime() + durationInMinutes * 60 * 1000, | ||
); | ||
// console.log( | ||
// `creationTimestamp: ${toDateTime( | ||
// creationTimestamp, | ||
// )} / completionTimestamp: ${toDateTime( | ||
// completionTimestamp, | ||
// )} / durationInMinutes: ${durationInMinutes} / failure: ${failure}`, | ||
// ); | ||
|
||
const creationTimestampAsKey = creationTimestamp | ||
.toISOString() | ||
.replace(/[^0-9]/g, ''); | ||
const id = 'demo-data-' + creationTimestampAsKey; | ||
|
||
const pipelineName = 'demo-data'; | ||
const pipelineRunName = 'demo-data-' + creationTimestampAsKey; | ||
const pipelineRunWithVariables = failure | ||
? pipelineRunStrings.failed | ||
: pipelineRunStrings.succeeded; | ||
|
||
const pipelineRun = load( | ||
pipelineRunWithVariables | ||
.replace(/\$NAMESPACE/g, namespace) | ||
.replace(/\$PIPELINE_NAME/g, pipelineName) | ||
.replace(/\$PIPELINERUN_NAME/g, pipelineRunName) | ||
.replace( | ||
/\$CREATIONTIMESTAMP/g, | ||
'"' + creationTimestamp.toISOString() + '"', | ||
) | ||
.replace( | ||
/\$COMPLETIONTIMESTAMP/g, | ||
'"' + completionTimestamp.toISOString() + '"', | ||
), | ||
); | ||
// console.log(pipelineRun); | ||
|
||
await sql`INSERT INTO results ${sql({ | ||
parent: namespace, | ||
id: id, | ||
})}`; | ||
await sql`INSERT INTO records ${sql({ | ||
parent: namespace, | ||
id: id, | ||
result_id: id, | ||
// result_name: '', | ||
type: 'tekton.dev/v1beta1.PipelineRun', | ||
data: pipelineRun, | ||
// name: '', | ||
// etag: '', | ||
created_time: creationTimestamp, | ||
updated_time: completionTimestamp, | ||
})}`; | ||
|
||
createdNumberOfPipelineRuns++; | ||
} | ||
} | ||
|
||
console.log( | ||
`Create ${createdNumberOfPipelineRuns} / ${targetNumberOfPipelineRuns} PipelineRuns between ${toDate( | ||
startDay, | ||
)} and ${toDate(endDay)}`, | ||
); | ||
}; | ||
|
||
// Currently the UI can show the last 4 weeks only. | ||
const last4WeeksInMinutes = 4 * 7 * 24 * 60; | ||
const startDay = new Date( | ||
new Date().getTime() - last4WeeksInMinutes * 60 * 1000, | ||
); | ||
const endDay = new Date(); | ||
createPipelineRuns(startDay, endDay); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,15 @@ | ||
{ | ||
"private": true, | ||
"name": "db", | ||
"version": "0.1.0", | ||
"main": "index.js", | ||
"scripts": { | ||
"create-demo-data": "ts-node create-demo-data.ts", | ||
"show-stats": "ts-node show-stats.ts" | ||
}, | ||
"dependencies": { | ||
"js-yaml": "^4.1.0", | ||
"postgres": "^3.4.3", | ||
"ts-node": "^10.9.1" | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,55 @@ | ||
import postgres from 'postgres'; | ||
|
||
const stats = async () => { | ||
const sql = postgres(undefined, { idle_timeout: 5 }); | ||
|
||
const pipelineRunTypes = [ | ||
'tekton.dev/v1beta1.PipelineRun', | ||
'tekton.dev/v1.PipelineRun', | ||
]; | ||
const sinceDays = 7; | ||
const since = new Date( | ||
new Date().getTime() - sinceDays * 24 * 60 * 60 * 1000, | ||
); | ||
console.log('Number of PipelineRuns since', since); | ||
|
||
const [{ count: all }] = await sql` | ||
SELECT count(*) | ||
FROM records | ||
WHERE type in (${pipelineRunTypes}) | ||
AND (data->'status'->>'startTime')::TIMESTAMP WITH TIME ZONE > ${since}::TIMESTAMP WITH TIME ZONE | ||
`; | ||
console.log('- All PipelineRuns:', Number(all)); | ||
|
||
const [{ count: fromPipeline }] = await sql` | ||
SELECT count(*) | ||
FROM records | ||
WHERE type in (${pipelineRunTypes}) | ||
AND (data->'status'->>'startTime')::TIMESTAMP WITH TIME ZONE > ${since}::TIMESTAMP WITH TIME ZONE | ||
AND data->'spec'->'pipelineRef'->>'name' IS NOT NULL | ||
`; | ||
console.log('- Started from a Pipeline:', Number(fromPipeline)); | ||
|
||
const [{ count: fromRepository }] = await sql` | ||
SELECT count(*) | ||
FROM records | ||
WHERE type in (${pipelineRunTypes}) | ||
AND (data->'status'->>'startTime')::TIMESTAMP WITH TIME ZONE > ${since}::TIMESTAMP WITH TIME ZONE | ||
AND data->'metadata'->'labels'->>'pipelinesascode.tekton.dev/repository' IS NOT NULL | ||
`; | ||
console.log('- Started from a Repository:', Number(fromRepository)); | ||
|
||
const [{ count: standalonePipelineRun }] = await sql` | ||
SELECT count(*) | ||
FROM records | ||
WHERE type in ('tekton.dev/v1beta1.PipelineRun', 'tekton.dev/v1.PipelineRun') | ||
AND (data->'status'->>'startTime')::TIMESTAMP WITH TIME ZONE > ${since}::TIMESTAMP WITH TIME ZONE | ||
AND data->'spec'->'pipelineRef'->>'name' IS NULL | ||
AND data->'metadata'->'labels'->>'pipelinesascode.tekton.dev/repository' IS NULL | ||
`; | ||
console.log('- Standalone PipelineRuns:', Number(standalonePipelineRun)); | ||
|
||
await sql.end({ timeout: 5 }); | ||
}; | ||
|
||
stats(); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,5 @@ | ||
{ | ||
"compilerOptions": { | ||
"esModuleInterop": true | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,15 @@ | ||
export const toDate = (date: Date) => | ||
date.toLocaleDateString(undefined, { | ||
weekday: 'short', | ||
day: 'numeric', | ||
month: 'short', | ||
year: 'numeric', | ||
}); | ||
|
||
export const toDateTime = (date: Date) => date.toISOString(); | ||
|
||
export const copyDay = (date: Date) => | ||
new Date(date.getFullYear(), date.getMonth(), date.getDate()); | ||
|
||
export const nextDay = (date: Date) => | ||
new Date(date.getFullYear(), date.getMonth(), date.getDate() + 1); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,31 @@ | ||
// Contains 32 values so that index 0 can be ignored! | ||
// Values are between 0 and 3 will be used to | ||
// 1. add some ups and downs in the PipelineRuns per day | ||
// 2. define the number of failures per day | ||
const dateOfMonth = [ | ||
-1, 1, 0, 2, 0, 2, 3, 1, 2, 0, 3, 2, 1, 2, 3, 2, 0, 1, 1, 2, 0, 3, 0, 1, 2, 0, | ||
1, 0, 2, 1, 1, 0, | ||
]; | ||
console.assert(dateOfMonth.length == 32); | ||
|
||
export const getNumberOfPipelinesPerDay = (day: Date) => { | ||
// increase it over time: | ||
// - 1 for Jan 2020 | ||
// - 4 for Dec 2020 | ||
// - 8 for Dec 2021 | ||
// - 12 for Dec 2022 | ||
// - 16 for Dec 2023 | ||
const yearAddend = Math.max((day.getFullYear() - 2020) * 4, 0); | ||
const monthAddend = Math.floor((day.getMonth() + 1) / 4) + 1; | ||
const dayOfMonthAddend = dateOfMonth[day.getDate()]; | ||
let weekdayFactor = 1; | ||
if (day.getDay() === 6) weekdayFactor = 0.2; // Saturday | ||
if (day.getDay() === 0) weekdayFactor = 0.1; // Sunday | ||
return Math.ceil( | ||
(yearAddend + monthAddend + dayOfMonthAddend) * weekdayFactor, | ||
); | ||
}; | ||
|
||
export const getNumberOfFailuresPerDay = (day: Date) => { | ||
return dateOfMonth[day.getDate()]; | ||
}; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,63 @@ | ||
apiVersion: tekton.dev/v1 | ||
kind: PipelineRun | ||
metadata: | ||
namespace: $NAMESPACE | ||
name: $PIPELINERUN_NAME | ||
creationTimestamp: $CREATIONTIMESTAMP | ||
labels: | ||
tekton.dev/pipeline: $PIPELINE_NAME | ||
spec: | ||
params: | ||
- name: NAME | ||
value: world | ||
pipelineSpec: | ||
tasks: | ||
- name: echo-hello | ||
taskSpec: | ||
metadata: {} | ||
spec: null | ||
steps: | ||
- computeResources: {} | ||
image: ubuntu | ||
name: echo | ||
script: | | ||
#!/usr/bin/env bash | ||
echo "Hello" | ||
- name: echo-name | ||
runAfter: | ||
- echo-hello | ||
taskSpec: | ||
metadata: {} | ||
spec: null | ||
steps: | ||
- computeResources: {} | ||
image: ubuntu | ||
name: echo | ||
script: | | ||
#!/usr/bin/env bash | ||
echo "$(params.NAME)" | ||
- name: echo-bye | ||
taskSpec: | ||
metadata: {} | ||
spec: null | ||
steps: | ||
- computeResources: {} | ||
image: ubuntu | ||
name: echo | ||
script: | | ||
#!/usr/bin/env bash | ||
sleep 3 | ||
echo "Goodbye!" | ||
taskRunTemplate: | ||
serviceAccountName: pipeline | ||
timeouts: | ||
pipeline: 1h0m0s | ||
status: | ||
startTime: $CREATIONTIMESTAMP | ||
completionTime: $COMPLETIONTIMESTAMP | ||
conditions: | ||
- lastTransitionTime: $COMPLETIONTIMESTAMP | ||
message: 'Tasks Completed: 2 (Failed: 1, Cancelled 0), Skipped: 1' | ||
reason: Failed | ||
status: 'False' | ||
type: Succeeded |
Oops, something went wrong.