Merge pull request #89 from zotoio/feature/expose-logs

Feature/expose logs

.envExample

@@ -41,4 +41,5 @@ GTM_SONAR_MODULES=
 GTM_TASK_CONFIG_DEFAULT_URL=
 GTM_TASK_CONFIG_DEFAULT_MESSAGE_PATH=
 GTM_DYNAMO_TABLE_EVENTS=GtmEvents
-GTM_AWS_VPC_ID=<redacted>
+GTM_AWS_VPC_ID=<redacted>
+GTM_BASE_URL=http://localhost:9091

README.md

@@ -88,6 +88,9 @@ Create an asynchronous CI agnostic mechanism for running custom test stage gates
 |GTM_SONAR_GITHUB_ENDPOINT| optional enterprise github api url|
 |GTM_TASK_CONFIG_DEFAULT_URL| url to default sample config used when repo is missing .githubTaskManager.json|
 |GTM_TASK_CONFIG_DEFAULT_MESSAGE_PATH| path to markdown comment file added to PRs when repo is missing .githubTaskManager.json|
+|GTM_DYNAMO_TABLE_EVENTS| DynamoDB table to store event summaries |
+|GTM_AWS_VPC_ID| vpc id - only required for ddb endpoints |
+|GTM_BASE_URL| Base url used to render links to agent ui - eg elb cname |
 
 
 ## Configure and deploy

src/agent/AgentMetrics.js

@@ -5,6 +5,11 @@ import { default as json } from 'format-json';
 import { default as DynamoDBStream } from 'dynamodb-stream';
 import { default as elasticsearch } from 'elasticsearch';
 import { schedule } from 'tempus-fugit';
+import { AgentUtils } from './AgentUtils';
+import { default as rp } from 'request-promise-native';
+import { version as agentVersion } from '../../../package.json';
+const agentGroup = process.env.GTM_AGENT_GROUP || 'default';
+
 AWS.config.update({ region: process.env.GTM_AWS_REGION });
 
 let log = AgentLogger.log();
@@ -16,7 +21,7 @@ let elastic;
 if (process.env.GTM_ELASTIC_HOST && process.env.GTM_ELASTIC_PORT) {
     elastic = new elasticsearch.Client({
         host: `${process.env.GTM_ELASTIC_HOST}:${process.env.GTM_ELASTIC_PORT}`,
-        log: 'trace'
+        log: 'info'
     });
 }
 
@@ -62,35 +67,35 @@ async function configureRoutes(app) {
         app.get('/metrics', (req, res) => {
             res.render('metrics.html');
         });
+    });
 
-        app.get('/metrics/log/:ghEventId', async (req, res) => {
-            if (!elastic) {
-                res.json({ error: 'elasticsearch is not configured' });
-                res.end();
-                return;
-            }
-
-            let ghEventId = req.params.ghEventId;
-            let logs = await getEventLogs(ghEventId);
-            res.json(logs);
-        });
+    app.get('/metrics/log/gtm-:ghEventId.txt', async (req, res) => {
+        if (!elastic) {
+            res.write('elasticsearch is not configured');
+            res.end();
+            return;
+        }
 
-        app.get('/metrics/log/:ghEventId/text', async (req, res) => {
-            if (!elastic) {
-                res.write('elasticsearch is not configured');
-                res.end();
-                return;
-            }
+        let ghEventId = req.params.ghEventId;
+        let logs = await getEventLogs(ghEventId);
 
-            let ghEventId = req.params.ghEventId;
-            let logs = await getEventLogs(ghEventId);
+        logs.forEach(log => {
+            res.write(`${log._source['@timestamp']} ${log._source.message} \n`);
+        });
 
-            logs.forEach(log => {
-                res.write(`${log._source['@timestamp']} ${log._source.message} \n`);
-            });
+        res.end();
+    });
 
+    app.get('/metrics/log/gtm-:ghEventId.json', async (req, res) => {
+        if (!elastic) {
+            res.json({ error: 'elasticsearch is not configured' });
             res.end();
-        });
+            return;
+        }
+
+        let ghEventId = req.params.ghEventId;
+        let logs = await getEventLogs(ghEventId);
+        res.json(logs);
     });
 
     async function getEventLogs(ghEventId) {
@@ -111,18 +116,138 @@ async function configureRoutes(app) {
     }
 
     ddbStream.on('insert record', eventObject => {
-        log.info(`inserted ${json.plain(eventObject)}`);
+        log.debug(`inserted ${json.plain(eventObject)}`);
         INITIAL_DATA.push(eventObject);
         EventMetricStream.updateInit(INITIAL_DATA);
         EventMetricStream.send(eventObject);
     });
 
     ddbStream.on('modify record', eventObject => {
-        log.info(`updated ${json.plain(eventObject)}`);
+        log.debug(`updated ${json.plain(eventObject)}`);
         INITIAL_DATA.push(eventObject);
         EventMetricStream.updateInit(INITIAL_DATA);
         EventMetricStream.send(eventObject);
     });
+
+    app.get('/metrics/health', async (req, res) => {
+        let includeDetail = false;
+        let result = await getHealth(includeDetail);
+        res.json(result);
+        res.end();
+    });
+
+    app.get('/metrics/health/detail', async (req, res) => {
+        let includeDetail = true;
+        let result = await getHealth(includeDetail);
+        res.json(result);
+        res.end();
+    });
+
+    async function getHealth(includeDetails) {
+        return {
+            agent: getAgentInfo(includeDetails),
+            node: getProcessInfo(includeDetails),
+            elastic: await getElasticInfo(includeDetails),
+            dynamodb: await getDynamoInfo(includeDetails),
+            sqs: await getSQSInfo(includeDetails)
+        };
+    }
+
+    function getAgentInfo(includeDetails) {
+        let result = {
+            id: AgentUtils.agentId(),
+            version: agentVersion,
+            group: agentGroup
+        };
+        if (includeDetails) {
+            result.env = getEnvParams();
+        }
+        return result;
+    }
+
+    async function getElasticInfo(includeDetails) {
+        let result = 'not configured';
+        if (process.env.GTM_ELASTIC_HOST && process.env.GTM_ELASTIC_PORT) {
+            result = await rp({
+                json: true,
+                uri: `http://${process.env.GTM_ELASTIC_HOST}:${process.env.GTM_ELASTIC_PORT}`
+            });
+            if (!includeDetails) {
+                result = 'found';
+            }
+        }
+        return result;
+    }
+
+    async function getSQSInfo(includeDetails) {
+        let sqsPendingStats = await describeQueue(process.env.GTM_SQS_PENDING_QUEUE, ['All'], true);
+        let sqsResultsStats = await describeQueue(process.env.GTM_SQS_RESULTS_QUEUE, ['All'], true);
+        if (!includeDetails) {
+            sqsPendingStats = 'found';
+            sqsResultsStats = 'found';
+        }
+        return {
+            pending: sqsPendingStats,
+            results: sqsResultsStats
+        };
+    }
+
+    async function getDynamoInfo(includeDetails) {
+        let result = 'not configured';
+        if (EVENTS_TABLE) {
+            result = {
+                events: await ddb.describeTable({ TableName: EVENTS_TABLE }).promise()
+            };
+            if (!includeDetails) {
+                result = 'found';
+            }
+        }
+        return result;
+    }
+
+    function getProcessInfo() {
+        return {
+            version: process.version,
+            pid: process.pid,
+            uptime: process.uptime(),
+            cpuUsage: process.cpuUsage(),
+            memoryUsage: process.memoryUsage()
+        };
+    }
+
+    function getEnvParams() {
+        let env = {};
+        Object.keys(process.env)
+            .sort()
+            .forEach(key => {
+                if (key.startsWith('GTM')) {
+                    env[key] = AgentUtils.varMask(key, process.env[key]);
+                }
+            });
+        return env;
+    }
+
+    async function describeQueue(queueName, attributeNameArray, includeDetails) {
+        let sqs = new AWS.SQS();
+        if (!includeDetails) {
+            return 'found';
+        }
+        let queueUrl = await sqs.getQueueUrl({ QueueName: queueName }).promise();
+        log.debug(`sqs queue url ${queueName}: ${json.plain(queueUrl)}`);
+        if (!attributeNameArray) attributeNameArray = ['All'];
+        let sqsQueueParams = {
+            QueueUrl: queueUrl.QueueUrl,
+            AttributeNames: attributeNameArray
+        };
+        let queueDetails = await sqs.getQueueAttributes(sqsQueueParams).promise();
+
+        let result = {};
+        result.name = queueName;
+        result.url = queueUrl.QueueUrl;
+        result.attributes = queueDetails.Attributes;
+        log.debug(`sqs queue details: ${result}`);
+        return result;
+    }
 }
 
 module.exports = {

src/agent/AgentUtils.js

@@ -212,12 +212,12 @@ export class AgentUtils {
                 return sns.publish(params).promise();
             })
             .then(data => {
-                log.info(`Published Message '${message}' to Queue`);
-                log.debug(data);
+                log && log.info(`Published Message '${message}' to Queue`);
+                log && log.debug(data);
                 return Promise.resolve(true);
             })
             .catch(e => {
-                log.error(e);
+                log && log.error(e);
                 throw e;
             });
     }

src/executors/ExecutorDocker.js

@@ -223,6 +223,9 @@ export class ExecutorDocker extends Executor {
                         logBuffer = [];
                         logStream.end('!stop!');
 
+                        log.info('container stopped, removing..');
+                        container.remove();
+
                         resolve(executor.taskOutputTail);
                     });
                 }

src/handlers/EventHandlerPullRequest.js

@@ -34,6 +34,21 @@ export class EventHandlerPullRequest extends EventHandler {
 
         return this.handleTasks(this, this).then(() => {
             return this.addPullRequestSummaryComment(this).then(event => {
+                let url;
+                if (process.env.GTM_ELASTIC_HOST && process.env.GTM_ELASTIC_PORT) {
+                    let baseUrl = process.env.GTM_BASE_URL || 'http://localhost:9091';
+                    url = `${baseUrl}/metrics/log/gtm-${event.eventId}.txt`;
+                }
+                let status = event.failed ? 'failure' : 'success';
+                let eventStatus = AgentUtils.createPullRequestStatus(
+                    this.eventData,
+                    status,
+                    'GitHub Task Manager',
+                    `Completed ${event.eventId}`,
+                    url
+                );
+
+                AgentUtils.postResultsAndTrigger(eventStatus, `Executing event: ${event.eventId}`);
                 let endTime = new Date().getTime();
                 let duration = endTime - startTime;
                 log.info({
@@ -69,6 +84,22 @@ export class EventHandlerPullRequest extends EventHandler {
         let promises = [];
         let log = this.log;
 
+        let url;
+        if (process.env.GTM_ELASTIC_HOST && process.env.GTM_ELASTIC_PORT) {
+            let baseUrl = process.env.GTM_BASE_URL || 'http://localhost:9091';
+            url = `${baseUrl}/metrics/log/gtm-${event.eventId}.txt`;
+        }
+
+        let eventStatus = AgentUtils.createPullRequestStatus(
+            this.eventData,
+            'pending',
+            'GitHub Task Manager',
+            `Executing ${event.eventId}`,
+            url
+        );
+
+        promises.push(AgentUtils.postResultsAndTrigger(eventStatus, `Executing event: ${event.eventId}`, log));
+
         if (event.taskConfig.pull_request.isDefaultConfig) {
             let warningPath =
                 process.env.GTM_TASK_CONFIG_DEFAULT_MESSAGE_PATH || __dirname + '/PullRequestDefaultConfigWarning.md';
@@ -371,6 +402,11 @@ export class EventHandlerPullRequest extends EventHandler {
             commentBody += `<details>${EventHandlerPullRequest.buildEventSummary(task, 0, '')}</details>`;
         });
 
+        // if elk stack is configured, link to rehydrated logs
+        if (process.env.GTM_ELASTIC_HOST && process.env.GTM_ELASTIC_PORT) {
+            let baseUrl = process.env.GTM_BASE_URL || 'http://localhost:9091';
+            commentBody += `<a href="${baseUrl}/metrics/log/gtm-${event.eventId}.txt">View full log</a>`;
+        }
         return this.addPullRequestComment(event, commentBody);
     }
 

src/serverless/gtmGithubHook/gtmGithubHook.js

@@ -101,6 +101,11 @@ async function handleEvent(type, body, signature) {
     producer.send(event, function(err) {
         if (err) console.log(err);
     });
+
+    // set pull request status to pending for each task
+    if (type === 'pull_request') {
+        setPullRequestEventStatus(ghEventId, body);
+    }
 }
 
 async function getTaskConfig(type, body) {
@@ -170,6 +175,23 @@ function decodeEventBody(event) {
     return JSON.parse(decodeURIComponent(event.body.replace(/\+/g, ' ')).replace('payload={', '{'));
 }
 
+function setPullRequestEventStatus(ghEventId, eventBody) {
+    let url;
+    if (process.env.GTM_ELASTIC_HOST && process.env.GTM_ELASTIC_PORT) {
+        let baseUrl = process.env.GTM_BASE_URL || 'http://localhost:9091';
+        url = `${baseUrl}/metrics/log/${ghEventId}/text`;
+    }
+
+    let status = githubUtils.createPullRequestStatus(
+        eventBody,
+        'pending',
+        'GitHub Task Manager',
+        `Queued ${ghEventId}`,
+        url
+    );
+    githubUtils.updateGitHubPullRequestStatus(status, () => {});
+}
+
 module.exports = {
     listener: listener,
     getTaskConfig: getTaskConfig,

src/serverless/gtmGithubUtils.js

@@ -130,6 +130,29 @@ async function updateGitHubPullRequest(status, done) {
     }
 }
 
+/**
+ * Create a Status Object to Send to GitHub
+ * @param {object} eventData - Data from GitHub Event
+ * @param {string} state - Current Task State (pending, passed, failed)
+ * @param {string} context - Content Name to Display in GitHub
+ * @param {string} description - Short Description to Display in GitHub
+ * @param {string} url - Link to more detail
+ *
+ */
+function createPullRequestStatus(eventData, state, context, description, url) {
+    return {
+        owner: eventData.repository.owner.login || 'Default_Owner',
+        repo: eventData.repository.name || 'Default_Repository',
+        sha: eventData.pull_request.head.sha || 'Missing SHA',
+        number: eventData.pull_request.number,
+        state: state,
+        target_url: url ? url : 'https://github.com/zotoio/github-task-manager',
+        description: description,
+        context: context,
+        eventData: eventData
+    };
+}
+
 /**
  * export type ReposCreateStatusParams =
  & {
@@ -224,5 +247,7 @@ module.exports = {
     invalidHook: invalidHook,
     decodeFileResponse: decodeFileResponse,
     getFile: getFile,
-    handleEventTaskResult: handleEventTaskResult
+    handleEventTaskResult: handleEventTaskResult,
+    updateGitHubPullRequestStatus: updateGitHubPullRequestStatus,
+    createPullRequestStatus: createPullRequestStatus
 };