Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix CDK deployment #34

Merged
merged 1 commit into from
Oct 17, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions src/CDKPipelineApp.js
Original file line number Diff line number Diff line change
Expand Up @@ -167,6 +167,10 @@ async function createAWSpipelineCDK({
CDKFile = CDKFile.replace( "const NEPTUNE_DB_NAME = '';", `const NEPTUNE_DB_NAME = '${NEPTUNE_DB_NAME}';` );
CDKFile = CDKFile.replace( "const NEPTUNE_TYPE = '';", `const NEPTUNE_TYPE = '${NEPTUNE_TYPE}';` );
CDKFile = CDKFile.replace( "const NEPTUNE_DBSubnetGroup = null;", `const NEPTUNE_DBSubnetGroup = '${NEPTUNE_DBSubnetGroup}';` );
if (neptuneClusterInfo) {
CDKFile = CDKFile.replace("const NEPTUNE_DBSubnetIds = null;", `const NEPTUNE_DBSubnetIds = '${neptuneClusterInfo.dbSubnetIds}';`);
CDKFile = CDKFile.replace("const NEPTUNE_VpcSecurityGroupId = null;",`const NEPTUNE_VpcSecurityGroupId = '${neptuneClusterInfo.vpcSecurityGroupId}';`);
}
CDKFile = CDKFile.replace( "const NEPTUNE_IAM_AUTH = false;", `const NEPTUNE_IAM_AUTH = ${isNeptuneIAMAuth};` );
CDKFile = CDKFile.replace( "const NEPTUNE_IAM_POLICY_RESOURCE = '*';", `const NEPTUNE_IAM_POLICY_RESOURCE = '${NEPTUNE_IAM_POLICY_RESOURCE}';` );

Expand Down
19 changes: 10 additions & 9 deletions src/NeptuneSchema.js
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@ import { aws4Interceptor } from "aws4-axios";
import { fromNodeProviderChain } from "@aws-sdk/credential-providers";
import { NeptunedataClient, ExecuteOpenCypherQueryCommand } from "@aws-sdk/client-neptunedata";
import { loggerDebug, loggerError, loggerInfo, yellow } from "./logger.js";
import { parseNeptuneDomainFromHost, parseNeptuneGraphName } from "./util.js";
import { ExecuteQueryCommand, GetGraphSummaryCommand, NeptuneGraphClient } from "@aws-sdk/client-neptune-graph";

const NEPTUNE_DB = 'neptune-db';
Expand All @@ -25,7 +24,8 @@ const HTTP_LANGUAGE = 'openCypher';
const NEPTUNE_GRAPH_LANGUAGE = 'OPEN_CYPHER';
let HOST = '';
let PORT = 8182;
let REGION = ''
let REGION = '';
let DOMAIN = '';
let SAMPLE = 5000;
let NEPTUNE_TYPE = NEPTUNE_DB;
let NAME = '';
Expand Down Expand Up @@ -331,12 +331,13 @@ async function getEdgesDirectionsCardinality() {
}


function setGetNeptuneSchemaParameters(host, port, region, neptuneType) {
HOST = host;
PORT = port;
REGION = region;
NEPTUNE_TYPE = neptuneType;
NAME = parseNeptuneGraphName(host);
function setGetNeptuneSchemaParameters(neptuneInfo) {
HOST = neptuneInfo.host;
PORT = neptuneInfo.port;
REGION = neptuneInfo.region;
NEPTUNE_TYPE = neptuneInfo.neptuneType;
NAME = neptuneInfo.graphName;
DOMAIN = neptuneInfo.domain;
}

function getNeptunedataClient() {
Expand All @@ -354,7 +355,7 @@ function getNeptuneGraphClient() {
loggerInfo('Instantiating NeptuneGraphClient')
neptuneGraphClient = new NeptuneGraphClient({
port: PORT,
host: parseNeptuneDomainFromHost(HOST),
host: DOMAIN,
region: REGION,
protocol: NEPTUNE_GRAPH_PROTOCOL,
});
Expand Down
107 changes: 53 additions & 54 deletions src/main.js
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ let spinner = null;
// find global installation dir
import path from 'path';
import { fileURLToPath } from 'url';
import { parseNeptuneDomainFromEndpoint } from "./util.js";
import { parseNeptuneEndpoint } from "./util.js";
const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);

Expand Down Expand Up @@ -63,7 +63,7 @@ let createUpdatePipelineNeptuneDatabaseName = '';
let removePipelineName = '';
let inputCDKpipeline = false;
let inputCDKpipelineName = '';
let inputCDKpipelineEnpoint = '';
let inputCDKpipelineEndpoint = '';
let inputCDKpipelineFile = '';
let inputCDKpipelineRegion = '';
let inputCDKpipelineDatabaseName = '';
Expand Down Expand Up @@ -212,10 +212,14 @@ function processArgs() {
inputCDKpipeline = true;
break;
case '-ce':
// support miss-spelled option for backwards compatibility - could be removed for next major release
case '--output-aws-pipeline-cdk-neptume-endpoint':
case '--output-aws-pipeline-cdk-neptune-endpoint':
inputCDKpipelineEnpoint = array[index + 1];
inputCDKpipelineEndpoint = array[index + 1];
break;
case '-cd':
// support miss-spelled option for backwards compatibility - could be removed for next major release
case '--output-aws-pipeline-cdk-neptume-database-name':

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nice

case '--output-aws-pipeline-cdk-neptune-database-name':
inputCDKpipelineDatabaseName = array[index + 1];
break;
Expand Down Expand Up @@ -285,37 +289,31 @@ async function main() {
}
}


let neptuneInfo;
// Check if any of the Neptune endpoints are a neptune analytic endpoint and if so, set the neptuneType and IAM to required
const nonEmptyEndpoints = [inputGraphDBSchemaNeptuneEndpoint, createUpdatePipelineEndpoint, inputCDKpipelineEnpoint].filter(endpoint => endpoint !== '');
const isNeptuneAnalyticsGraph = nonEmptyEndpoints.length > 0 && parseNeptuneDomainFromEndpoint(nonEmptyEndpoints[0]).includes(NEPTUNE_GRAPH);
if (isNeptuneAnalyticsGraph) {
neptuneType = NEPTUNE_GRAPH;
// neptune analytics requires IAM
loggerInfo("Detected neptune-graph from input endpoint - setting IAM auth to true as it is required for neptune analytics")
isNeptuneIAMAuth = true;
// only one of these endpoints are expected to be non-empty at the same time
const nonEmptyEndpoints = [inputGraphDBSchemaNeptuneEndpoint, createUpdatePipelineEndpoint, inputCDKpipelineEndpoint].filter(endpoint => endpoint !== '');
if (nonEmptyEndpoints.length > 0) {
neptuneInfo = parseNeptuneEndpoint(nonEmptyEndpoints[0]);
neptuneType = neptuneInfo.neptuneType;
if (neptuneType === NEPTUNE_GRAPH) {
// neptune analytics requires IAM
loggerInfo("Detected neptune-graph from input endpoint - setting IAM auth to true as it is required for neptune analytics")
isNeptuneIAMAuth = true;
}
}

// Get Neptune schema from endpoint
if (inputGraphDBSchemaNeptuneEndpoint != '' && inputGraphDBSchema == '' && inputGraphDBSchemaFile == '') {
let endpointParts = inputGraphDBSchemaNeptuneEndpoint.split(':');
if (endpointParts.length != 2) {
loggerError('Neptune endpoint must be in the form of host:port');
process.exit(1);
if (!neptuneInfo) {
neptuneInfo = parseNeptuneEndpoint(inputGraphDBSchemaNeptuneEndpoint);
}
let neptuneHost = endpointParts[0];
let neptunePort = endpointParts[1];

let neptuneRegionParts = inputGraphDBSchemaNeptuneEndpoint.split('.');
let neptuneRegion = '';
if (neptuneType === NEPTUNE_DB)
neptuneRegion = neptuneRegionParts[2];
else
neptuneRegion = neptuneRegionParts[1];

loggerInfo('Retrieving Neptune schema');
loggerDebug('Getting Neptune schema from endpoint: ' + yellow(neptuneHost + ':' + neptunePort), {toConsole: true});
loggerDebug('Getting Neptune schema from endpoint: ' + yellow(inputGraphDBSchemaNeptuneEndpoint), {toConsole: true});

setGetNeptuneSchemaParameters(neptuneHost, neptunePort, neptuneRegion, neptuneType);
setGetNeptuneSchemaParameters(neptuneInfo);
let startTime = performance.now();
inputGraphDBSchema = await getNeptuneSchema();
let endTime = performance.now();
Expand Down Expand Up @@ -372,15 +370,11 @@ async function main() {
process.exit(1);
}
if (createUpdatePipelineEndpoint != '') {
let parts = createUpdatePipelineEndpoint.split('.');
createUpdatePipelineNeptuneDatabaseName = parts[0];

let parsedRegion;
if (neptuneType === NEPTUNE_DB) {
parsedRegion = parts[2];
} else {
parsedRegion = parts[1];
if (!neptuneInfo) {
neptuneInfo = parseNeptuneEndpoint(createUpdatePipelineEndpoint);
}
createUpdatePipelineNeptuneDatabaseName = neptuneInfo.graphName;
const parsedRegion = neptuneInfo.region;

if (createUpdatePipelineRegion !== parsedRegion) {
if (createUpdatePipelineRegion !== '') {
Expand All @@ -399,27 +393,37 @@ async function main() {
// CDK
if (inputCDKpipeline) {
if (!inputGraphDBSchemaNeptuneEndpoint == '') {
inputCDKpipelineEnpoint = inputGraphDBSchemaNeptuneEndpoint;
inputCDKpipelineEndpoint = inputGraphDBSchemaNeptuneEndpoint;
}
if (inputCDKpipelineEnpoint == '' &&
if (inputCDKpipelineEndpoint == '' &&
inputCDKpipelineRegion == '' && inputCDKpipelineDatabaseName == '') {
loggerError('AWS CDK: is required a Neptune endpoint, or a Neptune database name and region.');
process.exit(1);
}
if (inputCDKpipelineEnpoint == '' &&
if (inputCDKpipelineEndpoint == '' &&
!inputCDKpipelineRegion == '' && inputCDKpipelineDatabaseName == '') {
loggerError('AWS CDK: a Neptune database name is required.');
process.exit(1);
}
if (inputCDKpipelineEnpoint == '' &&
if (inputCDKpipelineEndpoint == '' &&
inputCDKpipelineRegion == '' && !inputCDKpipelineDatabaseName == '') {
loggerError('AWS CDK: a Neptune database region is required.');
process.exit(1);
}
if (inputCDKpipelineEnpoint != '') {
let parts = inputCDKpipelineEnpoint.split('.');
inputCDKpipelineDatabaseName = parts[0];
inputCDKpipelineRegion = parts[2];
if (inputCDKpipelineEndpoint != '') {
if (!neptuneInfo) {
neptuneInfo = parseNeptuneEndpoint(inputCDKpipelineEndpoint);
}
inputCDKpipelineDatabaseName = neptuneInfo.graphName;
const parsedRegion = neptuneInfo.region;
if (inputCDKpipelineRegion !== parsedRegion) {
if (inputCDKpipelineRegion !== '') {
loggerInfo('Switching CDK region from ' + inputCDKpipelineRegion + ' to region parsed from endpoint: ' + parsedRegion);
} else {
loggerInfo('Region parsed from CDK endpoint: ' + parsedRegion);
}
inputCDKpipelineRegion = parsedRegion;
}
}
if (inputCDKpipelineName == '') {
inputCDKpipelineName = inputCDKpipelineDatabaseName;
Expand Down Expand Up @@ -576,13 +580,11 @@ async function main() {
// Create Update AWS Pipeline
if (createUpdatePipeline) {
try {
let endpointParts = createUpdatePipelineEndpoint.split(':');
if (endpointParts.length < 2) {
loggerError('Neptune endpoint must be in the form of host:port');
process.exit(1);
if (!neptuneInfo) {
neptuneInfo = parseNeptuneEndpoint(createUpdatePipelineEndpoint);
}
let neptuneHost = endpointParts[0];
let neptunePort = endpointParts[1];
let neptuneHost = neptuneInfo.host;
let neptunePort = neptuneInfo.port;

await createUpdateAWSpipeline( createUpdatePipelineName,
createUpdatePipelineNeptuneDatabaseName,
Expand All @@ -608,14 +610,11 @@ async function main() {
try {
loggerInfo('Creating CDK File', {toConsole: true});

let endpointParts = inputCDKpipelineEnpoint.split(':');
if (endpointParts.length < 2) {
loggerError('Neptune endpoint must be in the form of host:port');
process.exit(1);
if (!neptuneInfo) {
neptuneInfo = parseNeptuneEndpoint(inputCDKpipelineEndpoint);
}
let neptuneHost = endpointParts[0];
let neptunePort = endpointParts[1];

let neptuneHost = neptuneInfo.host;
let neptunePort = neptuneInfo.port;

if (inputCDKpipelineFile == '') {
inputCDKpipelineFile = `${outputFolderPath}/${inputCDKpipelineName}-cdk.js`;
Expand Down
20 changes: 6 additions & 14 deletions src/pipelineResources.js
Original file line number Diff line number Diff line change
Expand Up @@ -330,19 +330,6 @@ async function createLambdaRole() {


if (NEPTUNE_IAM_AUTH) {

let action = [];
if (NEPTUNE_TYPE === NEPTUNE_DB) {
action = [
"neptune-db:DeleteDataViaQuery",
"neptune-db:connect",
"neptune-db:ReadDataViaQuery",
"neptune-db:WriteDataViaQuery"
];
} else {
action = ["neptune-graph:*"]
}

// Create Neptune query policy
startSpinner('Creating policy for Neptune queries', true);
let policyName = NAME+"NeptuneQueryPolicy";
Expand All @@ -352,7 +339,12 @@ async function createLambdaRole() {
Statement: [
{
Effect: "Allow",
Action: action,
Action: [
NEPTUNE_TYPE + ':connect',
NEPTUNE_TYPE + ':DeleteDataViaQuery',
NEPTUNE_TYPE + ':ReadDataViaQuery',
NEPTUNE_TYPE + ':WriteDataViaQuery'
],
Resource: NEPTUNE_IAM_POLICY_RESOURCE
},
],
Expand Down
49 changes: 29 additions & 20 deletions src/test/util.test.js
Original file line number Diff line number Diff line change
@@ -1,37 +1,46 @@
import {parseNeptuneDomainFromEndpoint, parseNeptuneDomainFromHost, parseNeptuneGraphName} from '../util.js';
import {parseNeptuneDomainFromHost, parseNeptuneEndpoint} from '../util.js';

test('parse domain from neptune cluster host', () => {
expect(parseNeptuneDomainFromHost('db-neptune-abc-def.cluster-xyz.us-west-2.neptune.amazonaws.com')).toBe('neptune.amazonaws.com');
expect(parseNeptuneDomainFromHost('db-neptune-abc-def.cluster-xyz.us-west-2.neptune.amazonaws.com'))
.toBe('neptune.amazonaws.com');
});

test('parse domain from neptune analytics host', () => {
expect(parseNeptuneDomainFromHost('g-abcdef.us-west-2.neptune-graph.amazonaws.com')).toBe('neptune-graph.amazonaws.com');
expect(parseNeptuneDomainFromHost('g-abcdef.us-west-2.neptune-graph.amazonaws.com'))
.toBe('neptune-graph.amazonaws.com');
});

test('parse domain from host without enough parts throws error', () => {
expect(() => parseNeptuneDomainFromHost('invalid.com')).toThrow('Cannot parse neptune host invalid.com because it has 2 part(s) delimited by . but expected at least 5');
expect(() => parseNeptuneDomainFromHost('invalid.com'))
.toThrow('Cannot parse neptune host invalid.com because it has 2 part(s) delimited by . but expected at least 5');
});

test('parse domain from neptune cluster endpoint', () => {
expect(parseNeptuneDomainFromEndpoint('db-neptune-abc-def.cluster-xyz.us-west-2.neptune.amazonaws.com:8182')).toBe('neptune.amazonaws.com');
test('parse neptune db endpoint', () => {
let neptuneInfo = parseNeptuneEndpoint('db-neptune-abc-def.cluster-xyz.us-west-2.neptune.amazonaws.com:8182');
expect(neptuneInfo).toHaveProperty('port', '8182');
expect(neptuneInfo).toHaveProperty('host', 'db-neptune-abc-def.cluster-xyz.us-west-2.neptune.amazonaws.com');
expect(neptuneInfo).toHaveProperty('domain', 'neptune.amazonaws.com');
expect(neptuneInfo).toHaveProperty('region', 'us-west-2');
expect(neptuneInfo).toHaveProperty('graphName', 'db-neptune-abc-def');
expect(neptuneInfo).toHaveProperty('neptuneType', 'neptune-db');
});

test('parse domain from neptune analytics endpoint', () => {
expect(parseNeptuneDomainFromEndpoint('g-abcdef.us-west-2.neptune-graph.amazonaws.com:8182')).toBe('neptune-graph.amazonaws.com');
test('parse neptune analytics endpoint', () => {
let neptuneInfo = parseNeptuneEndpoint('g-abcdef.us-east-1.neptune-graph.amazonaws.com:8183');
expect(neptuneInfo).toHaveProperty('port', '8183');
expect(neptuneInfo).toHaveProperty('host', 'g-abcdef.us-east-1.neptune-graph.amazonaws.com');
expect(neptuneInfo).toHaveProperty('domain', 'neptune-graph.amazonaws.com');
expect(neptuneInfo).toHaveProperty('region', 'us-east-1');
expect(neptuneInfo).toHaveProperty('graphName', 'g-abcdef');
expect(neptuneInfo).toHaveProperty('neptuneType', 'neptune-graph');
});

test('parse domain from endpoint without enough parts throws error', () => {
expect(() => parseNeptuneDomainFromEndpoint('g-abcdef.us-west-2.neptune-graph.amazonaws.com')).toThrow('Cannot parse domain from neptune endpoint g-abcdef.us-west-2.neptune-graph.amazonaws.com because it has 1 part(s) delimited by : but expected 2');
test('parse neptune endpoint without port throws error', () => {
expect(() => parseNeptuneEndpoint('db-neptune-abc-def.cluster-xyz.us-west-2.neptune.amazonaws.com'))
.toThrow('Cannot parse neptune endpoint db-neptune-abc-def.cluster-xyz.us-west-2.neptune.amazonaws.com because it is not in expected format of host:port');
});

test('parse name from neptune cluster host', () => {
expect(parseNeptuneGraphName('db-neptune-abc-def.cluster-xyz.us-west-2.neptune.amazonaws.com')).toBe('db-neptune-abc-def');
});

test('parse name from neptune analytics host', () => {
expect(parseNeptuneGraphName('g-abcdef.us-west-2.neptune-graph.amazonaws.com')).toBe('g-abcdef');
});

test('parse name from host without enough parts throws error', () => {
expect(() => parseNeptuneGraphName('invalid.com')).toThrow('Cannot parse neptune host invalid.com because it has 2 part(s) delimited by . but expected at least 5');
test('parse neptune endpoint not enough parts in domain throws error', () => {
expect(() => parseNeptuneEndpoint('invalid.com:1234'))
.toThrow('Cannot parse neptune host invalid.com because it has 2 part(s) delimited by . but expected at least 5');
});
Loading