-
Notifications
You must be signed in to change notification settings - Fork 1
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
40 changed files
with
9,654 additions
and
2,013 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,35 @@ | ||
| version: 2 | ||
| jobs: | ||
| build: | ||
|
|
||
| branches: | ||
| only: | ||
| - master | ||
|
|
||
| working_directory: ~/neuronbridge-services | ||
|
|
||
| docker: | ||
| - image: circleci/openjdk:8u171-jdk | ||
|
|
||
| environment: | ||
| TERM: dumb | ||
|
|
||
| steps: | ||
|
|
||
| - checkout | ||
|
|
||
| - restore_cache: | ||
| keys: | ||
| - neuronbridge-{{ checksum "search/pom.xml" }} | ||
| # fallback to using the latest cache if no exact match is found | ||
| - neuronbridge | ||
|
|
||
| - run: mvn clean install -f search/pom.xml | ||
|
|
||
| - save_cache: | ||
| key: neuronbridge-{{ checksum "search/pom.xml" }} | ||
| paths: | ||
| - ~/.m2 | ||
|
|
||
| - run: mvn package -f search/pom.xml | ||
|
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,3 +1,4 @@ | ||
| .idea | ||
| .serverless | ||
| node_modules | ||
| local |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,10 +1,12 @@ | ||
| config: | ||
| project: NeuronBridge | ||
| stage: ${opt:stage, "tkjsdev"} | ||
| stage: ${opt:stage, tkjsdev} | ||
| version: 2.0.0 | ||
| libraryBucket: janelia-flylight-color-depth-dev | ||
| libraryThumbnailsBucket: janelia-flylight-color-depth-thumbnails-dev | ||
| alignmentTemplatesBucket: janelia-flylight-color-depth-dev | ||
| libraryBucket: janelia-flylight-color-depth-${env:IMAGE_DATA_LEVEL, 'dev'} | ||
| libraryThumbnailsBucket: janelia-flylight-color-depth-thumbnails-${env:IMAGE_DATA_LEVEL, 'dev'} | ||
| alignmentTemplatesBucket: janelia-flylight-color-depth-${env:IMAGE_DATA_LEVEL, 'dev'} | ||
| searchBucket: janelia-neuronbridge-masks-tkdev | ||
| perDaySearchLimits: ${env:PER_DAY_SEARCH_LIMITS, 100} | ||
| concurrentSearchLimits: ${env:CONCURRENT_SEARCH_LIMITS, 10} | ||
| tracing: true | ||
| debug: true |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,13 @@ | ||
| { | ||
| "env": { | ||
| "commonjs": true, | ||
| "es2021": true, | ||
| "node": true | ||
| }, | ||
| "extends": "eslint:recommended", | ||
| "parserOptions": { | ||
| "ecmaVersion": 12 | ||
| }, | ||
| "rules": { | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,51 @@ | ||
| ## Test Event for Color Depth Search Dispatch: | ||
|
|
||
| ``` | ||
| { | ||
| "libraryAlignmentSpace": "JRC2018_Unisex_20x_HR", | ||
| "searchableMIPSFolder": "searchable_neurons", | ||
| "libraries": [ | ||
| "FlyEM_Hemibrain_v1.1" | ||
| ], | ||
| "searchInputName": "mask1417367048598452966.png", | ||
| "searchInputFolder": "colorDepthTestData/test1", | ||
| "dataThreshold": 100, | ||
| "maskThreshold": 100, | ||
| "pixColorFluctuation": 2, | ||
| "xyShift": 2, | ||
| "mirrorMask": true | ||
| } | ||
| ``` | ||
|
|
||
| Note that libraryAlignmentSpace and searchableMIPSFolder, if specified will apply to all libraries from the list. | ||
| It is also possible to simply invoke it without specifying the libraryAlignmentSpace and/or the searchableMIPFolder | ||
| as below: | ||
| ``` | ||
| { | ||
| "libraries": [ | ||
| "JRC2018_Unisex_20x_HR/FlyEM_Hemibrain_v1.1/searchable_neurons" | ||
| ], | ||
| "searchInputName": "mask3900813784977233932.png", | ||
| "searchInputFolder": "colorDepthTestData/test2", | ||
| "dataThreshold": 100, | ||
| "maskThreshold": 100, | ||
| "pixColorFluctuation": 2, | ||
| "xyShift": 2, | ||
| "mirrorMask": true | ||
| } | ||
| ``` | ||
|
|
||
| ## Unit testing | ||
|
|
||
| Simply run: | ||
| ``` | ||
| npm test | ||
| ``` | ||
|
|
||
| ## Integration testing | ||
|
|
||
| This runs a local client which invokes the search dispatch, waits for the search to run, and then analyzes the search performance in various ways. | ||
|
|
||
| ``` | ||
| npm run-script search janelia-neuronbridge-cds-dev src/test/resources/test1.json | ||
| ``` |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
24 changes: 24 additions & 0 deletions
24
search/graphql/mapping-templates/dynamodb-listAllSearches-request-template.vtl
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,24 @@ | ||
| #set( $limit = $util.defaultIfNull($context.args.limit, 10) ) | ||
| #set( $ListRequest = { | ||
| "version": "2017-02-28", | ||
| "limit": $limit | ||
| } ) | ||
| #if( $context.args.nextToken ) | ||
| #set( $ListRequest.nextToken = $context.args.nextToken ) | ||
| #end | ||
| #if( $context.args.filter ) | ||
| #set( $ListRequest.filter = $util.parseJson("$util.transform.toDynamoDBFilterExpression($ctx.args.filter)") ) | ||
| #end | ||
| #if( !$util.isNull($modelQueryExpression) | ||
| && !$util.isNullOrEmpty($modelQueryExpression.expression) ) | ||
| $util.qr($ListRequest.put("operation", "Query")) | ||
| $util.qr($ListRequest.put("query", $modelQueryExpression)) | ||
| #if( !$util.isNull($ctx.args.sortDirection) && $ctx.args.sortDirection == "DESC" ) | ||
| #set( $ListRequest.scanIndexForward = false ) | ||
| #else | ||
| #set( $ListRequest.scanIndexForward = true ) | ||
| #end | ||
| #else | ||
| $util.qr($ListRequest.put("operation", "Scan")) | ||
| #end | ||
| $util.toJson($ListRequest) |
59 changes: 59 additions & 0 deletions
59
search/graphql/mapping-templates/dynamodb-listAllSearches-response-template.vtl
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,59 @@ | ||
| ## [Start] Determine request authentication mode ** | ||
| #if( $util.isNullOrEmpty($authMode) && !$util.isNull($ctx.identity) && !$util.isNull($ctx.identity.sub) && !$util.isNull($ctx.identity.issuer) && !$util.isNull($ctx.identity.username) && !$util.isNull($ctx.identity.claims) && !$util.isNull($ctx.identity.sourceIp) && !$util.isNull($ctx.identity.defaultAuthStrategy) ) | ||
| #set( $authMode = "userPools" ) | ||
| #end | ||
| ## [End] Determine request authentication mode ** | ||
| ## [Start] Check authMode and execute owner/group checks ** | ||
| #if( $authMode == "userPools" ) | ||
| ## [Start] Static Group Authorization Checks ** | ||
| #set($isStaticGroupAuthorized = $util.defaultIfNull( | ||
| $isStaticGroupAuthorized, false)) | ||
| ## Authorization rule: { allow: groups, groups: ["neuronbridge-admin"], groupClaim: "cognito:groups" } ** | ||
| #set( $userGroups = $util.defaultIfNull($ctx.identity.claims.get("cognito:groups"), []) ) | ||
| #set( $allowedGroups = ["neuronbridge-admin"] ) | ||
| #foreach( $userGroup in $userGroups ) | ||
| #if( $allowedGroups.contains($userGroup) ) | ||
| #set( $isStaticGroupAuthorized = true ) | ||
| #break | ||
| #end | ||
| #end | ||
| ## [End] Static Group Authorization Checks ** | ||
|
|
||
| ## [Start] If not static group authorized, filter items ** | ||
| #if( !$isStaticGroupAuthorized ) | ||
| #set( $items = [] ) | ||
| #foreach( $item in $ctx.result.items ) | ||
| ## No Dynamic Group Authorization Rules ** | ||
|
|
||
|
|
||
| ## [Start] Owner Authorization Checks ** | ||
| #set( $isLocalOwnerAuthorized = false ) | ||
| ## Authorization rule: { allow: owner, ownerField: "owner", identityClaim: "cognito:username" } ** | ||
| #set( $allowedOwners0 = $item.owner ) | ||
| #set( $identityValue = $util.defaultIfNull($ctx.identity.claims.get("username"), $util.defaultIfNull($ctx.identity.claims.get("cognito:username"), "___xamznone____")) ) | ||
| #if( $util.isList($allowedOwners0) ) | ||
| #foreach( $allowedOwner in $allowedOwners0 ) | ||
| #if( $allowedOwner == $identityValue ) | ||
| #set( $isLocalOwnerAuthorized = true ) | ||
| #end | ||
| #end | ||
| #end | ||
| #if( $util.isString($allowedOwners0) ) | ||
| #if( $allowedOwners0 == $identityValue ) | ||
| #set( $isLocalOwnerAuthorized = true ) | ||
| #end | ||
| #end | ||
| ## [End] Owner Authorization Checks ** | ||
|
|
||
|
|
||
| #if( ($isLocalDynamicGroupAuthorized == true || $isLocalOwnerAuthorized == true) ) | ||
| $util.qr($items.add($item)) | ||
| #end | ||
| #end | ||
| #set( $ctx.result.items = $items ) | ||
| #end | ||
| ## [End] If not static group authorized, filter items ** | ||
| #end | ||
| ## [End] Check authMode and execute owner/group checks ** | ||
|
|
||
| $util.toJson($context.result) |
Oops, something went wrong.