From 9fa1d5c2564d024ba6896d35c54fbabbd95863cb Mon Sep 17 00:00:00 2001 From: Richard Rodger Date: Thu, 13 Jun 2024 14:09:02 +0100 Subject: [PATCH] Initial commit --- .eslintignore | 2 + .eslintrc.js | 18 ++ .github/workflows/build.yml | 40 ++++ .github/workflows/todo.yml | 18 ++ .gitignore | 28 +++ .npmignore | 3 + .prettierrc | 4 + LICENSE | 23 ++ README.md | 152 +++++++++++++ dist/OpensearchStore.d.ts | 42 ++++ dist/OpensearchStore.js | 298 ++++++++++++++++++++++++++ dist/OpensearchStore.js.map | 1 + dist/OpensearchStoreDoc.d.ts | 4 + dist/OpensearchStoreDoc.js | 11 + dist/OpensearchStoreDoc.js.map | 1 + jest.config.js | 10 + package.json | 69 ++++++ src/OpensearchStore.ts | 378 +++++++++++++++++++++++++++++++++ src/OpensearchStoreDoc.ts | 11 + test/Opensearch.test.ts | 248 +++++++++++++++++++++ test/quick.js | 61 ++++++ tsconfig.json | 20 ++ tsfmt.json | 4 + 23 files changed, 1446 insertions(+) create mode 100644 .eslintignore create mode 100644 .eslintrc.js create mode 100644 .github/workflows/build.yml create mode 100644 .github/workflows/todo.yml create mode 100644 .gitignore create mode 100644 .npmignore create mode 100644 .prettierrc create mode 100644 LICENSE create mode 100644 README.md create mode 100644 dist/OpensearchStore.d.ts create mode 100644 dist/OpensearchStore.js create mode 100644 dist/OpensearchStore.js.map create mode 100644 dist/OpensearchStoreDoc.d.ts create mode 100644 dist/OpensearchStoreDoc.js create mode 100644 dist/OpensearchStoreDoc.js.map create mode 100644 jest.config.js create mode 100644 package.json create mode 100644 src/OpensearchStore.ts create mode 100644 src/OpensearchStoreDoc.ts create mode 100644 test/Opensearch.test.ts create mode 100644 test/quick.js create mode 100644 tsconfig.json create mode 100644 tsfmt.json diff --git a/.eslintignore b/.eslintignore new file mode 100644 index 0000000..d6f3562 --- /dev/null +++ b/.eslintignore @@ -0,0 +1,2 @@ +tmp +dist diff --git a/.eslintrc.js b/.eslintrc.js new file mode 100644 index 0000000..9ead932 --- /dev/null +++ b/.eslintrc.js @@ -0,0 +1,18 @@ +module.exports = { + extends: 'eslint:recommended', + env: { + node: true + }, + parserOptions: { + ecmaVersion: 8 + }, + rules: { + 'no-console': 0, + 'no-unused-vars': ['error', { 'args': 'none' }], + 'yoda': ["error", "always"], + 'max-len': ["error", { "code": 80 }], + }, + globals: { + Promise: 'readonly' + } +} diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..4efa263 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,40 @@ +# This workflow will do a clean install of node dependencies, build the source code and run tests across different versions of node +# For more information see: https://help.github.com/actions/language-and-framework-guides/using-nodejs-with-github-actions + +name: build + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + +jobs: + build: + timeout-minutes: 6 + + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest, macos-latest] + node-version: [20.x, 18.x, 16.x, 14.x] + + runs-on: ${{ matrix.os }} + + steps: + - uses: actions/checkout@v2 + - name: Use Node.js ${{ matrix.node-version }} + uses: actions/setup-node@v1 + with: + node-version: ${{ matrix.node-version }} + - run: npm install + # manually install peerdeps for node 12,14 + - run: npm i seneca seneca-entity + - run: npm run build --if-present + - run: npm test + + - name: Coveralls + uses: coverallsapp/github-action@master + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + path-to-lcov: ./test/lcov.info diff --git a/.github/workflows/todo.yml b/.github/workflows/todo.yml new file mode 100644 index 0000000..94a085c --- /dev/null +++ b/.github/workflows/todo.yml @@ -0,0 +1,18 @@ +name: 'TODO' +on: ['push'] +jobs: + build: + runs-on: 'ubuntu-latest' + steps: + - uses: 'actions/checkout@master' + - name: 'todo-to-issue' + uses: 'senecajs/todo-to-issue-action@master' + with: + REPO: ${{ github.repository }} + BEFORE: ${{ github.event.before }} + SHA: ${{ github.sha }} + TOKEN: ${{ secrets.GITHUB_TOKEN }} + LABEL: 'TODO:' + COMMENT_MARKER: '//' + INCLUDE_EXT: '.js,.md,.ts' + id: 'todo' diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..c7c462a --- /dev/null +++ b/.gitignore @@ -0,0 +1,28 @@ +lib-cov +*.seed +*.log +*.csv +*.dat +*.out +*.pid +*.gz + +pids +logs +results + +npm-debug.log +node_modules +*~ +.DS_Store +coverage.html + +.history +yarn.lock +package-lock.json +lcov.info +test/coverage.html + + +.env.local +coverage/ diff --git a/.npmignore b/.npmignore new file mode 100644 index 0000000..adfb6c3 --- /dev/null +++ b/.npmignore @@ -0,0 +1,3 @@ +*~ +*.off +*-off diff --git a/.prettierrc b/.prettierrc new file mode 100644 index 0000000..00fbdb1 --- /dev/null +++ b/.prettierrc @@ -0,0 +1,4 @@ +{ + "semi": false, + "singleQuote": true +} \ No newline at end of file diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..a30c387 --- /dev/null +++ b/LICENSE @@ -0,0 +1,23 @@ +The MIT License (MIT) + +Copyright (c) 2015-2016, Richard Rodger and other contributors. +Copyright (c) 2010-2014, Richard Rodger. + + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..12841cd --- /dev/null +++ b/README.md @@ -0,0 +1,152 @@ +![Seneca](http://senecajs.org/files/assets/seneca-logo.png) +> A [Seneca.js][] data storage plugin. + +# SenecaOpensearchStore +[![npm version][npm-badge]][npm-url] +[![Build](https://github.com/senecajs/SenecaOpensearchStore/actions/workflows/build.yml/badge.svg)](https://github.com/senecajs/seneca-OpensearchStore/actions/workflows/build.yml) +[![Dependency Status][david-badge]][david-url] +[![Maintainability](https://api.codeclimate.com/v1/badges/e2cdcc5415161cb378b0/maintainability)](https://codeclimate.com/github/senecajs/SenecaOpensearchStore/maintainability) +[![DeepScan grade](https://deepscan.io/api/teams/5016/projects/17225/branches/388415/badge/grade.svg)](https://deepscan.io/dashboard#view=project&tid=5016&pid=17225&bid=388415) +[![Coveralls][BadgeCoveralls]][Coveralls] + + + +| ![Voxgig](https://www.voxgig.com/res/img/vgt01r.png) | This open source module is sponsored and supported by [Voxgig](https://www.voxgig.com). | +|---|---| + + +## Description + +This module is a plugin for the Seneca framework. It provides an +in-memory storage engine that provides a set of data storage action +patterns. *Data does not persist betweens runs*. This plugin is most +useful for early development and unit testing. It also provides an +example of a document-oriented storage plugin code-base. + +The Seneca framework provides an [ActiveRecord-style data storage API][]. +Each supported database has a plugin, such as this one, that provides +the underlying Seneca plugin actions required for data persistence. + +This plugin is loaded by default by the [seneca-entity][seneca-entity-url] plugin that also needs the [seneca-basic][seneca-basic-url] plugin to function properly. + +If you're using this module, and need help, you can: + +- Post a [github issue][], +- Tweet to [@senecajs][], +- Ask on the [Gitter][gitter-url]. + +If you are new to Seneca in general, please take a look at [senecajs.org][]. We have everything from +tutorials to sample apps to help get you up and running quickly. + + +## Code examples + +For code samples, please see the [tests][OpensearchStore-tests] for this plugin. + +### Seneca compatibility +Supports Seneca versions **2.x** and above + + +### Supported functionality +All Seneca data store supported functionality is implemented in [seneca-store-test](https://github.com/senecajs/seneca-store-test) as a test suite. The tests represent the store functionality specifications. + +## Install + +```sh +npm install seneca +npm install SenecaOpensearchStore +``` + +You'll need the [seneca](http://github.com/senecajs/seneca) toolkit to use this module - it's just a plugin. + +## Quick Example + +```js +var seneca = require('seneca')() + +seneca.use('basic') +.use('entity') + +// Since OpensearchStore is a default plugin, it does not need to be +// added with .use(). You can just go ahead and use it. +seneca.ready(function () { + var apple = seneca.make$('fruit') + apple.name = 'Pink Lady' + apple.price = 0.99 + + apple.save$(function (err, apple) { + console.log("apple.id = " + apple.id) + }) +}) +``` + +## Usage +You don't use this module directly. It provides an underlying data storage engine for the Seneca entity API: + +```js +var entity = seneca.make$('typename') +entity.someproperty = "something" +entity.anotherproperty = 100 + +entity.save$(function (err, entity) { ... }) +entity.load$({id: ... }, function (err, entity) { ... }) +entity.list$({property: ... }, function (err, entity) { ... }) +entity.remove$({id: ... }, function (err, entity) { ... }) +``` + +### Query Support +The standard Seneca query format is supported: + +- `.list$({f1:v1, f2:v2, ...})` implies pseudo-query `f1==v1 AND f2==v2, ...`. + +- `.list$({f1:v1,...}, {sort$:{field1:1}})` means sort by f1, ascending. + +- `.list$({f1:v1,...}, {sort$:{field1:-1}})` means sort by f1, descending. + +- `.list$({f1:v1,...}, {limit$:10})` means only return 10 results. + +- `.list$({f1:v1,...}, {skip$:5})` means skip the first 5. + +- `.list$({f1:v1,...}, {fields$:['fd1','f2']})` means only return the listed fields. + +Note: you can use `sort$`, `limit$`, `skip$` and `fields$` together. + +### Native Driver +This store is an in memory store and as such does not require the need of a native driver. + +## Contributing +The [Senecajs org][] encourages open participation. If you feel you can help in any way, be it with +documentation, examples, extra testing, or new features please get in touch. + +## Test +To run tests, simply use npm: + +```sh +npm run test +``` + +## License +Copyright (c) 2015-2016, Richard Rodger and other contributors. +Copyright (c) 2010-2014, Richard Rodger. +Licensed under [MIT][]. + +[MIT]: ./LICENSE +[npm-badge]: https://badge.fury.io/js/SenecaOpensearchStore.svg +[npm-url]: https://badge.fury.io/js/SenecaOpensearchStore +[Senecajs org]: https://github.com/senecajs/ +[Seneca.js]: https://www.npmjs.com/package/seneca +[@senecajs]: http://twitter.com/senecajs +[senecajs.org]: http://senecajs.org/ +[travis-badge]: https://travis-ci.org/senecajs/SenecaOpensearchStore.svg +[travis-url]: https://travis-ci.org/senecajs/SenecaOpensearchStore +[gitter-badge]: https://badges.gitter.im/Join%20Chat.svg +[gitter-url]: https://gitter.im/senecajs/seneca +[github issue]: https://github.com/senecajs/SenecaOpensearchStore/issues +[ActiveRecord-style data storage API]:http://senecajs.org/tutorials/understanding-data-entities.html +[david-badge]: https://david-dm.org/senecajs/SenecaOpensearchStore.svg +[david-url]: https://david-dm.org/senecajs/SenecaOpensearchStore +[Coveralls]: https://coveralls.io/github/senecajs/SenecaOpensearchStore?branch=master +[BadgeCoveralls]: https://coveralls.io/repos/github/senecajs/SenecaOpensearchStore/badge.svg?branch=master +[seneca-basic-url]: https://github.com/senecajs/seneca-basic +[seneca-entity-url]: https://github.com/senecajs/seneca-entity +[OpensearchStore-tests]: https://github.com/senecajs/SenecaOpensearchStore/tree/master/test diff --git a/dist/OpensearchStore.d.ts b/dist/OpensearchStore.d.ts new file mode 100644 index 0000000..22a9f9a --- /dev/null +++ b/dist/OpensearchStore.d.ts @@ -0,0 +1,42 @@ +type Options = { + debug: boolean; + map?: any; + index: { + prefix: string; + suffix: string; + map: Record; + exact: string; + }; + field: { + zone: { + name: string; + }; + base: { + name: string; + }; + name: { + name: string; + }; + vector: { + name: string; + }; + }; + cmd: { + list: { + size: number; + }; + }; + aws: any; + opensearch: any; +}; +export type OpensearchStoreOptions = Partial; +declare function OpensearchStore(this: any, options: Options): { + name: string; + tag: any; + exportmap: { + native: () => { + client: any; + }; + }; +}; +export default OpensearchStore; diff --git a/dist/OpensearchStore.js b/dist/OpensearchStore.js new file mode 100644 index 0000000..e70d2e7 --- /dev/null +++ b/dist/OpensearchStore.js @@ -0,0 +1,298 @@ +"use strict"; +/* Copyright (c) 2024 Seneca contributors, MIT License */ +Object.defineProperty(exports, "__esModule", { value: true }); +const aws_1 = require("@opensearch-project/opensearch/aws"); +const opensearch_1 = require("@opensearch-project/opensearch"); +const credential_provider_node_1 = require("@aws-sdk/credential-provider-node"); +const gubu_1 = require("gubu"); +const { Open, Any } = gubu_1.Gubu; +function OpensearchStore(options) { + const seneca = this; + const init = seneca.export('entity/init'); + let desc = 'OpensearchStore'; + let client; + let store = { + name: 'OpensearchStore', + save: function (msg, reply) { + // const seneca = this + const ent = msg.ent; + const canon = ent.canon$({ object: true }); + const index = resolveIndex(ent, options); + const body = ent.data$(false); + const fieldOpts = options.field; + ['zone', 'base', 'name'].forEach((n) => { + if ('' != fieldOpts[n].name && null != canon[n] && '' != canon[n]) { + body[fieldOpts[n].name] = canon[n]; + } + }); + const req = { + index, + body, + }; + client + .index(req) + .then((res) => { + const body = res.body; + ent.data$(body._source); + ent.id = body._id; + reply(ent); + }) + .catch((err) => reply(err)); + }, + load: function (msg, reply) { + // const seneca = this + const ent = msg.ent; + // const canon = ent.canon$({ object: true }) + const index = resolveIndex(ent, options); + let q = msg.q || {}; + if (null != q.id) { + client + .get({ + index, + id: q.id, + }) + .then((res) => { + const body = res.body; + ent.data$(body._source); + ent.id = body._id; + reply(ent); + }) + .catch((err) => { + // Not found + if (err.meta && 404 === err.meta.statusCode) { + reply(null); + } + reply(err); + }); + } + else { + reply(); + } + }, + list: function (msg, reply) { + // const seneca = this + const ent = msg.ent; + const index = resolveIndex(ent, options); + const query = buildQuery({ index, options, msg }); + // console.log('LISTQ') + // console.dir(query, { depth: null }) + if (null == query) { + return reply([]); + } + client + .search(query) + .then((res) => { + const hits = res.body.hits; + const list = hits.hits.map((entry) => { + let item = ent.make$().data$(entry._source); + item.id = entry._id; + item.custom$ = { score: entry._score }; + return item; + }); + reply(list); + }) + .catch((err) => { + reply(err); + }); + }, + // NOTE: all$:true is REQUIRED for deleteByQuery + remove: function (msg, reply) { + // const seneca = this + const ent = msg.ent; + const index = resolveIndex(ent, options); + const q = msg.q || {}; + let id = q.id; + let query; + if (null == id) { + query = buildQuery({ index, options, msg }); + if (null == query || true !== q.all$) { + return reply(null); + } + } + // console.log('REMOVE', id) + // console.dir(query, { depth: null }) + if (null != id) { + client + .delete({ + index, + id, + // refresh: true, + }) + .then((_res) => { + reply(null); + }) + .catch((err) => { + // Not found + if (err.meta && 404 === err.meta.statusCode) { + return reply(null); + } + reply(err); + }); + } + else if (null != query && true === q.all$) { + client + .deleteByQuery({ + index, + body: { + query, + }, + // refresh: true, + }) + .then((_res) => { + reply(null); + }) + .catch((err) => { + // console.log('REM ERR', err) + reply(err); + }); + } + else { + reply(null); + } + }, + close: function (_msg, reply) { + this.log.debug('close', desc); + reply(); + }, + // TODO: obsolete - remove from seneca entity + native: function (_msg, reply) { + reply(null, { + client: () => client, + }); + }, + }; + let meta = init(seneca, options, store); + desc = meta.desc; + seneca.prepare(async function () { + const region = options.aws.region; + const node = options.opensearch.node; + client = new opensearch_1.Client({ + ...(0, aws_1.AwsSigv4Signer)({ + region, + service: 'aoss', + getCredentials: () => { + const credentialsProvider = (0, credential_provider_node_1.defaultProvider)(); + return credentialsProvider(); + }, + }), + node, + }); + }); + return { + name: store.name, + tag: meta.tag, + exportmap: { + native: () => { + return { client }; + }, + }, + }; +} +function buildQuery(spec) { + var _a; + const { index, options, msg } = spec; + const q = msg.q || {}; + let query = { + index, + body: { + size: msg.size$ || options.cmd.list.size, + _source: { + excludes: [options.field.vector.name].filter((n) => '' !== n), + }, + query: {}, + }, + }; + let excludeKeys = { vector: 1 }; + const parts = []; + for (let k in q) { + if (!excludeKeys[k] && !k.match(/\$/)) { + parts.push({ + match: { [k]: q[k] }, + }); + } + } + const vector$ = msg.vector$ || ((_a = q.directive$) === null || _a === void 0 ? void 0 : _a.vector$); + if (vector$) { + parts.push({ + knn: { + vector: { + vector: q.vector, + k: null == vector$.k ? 11 : vector$.k, + }, + }, + }); + } + if (0 === parts.length) { + query = null; + } + else if (1 === parts.length) { + query.body.query = parts[0]; + } + else { + query.body.query = { + bool: { + must: parts, + }, + }; + } + return query; +} +function resolveIndex(ent, options) { + let indexOpts = options.index; + if ('' != indexOpts.exact && null != indexOpts.exact) { + return indexOpts.exact; + } + let canonstr = ent.canon$({ string: true }); + indexOpts.map = indexOpts.map || {}; + if ('' != indexOpts.map[canonstr] && null != indexOpts.map[canonstr]) { + return indexOpts.map[canonstr]; + } + let prefix = indexOpts.prefix; + let suffix = indexOpts.suffix; + prefix = '' == prefix || null == prefix ? '' : prefix + '_'; + suffix = '' == suffix || null == suffix ? '' : '_' + suffix; + // TOOD: need ent.canon$({ external: true }) : foo/bar -> foo_bar + let infix = ent + .canon$({ string: true }) + .replace(/-\//g, '') + .replace(/\//g, '_'); + return prefix + infix + suffix; +} +// Default options. +const defaults = { + debug: false, + map: Any(), + index: { + prefix: '', + suffix: '', + map: {}, + exact: '', + }, + // '' === name => do not inject + field: { + zone: { name: 'zone' }, + base: { name: 'base' }, + name: { name: 'name' }, + vector: { name: 'vector' }, + }, + cmd: { + list: { + size: 11, + }, + }, + aws: Open({ + region: 'us-east-1', + }), + opensearch: Open({ + node: 'NODE-URL', + }), +}; +Object.assign(OpensearchStore, { + defaults, + utils: { resolveIndex }, +}); +exports.default = OpensearchStore; +if ('undefined' !== typeof module) { + module.exports = OpensearchStore; +} +//# sourceMappingURL=OpensearchStore.js.map \ No newline at end of file diff --git a/dist/OpensearchStore.js.map b/dist/OpensearchStore.js.map new file mode 100644 index 0000000..9b6f0a5 --- /dev/null +++ b/dist/OpensearchStore.js.map @@ -0,0 +1 @@ +{"version":3,"file":"OpensearchStore.js","sourceRoot":"","sources":["../src/OpensearchStore.ts"],"names":[],"mappings":";AAAA,yDAAyD;;AAEzD,4DAAmE;AACnE,+DAAuD;AACvD,gFAAmE;AAEnE,+BAA2B;AAE3B,MAAM,EAAE,IAAI,EAAE,GAAG,EAAE,GAAG,WAAI,CAAA;AA4B1B,SAAS,eAAe,CAAY,OAAgB;IAClD,MAAM,MAAM,GAAQ,IAAI,CAAA;IAExB,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,CAAA;IAEzC,IAAI,IAAI,GAAQ,iBAAiB,CAAA;IAEjC,IAAI,MAAW,CAAA;IAEf,IAAI,KAAK,GAAG;QACV,IAAI,EAAE,iBAAiB;QAEvB,IAAI,EAAE,UAAqB,GAAQ,EAAE,KAAU;YAC7C,sBAAsB;YACtB,MAAM,GAAG,GAAG,GAAG,CAAC,GAAG,CAAA;YAEnB,MAAM,KAAK,GAAG,GAAG,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAA;YAC1C,MAAM,KAAK,GAAG,YAAY,CAAC,GAAG,EAAE,OAAO,CAAC,CAAA;YAExC,MAAM,IAAI,GAAG,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,CAAA;YAE7B,MAAM,SAAS,GAAQ,OAAO,CAAC,KAAK,CAEnC;YAAA,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC,OAAO,CAAC,CAAC,CAAS,EAAE,EAAE;gBAC9C,IAAI,EAAE,IAAI,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,IAAI,IAAI,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,IAAI,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;oBAClE,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAA;gBACpC,CAAC;YACH,CAAC,CAAC,CAAA;YAEF,MAAM,GAAG,GAAG;gBACV,KAAK;gBACL,IAAI;aACL,CAAA;YAED,MAAM;iBACH,KAAK,CAAC,GAAG,CAAC;iBACV,IAAI,CAAC,CAAC,GAAQ,EAAE,EAAE;gBACjB,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,CAAA;gBACrB,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;gBACvB,GAAG,CAAC,EAAE,GAAG,IAAI,CAAC,GAAG,CAAA;gBACjB,KAAK,CAAC,GAAG,CAAC,CAAA;YACZ,CAAC,CAAC;iBACD,KAAK,CAAC,CAAC,GAAQ,EAAE,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAA;QACpC,CAAC;QAED,IAAI,EAAE,UAAqB,GAAQ,EAAE,KAAU;YAC7C,sBAAsB;YACtB,MAAM,GAAG,GAAG,GAAG,CAAC,GAAG,CAAA;YAEnB,6CAA6C;YAC7C,MAAM,KAAK,GAAG,YAAY,CAAC,GAAG,EAAE,OAAO,CAAC,CAAA;YAExC,IAAI,CAAC,GAAG,GAAG,CAAC,CAAC,IAAI,EAAE,CAAA;YAEnB,IAAI,IAAI,IAAI,CAAC,CAAC,EAAE,EAAE,CAAC;gBACjB,MAAM;qBACH,GAAG,CAAC;oBACH,KAAK;oBACL,EAAE,EAAE,CAAC,CAAC,EAAE;iBACT,CAAC;qBACD,IAAI,CAAC,CAAC,GAAQ,EAAE,EAAE;oBACjB,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,CAAA;oBACrB,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;oBACvB,GAAG,CAAC,EAAE,GAAG,IAAI,CAAC,GAAG,CAAA;oBACjB,KAAK,CAAC,GAAG,CAAC,CAAA;gBACZ,CAAC,CAAC;qBACD,KAAK,CAAC,CAAC,GAAQ,EAAE,EAAE;oBAClB,YAAY;oBACZ,IAAI,GAAG,CAAC,IAAI,IAAI,GAAG,KAAK,GAAG,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC;wBAC5C,KAAK,CAAC,IAAI,CAAC,CAAA;oBACb,CAAC;oBAED,KAAK,CAAC,GAAG,CAAC,CAAA;gBACZ,CAAC,CAAC,CAAA;YACN,CAAC;iBAAM,CAAC;gBACN,KAAK,EAAE,CAAA;YACT,CAAC;QACH,CAAC;QAED,IAAI,EAAE,UAAU,GAAQ,EAAE,KAAU;YAClC,sBAAsB;YACtB,MAAM,GAAG,GAAG,GAAG,CAAC,GAAG,CAAA;YAEnB,MAAM,KAAK,GAAG,YAAY,CAAC,GAAG,EAAE,OAAO,CAAC,CAAA;YACxC,MAAM,KAAK,GAAG,UAAU,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC,CAAA;YAEjD,uBAAuB;YACvB,sCAAsC;YAEtC,IAAI,IAAI,IAAI,KAAK,EAAE,CAAC;gBAClB,OAAO,KAAK,CAAC,EAAE,CAAC,CAAA;YAClB,CAAC;YAED,MAAM;iBACH,MAAM,CAAC,KAAK,CAAC;iBACb,IAAI,CAAC,CAAC,GAAQ,EAAE,EAAE;gBACjB,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC,IAAI,CAAA;gBAC1B,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,KAAU,EAAE,EAAE;oBACxC,IAAI,IAAI,GAAG,GAAG,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,CAAA;oBAC3C,IAAI,CAAC,EAAE,GAAG,KAAK,CAAC,GAAG,CAAA;oBACnB,IAAI,CAAC,OAAO,GAAG,EAAE,KAAK,EAAE,KAAK,CAAC,MAAM,EAAE,CAAA;oBACtC,OAAO,IAAI,CAAA;gBACb,CAAC,CAAC,CAAA;gBACF,KAAK,CAAC,IAAI,CAAC,CAAA;YACb,CAAC,CAAC;iBACD,KAAK,CAAC,CAAC,GAAQ,EAAE,EAAE;gBAClB,KAAK,CAAC,GAAG,CAAC,CAAA;YACZ,CAAC,CAAC,CAAA;QACN,CAAC;QAED,gDAAgD;QAChD,MAAM,EAAE,UAAqB,GAAQ,EAAE,KAAU;YAC/C,sBAAsB;YACtB,MAAM,GAAG,GAAG,GAAG,CAAC,GAAG,CAAA;YAEnB,MAAM,KAAK,GAAG,YAAY,CAAC,GAAG,EAAE,OAAO,CAAC,CAAA;YAExC,MAAM,CAAC,GAAG,GAAG,CAAC,CAAC,IAAI,EAAE,CAAA;YACrB,IAAI,EAAE,GAAG,CAAC,CAAC,EAAE,CAAA;YACb,IAAI,KAAK,CAAA;YAET,IAAI,IAAI,IAAI,EAAE,EAAE,CAAC;gBACf,KAAK,GAAG,UAAU,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC,CAAA;gBAE3C,IAAI,IAAI,IAAI,KAAK,IAAI,IAAI,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC;oBACrC,OAAO,KAAK,CAAC,IAAI,CAAC,CAAA;gBACpB,CAAC;YACH,CAAC;YAED,4BAA4B;YAC5B,sCAAsC;YAEtC,IAAI,IAAI,IAAI,EAAE,EAAE,CAAC;gBACf,MAAM;qBACH,MAAM,CAAC;oBACN,KAAK;oBACL,EAAE;oBACF,iBAAiB;iBAClB,CAAC;qBACD,IAAI,CAAC,CAAC,IAAS,EAAE,EAAE;oBAClB,KAAK,CAAC,IAAI,CAAC,CAAA;gBACb,CAAC,CAAC;qBACD,KAAK,CAAC,CAAC,GAAQ,EAAE,EAAE;oBAClB,YAAY;oBACZ,IAAI,GAAG,CAAC,IAAI,IAAI,GAAG,KAAK,GAAG,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC;wBAC5C,OAAO,KAAK,CAAC,IAAI,CAAC,CAAA;oBACpB,CAAC;oBAED,KAAK,CAAC,GAAG,CAAC,CAAA;gBACZ,CAAC,CAAC,CAAA;YACN,CAAC;iBAAM,IAAI,IAAI,IAAI,KAAK,IAAI,IAAI,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC;gBAC5C,MAAM;qBACH,aAAa,CAAC;oBACb,KAAK;oBACL,IAAI,EAAE;wBACJ,KAAK;qBACN;oBACD,iBAAiB;iBAClB,CAAC;qBACD,IAAI,CAAC,CAAC,IAAS,EAAE,EAAE;oBAClB,KAAK,CAAC,IAAI,CAAC,CAAA;gBACb,CAAC,CAAC;qBACD,KAAK,CAAC,CAAC,GAAQ,EAAE,EAAE;oBAClB,8BAA8B;oBAC9B,KAAK,CAAC,GAAG,CAAC,CAAA;gBACZ,CAAC,CAAC,CAAA;YACN,CAAC;iBAAM,CAAC;gBACN,KAAK,CAAC,IAAI,CAAC,CAAA;YACb,CAAC;QACH,CAAC;QAED,KAAK,EAAE,UAAqB,IAAS,EAAE,KAAU;YAC/C,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,EAAE,IAAI,CAAC,CAAA;YAC7B,KAAK,EAAE,CAAA;QACT,CAAC;QAED,6CAA6C;QAC7C,MAAM,EAAE,UAAqB,IAAS,EAAE,KAAU;YAChD,KAAK,CAAC,IAAI,EAAE;gBACV,MAAM,EAAE,GAAG,EAAE,CAAC,MAAM;aACrB,CAAC,CAAA;QACJ,CAAC;KACF,CAAA;IAED,IAAI,IAAI,GAAG,IAAI,CAAC,MAAM,EAAE,OAAO,EAAE,KAAK,CAAC,CAAA;IAEvC,IAAI,GAAG,IAAI,CAAC,IAAI,CAAA;IAEhB,MAAM,CAAC,OAAO,CAAC,KAAK;QAClB,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAA;QACjC,MAAM,IAAI,GAAG,OAAO,CAAC,UAAU,CAAC,IAAI,CAAA;QAEpC,MAAM,GAAG,IAAI,mBAAM,CAAC;YAClB,GAAG,IAAA,oBAAc,EAAC;gBAChB,MAAM;gBACN,OAAO,EAAE,MAAM;gBACf,cAAc,EAAE,GAAG,EAAE;oBACnB,MAAM,mBAAmB,GAAG,IAAA,0CAAe,GAAE,CAAA;oBAC7C,OAAO,mBAAmB,EAAE,CAAA;gBAC9B,CAAC;aACF,CAAC;YACF,IAAI;SACL,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;IAEF,OAAO;QACL,IAAI,EAAE,KAAK,CAAC,IAAI;QAChB,GAAG,EAAE,IAAI,CAAC,GAAG;QACb,SAAS,EAAE;YACT,MAAM,EAAE,GAAG,EAAE;gBACX,OAAO,EAAE,MAAM,EAAE,CAAA;YACnB,CAAC;SACF;KACF,CAAA;AACH,CAAC;AAED,SAAS,UAAU,CAAC,IAA+C;;IACjE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,EAAE,GAAG,IAAI,CAAA;IAEpC,MAAM,CAAC,GAAG,GAAG,CAAC,CAAC,IAAI,EAAE,CAAA;IAErB,IAAI,KAAK,GAAQ;QACf,KAAK;QACL,IAAI,EAAE;YACJ,IAAI,EAAE,GAAG,CAAC,KAAK,IAAI,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI;YACxC,OAAO,EAAE;gBACP,QAAQ,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,KAAK,CAAC,CAAC;aAC9D;YACD,KAAK,EAAE,EAAE;SACV;KACF,CAAA;IAED,IAAI,WAAW,GAAQ,EAAE,MAAM,EAAE,CAAC,EAAE,CAAA;IAEpC,MAAM,KAAK,GAAG,EAAE,CAAA;IAEhB,KAAK,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QAChB,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;YACtC,KAAK,CAAC,IAAI,CAAC;gBACT,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE;aACrB,CAAC,CAAA;QACJ,CAAC;IACH,CAAC;IAED,MAAM,OAAO,GAAG,GAAG,CAAC,OAAO,KAAI,MAAA,CAAC,CAAC,UAAU,0CAAE,OAAO,CAAA,CAAA;IACpD,IAAI,OAAO,EAAE,CAAC;QACZ,KAAK,CAAC,IAAI,CAAC;YACT,GAAG,EAAE;gBACH,MAAM,EAAE;oBACN,MAAM,EAAE,CAAC,CAAC,MAAM;oBAChB,CAAC,EAAE,IAAI,IAAI,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;iBACtC;aACF;SACF,CAAC,CAAA;IACJ,CAAC;IAED,IAAI,CAAC,KAAK,KAAK,CAAC,MAAM,EAAE,CAAC;QACvB,KAAK,GAAG,IAAI,CAAA;IACd,CAAC;SAAM,IAAI,CAAC,KAAK,KAAK,CAAC,MAAM,EAAE,CAAC;QAC9B,KAAK,CAAC,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAA;IAC7B,CAAC;SAAM,CAAC;QACN,KAAK,CAAC,IAAI,CAAC,KAAK,GAAG;YACjB,IAAI,EAAE;gBACJ,IAAI,EAAE,KAAK;aACZ;SACF,CAAA;IACH,CAAC;IAED,OAAO,KAAK,CAAA;AACd,CAAC;AAED,SAAS,YAAY,CAAC,GAAQ,EAAE,OAAgB;IAC9C,IAAI,SAAS,GAAG,OAAO,CAAC,KAAK,CAAA;IAC7B,IAAI,EAAE,IAAI,SAAS,CAAC,KAAK,IAAI,IAAI,IAAI,SAAS,CAAC,KAAK,EAAE,CAAC;QACrD,OAAO,SAAS,CAAC,KAAK,CAAA;IACxB,CAAC;IAED,IAAI,QAAQ,GAAG,GAAG,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAA;IAC3C,SAAS,CAAC,GAAG,GAAG,SAAS,CAAC,GAAG,IAAI,EAAE,CAAA;IACnC,IAAI,EAAE,IAAI,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,IAAI,IAAI,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;QACrE,OAAO,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAA;IAChC,CAAC;IAED,IAAI,MAAM,GAAG,SAAS,CAAC,MAAM,CAAA;IAC7B,IAAI,MAAM,GAAG,SAAS,CAAC,MAAM,CAAA;IAE7B,MAAM,GAAG,EAAE,IAAI,MAAM,IAAI,IAAI,IAAI,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,GAAG,CAAA;IAC3D,MAAM,GAAG,EAAE,IAAI,MAAM,IAAI,IAAI,IAAI,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,GAAG,MAAM,CAAA;IAE3D,iEAAiE;IACjE,IAAI,KAAK,GAAG,GAAG;SACZ,MAAM,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;SACxB,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;SACnB,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAA;IAEtB,OAAO,MAAM,GAAG,KAAK,GAAG,MAAM,CAAA;AAChC,CAAC;AAED,mBAAmB;AACnB,MAAM,QAAQ,GAAY;IACxB,KAAK,EAAE,KAAK;IACZ,GAAG,EAAE,GAAG,EAAE;IACV,KAAK,EAAE;QACL,MAAM,EAAE,EAAE;QACV,MAAM,EAAE,EAAE;QACV,GAAG,EAAE,EAAE;QACP,KAAK,EAAE,EAAE;KACV;IAED,+BAA+B;IAC/B,KAAK,EAAE;QACL,IAAI,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE;QACtB,IAAI,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE;QACtB,IAAI,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE;QACtB,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;KAC3B;IAED,GAAG,EAAE;QACH,IAAI,EAAE;YACJ,IAAI,EAAE,EAAE;SACT;KACF;IAED,GAAG,EAAE,IAAI,CAAC;QACR,MAAM,EAAE,WAAW;KACpB,CAAC;IAEF,UAAU,EAAE,IAAI,CAAC;QACf,IAAI,EAAE,UAAU;KACjB,CAAC;CACH,CAAA;AAED,MAAM,CAAC,MAAM,CAAC,eAAe,EAAE;IAC7B,QAAQ;IACR,KAAK,EAAE,EAAE,YAAY,EAAE;CACxB,CAAC,CAAA;AAEF,kBAAe,eAAe,CAAA;AAE9B,IAAI,WAAW,KAAK,OAAO,MAAM,EAAE,CAAC;IAClC,MAAM,CAAC,OAAO,GAAG,eAAe,CAAA;AAClC,CAAC"} \ No newline at end of file diff --git a/dist/OpensearchStoreDoc.d.ts b/dist/OpensearchStoreDoc.d.ts new file mode 100644 index 0000000..fa355c6 --- /dev/null +++ b/dist/OpensearchStoreDoc.d.ts @@ -0,0 +1,4 @@ +declare const docs: { + messages: {}; +}; +export default docs; diff --git a/dist/OpensearchStoreDoc.js b/dist/OpensearchStoreDoc.js new file mode 100644 index 0000000..00b9ab3 --- /dev/null +++ b/dist/OpensearchStoreDoc.js @@ -0,0 +1,11 @@ +"use strict"; +/* Copyright © 2024 Seneca Project Contributors, MIT License. */ +Object.defineProperty(exports, "__esModule", { value: true }); +const docs = { + messages: {}, +}; +exports.default = docs; +if ('undefined' !== typeof module) { + module.exports = docs; +} +//# sourceMappingURL=OpensearchStoreDoc.js.map \ No newline at end of file diff --git a/dist/OpensearchStoreDoc.js.map b/dist/OpensearchStoreDoc.js.map new file mode 100644 index 0000000..5384f7f --- /dev/null +++ b/dist/OpensearchStoreDoc.js.map @@ -0,0 +1 @@ +{"version":3,"file":"OpensearchStoreDoc.js","sourceRoot":"","sources":["../src/OpensearchStoreDoc.ts"],"names":[],"mappings":";AAAA,gEAAgE;;AAEhE,MAAM,IAAI,GAAG;IACX,QAAQ,EAAE,EAAE;CACb,CAAA;AAED,kBAAe,IAAI,CAAA;AAEnB,IAAI,WAAW,KAAK,OAAO,MAAM,EAAE,CAAC;IAClC,MAAM,CAAC,OAAO,GAAG,IAAI,CAAA;AACvB,CAAC"} \ No newline at end of file diff --git a/jest.config.js b/jest.config.js new file mode 100644 index 0000000..a75401e --- /dev/null +++ b/jest.config.js @@ -0,0 +1,10 @@ +module.exports = { + transform: { + '^.+\\.tsx?$': ['esbuild-jest', { sourcemap: true }], + }, + testEnvironment: 'node', + testMatch: ['**/test/**/*.test.ts'], + watchPathIgnorePatterns: ['dist\\/'], + collectCoverageFrom: ['src/**/*.ts'], + coverageProvider: 'v8', +} diff --git a/package.json b/package.json new file mode 100644 index 0000000..24b28da --- /dev/null +++ b/package.json @@ -0,0 +1,69 @@ +{ + "name": "@seneca/opensearch-store", + "version": "0.0.1", + "description": "Seneca OpenSearch data storage plugin.", + "main": "dist/OpensearchStore.js", + "type": "commonjs", + "types": "dist/OpensearchStore.d.ts", + "license": "MIT", + "author": "Richard Rodger (http://richardrodger.com)", + "contributors": [ + "Richard Rodger (http://richardrodger.com)" + ], + "scripts": { + "test": "jest --coverage", + "test-some": "jest -t", + "test-watch": "jest --coverage --watchAll", + "watch": "tsc -w -d", + "build": "tsc -d", + "doc": "seneca-doc -p seneca-entity", + "prettier": "prettier --write --no-semi --single-quote src/**/*.ts test/*.js", + "reset": "npm run clean && npm i && npm run build && npm test", + "clean": "rm -rf node_modules dist package-lock.json yarn.lock", + "repo-tag": "REPO_VERSION=`node -e \"console.log(require('./package').version)\"` && echo TAG: v$REPO_VERSION && git commit -a -m v$REPO_VERSION && git push && git tag v$REPO_VERSION && git push --tags;", + "repo-publish": "npm run clean && npm i --registry=http://registry.npmjs.org && npm run repo-publish-quick", + "repo-publish-quick": "npm run prettier && npm run build && npm run doc && npm test && npm run repo-tag && npm publish --access public --registry=https://registry.npmjs.org" + }, + "repository": { + "type": "git", + "url": "https://github.com/senecajs/SenecaOpensearchStore" + }, + "keywords": [ + "seneca", + "plugin", + "store", + "mem", + "memory" + ], + "peerDependencies": { + "@seneca/entity-util": ">=2", + "seneca": ">=3", + "seneca-entity": ">=25", + "seneca-promisify": ">=3" + }, + "devDependencies": { + "@seneca/doc": "^7.2.0", + "@seneca/maintain": "^0.1.0", + "@types/jest": "^29.5.12", + "@types/node": "^20.11.24", + "esbuild": "^0.20.1", + "esbuild-jest": "^0.5.0", + "jest": "^29.7.0", + "dotenv": "^16.4.5", + "prettier": "^3.2.5", + "seneca-msg-test": "^4.1.0", + "seneca-store-test": "^5.2.0", + "typescript": "^5.3.3" + }, + "files": [ + "README.md", + "CHANGES.md", + "LICENSE", + "src", + "dist" + ], + "dependencies": { + "@aws-sdk/credential-provider-node": "^3.525.0", + "@opensearch-project/opensearch": "^2.5.0" + } +} diff --git a/src/OpensearchStore.ts b/src/OpensearchStore.ts new file mode 100644 index 0000000..6bd4fc8 --- /dev/null +++ b/src/OpensearchStore.ts @@ -0,0 +1,378 @@ +/* Copyright (c) 2024 Seneca contributors, MIT License */ + +import { AwsSigv4Signer } from '@opensearch-project/opensearch/aws' +import { Client } from '@opensearch-project/opensearch' +import { defaultProvider } from '@aws-sdk/credential-provider-node' + +import { Gubu } from 'gubu' + +const { Open, Any } = Gubu + +type Options = { + debug: boolean + map?: any + index: { + prefix: string + suffix: string + map: Record + exact: string + } + field: { + zone: { name: string } + base: { name: string } + name: { name: string } + vector: { name: string } + } + cmd: { + list: { + size: number + } + } + aws: any + opensearch: any +} + +export type OpensearchStoreOptions = Partial + +function OpensearchStore(this: any, options: Options) { + const seneca: any = this + + const init = seneca.export('entity/init') + + let desc: any = 'OpensearchStore' + + let client: any + + let store = { + name: 'OpensearchStore', + + save: function (this: any, msg: any, reply: any) { + // const seneca = this + const ent = msg.ent + + const canon = ent.canon$({ object: true }) + const index = resolveIndex(ent, options) + + const body = ent.data$(false) + + const fieldOpts: any = options.field + + ;['zone', 'base', 'name'].forEach((n: string) => { + if ('' != fieldOpts[n].name && null != canon[n] && '' != canon[n]) { + body[fieldOpts[n].name] = canon[n] + } + }) + + const req = { + index, + body, + } + + client + .index(req) + .then((res: any) => { + const body = res.body + ent.data$(body._source) + ent.id = body._id + reply(ent) + }) + .catch((err: any) => reply(err)) + }, + + load: function (this: any, msg: any, reply: any) { + // const seneca = this + const ent = msg.ent + + // const canon = ent.canon$({ object: true }) + const index = resolveIndex(ent, options) + + let q = msg.q || {} + + if (null != q.id) { + client + .get({ + index, + id: q.id, + }) + .then((res: any) => { + const body = res.body + ent.data$(body._source) + ent.id = body._id + reply(ent) + }) + .catch((err: any) => { + // Not found + if (err.meta && 404 === err.meta.statusCode) { + reply(null) + } + + reply(err) + }) + } else { + reply() + } + }, + + list: function (msg: any, reply: any) { + // const seneca = this + const ent = msg.ent + + const index = resolveIndex(ent, options) + const query = buildQuery({ index, options, msg }) + + // console.log('LISTQ') + // console.dir(query, { depth: null }) + + if (null == query) { + return reply([]) + } + + client + .search(query) + .then((res: any) => { + const hits = res.body.hits + const list = hits.hits.map((entry: any) => { + let item = ent.make$().data$(entry._source) + item.id = entry._id + item.custom$ = { score: entry._score } + return item + }) + reply(list) + }) + .catch((err: any) => { + reply(err) + }) + }, + + // NOTE: all$:true is REQUIRED for deleteByQuery + remove: function (this: any, msg: any, reply: any) { + // const seneca = this + const ent = msg.ent + + const index = resolveIndex(ent, options) + + const q = msg.q || {} + let id = q.id + let query + + if (null == id) { + query = buildQuery({ index, options, msg }) + + if (null == query || true !== q.all$) { + return reply(null) + } + } + + // console.log('REMOVE', id) + // console.dir(query, { depth: null }) + + if (null != id) { + client + .delete({ + index, + id, + // refresh: true, + }) + .then((_res: any) => { + reply(null) + }) + .catch((err: any) => { + // Not found + if (err.meta && 404 === err.meta.statusCode) { + return reply(null) + } + + reply(err) + }) + } else if (null != query && true === q.all$) { + client + .deleteByQuery({ + index, + body: { + query, + }, + // refresh: true, + }) + .then((_res: any) => { + reply(null) + }) + .catch((err: any) => { + // console.log('REM ERR', err) + reply(err) + }) + } else { + reply(null) + } + }, + + close: function (this: any, _msg: any, reply: any) { + this.log.debug('close', desc) + reply() + }, + + // TODO: obsolete - remove from seneca entity + native: function (this: any, _msg: any, reply: any) { + reply(null, { + client: () => client, + }) + }, + } + + let meta = init(seneca, options, store) + + desc = meta.desc + + seneca.prepare(async function (this: any) { + const region = options.aws.region + const node = options.opensearch.node + + client = new Client({ + ...AwsSigv4Signer({ + region, + service: 'aoss', + getCredentials: () => { + const credentialsProvider = defaultProvider() + return credentialsProvider() + }, + }), + node, + }) + }) + + return { + name: store.name, + tag: meta.tag, + exportmap: { + native: () => { + return { client } + }, + }, + } +} + +function buildQuery(spec: { index: string; options: any; msg: any }) { + const { index, options, msg } = spec + + const q = msg.q || {} + + let query: any = { + index, + body: { + size: msg.size$ || options.cmd.list.size, + _source: { + excludes: [options.field.vector.name].filter((n) => '' !== n), + }, + query: {}, + }, + } + + let excludeKeys: any = { vector: 1 } + + const parts = [] + + for (let k in q) { + if (!excludeKeys[k] && !k.match(/\$/)) { + parts.push({ + match: { [k]: q[k] }, + }) + } + } + + const vector$ = msg.vector$ || q.directive$?.vector$ + if (vector$) { + parts.push({ + knn: { + vector: { + vector: q.vector, + k: null == vector$.k ? 11 : vector$.k, + }, + }, + }) + } + + if (0 === parts.length) { + query = null + } else if (1 === parts.length) { + query.body.query = parts[0] + } else { + query.body.query = { + bool: { + must: parts, + }, + } + } + + return query +} + +function resolveIndex(ent: any, options: Options) { + let indexOpts = options.index + if ('' != indexOpts.exact && null != indexOpts.exact) { + return indexOpts.exact + } + + let canonstr = ent.canon$({ string: true }) + indexOpts.map = indexOpts.map || {} + if ('' != indexOpts.map[canonstr] && null != indexOpts.map[canonstr]) { + return indexOpts.map[canonstr] + } + + let prefix = indexOpts.prefix + let suffix = indexOpts.suffix + + prefix = '' == prefix || null == prefix ? '' : prefix + '_' + suffix = '' == suffix || null == suffix ? '' : '_' + suffix + + // TOOD: need ent.canon$({ external: true }) : foo/bar -> foo_bar + let infix = ent + .canon$({ string: true }) + .replace(/-\//g, '') + .replace(/\//g, '_') + + return prefix + infix + suffix +} + +// Default options. +const defaults: Options = { + debug: false, + map: Any(), + index: { + prefix: '', + suffix: '', + map: {}, + exact: '', + }, + + // '' === name => do not inject + field: { + zone: { name: 'zone' }, + base: { name: 'base' }, + name: { name: 'name' }, + vector: { name: 'vector' }, + }, + + cmd: { + list: { + size: 11, + }, + }, + + aws: Open({ + region: 'us-east-1', + }), + + opensearch: Open({ + node: 'NODE-URL', + }), +} + +Object.assign(OpensearchStore, { + defaults, + utils: { resolveIndex }, +}) + +export default OpensearchStore + +if ('undefined' !== typeof module) { + module.exports = OpensearchStore +} diff --git a/src/OpensearchStoreDoc.ts b/src/OpensearchStoreDoc.ts new file mode 100644 index 0000000..aceb7b2 --- /dev/null +++ b/src/OpensearchStoreDoc.ts @@ -0,0 +1,11 @@ +/* Copyright © 2024 Seneca Project Contributors, MIT License. */ + +const docs = { + messages: {}, +} + +export default docs + +if ('undefined' !== typeof module) { + module.exports = docs +} diff --git a/test/Opensearch.test.ts b/test/Opensearch.test.ts new file mode 100644 index 0000000..bdf22d3 --- /dev/null +++ b/test/Opensearch.test.ts @@ -0,0 +1,248 @@ +/* Copyright © 2024 Seneca Project Contributors, MIT License. */ + +require('dotenv').config({ path: '.env.local' }) +// console.log(process.env) // remove this + + +import Seneca from 'seneca' +// import SenecaMsgTest from 'seneca-msg-test' +// import { Maintain } from '@seneca/maintain' + +import OpensearchStoreDoc from '../src/OpensearchStoreDoc' +import OpensearchStore from '../src/OpensearchStore' + + + +describe('OpensearchStore', () => { + test('load-plugin', async () => { + expect(OpensearchStore).toBeDefined() + expect(OpensearchStoreDoc).toBeDefined() + + const seneca = Seneca({ legacy: false }) + .test() + .use('promisify') + .use('entity') + .use(OpensearchStore) + await seneca.ready() + + expect(seneca.export('OpensearchStore/native')).toBeDefined() + }) + + + test('utils.resolveIndex', () => { + const utils = OpensearchStore['utils'] + const resolveIndex = utils.resolveIndex + const seneca = makeSeneca() + const ent0 = seneca.make('foo') + const ent1 = seneca.make('foo/bar') + + expect(resolveIndex(ent0, { index: {} })).toEqual('foo') + expect(resolveIndex(ent0, { index: { exact: 'qaz' } })).toEqual('qaz') + + expect(resolveIndex(ent1, { index: {} })).toEqual('foo_bar') + expect(resolveIndex(ent1, { index: { prefix: 'p0', suffix: 's0' } })).toEqual('p0_foo_bar_s0') + expect(resolveIndex(ent1, { + index: { map: { '-/foo/bar': 'FOOBAR' }, prefix: 'p0', suffix: 's0' } + })) + .toEqual('FOOBAR') + }, 22222) + + + test('insert-remove', async () => { + const seneca = await makeSeneca() + await seneca.ready() + + + // no query params means no results + const list0 = await seneca.entity('foo/chunk').list$() + expect(0 === list0.length) + + const list1 = await seneca.entity('foo/chunk').list$({ test: 'insert-remove' }) + // console.log(list1) + + let ent0: any + + if (0 === list1.length) { + ent0 = await seneca.entity('foo/chunk') + .make$() + .data$({ + test: 'insert-remove', + text: 't01', + vector: [0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7], + directive$: { vector$: true }, + }) + .save$() + expect(ent0).toMatchObject({ test: 'insert-remove' }) + await new Promise((r) => setTimeout(r, 2222)) + } + else { + ent0 = list1[0] + } + + await seneca.entity('foo/chunk').remove$(ent0.id) + + await new Promise((r) => setTimeout(r, 2222)) + + const list2 = await seneca.entity('foo/chunk').list$({ test: 'insert-remove' }) + // console.log(list2) + expect(list2.filter((n: any) => n.id === ent0.id)).toEqual([]) + }, 22222) + + + test('vector-cat', async () => { + const seneca = await makeSeneca() + await seneca.ready() + + // const list0 = await seneca.entity('foo/chunk').list$({ test: 'vector-cat' }) + // console.log('list0', list0) + + // NOT AVAILABLE ON AWS + // await seneca.entity('foo/chunk').remove$({ all$: true, test: 'vector-cat' }) + + const list1 = await seneca.entity('foo/chunk').list$({ test: 'vector-cat' }) + // console.log('list1', list1) + + /* + for (let i = 0; i < list1.length; i++) { + await list1[i].remove$() + } + + await new Promise((r) => setTimeout(r, 2222)) + + const list1r = await seneca.entity('foo/chunk').list$({ test: 'vector-cat' }) + // console.log('list1r', list1r) + */ + + if (!list1.find((n: any) => 'code0' === n.code)) { + await seneca.entity('foo/chunk') + .make$() + .data$({ + code: 'code0', + test: 'vector-cat', + text: 't01', + vector: [0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1], + directive$: { vector$: true }, + }) + .save$() + } + + if (!list1.find((n: any) => 'code1' === n.code)) { + await seneca.entity('foo/chunk') + .make$() + .data$({ + code: 'code1', + test: 'vector-cat', + text: 't01', + vector: [0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1], + directive$: { vector$: true }, + }) + .save$() + } + + await new Promise((r) => setTimeout(r, 2222)) + + const list2 = await seneca.entity('foo/chunk').list$({ + directive$: { vector$: { k: 2 } }, + vector: [0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1], + }) + // console.log('list2', list2.map((n: any) => ({ ...n }))) + expect(1 < list2.length).toEqual(true) + + const list3 = await seneca.entity('foo/chunk').list$({ + directive$: { vector$: { k: 2 } }, + vector: [0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1], + code: 'code0' + }) + // console.log('list3', list3.map((n: any) => ({ ...n }))) + expect(list3.length).toEqual(1) + + }, 22222) + + + +}) + + +function makeSeneca() { + return Seneca({ legacy: false }) + .test() + .use('promisify') + .use('entity') + .use(OpensearchStore, { + map: { + 'foo/chunk': '*' + }, + index: { + exact: process.env.SENECA_OPENSEARCH_TEST_INDEX, + }, + opensearch: { + node: process.env.SENECA_OPENSEARCH_TEST_NODE, + } + }) +} + + +const index_test01 = { + "mappings": { + "properties": { + "text": { "type": "text" }, + "vector": { + "type": "knn_vector", + "dimension": 8, // 1536, + "method": { + "engine": "nmslib", + "space_type": "cosinesimil", + "name": "hnsw", + "parameters": { "ef_construction": 512, "m": 16 } + } + } + } + }, + "settings": { + "index": { + "number_of_shards": 2, + "knn.algo_param": { "ef_search": 512 }, + "knn": true + } + } +} + + +/* + [ + { + "Rules": [ + { + "Resource": [ + "collection/podmind03a" + ], + "Permission": [ + "aoss:CreateCollectionItems", + "aoss:DeleteCollectionItems", + "aoss:UpdateCollectionItems", + "aoss:DescribeCollectionItems" + ], + "ResourceType": "collection" + }, + { + "Resource": [ + "index/podmind03a/*" + ], + "Permission": [ + "aoss:CreateIndex", + "aoss:DeleteIndex", + "aoss:UpdateIndex", + "aoss:DescribeIndex", + "aoss:ReadDocument", + "aoss:WriteDocument" + ], + "ResourceType": "index" + } + ], + "Principal": [ + "arn:aws:iam::...:role/...LambdaRole..." + ], + "Description": "Easy data policy" + } +] + */ diff --git a/test/quick.js b/test/quick.js new file mode 100644 index 0000000..dd121de --- /dev/null +++ b/test/quick.js @@ -0,0 +1,61 @@ +require('dotenv').config({ path: '.env.local' }) +// console.log(process.env) // remove this + +const Seneca = require('seneca') + +run() + +async function run() { + const seneca = Seneca({ legacy: false }) + .test() + .use('promisify') + .use('entity') + .use('..', { + map: { + 'foo/chunk': '*', + }, + index: { + exact: process.env.SENECA_OPENSEARCH_TEST_INDEX, + }, + opensearch: { + node: process.env.SENECA_OPENSEARCH_TEST_NODE, + }, + }) + + await seneca.ready() + + // console.log(await seneca.entity('bar/qaz').data$({q:1}).save$()) + + /* + const save0 = await seneca.entity('foo/chunk') + .make$() + .data$({ + x:3, + o:{m:'M2',n:3}, + text: 't03', + vector: [0.0,0.1,0.2,0.3,0.4,0.5,0.6,0.6], + directive$:{vector$:true}, + }) + .save$() + console.log('save0', save0) + */ + + // const id = '1%3A0%3Au0rACY4BB33NxQZdwDrQ' + // const id = 'notanid' + //const id = '1%3A0%3AvUrfCY4BB33NxQZd-DrZ' + const id = '1%3A0%3AvUrfCY4BB33NxQZd-DrQ' + const load0 = await seneca.entity('foo/chunk').load$(id) + console.log('load0', load0) + + /* + const list0 = await seneca.entity('foo/chunk').list$({ + // x:2 + directive$:{vector$:true}, + vector:[0.1,0.1,0.2,0.3,0.4,0.5,0.6,0.7], + }) + console.log('list0', list0) + + + console.log(await seneca.entity('bar/qaz').list$()) + */ +} diff --git a/tsconfig.json b/tsconfig.json new file mode 100644 index 0000000..67c6dc9 --- /dev/null +++ b/tsconfig.json @@ -0,0 +1,20 @@ +{ + "compilerOptions": { + "esModuleInterop": true, + "isolatedModules": true, + "module": "commonjs", + "noEmitOnError": true, + "outDir":"dist", + "resolveJsonModule": true, + "rootDir": "src", + "sourceMap": true, + "strict": true, + "target": "ES2019" + }, + "exclude": [ + "dist", + "node_modules", + "test" + ] +} + diff --git a/tsfmt.json b/tsfmt.json new file mode 100644 index 0000000..86d198a --- /dev/null +++ b/tsfmt.json @@ -0,0 +1,4 @@ +{ + "indentSize": 2, + "insertSpaceAfterFunctionKeywordForAnonymousFunctions": true +}