diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 9493c9182..1fe6bfdb0 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -41,7 +41,8 @@ jobs: "postgres", "postgres13", "postgres12", "postgres11", "postgres10", "postgres9", "mysql", "mysql5", "mssql", "mssql17", - spanner + spanner, + "mongo", "mongo4", ] env: diff --git a/apps/velo-external-db/src/storage/factory.ts b/apps/velo-external-db/src/storage/factory.ts index fb5cbb05c..0c0d64f8c 100644 --- a/apps/velo-external-db/src/storage/factory.ts +++ b/apps/velo-external-db/src/storage/factory.ts @@ -23,10 +23,10 @@ export const engineConnectorFor = async(_type: string, config: any): Promise { + testIfSupportedOperationsIncludes(supportedOperations, [AtomicBulkInsert])('insert api should fail if item already exists', async() => { await schema.givenCollection(ctx.collectionName, [ctx.column], authOwner) await data.givenItems([ ctx.items[1] ], ctx.collectionName, authAdmin) @@ -98,6 +98,23 @@ describe(`Velo External DB Data REST API: ${currentDbImplementationName()}`, () ) }) + testIfSupportedOperationsIncludes(supportedOperations, [NonAtomicBulkInsert])('insert api should throw 409 error if item already exists and continue inserting the rest', async() => { + await schema.givenCollection(ctx.collectionName, [ctx.column], authOwner) + await data.givenItems([ ctx.items[1] ], ctx.collectionName, authAdmin) + + const response = axiosInstance.post('/data/insert', data.insertRequest(ctx.collectionName, ctx.items, false), { responseType: 'stream', ...authAdmin }) + + const expectedItems = ctx.items.map(i => dataSpi.QueryResponsePart.item(i)) + + await expect(response).rejects.toThrow('409') + await expect(data.queryCollectionAsArray(ctx.collectionName, [], undefined, authOwner)).resolves.toEqual(expect.toIncludeAllMembers( + [ + ...expectedItems, + data.pagingMetadata(expectedItems.length, expectedItems.length) + ]) + ) + }) + test('insert api should succeed if item already exists and overwriteExisting is on', async() => { await schema.givenCollection(ctx.collectionName, [ctx.column], authOwner) await data.givenItems([ ctx.item ], ctx.collectionName, authAdmin) @@ -347,8 +364,24 @@ describe(`Velo External DB Data REST API: ${currentDbImplementationName()}`, () }) }) + interface Ctx { + collectionName: string + column: InputField + numberColumns: InputField[] + objectColumn: InputField + item: Item + items: Item[] + modifiedItem: Item + modifiedItems: Item[] + anotherItem: Item + numberItem: Item + anotherNumberItem: Item + objectItem: Item + nestedFieldName: string + pastVeloDate: { $date: string; } + } - const ctx = { + const ctx: Ctx = { collectionName: Uninitialized, column: Uninitialized, numberColumns: Uninitialized, diff --git a/apps/velo-external-db/test/env/env.db.setup.js b/apps/velo-external-db/test/env/env.db.setup.js index d30e10447..c6a945002 100644 --- a/apps/velo-external-db/test/env/env.db.setup.js +++ b/apps/velo-external-db/test/env/env.db.setup.js @@ -8,7 +8,7 @@ const { testResources: mysql } = require ('@wix-velo/external-db-mysql') const { testResources: spanner } = require ('@wix-velo/external-db-spanner') // const { testResources: firestore } = require ('@wix-velo/external-db-firestore') const { testResources: mssql } = require ('@wix-velo/external-db-mssql') -// const { testResources: mongo } = require ('@wix-velo/external-db-mongo') +const { testResources: mongo } = require ('@wix-velo/external-db-mongo') // const { testResources: googleSheet } = require('@wix-velo/external-db-google-sheets') // const { testResources: airtable } = require('@wix-velo/external-db-airtable') // const { testResources: dynamoDb } = require('@wix-velo/external-db-dynamodb') @@ -39,9 +39,9 @@ const initEnv = async(testEngine) => { await mssql.initEnv() break - // case 'mongo': - // await mongo.initEnv() - // break + case 'mongo': + await mongo.initEnv() + break // case 'google-sheet': // await googleSheet.initEnv() // break @@ -86,9 +86,9 @@ const cleanup = async(testEngine) => { // await googleSheet.cleanup() // break - // case 'mongo': - // await mongo.cleanup() - // break + case 'mongo': + await mongo.cleanup() + break // case 'dynamodb': // await dynamoDb.cleanup() diff --git a/apps/velo-external-db/test/env/env.db.teardown.js b/apps/velo-external-db/test/env/env.db.teardown.js index 6e3bedcb5..7102ca700 100644 --- a/apps/velo-external-db/test/env/env.db.teardown.js +++ b/apps/velo-external-db/test/env/env.db.teardown.js @@ -3,7 +3,7 @@ const { testResources: mysql } = require ('@wix-velo/external-db-mysql') const { testResources: spanner } = require ('@wix-velo/external-db-spanner') // const { testResources: firestore } = require ('@wix-velo/external-db-firestore') const { testResources: mssql } = require ('@wix-velo/external-db-mssql') -// const { testResources: mongo } = require ('@wix-velo/external-db-mongo') +const { testResources: mongo } = require ('@wix-velo/external-db-mongo') // const { testResources: googleSheet } = require('@wix-velo/external-db-google-sheets') // const { testResources: airtable } = require('@wix-velo/external-db-airtable') // const { testResources: dynamo } = require('@wix-velo/external-db-dynamodb') @@ -45,9 +45,9 @@ const shutdownEnv = async(testEngine) => { // await dynamo.shutdownEnv() // break - // case 'mongo': - // await mongo.shutdownEnv() - // break + case 'mongo': + await mongo.shutdownEnv() + break // case 'bigquery': // await bigquery.shutdownEnv() diff --git a/apps/velo-external-db/test/resources/e2e_resources.ts b/apps/velo-external-db/test/resources/e2e_resources.ts index 24be281de..e101ad754 100644 --- a/apps/velo-external-db/test/resources/e2e_resources.ts +++ b/apps/velo-external-db/test/resources/e2e_resources.ts @@ -33,7 +33,7 @@ const testSuits = { spanner: new E2EResources(spanner, createAppWithWixDataBaseUrl), firestore: new E2EResources(firestore, createApp), mssql: new E2EResources(mssql, createAppWithWixDataBaseUrl), - mongo: new E2EResources(mongo, createApp), + mongo: new E2EResources(mongo, createAppWithWixDataBaseUrl), 'google-sheet': new E2EResources(googleSheet, createApp), airtable: new E2EResources(airtable, createApp), dynamodb: new E2EResources(dynamo, createApp), diff --git a/apps/velo-external-db/test/resources/provider_resources.ts b/apps/velo-external-db/test/resources/provider_resources.ts index 8ac2a8d67..b74e2cc39 100644 --- a/apps/velo-external-db/test/resources/provider_resources.ts +++ b/apps/velo-external-db/test/resources/provider_resources.ts @@ -74,7 +74,7 @@ const testSuits = { spanner: suiteDef('Spanner', spannerTestEnvInit, spanner.testResources), firestore: suiteDef('Firestore', firestoreTestEnvInit, firestore.testResources.supportedOperations), mssql: suiteDef('Sql Server', mssqlTestEnvInit, mssql.testResources), - mongo: suiteDef('Mongo', mongoTestEnvInit, mongo.testResources.supportedOperations), + mongo: suiteDef('Mongo', mongoTestEnvInit, mongo.testResources), airtable: suiteDef('Airtable', airTableTestEnvInit, airtable.testResources.supportedOperations), dynamodb: suiteDef('DynamoDb', dynamoTestEnvInit, dynamo.testResources.supportedOperations), bigquery: suiteDef('BigQuery', bigqueryTestEnvInit, bigquery.testResources.supportedOperations), diff --git a/libs/external-db-mongo/src/exception_translator.ts b/libs/external-db-mongo/src/exception_translator.ts index 7b65001d6..c35d8c59b 100644 --- a/libs/external-db-mongo/src/exception_translator.ts +++ b/libs/external-db-mongo/src/exception_translator.ts @@ -1,15 +1,17 @@ import { errors } from '@wix-velo/velo-external-db-commons' const { ItemAlreadyExists } = errors -const notThrowingTranslateErrorCodes = (err: any) => { +const extractItemIdFromError = (err: any) => err.message.split('"')[1] + +const notThrowingTranslateErrorCodes = (err: any, collectionName: string) => { switch (err.code) { - case 11000: - return new ItemAlreadyExists(`Item already exists: ${err.message}`) + case 11000: + return new ItemAlreadyExists(`Item already exists: ${err.message}`, collectionName, extractItemIdFromError(err)) default: return new Error (`default ${err.message}`) } } -export const translateErrorCodes = (err: any) => { - throw notThrowingTranslateErrorCodes(err) +export const translateErrorCodes = (err: any, collectionName: string) => { + throw notThrowingTranslateErrorCodes(err, collectionName) } diff --git a/libs/external-db-mongo/src/mongo_capabilities.ts b/libs/external-db-mongo/src/mongo_capabilities.ts new file mode 100644 index 000000000..104d6f5ce --- /dev/null +++ b/libs/external-db-mongo/src/mongo_capabilities.ts @@ -0,0 +1,19 @@ +import { AdapterOperators } from '@wix-velo/velo-external-db-commons' +import { CollectionOperation, DataOperation, FieldType } from '@wix-velo/velo-external-db-types' + +const { query, count, queryReferenced, aggregate, } = DataOperation +const { eq, ne, string_contains, string_begins, string_ends, gt, gte, lt, lte, include } = AdapterOperators + +export const ReadWriteOperations = Object.values(DataOperation) +export const ReadOnlyOperations = [query, count, queryReferenced, aggregate] +export const FieldTypes = Object.values(FieldType) +export const CollectionOperations = Object.values(CollectionOperation) +export const ColumnsCapabilities = { + text: { sortable: true, columnQueryOperators: [eq, ne, string_contains, string_begins, string_ends, include, gt, gte, lt, lte] }, + url: { sortable: true, columnQueryOperators: [eq, ne, string_contains, string_begins, string_ends, include, gt, gte, lt, lte] }, + number: { sortable: true, columnQueryOperators: [eq, ne, gt, gte, lt, lte, include] }, + boolean: { sortable: true, columnQueryOperators: [eq] }, + image: { sortable: false, columnQueryOperators: [] }, + object: { sortable: false, columnQueryOperators: [] }, + datetime: { sortable: true, columnQueryOperators: [eq, ne, gt, gte, lt, lte] }, +} diff --git a/libs/external-db-mongo/src/mongo_data_provider.ts b/libs/external-db-mongo/src/mongo_data_provider.ts index b99859c52..4a5abaa3a 100644 --- a/libs/external-db-mongo/src/mongo_data_provider.ts +++ b/libs/external-db-mongo/src/mongo_data_provider.ts @@ -1,6 +1,6 @@ import { translateErrorCodes } from './exception_translator' -import { unpackIdFieldForItem, updateExpressionFor, validateTable } from './mongo_utils' -import { IDataProvider, AdapterFilter as Filter, AdapterAggregation as Aggregation, Item } from '@wix-velo/velo-external-db-types' +import { insertExpressionFor, isEmptyObject, unpackIdFieldForItem, updateExpressionFor, validateTable } from './mongo_utils' +import { IDataProvider, AdapterFilter as Filter, AdapterAggregation as Aggregation, Item, Sort, } from '@wix-velo/velo-external-db-types' import FilterParser from './sql_filter_transformer' import { MongoClient } from 'mongodb' @@ -34,14 +34,14 @@ export default class DataProvider implements IDataProvider { .count(filterExpr) } - async insert(collectionName: string, items: Item[] ): Promise { + async insert(collectionName: string, items: Item[], _fields: any[], upsert = false): Promise { validateTable(collectionName) - const result = await this.client.db() - .collection(collectionName) - //@ts-ignore - Type 'string' is not assignable to type 'ObjectId', objectId Can be a 24 character hex string, 12 byte binary Buffer, or a number. and we cant assume that on the _id input - .insertMany(items) - .catch(translateErrorCodes) - return result.insertedCount + const { insertedCount, upsertedCount } = await this.client.db() + .collection(collectionName) + .bulkWrite(insertExpressionFor(items, upsert), { ordered: false }) + .catch(e => translateErrorCodes(e, collectionName)) + + return insertedCount + upsertedCount } async update(collectionName: string, items: Item[]): Promise { @@ -49,7 +49,7 @@ export default class DataProvider implements IDataProvider { const result = await this.client.db() .collection(collectionName) .bulkWrite( updateExpressionFor(items) ) - return result.nModified + return result.nModified } async delete(collectionName: string, itemIds: string[]): Promise { @@ -67,17 +67,26 @@ export default class DataProvider implements IDataProvider { .deleteMany({}) } - async aggregate(collectionName: string, filter: Filter, aggregation: Aggregation): Promise { + async aggregate(collectionName: string, filter: Filter, aggregation: Aggregation, sort: Sort[], skip: number, limit: number): Promise { validateTable(collectionName) + const additionalAggregationStages = [] const { fieldsStatement, havingFilter } = this.filterParser.parseAggregation(aggregation) const { filterExpr } = this.filterParser.transform(filter) + const sortExpr = this.filterParser.orderAggregationBy(sort) + + !isEmptyObject(sortExpr.$sort)? additionalAggregationStages.push(sortExpr) : null + skip? additionalAggregationStages.push({ $skip: skip }) : null + limit? additionalAggregationStages.push({ $limit: limit }) : null + const result = await this.client.db() - .collection(collectionName) - .aggregate( [ { $match: filterExpr }, - fieldsStatement, - havingFilter - ] ) - .toArray() + .collection(collectionName) + .aggregate([ + { $match: filterExpr }, + fieldsStatement, + havingFilter, + ...additionalAggregationStages + ]) + .toArray() return result.map( unpackIdFieldForItem ) } diff --git a/libs/external-db-mongo/src/mongo_schema_provider.ts b/libs/external-db-mongo/src/mongo_schema_provider.ts index 26d2aa9f9..3f7b899cf 100644 --- a/libs/external-db-mongo/src/mongo_schema_provider.ts +++ b/libs/external-db-mongo/src/mongo_schema_provider.ts @@ -1,33 +1,49 @@ -import { SystemFields, validateSystemFields, AllSchemaOperations } from '@wix-velo/velo-external-db-commons' -import { InputField, ResponseField, ISchemaProvider, SchemaOperations, Table } from '@wix-velo/velo-external-db-types' -const { CollectionDoesNotExists, FieldAlreadyExists, FieldDoesNotExist } = require('@wix-velo/velo-external-db-commons').errors -import { validateTable, SystemTable } from './mongo_utils' +import { MongoClient } from 'mongodb' +import { SystemFields, validateSystemFields, AllSchemaOperations, EmptyCapabilities, errors } from '@wix-velo/velo-external-db-commons' +import { InputField, ResponseField, ISchemaProvider, SchemaOperations, Table, CollectionCapabilities, Encryption } from '@wix-velo/velo-external-db-types' +import { validateTable, SystemTable, updateExpressionFor, CollectionObject } from './mongo_utils' +import { CollectionOperations, FieldTypes, ReadWriteOperations, ColumnsCapabilities } from './mongo_capabilities' +const { CollectionDoesNotExists, FieldAlreadyExists, FieldDoesNotExist } = errors + export default class SchemaProvider implements ISchemaProvider { - client: any + client: MongoClient constructor(client: any) { this.client = client } - reformatFields(field: InputField ) { + reformatFields(field: {name: string, type: string}): ResponseField { return { field: field.name, type: field.type, + capabilities: ColumnsCapabilities[field.type as keyof typeof ColumnsCapabilities] ?? EmptyCapabilities } } + private collectionCapabilities(): CollectionCapabilities { + return { + dataOperations: ReadWriteOperations, + fieldTypes: FieldTypes, + collectionOperations: CollectionOperations, + encryption: Encryption.notSupported, + indexing: [], + referenceCapabilities: { supportedNamespaces: [] } + } + } + async list(): Promise { await this.ensureSystemTableExists() const resp = await this.client.db() .collection(SystemTable) - .find({}) + .find({}) const l = await resp.toArray() const tables = l.reduce((o: any, d: { _id: string; fields: any }) => ({ ...o, [d._id]: { fields: d.fields } }), {}) return Object.entries(tables) .map(([collectionName, rs]: [string, any]) => ({ id: collectionName, - fields: [...SystemFields, ...rs.fields].map( this.reformatFields.bind(this) ) + fields: [...SystemFields, ...rs.fields].map( this.reformatFields.bind(this) ), + capabilities: this.collectionCapabilities() })) } @@ -37,7 +53,7 @@ export default class SchemaProvider implements ISchemaProvider { const resp = await this.client.db() .collection(SystemTable) - .find({}) + .find({}) const data = await resp.toArray() return data.map((rs: { _id: string }) => rs._id) } @@ -52,7 +68,7 @@ export default class SchemaProvider implements ISchemaProvider { if (!collection) { await this.client.db() .collection(SystemTable) - .insertOne( { _id: collectionName, fields: columns || [] }) + .insertOne({ _id: collectionName as any, fields: columns || [] }) await this.client.db() .createCollection(collectionName) } @@ -98,14 +114,33 @@ export default class SchemaProvider implements ISchemaProvider { { $pull: { fields: { name: { $eq: columnName } } } } ) } - async describeCollection(collectionName: string): Promise { - validateTable(collectionName) + async changeColumnType(collectionName: string, column: InputField): Promise { const collection = await this.collectionDataFor(collectionName) + if (!collection) { throw new CollectionDoesNotExists('Collection does not exists') } + + await this.client.db() + .collection(SystemTable) + .bulkWrite(updateExpressionFor([{ + _id: collection._id, + fields: [...collection.fields.filter((f: InputField) => f.name !== column.name), column] + }])) - return [...SystemFields, ...collection.fields].map( this.reformatFields.bind(this) ) + } + + async describeCollection(collectionName: string): Promise { + validateTable(collectionName) + const collection = await this.collectionDataFor(collectionName) + if (!collection) { + throw new CollectionDoesNotExists('Collection does not exists', collectionName) + } + return { + id: collectionName, + fields: [...SystemFields, ...collection.fields].map( this.reformatFields.bind(this) ), + capabilities: this.collectionCapabilities() + } } async drop(collectionName: string): Promise { @@ -120,11 +155,11 @@ export default class SchemaProvider implements ISchemaProvider { } } - async collectionDataFor(collectionName: string): Promise { //fixme: any + async collectionDataFor(collectionName: string) { validateTable(collectionName) return await this.client.db() .collection(SystemTable) - .findOne({ _id: collectionName }) + .findOne({ _id: collectionName }) } async ensureSystemTableExists(): Promise { diff --git a/libs/external-db-mongo/src/mongo_utils.spec.ts b/libs/external-db-mongo/src/mongo_utils.spec.ts index 8b31003c0..6ff585318 100644 --- a/libs/external-db-mongo/src/mongo_utils.spec.ts +++ b/libs/external-db-mongo/src/mongo_utils.spec.ts @@ -1,5 +1,5 @@ const { InvalidQuery } = require('@wix-velo/velo-external-db-commons').errors -import { unpackIdFieldForItem, validateTable } from './mongo_utils' +import { unpackIdFieldForItem, validateTable, insertExpressionFor, isEmptyObject } from './mongo_utils' describe('Mongo Utils', () => { describe('unpackIdFieldForItem', () => { @@ -48,4 +48,28 @@ describe('Mongo Utils', () => { expect(() => validateTable('someTable')).not.toThrow() }) }) + + describe('insertExpressionFor', () => { + test('insertExpressionFor with upsert set to false will return insert expression', () => { + expect(insertExpressionFor([{ _id: 'itemId' }], false)[0]).toEqual({ insertOne: { document: { _id: 'itemId' } } }) + }) + test('insertExpressionFor with upsert set to true will return update expression', () => { + expect(insertExpressionFor([{ _id: 'itemId' }], true)[0]).toEqual({ + updateOne: { + filter: { _id: 'itemId' }, + update: { $set: { _id: 'itemId' } }, + upsert: true + } + }) + }) + }) + + describe('isEmptyObject', () => { + test('isEmptyObject will return true for empty object', () => { + expect(isEmptyObject({})).toBe(true) + expect(isEmptyObject({ a: {} }.a)).toBe(true) + } + ) + + }) }) diff --git a/libs/external-db-mongo/src/mongo_utils.ts b/libs/external-db-mongo/src/mongo_utils.ts index 3ef8d2a1a..c970c911c 100644 --- a/libs/external-db-mongo/src/mongo_utils.ts +++ b/libs/external-db-mongo/src/mongo_utils.ts @@ -35,14 +35,28 @@ export const isConnected = (client: { topology: { isConnected: () => any } }) => return client && client.topology && client.topology.isConnected() } -const updateExpressionForItem = (item: { _id: any }) => ({ +const insertExpressionForItem = (item: { _id: any }) => ({ + insertOne: { + document: { ...item, _id: item._id as any } + } +}) + +const updateExpressionForItem = (item: { _id: any }, upsert: boolean) => ({ updateOne: { filter: { _id: item._id }, - update: { $set: { ...item } } + update: { $set: { ...item } }, + upsert } }) -export const updateExpressionFor = (items: any[]) => items.map(updateExpressionForItem) +export const insertExpressionFor = (items: any[], upsert: boolean) => { + return upsert? + items.map(i => updateExpressionForItem(i, upsert)): + items.map(i => insertExpressionForItem(i)) +} + + +export const updateExpressionFor = (items: any[], upsert = false) => items.map(i => updateExpressionForItem(i, upsert)) export const unpackIdFieldForItem = (item: { [x: string]: any, _id?: any }) => { if (isObject(item._id)) { @@ -56,3 +70,10 @@ export const unpackIdFieldForItem = (item: { [x: string]: any, _id?: any }) => { export const EmptySort = { sortExpr: { sort: [] }, } + +export interface CollectionObject { + _id: string, + fields: { name: string, type: string, subtype?: string }[] +} + +export const isEmptyObject = (obj: any) => Object.keys(obj).length === 0 && obj.constructor === Object diff --git a/libs/external-db-mongo/src/sql_filter_transformer.spec.ts b/libs/external-db-mongo/src/sql_filter_transformer.spec.ts index 28af5067a..e90566334 100644 --- a/libs/external-db-mongo/src/sql_filter_transformer.spec.ts +++ b/libs/external-db-mongo/src/sql_filter_transformer.spec.ts @@ -1,5 +1,5 @@ import each from 'jest-each' -import Chance = require('chance') +import * as Chance from 'chance' import { AdapterOperators, errors } from '@wix-velo/velo-external-db-commons' import { AdapterFunctions } from '@wix-velo/velo-external-db-types' import { Uninitialized, gen } from '@wix-velo/test-commons' @@ -416,6 +416,28 @@ describe('Sql Parser', () => { havingFilter: { $match: {} }, }) }) + + test('orderAggregationBy', () => { + expect(env.filterParser.orderAggregationBy([ + { fieldName: ctx.fieldName, direction: ctx.direction }, + ])).toEqual({ + $sort: { + [ctx.fieldName]: ctx.direction === 'asc' ? 1 : -1 + } + }) + + expect(env.filterParser.orderAggregationBy([ + { fieldName: ctx.fieldName, direction: ctx.direction }, + { fieldName: ctx.anotherFieldName, direction: ctx.anotherDirection }, + ])).toEqual({ + $sort: { + [ctx.fieldName]: ctx.direction === 'asc' ? 1 : -1, + [ctx.anotherFieldName]: ctx.anotherDirection === 'asc' ? 1 : -1 + } + }) + + + }) }) }) @@ -423,16 +445,18 @@ describe('Sql Parser', () => { }) interface Context { - fieldName: any - fieldValue: any - anotherValue: any - moreValue: any - fieldListValue: any - anotherFieldName: any - moreFieldName: any - filter: any - anotherFilter: any - offset: any + fieldName: string + fieldValue: string + anotherValue: string + moreValue: string + fieldListValue: string[] + anotherFieldName: string + moreFieldName: string + filter: { fieldName: string; operator: string; value: string | string[] } + anotherFilter: { fieldName: string; operator: string; value: string | string[]; } + offset: number + direction: 'asc' | 'desc' + anotherDirection: 'asc' | 'desc' } const ctx : Context = { @@ -446,6 +470,8 @@ describe('Sql Parser', () => { filter: Uninitialized, anotherFilter: Uninitialized, offset: Uninitialized, + direction: Uninitialized, + anotherDirection: Uninitialized, } interface Enviorment { @@ -470,6 +496,9 @@ describe('Sql Parser', () => { ctx.anotherFilter = gen.randomWrappedFilter() ctx.offset = chance.natural({ min: 2, max: 20 }) + + ctx.direction = chance.pickone(['asc', 'desc']) + ctx.anotherDirection = chance.pickone(['asc', 'desc']) }) beforeAll(function() { diff --git a/libs/external-db-mongo/src/sql_filter_transformer.ts b/libs/external-db-mongo/src/sql_filter_transformer.ts index eebcbd7ae..308867257 100644 --- a/libs/external-db-mongo/src/sql_filter_transformer.ts +++ b/libs/external-db-mongo/src/sql_filter_transformer.ts @@ -141,6 +141,15 @@ export default class FilterParser { } } + orderAggregationBy(sort: Sort[]) { + return { + $sort: sort.reduce((acc, s) => { + const direction = s.direction === 'asc'? 1 : -1 + return { ...acc, [s.fieldName]: direction } + }, {}) + } + } + parseSort({ fieldName, direction }: Sort): { expr: MongoFieldSort } | [] { if (typeof fieldName !== 'string') { return [] diff --git a/libs/external-db-mongo/src/supported_operations.ts b/libs/external-db-mongo/src/supported_operations.ts index 1fb2f6506..c266ba31d 100644 --- a/libs/external-db-mongo/src/supported_operations.ts +++ b/libs/external-db-mongo/src/supported_operations.ts @@ -1,5 +1,5 @@ import { AllSchemaOperations } from '@wix-velo/velo-external-db-commons' import { SchemaOperations } from '@wix-velo/velo-external-db-types' -const notSupportedOperations = [SchemaOperations.QueryNestedFields] +const notSupportedOperations = [SchemaOperations.AtomicBulkInsert] export const supportedOperations = AllSchemaOperations.filter(op => !notSupportedOperations.includes(op)) diff --git a/libs/external-db-mongo/tests/drivers/sql_filter_transformer_test_support.ts b/libs/external-db-mongo/tests/drivers/sql_filter_transformer_test_support.ts index e20d595b7..0a40fdb48 100644 --- a/libs/external-db-mongo/tests/drivers/sql_filter_transformer_test_support.ts +++ b/libs/external-db-mongo/tests/drivers/sql_filter_transformer_test_support.ts @@ -7,6 +7,7 @@ export const filterParser = { parseFilter: jest.fn(), orderBy: jest.fn(), parseAggregation: jest.fn(), + orderAggregationBy: jest.fn(), selectFieldsFor: jest.fn() } @@ -23,6 +24,8 @@ export const stubEmptyFilterFor = (filter: any) => { export const stubEmptyOrderByFor = (sort: any) => { when(filterParser.orderBy).calledWith(sort) .mockReturnValue(EmptySort) + when(filterParser.orderAggregationBy).calledWith(sort) + .mockReturnValue({ $sort: {} }) } export const givenOrderByFor = (column: any, sort: any) => { @@ -97,6 +100,7 @@ export const givenIncludeFilterForIdColumn = (filter: any, value: any) => export const reset = () => { filterParser.transform.mockClear() filterParser.orderBy.mockClear() + filterParser.orderAggregationBy.mockClear() filterParser.parseAggregation.mockClear() filterParser.parseFilter.mockClear() filterParser.selectFieldsFor.mockClear() diff --git a/libs/external-db-mongo/tests/e2e-testkit/mongo_resources.ts b/libs/external-db-mongo/tests/e2e-testkit/mongo_resources.ts index d68464377..f1d17243e 100644 --- a/libs/external-db-mongo/tests/e2e-testkit/mongo_resources.ts +++ b/libs/external-db-mongo/tests/e2e-testkit/mongo_resources.ts @@ -2,6 +2,8 @@ import * as compose from 'docker-compose' import init from '../../src/connection_provider' export { supportedOperations } from '../../src/supported_operations' +export * as capabilities from '../../src/mongo_capabilities' + export const connection = async() => { const { connection, schemaProvider, cleanup } = await init({ connectionUri: 'mongodb://root:pass@localhost/testdb' }) diff --git a/libs/external-db-mssql/src/supported_operations.ts b/libs/external-db-mssql/src/supported_operations.ts index 9dd0a149d..b1c9e223b 100644 --- a/libs/external-db-mssql/src/supported_operations.ts +++ b/libs/external-db-mssql/src/supported_operations.ts @@ -1,5 +1,5 @@ import { AllSchemaOperations } from '@wix-velo/velo-external-db-commons' import { SchemaOperations } from '@wix-velo/velo-external-db-types' -const notSupportedOperations = [SchemaOperations.QueryNestedFields, SchemaOperations.FindObject] +const notSupportedOperations = [SchemaOperations.QueryNestedFields, SchemaOperations.FindObject, SchemaOperations.NonAtomicBulkInsert] export const supportedOperations = AllSchemaOperations.filter(op => !notSupportedOperations.includes(op)) diff --git a/libs/external-db-mysql/src/supported_operations.ts b/libs/external-db-mysql/src/supported_operations.ts index 642b18976..a2dc49fa4 100644 --- a/libs/external-db-mysql/src/supported_operations.ts +++ b/libs/external-db-mysql/src/supported_operations.ts @@ -1 +1,5 @@ -export { AllSchemaOperations as supportedOperations } from '@wix-velo/velo-external-db-commons' +import { AllSchemaOperations } from '@wix-velo/velo-external-db-commons' +import { SchemaOperations } from '@wix-velo/velo-external-db-types' +const notSupportedOperations = [SchemaOperations.NonAtomicBulkInsert] + +export const supportedOperations = AllSchemaOperations.filter(op => !notSupportedOperations.includes(op)) diff --git a/libs/external-db-postgres/src/supported_operations.ts b/libs/external-db-postgres/src/supported_operations.ts index 642b18976..a2dc49fa4 100644 --- a/libs/external-db-postgres/src/supported_operations.ts +++ b/libs/external-db-postgres/src/supported_operations.ts @@ -1 +1,5 @@ -export { AllSchemaOperations as supportedOperations } from '@wix-velo/velo-external-db-commons' +import { AllSchemaOperations } from '@wix-velo/velo-external-db-commons' +import { SchemaOperations } from '@wix-velo/velo-external-db-types' +const notSupportedOperations = [SchemaOperations.NonAtomicBulkInsert] + +export const supportedOperations = AllSchemaOperations.filter(op => !notSupportedOperations.includes(op)) diff --git a/libs/external-db-spanner/src/supported_operations.ts b/libs/external-db-spanner/src/supported_operations.ts index 77602c18e..1a43303df 100644 --- a/libs/external-db-spanner/src/supported_operations.ts +++ b/libs/external-db-spanner/src/supported_operations.ts @@ -1,6 +1,6 @@ import { AllSchemaOperations } from '@wix-velo/velo-external-db-commons' import { SchemaOperations } from '@wix-velo/velo-external-db-types' //change column types - https://cloud.google.com/spanner/docs/schema-updates#supported_schema_updates -const notSupportedOperations = [SchemaOperations.ChangeColumnType] +const notSupportedOperations = [SchemaOperations.ChangeColumnType, SchemaOperations.NonAtomicBulkInsert] export const supportedOperations = AllSchemaOperations.filter(op => !notSupportedOperations.includes(op)) diff --git a/libs/test-commons/src/libs/gen.ts b/libs/test-commons/src/libs/gen.ts index fdf332d4f..fa8416ae5 100644 --- a/libs/test-commons/src/libs/gen.ts +++ b/libs/test-commons/src/libs/gen.ts @@ -1,5 +1,6 @@ import * as Chance from 'chance' import { AdapterOperators } from '@wix-velo/velo-external-db-commons' +import { Item } from '@wix-velo/velo-external-db-types' const { eq, gt, gte, include, lt, lte, ne, string_begins, string_ends, string_contains } = AdapterOperators const chance = Chance() @@ -48,8 +49,8 @@ export const randomCollections = () => randomArrayOf( randomCollectionName ) export const randomFieldName = () => chance.word({ length: 5 }) -export const randomEntity = (columns?: any[]) => { - const entity : {[x:string]: any} = { +export const randomEntity = (columns?: string[]) => { + const entity : Item = { _id: chance.guid(), _createdDate: veloDate(), _updatedDate: veloDate(), @@ -65,7 +66,7 @@ export const randomEntity = (columns?: any[]) => { } export const randomNumberEntity = (columns: any[]) => { - const entity : {[x:string]: any} = { + const entity : Item = { _id: chance.guid(), _createdDate: veloDate(), _updatedDate: veloDate(), diff --git a/libs/velo-external-db-types/src/collection_types.ts b/libs/velo-external-db-types/src/collection_types.ts index a6f187df2..cf56870f6 100644 --- a/libs/velo-external-db-types/src/collection_types.ts +++ b/libs/velo-external-db-types/src/collection_types.ts @@ -85,6 +85,8 @@ export enum SchemaOperations { IncludeOperator = 'include', FilterByEveryField = 'filterByEveryField', QueryNestedFields = 'queryNestedFields', + NonAtomicBulkInsert = 'NonAtomicBulkInsert', + AtomicBulkInsert = 'AtomicBulkInsert' } export type InputField = FieldAttributes & { name: string } diff --git a/package.json b/package.json index a9f14d2ad..d3bb83897 100644 --- a/package.json +++ b/package.json @@ -7,7 +7,7 @@ "build:full-image": "nx run velo-external-db:build-image ", "lint": "eslint --cache ./", "lint:fix": "eslint --cache --fix ./", - "test": "npm run test:core; npm run test:mysql; npm run test:postgres; npm run test:mssql; npm run test:spanner", + "test": "npm run test:core; npm run test:mysql; npm run test:postgres; npm run test:mssql; npm run test:spanner; npm run test:mongo;", "test:core": "nx run-many --skip-nx-cache --target=test --projects=@wix-velo/external-db-config,@wix-velo/velo-external-db-core,@wix-velo/external-db-security", "test:postgres": "TEST_ENGINE=postgres nx run-many --skip-nx-cache --target=test --projects=@wix-velo/external-db-postgres,velo-external-db", "test:postgres13": "npm run test:postgres", diff --git a/workspace.json b/workspace.json index ae7677764..d3a5c9c92 100644 --- a/workspace.json +++ b/workspace.json @@ -9,6 +9,7 @@ "@wix-velo/external-db-security": "libs/external-db-security", "@wix-velo/test-commons": "libs/test-commons", "@wix-velo/velo-external-db-commons": "libs/velo-external-db-commons", + "@wix-velo/external-db-mongo": "libs/external-db-mongo", "@wix-velo/velo-external-db-core": "libs/velo-external-db-core", "@wix-velo/velo-external-db-types": "libs/velo-external-db-types", "@wix-velo/external-db-testkit": "libs/external-db-testkit",