diff --git a/src/datasources/tag/TagDataSource.test.ts b/src/datasources/tag/TagDataSource.test.ts index ff059cc..47d47bb 100644 --- a/src/datasources/tag/TagDataSource.test.ts +++ b/src/datasources/tag/TagDataSource.test.ts @@ -12,7 +12,7 @@ import { setupDataSource, } from 'test/fixtures'; import { TagDataSource } from './TagDataSource'; -import { TagQuery, TagQueryType, TagWithValue } from './types'; +import { TagQuery, TagQueryType, TagWithValue, TagDataType } from './types'; let ds: TagDataSource, backendSrv: MockProxy, templateSrv: MockProxy; @@ -124,24 +124,24 @@ describe('queries', () => { test('current value for all data types', async () => { backendSrv.fetch .mockReturnValueOnce(createQueryTagsResponse([{ - tag: { type: 'INT', path: 'tag1' }, + tag: { type: TagDataType.INT, path: 'tag1' }, current: { value: { value: '3' } } }])) .mockReturnValueOnce(createQueryTagsResponse([{ - tag: { type: 'DOUBLE', path: 'tag2' }, + tag: { type: TagDataType.DOUBLE, path: 'tag2' }, current: { value: { value: '3.3' } } }])) .mockReturnValueOnce(createQueryTagsResponse([{ - tag: { type: 'STRING', path: 'tag3' }, + tag: { type: TagDataType.STRING, path: 'tag3' }, current: { value: { value: 'foo' } } }])) .mockReturnValueOnce(createQueryTagsResponse([{ - tag: { type: 'BOOLEAN', path: 'tag4' }, + tag: { type: TagDataType.BOOLEAN, path: 'tag4' }, current: { value: { value: 'True' } } }])) .mockReturnValueOnce( createQueryTagsResponse([{ - tag: { type: 'U_INT64', path: 'tag5' }, + tag: { type: TagDataType.U_INT64, path: 'tag5' }, current: { value: { value: '2147483648' } } }]) ); @@ -180,9 +180,73 @@ describe('queries', () => { }) ) .mockReturnValue( - createTagHistoryResponse('my.tag', 'DOUBLE', [ - { timestamp: '2023-01-01T00:00:00Z', value: '1' }, - { timestamp: '2023-01-01T00:01:00Z', value: '2' }, + createTagHistoryResponse([ + { + path: 'my.tag', + type: TagDataType.DOUBLE, + values: [ + { timestamp: '2023-01-01T00:00:00Z', value: '1' }, + { timestamp: '2023-01-01T00:01:00Z', value: '2' }, + ] + } + ]) + ); + + const result = await ds.query(queryRequest); + + expect(result.data).toMatchSnapshot(); + }); + + test('numeric multiple tags history', async () => { + const queryRequest = buildQuery({ type: TagQueryType.History, path: 'my.tag.*' }); + + backendSrv.fetch + .calledWith(requestMatching({ url: '/nitag/v2/query-tags-with-values', data: { filter: 'path = "my.tag.*"' } })) + .mockReturnValue(createQueryTagsResponse([ + { tag: { path: 'my.tag.1' } }, + { tag: { path: 'my.tag.2' } }, + { tag: { path: 'my.tag.3' } } + ])); + + backendSrv.fetch + .calledWith( + requestMatching({ + url: '/nitaghistorian/v2/tags/query-decimated-history', + data: { + paths: ['my.tag.1', 'my.tag.2', 'my.tag.3'], + workspace: '1', + startTime: queryRequest.range.from.toISOString(), + endTime: queryRequest.range.to.toISOString(), + decimation: 300, + }, + }) + ) + .mockReturnValue( + createTagHistoryResponse([ + { + path: 'my.tag.1', + type: TagDataType.DOUBLE, + values: [ + { timestamp: '2023-01-01T00:00:00Z', value: '1' }, + { timestamp: '2023-01-01T00:01:00Z', value: '2' }, + ] + }, + { + path: 'my.tag.2', + type: TagDataType.DOUBLE, + values: [ + { timestamp: '2023-01-01T00:00:00Z', value: '2' }, + { timestamp: '2023-01-01T00:01:00Z', value: '3' }, + ] + }, + { + path: 'my.tag.3', + type: TagDataType.DOUBLE, + values: [ + { timestamp: '2023-01-01T00:00:00Z', value: '3' }, + { timestamp: '2023-01-01T00:02:00Z', value: '4' }, + ] + } ]) ); @@ -194,9 +258,15 @@ describe('queries', () => { test('string tag history', async () => { backendSrv.fetch.mockReturnValueOnce(createQueryTagsResponse()); backendSrv.fetch.mockReturnValueOnce( - createTagHistoryResponse('my.tag', 'STRING', [ - { timestamp: '2023-01-01T00:00:00Z', value: '3.14' }, - { timestamp: '2023-01-01T00:01:00Z', value: 'foo' }, + createTagHistoryResponse([ + { + path: 'my.tag', + type: TagDataType.STRING, + values: [ + { timestamp: '2023-01-01T00:00:00Z', value: '3.14' }, + { timestamp: '2023-01-01T00:01:00Z', value: 'foo' }, + ] + } ]) ); @@ -205,6 +275,108 @@ describe('queries', () => { expect(result.data).toMatchSnapshot(); }); + test('add workspace prefix only when a tag with the same path exists in multiple workspaces', async () => { + const queryRequest = buildQuery({ type: TagQueryType.History, path: 'my.tag.*' }); + + backendSrv.fetch + .calledWith(requestMatching({ url: '/nitag/v2/query-tags-with-values', data: { filter: 'path = "my.tag.*"' } })) + .mockReturnValue(createQueryTagsResponse([ + { tag: { path: 'my.tag.1', workspace: '1' } }, + { tag: { path: 'my.tag.2', workspace: '1' } }, + { tag: { path: 'my.tag.1', workspace: '2' } }, + { tag: { path: 'my.tag.2', workspace: '2' } }, + { tag: { path: 'my.tag.3', workspace: '2' } }, + { tag: { path: 'my.tag.4', workspace: '2' } } + ])); + + backendSrv.fetch + .calledWith( + requestMatching({ + url: '/nitaghistorian/v2/tags/query-decimated-history', + data: { + paths: ['my.tag.1', 'my.tag.2'], + workspace: '1', + startTime: queryRequest.range.from.toISOString(), + endTime: queryRequest.range.to.toISOString(), + decimation: 300, + }, + }) + ) + .mockReturnValue( + createTagHistoryResponse([ + { + path: 'my.tag.1', + type: TagDataType.DOUBLE, + values: [ + { timestamp: '2023-01-01T00:00:00Z', value: '1' }, + { timestamp: '2023-01-01T00:01:00Z', value: '2' }, + ], + }, + { + path: 'my.tag.2', + type: TagDataType.DOUBLE, + values: [ + { timestamp: '2023-01-01T00:00:00Z', value: '2' }, + { timestamp: '2023-01-01T00:01:00Z', value: '3' }, + ] + } + ]) + ) + + backendSrv.fetch.calledWith( + requestMatching({ + url: '/nitaghistorian/v2/tags/query-decimated-history', + data: { + paths: ['my.tag.1', 'my.tag.2', 'my.tag.3', 'my.tag.4'], + workspace: '2', + startTime: queryRequest.range.from.toISOString(), + endTime: queryRequest.range.to.toISOString(), + decimation: 300, + }, + }) + ) + .mockReturnValue( + createTagHistoryResponse([ + { + path: 'my.tag.1', + type: TagDataType.DOUBLE, + values: [ + { timestamp: '2023-01-01T00:00:00Z', value: '1' }, + { timestamp: '2023-01-01T00:01:00Z', value: '2' }, + ], + }, + { + path: 'my.tag.2', + type: TagDataType.DOUBLE, + values: [ + { timestamp: '2023-01-01T00:00:00Z', value: '2' }, + { timestamp: '2023-01-01T00:01:00Z', value: '3' }, + ] + }, + { + path: 'my.tag.3', + type: TagDataType.DOUBLE, + values: [ + { timestamp: '2023-01-01T00:00:00Z', value: '3' }, + { timestamp: '2023-01-01T00:01:00Z', value: '4' }, + ], + }, + { + path: 'my.tag.4', + type: TagDataType.DOUBLE, + values: [ + { timestamp: '2023-01-01T00:00:00Z', value: '4' }, + { timestamp: '2023-01-01T00:01:00Z', value: '5' }, + ] + } + ]) + ) + + const result = await ds.query(queryRequest); + + expect(result.data).toMatchSnapshot(); + }); + test('decimation parameter does not go above 1000', async () => { const queryRequest = buildQuery({ type: TagQueryType.History, path: 'my.tag' }); queryRequest.maxDataPoints = 1500; @@ -212,9 +384,15 @@ describe('queries', () => { backendSrv.fetch.mockReturnValueOnce(createQueryTagsResponse()); backendSrv.fetch.mockReturnValueOnce( - createTagHistoryResponse('my.tag', 'INT', [ - { timestamp: '2023-01-01T00:00:00Z', value: '1' }, - { timestamp: '2023-01-01T00:01:00Z', value: '2' }, + createTagHistoryResponse([ + { + path: 'my.tag', + type: TagDataType.INT, + values: [ + { timestamp: '2023-01-01T00:00:00Z', value: '1' }, + { timestamp: '2023-01-01T00:01:00Z', value: '2' }, + ] + } ]) ); @@ -236,7 +414,12 @@ describe('queries', () => { test('filters by workspace if provided', async () => { backendSrv.fetch.mockReturnValueOnce(createQueryTagsResponse([{ tag: { workspace: '2' } }])); - backendSrv.fetch.mockReturnValueOnce(createTagHistoryResponse('my.tag', 'DOUBLE', [])); + backendSrv.fetch.mockReturnValueOnce(createTagHistoryResponse([{ + path: 'my.tag', + type: TagDataType.DOUBLE, + values: [] + } + ])); await ds.query(buildQuery({ type: TagQueryType.History, path: 'my.tag', workspace: '2' })); @@ -247,7 +430,11 @@ describe('queries', () => { test('retries failed request with 429 status', async () => { backendSrv.fetch.mockReturnValueOnce(createQueryTagsResponse()); backendSrv.fetch.mockReturnValueOnce(createFetchError(429)); - backendSrv.fetch.mockReturnValueOnce(createTagHistoryResponse('my.tag', 'INT', [])); + backendSrv.fetch.mockReturnValueOnce(createTagHistoryResponse([{ + path: 'my.tag', + type: TagDataType.INT, + values: [] + }])); await ds.query(buildQuery({ type: TagQueryType.History, path: 'my.tag' })); @@ -299,7 +486,7 @@ describe('queries', () => { tagsWithValues: [{ tag: { datatype: 'DOUBLE', path: 'my.tag', workspace_id: '1' } }], }) ) - .mockReturnValueOnce(createTagHistoryResponse('my.tag', 'DOUBLE', [])); + .mockReturnValueOnce(createTagHistoryResponse([{ path: 'my.tag', type: TagDataType.DOUBLE, values: [] }])); await ds.query(buildQuery({ path: 'my.tag', type: TagQueryType.History })); @@ -352,12 +539,21 @@ function createQueryTagsResponse( return createFetchResponse({ tagsWithValues: [{ current: { value: { value: '3.14' }, timestamp: '2023-10-04T00:00:00.000000Z' }, - tag: { type: 'DOUBLE', path: 'my.tag', properties: {}, workspace: '1' }, + tag: { type: TagDataType.DOUBLE, path: 'my.tag', properties: {}, workspace: '1' }, }], }); } } -function createTagHistoryResponse(path: string, type: string, values: Array<{ timestamp: string; value: string }>) { - return createFetchResponse({ results: { [path]: { type, values } } }); + +function createTagHistoryResponse(tagsHistory: Array<{ + path: string, + type: TagDataType, + values: Array<{ timestamp: string; value: string }> +}>) { + const results: { [key: string]: { type: string, values: any[] } } = {}; + tagsHistory.forEach(({ path, type, values }) => { + results[path] = { type, values }; + }); + return createFetchResponse({ results }); } diff --git a/src/datasources/tag/TagDataSource.ts b/src/datasources/tag/TagDataSource.ts index aceb1c9..73a1b75 100644 --- a/src/datasources/tag/TagDataSource.ts +++ b/src/datasources/tag/TagDataSource.ts @@ -1,17 +1,25 @@ import { DataFrameDTO, DataSourceInstanceSettings, - dateTime, DataQueryRequest, TimeRange, - FieldDTO, FieldType, TestDataSourceResponse, + FieldConfig, + dateTime, } from '@grafana/data'; import { BackendSrv, TemplateSrv, getBackendSrv, getTemplateSrv } from '@grafana/runtime'; import { DataSourceBase } from 'core/DataSourceBase'; -import { TagHistoryResponse, TagQuery, TagQueryType, TagsWithValues, TagWithValue } from './types'; -import { Throw } from 'core/utils'; +import { + TagHistoryResponse, + TagQuery, + TagQueryType, + TagsWithValues, + TagWithValue, + TimeAndTagTypeValues, + TypeAndValues, +} from './types'; +import { Throw, getWorkspaceName } from 'core/utils'; export class TagDataSource extends DataSourceBase { constructor( @@ -37,9 +45,7 @@ export class TagDataSource extends DataSourceBase { this.templateSrv.replace(query.path, scopedVars), this.templateSrv.replace(query.workspace, scopedVars) ); - - const tag = tagsWithValues[0].tag - const name = tag.properties?.displayName ?? tag.path; + const workspaces = await this.getWorkspaces(); const result: DataFrameDTO = { refId: query.refId, fields: [] }; if (query.type === TagQueryType.Current) { @@ -73,18 +79,69 @@ export class TagDataSource extends DataSourceBase { return result } else { - const history = await this.getTagHistoryValues(tag.path, tag.workspace ?? tag.workspace_id, range, maxDataPoints); - result.fields = [ - { name: 'time', values: history.datetimes }, - { name, values: history.values }, - ]; - } + const workspaceTagMap: Record = {}; + const tagPropertiesMap: Record | null> = {}; - if (query.properties) { - result.fields = result.fields.concat(this.getPropertiesAsFields(tag.properties)); - } + // Identify tags that exist in more than one workspace + const tagPathCount: Record = {}; + for (const tagWithValue of tagsWithValues) { + tagPathCount[tagWithValue.tag.path] = (tagPathCount[tagWithValue.tag.path] || 0) + 1; + } + + for (const tagWithValue of tagsWithValues) { + const workspace = tagWithValue.tag.workspace ?? tagWithValue.tag.workspace_id; + if (!workspaceTagMap[workspace]) { + workspaceTagMap[workspace] = []; + } + workspaceTagMap[workspace].push(tagWithValue); + const prefixedPath = tagPathCount[tagWithValue.tag.path] > 1 + ? `${getWorkspaceName(workspaces, workspace)}.${tagWithValue.tag.path}` + : tagWithValue.tag.path; + tagPropertiesMap[prefixedPath] = tagWithValue.tag.properties; + } - return result; + let tagsDecimatedHistory: { [key: string]: TypeAndValues } = {}; + for (const workspace in workspaceTagMap) { + const tagHistoryResponse = await this.getTagHistoryWithChunks( + workspaceTagMap[workspace], + workspace, + range, + maxDataPoints + ) + for (const path in tagHistoryResponse.results) { + const prefixedPath = tagPathCount[path] > 1 + ? `${getWorkspaceName(workspaces, workspace)}.${path}` + : path; + tagsDecimatedHistory[prefixedPath] = tagHistoryResponse.results[path]; + } + } + + const mergedTagValuesWithType = this.mergeTagsHistoryValues(tagsDecimatedHistory); + result.fields.push({ + name: 'time', values: mergedTagValuesWithType.timestamps.map(v => dateTime(v).valueOf()), type: FieldType.time + }); + + for (const path in mergedTagValuesWithType.values) { + const config: FieldConfig = {}; + const tagProps = tagPropertiesMap[path] + if (tagProps?.units) { + config.unit = tagProps.units + } + if (tagProps?.displayName) { + config.displayName = tagProps.displayName + config.displayNameFromDS = tagProps.displayName + } + result.fields.push({ + name: path, + values: mergedTagValuesWithType.values[path].values.map((value) => { + return this.convertTagValue(mergedTagValuesWithType.values[path].type, value) + }), + config + }); + } + + return result + } } private async getMostRecentTags(path: string, workspace: string) { @@ -103,36 +160,42 @@ export class TagDataSource extends DataSourceBase { return response.tagsWithValues.length ? response.tagsWithValues : Throw(`No tags matched the path '${path}'`) } - private async getTagHistoryValues(path: string, workspace: string, range: TimeRange, intervals?: number) { - const response = await this.post(this.tagHistoryUrl + '/query-decimated-history', { - paths: [path], + private async getTagHistoryWithChunks(paths: TagWithValue[], workspace: string, range: TimeRange, intervals?: number): Promise { + const pathChunks: TagWithValue[][] = []; + for (let i = 0; i < paths.length; i += 10) { + pathChunks.push(paths.slice(i, i + 10)); + } + // Fetch and aggregate the data from each chunk + const aggregatedResults: TagHistoryResponse = { results: {} }; + for (const chunk of pathChunks) { + const chunkResult = await this.getTagHistoryValues(chunk.map(tag => tag.tag.path), workspace, range, intervals); + // Merge the results from the current chunk with the aggregated results + for (const [path, data] of Object.entries(chunkResult.results)) { + if (!aggregatedResults.results[path]) { + aggregatedResults.results[path] = data; + } else { + aggregatedResults.results[path].values = aggregatedResults.results[path].values.concat(data.values); + } + } + } + + return aggregatedResults; + } + + private async getTagHistoryValues(paths: string[], workspace: string, range: TimeRange, intervals?: number): Promise { + return await this.post(`${this.tagHistoryUrl}/query-decimated-history`, { + paths, workspace, startTime: range.from.toISOString(), endTime: range.to.toISOString(), decimation: intervals ? Math.min(intervals, 1000) : 500, }); - - const { type, values } = response.results[path]; - return { - datetimes: values.map(v => dateTime(v.timestamp).valueOf()), - values: values.map(v => this.convertTagValue(type, v.value)), - }; - } + }; private convertTagValue(type: string, value?: string) { return value && ['DOUBLE', 'INT', 'U_INT64'].includes(type) ? Number(value) : value; } - private getPropertiesAsFields(properties: Record | null): FieldDTO[] { - if (!properties) { - return []; - } - - return Object.keys(properties) - .filter(name => !name.startsWith('nitag')) - .map(name => ({ name, values: [properties[name]] })); - } - private getAllProperties(data: TagWithValue[]) { const props: Set = new Set(); data.forEach((tag) => { @@ -152,6 +215,38 @@ export class TagDataSource extends DataSourceBase { return Boolean(query.path); } + mergeTagsHistoryValues = (history: Record): TimeAndTagTypeValues => { + const timestampsSet: Set = new Set(); + const values: TimeAndTagTypeValues = { + timestamps: [], + values: {} + }; + for (const path in history) { + for (const { timestamp } of history[path].values) { + timestampsSet.add(timestamp); + } + } + // Uniq timestamps from history data + const timestamps = [...timestampsSet]; + // Sort timestamps to ensure a consistent order + timestamps.sort(); + values.timestamps = timestamps; + + // Initialize arrays for each key + for (const path in history) { + values.values[path] = { 'type': history[path].type, 'values': new Array(timestamps.length).fill(null) }; + } + // Populate the values arrays + for (const path in history) { + for (const historicalValue of history[path].values) { + const index = timestamps.indexOf(historicalValue.timestamp); + values.values[path]['values'][index] = historicalValue.value; + } + } + + return values; + } + async testDatasource(): Promise { await this.get(this.tagUrl + '/tags-count'); return { status: 'success', message: 'Data source connected and authentication successful!' }; diff --git a/src/datasources/tag/__snapshots__/TagDataSource.test.ts.snap b/src/datasources/tag/__snapshots__/TagDataSource.test.ts.snap index dfca11b..1a7c73b 100644 --- a/src/datasources/tag/__snapshots__/TagDataSource.test.ts.snap +++ b/src/datasources/tag/__snapshots__/TagDataSource.test.ts.snap @@ -430,12 +430,14 @@ exports[`queries numeric tag history 1`] = ` "fields": [ { "name": "time", + "type": "time", "values": [ 1672531200000, 1672531260000, ], }, { + "config": {}, "name": "my.tag", "values": [ 1, @@ -448,6 +450,52 @@ exports[`queries numeric tag history 1`] = ` ] `; +exports[`queries numeric multiple tags history 1`] = ` +[ + { + "fields": [ + { + "name": "time", + "type": "time", + "values": [ + 1672531200000, + 1672531260000, + 1672531320000, + ], + }, + { + "config": {}, + "name": "my.tag.1", + "values": [ + 1, + 2, + null, + ], + }, + { + "config": {}, + "name": "my.tag.2", + "values": [ + 2, + 3, + null, + ], + }, + { + "config": {}, + "name": "my.tag.3", + "values": [ + 3, + null, + 4, + ], + }, + ], + "refId": "A", + }, +] +`; + exports[`queries replaces tag path with variable 1`] = ` [ { @@ -486,12 +534,14 @@ exports[`queries string tag history 1`] = ` "fields": [ { "name": "time", + "type": "time", "values": [ 1672531200000, 1672531260000, ], }, { + "config": {}, "name": "my.tag", "values": [ "3.14", @@ -599,3 +649,69 @@ exports[`queries uses displayName property 1`] = ` }, ] `; + +exports[`queries add workspace prefix only when a tag with the same path exists in multiple workspaces 1`] = ` +[ + { + "fields": [ + { + "name": "time", + "type": "time", + "values": [ + 1672531200000, + 1672531260000, + ], + }, + { + "config": {}, + "name": "Default workspace.my.tag.1", + "values": [ + 1, + 2, + ], + }, + { + "config": {}, + "name": "Default workspace.my.tag.2", + "values": [ + 2, + 3, + ], + }, + { + "config": {}, + "name": "Other workspace.my.tag.1", + "values": [ + 1, + 2, + ], + }, + { + "config": {}, + "name": "Other workspace.my.tag.2", + "values": [ + 2, + 3, + ], + }, + { + "config": {}, + "name": "my.tag.3", + "values": [ + 3, + 4, + ], + }, + { + "config": {}, + "name": "my.tag.4", + "values": [ + 4, + 5, + ], + }, + ], + "refId": "A", + }, +] +`; diff --git a/src/datasources/tag/components/TagQueryEditor.test.tsx b/src/datasources/tag/components/TagQueryEditor.test.tsx index c07b1ea..ceeb228 100644 --- a/src/datasources/tag/components/TagQueryEditor.test.tsx +++ b/src/datasources/tag/components/TagQueryEditor.test.tsx @@ -27,9 +27,11 @@ it('renders with initial query and updates when user makes changes', async () => expect(screen.getByRole('radio', { name: 'History' })).toBeChecked(); expect(screen.getByLabelText('Tag path')).toHaveValue('my.tag'); expect(screen.getByText('Default workspace')).toBeInTheDocument(); + expect(screen.queryByRole('checkbox')).not.toBeInTheDocument(); // Users changes query type await userEvent.click(screen.getByRole('radio', { name: 'Current' })); + expect(screen.queryByRole('checkbox')).toBeInTheDocument(); expect(onChange).toHaveBeenCalledWith(expect.objectContaining({ type: TagQueryType.Current })); // User types in new tag path diff --git a/src/datasources/tag/components/TagQueryEditor.tsx b/src/datasources/tag/components/TagQueryEditor.tsx index ec6c08a..0aab755 100644 --- a/src/datasources/tag/components/TagQueryEditor.tsx +++ b/src/datasources/tag/components/TagQueryEditor.tsx @@ -10,7 +10,6 @@ type Props = QueryEditorProps; export function TagQueryEditor({ query, onChange, onRunQuery, datasource }: Props) { query = datasource.prepareQuery(query); - const workspaces = useWorkspaceOptions(datasource); const onTypeChange = (value: TagQueryType) => { @@ -36,10 +35,10 @@ export function TagQueryEditor({ query, onChange, onRunQuery, datasource }: Prop return ( <> - + - +