Skip to content

Commit

Permalink
feat(lineage-query): impl of lineage query
Browse files Browse the repository at this point in the history
  • Loading branch information
EagleoutIce committed Oct 13, 2024
1 parent 4ea237a commit 7cf9c3c
Show file tree
Hide file tree
Showing 6 changed files with 85 additions and 3 deletions.
7 changes: 7 additions & 0 deletions src/documentation/doc-util/doc-query.ts
Original file line number Diff line number Diff line change
Expand Up @@ -147,6 +147,13 @@ export function asciiSummaryOfQueryResult(formatter: OutputFormatter, totalInMs:
}))`);
}
continue;
} else if(query === 'lineage') {
const out = queryResults as QueryResults<'lineage'>['lineage'];
result.push(`Query: ${bold(query, formatter)} (${printAsMs(out['.meta'].timing, 0)})`);
for(const [criteria, lineage] of Object.entries(out.lineages)) {
result.push(` ╰ ${criteria}: {${summarizeIdsIfTooLong([...lineage])}}`);
}
continue;
}

result.push(`Query: ${bold(query, formatter)}`);
Expand Down
26 changes: 26 additions & 0 deletions src/queries/catalog/lineage-query/lineage-query-executor.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
import type { BasicQueryData } from '../../query';
import type {
LineageQuery,
LineageQueryResult
} from './lineage-query-format';
import { log } from '../../../util/log';
import { getLineage } from '../../../cli/repl/commands/repl-lineage';

/* TODO: deprecate the request */
export function executeLineageQuery({ graph, ast }: BasicQueryData, queries: readonly LineageQuery[]): LineageQueryResult {
const start = Date.now();
const result: LineageQueryResult['lineages'] = {};
for(const { criterion } of queries) {
if(result[criterion]) {
log.warn('Duplicate criterion in lineage query:', criterion);
}
result[criterion] = getLineage(criterion, graph, ast.idMap);
}

return {
'.meta': {
timing: Date.now() - start
},
lineages: result
};
}
16 changes: 16 additions & 0 deletions src/queries/catalog/lineage-query/lineage-query-format.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
import type { BaseQueryFormat, BaseQueryResult } from '../../base-query-format';
import type { SingleSlicingCriterion } from '../../../slicing/criterion/parse';
import type { NodeId } from '../../../r-bridge/lang-4.x/ast/model/processing/node-id';

/**
* Calculates the lineage of the given criterion.
*/
export interface LineageQuery extends BaseQueryFormat {
readonly type: 'lineage';
readonly criterion: SingleSlicingCriterion;
}

export interface LineageQueryResult extends BaseQueryResult {
/** Maps each criterion to the found lineage, duplicates are ignored. */
readonly lineages: Record<SingleSlicingCriterion, Set<NodeId>>;
}
8 changes: 7 additions & 1 deletion src/queries/query-schema.ts
Original file line number Diff line number Diff line change
Expand Up @@ -38,14 +38,20 @@ export const StaticSliceQuerySchema = Joi.object({
noMagicComments: Joi.boolean().optional().description('Should the magic comments (force-including lines within the slice) be ignored?')
}).description('Slice query used to slice the dataflow graph');

export const LineageQuerySchema = Joi.object({
type: Joi.string().valid('lineage').required().description('The type of the query.'),
id: Joi.string().required().description('The ID of the node to get the lineage of.')
}).description('Lineage query used to find the lineage of a node in the dataflow graph');


export const SupportedQueriesSchema = Joi.alternatives(
CallContextQuerySchema,
DataflowQuerySchema,
IdMapQuerySchema,
NormalizedAstQuerySchema,
DataflowClusterQuerySchema,
StaticSliceQuerySchema
StaticSliceQuerySchema,
LineageQuerySchema
).description('Supported queries');

export const CompoundQuerySchema = Joi.object({
Expand Down
7 changes: 5 additions & 2 deletions src/queries/query.ts
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,10 @@ import type { DataflowClusterQuery } from './catalog/cluster-query/cluster-query
import { executeDataflowClusterQuery } from './catalog/cluster-query/cluster-query-executor';
import type { StaticSliceQuery } from './catalog/static-slice-query/static-slice-query-format';
import { executeStaticSliceClusterQuery } from './catalog/static-slice-query/static-slice-query-executor';
import type { LineageQuery } from './catalog/lineage-query/lineage-query-format';
import { executeLineageQuery } from './catalog/lineage-query/lineage-query-executor';

export type Query = CallContextQuery | DataflowQuery | NormalizedAstQuery | IdMapQuery | DataflowClusterQuery | StaticSliceQuery;
export type Query = CallContextQuery | DataflowQuery | NormalizedAstQuery | IdMapQuery | DataflowClusterQuery | StaticSliceQuery | LineageQuery;

export type QueryArgumentsWithType<QueryType extends BaseQueryFormat['type']> = Query & { type: QueryType };

Expand All @@ -41,7 +43,8 @@ export const SupportedQueries = {
'id-map': executeIdMapQuery,
'normalized-ast': executeNormalizedAstQuery,
'dataflow-cluster': executeDataflowClusterQuery,
'static-slice': executeStaticSliceClusterQuery
'static-slice': executeStaticSliceClusterQuery,
'lineage': executeLineageQuery
} as const satisfies SupportedQueries;

export type SupportedQueryTypes = keyof typeof SupportedQueries;
Expand Down
24 changes: 24 additions & 0 deletions test/functionality/dataflow/query/lineage-query-tests.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
import { assertQuery } from '../../_helper/query';
import { label } from '../../_helper/label';
import { withShell } from '../../_helper/shell';
import type {
LineageQuery,
LineageQueryResult
} from '../../../../src/queries/catalog/lineage-query/lineage-query-format';
import { getLineage } from '../../../../src/cli/repl/commands/repl-lineage';

describe('Lineage Query', withShell(shell => {
function testQuery(name: string, code: string, query: readonly LineageQuery[]) {
assertQuery(label(name), shell, code, query, ({ dataflow }) => ({
'lineage': {
lineages: query.reduce((acc, { criterion }) => {
acc[criterion] = getLineage(criterion, dataflow.graph);
return acc;
}, {} as LineageQueryResult['lineages'])
}
}));
}

testQuery('Single Expression', 'x + 1', [{ type: 'lineage', criterion: '1@x' }]);
testQuery('Multiple Queries', 'x + 1', [{ type: 'lineage', criterion: '1@x' }, { type: 'lineage', criterion: '1@x' }, { type: 'lineage', criterion: '1@x' }]);
}));

0 comments on commit 7cf9c3c

Please sign in to comment.