diff --git a/examples/calculator/src/index.ts b/examples/calculator/src/index.ts index 9967b64..0c799ba 100644 --- a/examples/calculator/src/index.ts +++ b/examples/calculator/src/index.ts @@ -1,4 +1,4 @@ -import { token as T } from 'terrario'; +import { tokenParsers as T } from 'terrario'; import { Operator, buildPrattParser } from './pratt.js'; type Expr = { kind: 'number', value: number } | Operator; diff --git a/examples/calculator/src/pratt.ts b/examples/calculator/src/pratt.ts index c1b7482..a346ed0 100644 --- a/examples/calculator/src/pratt.ts +++ b/examples/calculator/src/pratt.ts @@ -1,4 +1,4 @@ -import { token as T } from 'terrario'; +import { tokenParsers as T } from 'terrario'; type OperatorInfo = { kind: 'prefix' | 'postfix', diff --git a/examples/json/src/index.ts b/examples/json/src/index.ts index 6617ca1..9549696 100644 --- a/examples/json/src/index.ts +++ b/examples/json/src/index.ts @@ -1,6 +1,6 @@ -import { string as T } from 'terrario'; +import * as T from 'terrario'; -const spaces = T.str(/[ \t\r\n]/).many(); +const spaces = T.token(/[ \t\r\n]/).many(); type JsonValue = null | boolean | string | number | Record | unknown[]; @@ -31,35 +31,35 @@ const lang = T.language({ r.number, ]), - null: r => T.str('null').map(() => null), + null: r => T.token('null').map(() => null), bool: r => T.alt([ - T.str('true'), - T.str('false'), + T.token('true'), + T.token('false'), ]).map(value => (value === 'true')), string: r => T.seq([ - T.str('"'), - T.char.many({ notMatch: T.alt([T.str('"'), T.cr, T.lf]) }).text(), - T.str('"'), + T.token('"'), + T.any.many({ notMatch: T.alt([T.token('"'), T.cr, T.lf]) }).span(), + T.token('"'), ], 1), number: r => T.alt([ T.seq([ - T.str(/[+-]/).option(), - T.str(/[0-9]/).many(1), + T.token(/[+-]/).option(), + T.token(/[0-9]/).many(1), T.seq([ - T.str('.'), - T.str(/[0-9]/).many(1), + T.token('.'), + T.token(/[0-9]/).many(1), ]).option(), - ]).text(), + ]).span(), ]).map(value => Number(value)), object: r => { const entry = T.seq([ r.string, spaces, - T.str(':'), + T.token(':'), spaces, r.value, ]).map(value => { @@ -67,11 +67,11 @@ const lang = T.language({ }); const separator = T.seq([ spaces, - T.str(','), + T.token(','), spaces, ]); return T.seq([ - T.str('{'), + T.token('{'), spaces, T.seq([ entry, @@ -80,7 +80,7 @@ const lang = T.language({ ], 1).many(), ]).map(x => [x[0], ...x[1]]).option(), spaces, - T.str('}'), + T.token('}'), ], 2).map(value => { if (value == null) { return {}; @@ -96,11 +96,11 @@ const lang = T.language({ array: r => { const separator = T.seq([ spaces, - T.str(','), + T.token(','), spaces, ]); return T.seq([ - T.str('['), + T.token('['), spaces, T.seq([ r.value, @@ -110,7 +110,7 @@ const lang = T.language({ ], 1).many(), ]).map(x => [x[0], ...x[1]]).option(), spaces, - T.str(']'), + T.token(']'), ], 2).map(value => { return (value != null ? value : []); }); diff --git a/src/index.ts b/src/index.ts index 78b7852..c0e7940 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,7 +1,79 @@ -import * as string from './string-parser.js'; -import * as token from './token-parser.js'; +import { + Failure, + Language, + LanguageSource, + LazyContext, + LazyParserOpts, + ParserContext, + ParserHandler, + Result, + ResultType, + ResultTypes, + StrictParserOpts, + Success, + Parser, + alt, + any, + cr, + crlf, + eof, + failure, + language, + lazy, + lf, + lineBegin, + lineEnd, + match, + newline, + notMatch, + parser, + seq, + sof, + succeeded, + success, + token, + where, +} from './string-parsers.js'; export { - string, + Failure, + Language, + LanguageSource, + LazyContext, + LazyParserOpts, + ParserContext, + ParserHandler, + Result, + ResultType, + ResultTypes, + StrictParserOpts, + Success, + Parser, + alt, + any, + cr, + crlf, + eof, + failure, + language, + lazy, + lf, + lineBegin, + lineEnd, + match, + newline, + notMatch, + parser, + seq, + sof, + succeeded, + success, token, + where, +}; + +import * as tokenParsers from './token-parsers.js'; + +export { + tokenParsers, }; diff --git a/src/string-parser.ts b/src/string-parsers.ts similarity index 80% rename from src/string-parser.ts rename to src/string-parsers.ts index 9982fb8..2198efe 100644 --- a/src/string-parser.ts +++ b/src/string-parsers.ts @@ -1,11 +1,25 @@ -import { success, failure, Result } from './result.js'; +import { + Failure, + Result, + Success, + failure, + success, +} from './result.js'; + +export { + Failure, + Result, + Success, + failure, + success, +}; /** * String Parser * * @public */ -export class StringParser { +export class Parser { tag: string; ctx: ParserContext | LazyContext; @@ -109,7 +123,7 @@ export class StringParser { * * @public */ - map(fn: (value: U) => V): StringParser { + map(fn: (value: U) => V): Parser { return createParser((input, index, state) => { const result = this.exec(input, state, index); if (!result.success) { @@ -124,14 +138,14 @@ export class StringParser { * * @public */ - many(min?: number, max?: number): StringParser + many(min?: number, max?: number): Parser /** * Create a new parser that tries to apply the parser iteratively. * * @public */ - many(opts: { min?: number, max?: number, notMatch?: StringParser }): StringParser - many(arg1?: number | { min?: number, max?: number, notMatch?: StringParser }, arg2?: number): StringParser { + many(opts: { min?: number, max?: number, notMatch?: Parser }): Parser + many(arg1?: number | { min?: number, max?: number, notMatch?: Parser }, arg2?: number): Parser { if (typeof arg1 === 'number') { // with min, max return many(this, { min: arg1, max: arg2 }); @@ -148,7 +162,7 @@ export class StringParser { * * @public */ - option(): StringParser { + option(): Parser { return alt([ this, succeeded(null), @@ -163,7 +177,7 @@ export class StringParser { * * @public */ - state(key: string, value: (state: any) => any): StringParser { + state(key: string, value: (state: any) => any): Parser { return createParser((input, index, state) => { const storedValue = state[key]; state[key] = value(state); @@ -175,11 +189,11 @@ export class StringParser { /** * Create a new parser that wraps the current parser. - * The generated parser will return the text in the range matched by the inner parser. + * The generated parser will return the span in the range matched by the inner parser. * * @public */ - text(): StringParser { + span(): Parser { return createParser((input, index, state) => { const result = this.exec(input, state, index); if (!result.success) { @@ -187,7 +201,7 @@ export class StringParser { } const text = input.slice(index, result.index); return success(result.index, text); - }, 'text'); + }, 'span'); } } @@ -204,7 +218,7 @@ export type StrictParserOpts = { * @internal */ export type LazyParserOpts = { - lazy: () => StringParser, + lazy: () => Parser, tag?: string, handler?: undefined, }; @@ -229,7 +243,7 @@ export type ParserContext = { */ export type LazyContext = { kind: 'lazy', - handler: () => StringParser, + handler: () => Parser, }; /** @@ -237,7 +251,7 @@ export type LazyContext = { * * @internal */ -export type ResultType = U extends StringParser ? R : never; +export type ResultType = U extends Parser ? R : never; /** * Get result types of Parsers. @@ -265,7 +279,7 @@ function wrapByTraceHandler(handler: ParserHandler, tag: string): ParserHa }; } -function many(parser: StringParser, opts: { min?: number, max?: number, notMatch?: StringParser } = {}): StringParser { +function many(parser: Parser, opts: { min?: number, max?: number, notMatch?: Parser } = {}): Parser { if (opts.notMatch != null) { return many(seq([ notMatch(opts.notMatch), @@ -299,19 +313,19 @@ function many(parser: StringParser, opts: { min?: number, max?: number, no * * @public */ -export function seq[]>(parsers: [...U]): StringParser> +export function seq[]>(parsers: [...U]): Parser> /** * Create a new parser that sequentially applies an array of parser. * * @public * @param select - The index of the data returned in the result. */ -export function seq[], V extends number>(parsers: [...U], select: V): U[V] -export function seq(parsers: StringParser[], select?: number) { +export function seq[], V extends number>(parsers: [...U], select: V): U[V] +export function seq(parsers: Parser[], select?: number) { return (select == null) ? seqAll(parsers) : seqSelect(parsers, select); } -function seqAll[]>(parsers: [...U]): StringParser> { +function seqAll[]>(parsers: [...U]): Parser> { return createParser((input, index, state) => { let result; let latestIndex = index; @@ -328,7 +342,7 @@ function seqAll[]>(parsers: [...U]): StringParser[], V extends number>(parsers: [...U], select: V): U[V] { +function seqSelect[], V extends number>(parsers: [...U], select: V): U[V] { return seqAll(parsers).map(values => values[select]); } @@ -337,7 +351,7 @@ function seqSelect[], V extends number>(parsers: [.. * * @public */ -export function alt[]>(parsers: [...U]): StringParser[number]> { +export function alt[]>(parsers: [...U]): Parser[number]> { return createParser((input, index, state) => { let result; for (let i = 0; i < parsers.length; i++) { @@ -355,8 +369,8 @@ export function alt[]>(parsers: [...U]): StringP * * @public */ -function createParser(handler: ParserHandler, tag?: string): StringParser { - return new StringParser({ handler, tag }); +function createParser(handler: ParserHandler, tag?: string): Parser { + return new Parser({ handler, tag }); } export { createParser as parser }; @@ -365,8 +379,8 @@ export { createParser as parser }; * * @public */ -export function lazy(fn: () => StringParser, tag?: string): StringParser { - return new StringParser({ lazy: fn, tag }); +export function lazy(fn: () => Parser, tag?: string): Parser { + return new Parser({ lazy: fn, tag }); } /** @@ -374,7 +388,7 @@ export function lazy(fn: () => StringParser, tag?: string): StringParser(value: U): StringParser { +export function succeeded(value: U): Parser { return createParser((_input, index, _state) => { return success(index, value); }, 'succeeded'); @@ -385,7 +399,7 @@ export function succeeded(value: U): StringParser { * * @public */ -export function match(parser: StringParser): StringParser { +export function match(parser: Parser): Parser { return createParser((input, index, state) => { const result = parser.exec(input, state, index); return result.success @@ -399,7 +413,7 @@ export function match(parser: StringParser): StringParser { * * @public */ -export function notMatch(parser: StringParser): StringParser { +export function notMatch(parser: Parser): Parser { return createParser((input, index, state) => { const result = parser.exec(input, state, index); return !result.success @@ -413,7 +427,7 @@ export function notMatch(parser: StringParser): StringParser { * * @public */ -export function where(condition: (state: any) => boolean, parser: StringParser): StringParser { +export function where(condition: (state: any) => boolean, parser: Parser): Parser { return createParser((input, index, state) => { return condition(state) ? parser.exec(input, state, index) @@ -449,11 +463,11 @@ export const eof = createParser((input, index, _state) => { * @public */ export function language>(source: LanguageSource): U { - const lang: Record> = {}; + const lang: Record> = {}; for (const key of Object.keys(source)) { lang[key] = lazy(() => { const parser = (source as any)[key](lang); - if (parser == null || !(parser instanceof StringParser)) { + if (parser == null || !(parser instanceof Parser)) { throw new Error('syntax must return a Parser.'); } parser.tag = `${parser.tag} key=${key}`; @@ -468,7 +482,7 @@ export function language>(source: LanguageSource): U { * * @public */ -export type Language = {[K in keyof U]: U[K] extends StringParser ? U[K] : never }; +export type Language = {[K in keyof U]: U[K] extends Parser ? U[K] : never }; /** * Language source @@ -477,26 +491,23 @@ export type Language = {[K in keyof U]: U[K] extends StringParser ? */ export type LanguageSource> = { [K in keyof U]: (lang: U) => U[K] }; - - - /** * Create a new parser that matches the given string. * * @public */ -export function str(value: U): StringParser +export function token(value: U): Parser /** * Create a new parser that matches the given regular expression. * * @public */ -export function str(pattern: RegExp): StringParser -export function str(value: string | RegExp): StringParser { - return (typeof value === 'string') ? strWithString(value) : strWithRegExp(value); +export function token(pattern: RegExp): Parser +export function token(value: string | RegExp): Parser { + return (typeof value === 'string') ? tokenWithString(value) : tokenWithRegExp(value); } -function strWithString(value: U): StringParser { +function tokenWithString(value: U): Parser { return createParser((input, index, _state) => { if ((input.length - index) < value.length) { return failure(index); @@ -505,10 +516,10 @@ function strWithString(value: U): StringParser { return failure(index); } return success(index + value.length, value); - }, `str value=${value}`); + }, `token value=${value}`); } -function strWithRegExp(pattern: RegExp): StringParser { +function tokenWithRegExp(pattern: RegExp): Parser { const re = RegExp(`^(?:${pattern.source})`, pattern.flags); return createParser((input, index, _state) => { const text = input.slice(index); @@ -517,7 +528,7 @@ function strWithRegExp(pattern: RegExp): StringParser { return failure(index); } return success(index + result[0].length, result[0]); - }, `str pattern=${pattern}`); + }, `token pattern=${pattern}`); } /** @@ -525,18 +536,17 @@ function strWithRegExp(pattern: RegExp): StringParser { * * @public */ -export const char = createParser((input, index, _state) => { +export const any = createParser((input, index, _state) => { if ((input.length - index) < 1) { return failure(index); } const value = input.charAt(index); return success(index + 1, value); -}, 'char'); - +}, 'any'); -export const cr = str('\r'); -export const lf = str('\n'); -export const crlf = str('\r\n'); +export const cr = token('\r'); +export const lf = token('\n'); +export const crlf = token('\r\n'); /** * newline diff --git a/src/token-parser.ts b/src/token-parsers.ts similarity index 78% rename from src/token-parser.ts rename to src/token-parsers.ts index 70716b7..bc7fc86 100644 --- a/src/token-parser.ts +++ b/src/token-parsers.ts @@ -1,11 +1,25 @@ -import { success, failure, Result } from './result.js'; +import { + Failure, + Result, + Success, + failure, + success, +} from './result.js'; + +export { + Failure, + Result, + Success, + failure, + success, +}; /** * Token Parser * * @public */ -export class TokenParser { +export class Parser { tag: string; ctx: ParserContext | LazyContext; @@ -109,7 +123,7 @@ export class TokenParser { * * @public */ - map(fn: (value: U) => V): TokenParser { + map(fn: (value: U) => V): Parser { return createParser((input, index, state) => { const result = this.exec(input, state, index); if (!result.success) { @@ -124,14 +138,14 @@ export class TokenParser { * * @public */ - many(min?: number, max?: number): TokenParser + many(min?: number, max?: number): Parser /** * Create a new parser that tries to apply the parser iteratively. * * @public */ - many(opts: { min?: number, max?: number, notMatch?: TokenParser }): TokenParser - many(arg1?: number | { min?: number, max?: number, notMatch?: TokenParser }, arg2?: number): TokenParser { + many(opts: { min?: number, max?: number, notMatch?: Parser }): Parser + many(arg1?: number | { min?: number, max?: number, notMatch?: Parser }, arg2?: number): Parser { if (typeof arg1 === 'number') { // with min, max return many(this, { min: arg1, max: arg2 }); @@ -148,7 +162,7 @@ export class TokenParser { * * @public */ - option(): TokenParser { + option(): Parser { return alt([ this, succeeded(null), @@ -163,7 +177,7 @@ export class TokenParser { * * @public */ - state(key: string, value: (state: any) => any): TokenParser { + state(key: string, value: (state: any) => any): Parser { return createParser((input, index, state) => { const storedValue = state[key]; state[key] = value(state); @@ -172,6 +186,23 @@ export class TokenParser { return result; }, 'state'); } + + /** + * Create a new parser that wraps the current parser. + * The generated parser will return the span in the range matched by the inner parser. + * + * @public + */ + span(): Parser { + return createParser((input, index, state) => { + const result = this.exec(input, state, index); + if (!result.success) { + return result; + } + const text = input.slice(index, result.index); + return success(result.index, text); + }, 'span'); + } } /** @@ -187,7 +218,7 @@ export type StrictParserOpts = { * @internal */ export type LazyParserOpts = { - lazy: () => TokenParser, + lazy: () => Parser, tag?: string, handler?: undefined, }; @@ -212,7 +243,7 @@ export type ParserContext = { */ export type LazyContext = { kind: 'lazy', - handler: () => TokenParser, + handler: () => Parser, }; /** @@ -220,7 +251,7 @@ export type LazyContext = { * * @internal */ -export type ResultType = U extends TokenParser ? R : never; +export type ResultType = U extends Parser ? R : never; /** * Get result types of Parsers. @@ -248,7 +279,7 @@ function wrapByTraceHandler(handler: ParserHandler, tag: string): ParserHa }; } -function many(parser: TokenParser, opts: { min?: number, max?: number, notMatch?: TokenParser } = {}): TokenParser { +function many(parser: Parser, opts: { min?: number, max?: number, notMatch?: Parser } = {}): Parser { if (opts.notMatch != null) { return many(seq([ notMatch(opts.notMatch), @@ -282,19 +313,19 @@ function many(parser: TokenParser, opts: { min?: number, max?: number, not * * @public */ -export function seq[]>(parsers: [...U]): TokenParser> +export function seq[]>(parsers: [...U]): Parser> /** * Create a new parser that sequentially applies an array of parser. * * @public * @param select - The index of the data returned in the result. */ -export function seq[], V extends number>(parsers: [...U], select: V): U[V] -export function seq(parsers: TokenParser[], select?: number) { +export function seq[], V extends number>(parsers: [...U], select: V): U[V] +export function seq(parsers: Parser[], select?: number) { return (select == null) ? seqAll(parsers) : seqSelect(parsers, select); } -function seqAll[]>(parsers: [...U]): TokenParser> { +function seqAll[]>(parsers: [...U]): Parser> { return createParser((input, index, state) => { let result; let latestIndex = index; @@ -311,7 +342,7 @@ function seqAll[]>(parsers: [...U]): TokenParser[], V extends number>(parsers: [...U], select: V): U[V] { +function seqSelect[], V extends number>(parsers: [...U], select: V): U[V] { return seqAll(parsers).map(values => values[select]); } @@ -320,7 +351,7 @@ function seqSelect[], V extends number>(parsers: [... * * @public */ -export function alt[]>(parsers: [...U]): TokenParser[number]> { +export function alt[]>(parsers: [...U]): Parser[number]> { return createParser((input, index, state) => { let result; for (let i = 0; i < parsers.length; i++) { @@ -338,8 +369,8 @@ export function alt[]>(parsers: [...U]): TokenPar * * @public */ -function createParser(handler: ParserHandler, tag?: string): TokenParser { - return new TokenParser({ handler, tag }); +function createParser(handler: ParserHandler, tag?: string): Parser { + return new Parser({ handler, tag }); } export { createParser as parser }; @@ -348,8 +379,8 @@ export { createParser as parser }; * * @public */ -export function lazy(fn: () => TokenParser, tag?: string): TokenParser { - return new TokenParser({ lazy: fn, tag }); +export function lazy(fn: () => Parser, tag?: string): Parser { + return new Parser({ lazy: fn, tag }); } /** @@ -357,7 +388,7 @@ export function lazy(fn: () => TokenParser, tag?: string): TokenParser * * @public */ -export function succeeded(value: U): TokenParser { +export function succeeded(value: U): Parser { return createParser((_input, index, _state) => { return success(index, value); }, 'succeeded'); @@ -368,7 +399,7 @@ export function succeeded(value: U): TokenParser { * * @public */ -export function match(parser: TokenParser): TokenParser { +export function match(parser: Parser): Parser { return createParser((input, index, state) => { const result = parser.exec(input, state, index); return result.success @@ -382,7 +413,7 @@ export function match(parser: TokenParser): TokenParser { * * @public */ -export function notMatch(parser: TokenParser): TokenParser { +export function notMatch(parser: Parser): Parser { return createParser((input, index, state) => { const result = parser.exec(input, state, index); return !result.success @@ -396,7 +427,7 @@ export function notMatch(parser: TokenParser): TokenParser { * * @public */ -export function where(condition: (state: any) => boolean, parser: TokenParser): TokenParser { +export function where(condition: (state: any) => boolean, parser: Parser): Parser { return createParser((input, index, state) => { return condition(state) ? parser.exec(input, state, index) @@ -432,11 +463,11 @@ export const eof = createParser((input, index, _state) => { * @public */ export function language>(source: LanguageSource): U { - const lang: Record> = {}; + const lang: Record> = {}; for (const key of Object.keys(source)) { lang[key] = lazy(() => { const parser = (source as any)[key](lang); - if (parser == null || !(parser instanceof TokenParser)) { + if (parser == null || !(parser instanceof Parser)) { throw new Error('syntax must return a Parser.'); } parser.tag = `${parser.tag} key=${key}`; @@ -451,7 +482,7 @@ export function language>(source: LanguageSource): U { * * @public */ -export type Language = {[K in keyof U]: U[K] extends TokenParser ? U[K] : never }; +export type Language = {[K in keyof U]: U[K] extends Parser ? U[K] : never }; /** * Language source @@ -465,18 +496,18 @@ export type LanguageSource> = { [K in keyof U]: (lang: U) * * @public */ -export function token(value: U): TokenParser +export function token(value: U): Parser /** * Create a new parser that matches the given regular expression. * * @public */ -export function token(pattern: RegExp): TokenParser -export function token(value: string | RegExp): TokenParser { +export function token(pattern: RegExp): Parser +export function token(value: string | RegExp): Parser { return (typeof value === 'string') ? tokenWithString(value) : tokenWithRegExp(value); } -function tokenWithString(value: U): TokenParser { +function tokenWithString(value: U): Parser { return createParser((input, index, _state) => { if (index >= input.length) { return failure(index); @@ -485,10 +516,10 @@ function tokenWithString(value: U): TokenParser { return failure(index); } return success(index + 1, value); - }, `str value=${value}`); + }, `token value=${value}`); } -function tokenWithRegExp(pattern: RegExp): TokenParser { +function tokenWithRegExp(pattern: RegExp): Parser { const re = RegExp(`^(?:${pattern.source})$`, pattern.flags); return createParser((input, index, _state) => { if (index >= input.length) { @@ -499,9 +530,22 @@ function tokenWithRegExp(pattern: RegExp): TokenParser { return failure(index); } return success(index + 1, result[0]); - }, `str pattern=${pattern}`); + }, `token pattern=${pattern}`); } +/** + * any token + * + * @public +*/ +export const any = createParser((input, index, _state) => { + if ((input.length - index) < 1) { + return failure(index); + } + const value = input[index]; + return success(index + 1, value); +}, 'any'); + export const cr = token('\r'); export const lf = token('\n'); export const crlf = seq([cr, lf]);