From f8bd2139b635a82fde423e62553740caa81d928e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0t=C4=9Bp=C3=A1n=20Gran=C3=A1t?= Date: Tue, 4 Apr 2023 20:01:10 +0200 Subject: [PATCH 1/3] feat: add separate tag parser --- .../core/src/FormatSimple/FormatSimple.ts | 2 +- .../{FormatError.ts => FormatSimpleError.ts} | 2 +- .../core/src/FormatSimple/formatParser.ts | 6 +- .../core/src/FormatSimple/formatter.test.ts | 10 +- packages/core/src/TagParser/TagParser.ts | 13 ++ packages/core/src/TagParser/TagParserError.ts | 30 ++++ packages/core/src/TagParser/handleTag.ts | 72 ++++++++ packages/core/src/TagParser/parser.test.ts | 42 +++++ packages/core/src/TagParser/parser.ts | 10 ++ packages/core/src/TagParser/tokenizer.ts | 155 ++++++++++++++++++ packages/core/src/index.ts | 1 + testapps/react/src/App.tsx | 13 +- testapps/react/src/TranslationMethods.tsx | 10 +- 13 files changed, 348 insertions(+), 18 deletions(-) rename packages/core/src/FormatSimple/{FormatError.ts => FormatSimpleError.ts} (93%) create mode 100644 packages/core/src/TagParser/TagParser.ts create mode 100644 packages/core/src/TagParser/TagParserError.ts create mode 100644 packages/core/src/TagParser/handleTag.ts create mode 100644 packages/core/src/TagParser/parser.test.ts create mode 100644 packages/core/src/TagParser/parser.ts create mode 100644 packages/core/src/TagParser/tokenizer.ts diff --git a/packages/core/src/FormatSimple/FormatSimple.ts b/packages/core/src/FormatSimple/FormatSimple.ts index 1e76e746e1..c834b3e6fa 100644 --- a/packages/core/src/FormatSimple/FormatSimple.ts +++ b/packages/core/src/FormatSimple/FormatSimple.ts @@ -8,6 +8,6 @@ function createFormatSimple(): FinalFormatterMiddleware { } export const FormatSimple = (): TolgeePlugin => (tolgee, tools) => { - tools.setFinalFormatter(createFormatSimple()); + tools.addFormatter(createFormatSimple()); return tolgee; }; diff --git a/packages/core/src/FormatSimple/FormatError.ts b/packages/core/src/FormatSimple/FormatSimpleError.ts similarity index 93% rename from packages/core/src/FormatSimple/FormatError.ts rename to packages/core/src/FormatSimple/FormatSimpleError.ts index 9ca3c169b0..1364ba055e 100644 --- a/packages/core/src/FormatSimple/FormatError.ts +++ b/packages/core/src/FormatSimple/FormatSimpleError.ts @@ -7,7 +7,7 @@ export type ErrorCode = | typeof ERROR_UNEXPECTED_CHAR | typeof ERROR_UNEXPECTED_END; -export class FormatError extends Error { +export class FormatSimpleError extends Error { public readonly code: ErrorCode; public readonly index: number; constructor(code: ErrorCode, index: number, text: string) { diff --git a/packages/core/src/FormatSimple/formatParser.ts b/packages/core/src/FormatSimple/formatParser.ts index 89c8f3754a..eb9ae0a458 100644 --- a/packages/core/src/FormatSimple/formatParser.ts +++ b/packages/core/src/FormatSimple/formatParser.ts @@ -3,8 +3,8 @@ import { ERROR_PARAM_EMPTY, ERROR_UNEXPECTED_CHAR, ERROR_UNEXPECTED_END, - FormatError, -} from './FormatError'; + FormatSimpleError, +} from './FormatSimpleError'; function isWhitespace(ch: string) { return /\s/.test(ch); @@ -46,7 +46,7 @@ export function formatParser(translation: string) { let i = 0; function parsingError(code: ErrorCode): never { - throw new FormatError(code, i, translation); + throw new FormatSimpleError(code, i, translation); } const addText = () => { diff --git a/packages/core/src/FormatSimple/formatter.test.ts b/packages/core/src/FormatSimple/formatter.test.ts index da17c59b83..4052790a8f 100644 --- a/packages/core/src/FormatSimple/formatter.test.ts +++ b/packages/core/src/FormatSimple/formatter.test.ts @@ -6,8 +6,8 @@ import { ERROR_PARAM_EMPTY, ERROR_UNEXPECTED_CHAR, ERROR_UNEXPECTED_END, - FormatError, -} from './FormatError'; + FormatSimpleError, +} from './FormatSimpleError'; function icu(text: string, params?: TranslateParams) { return new IntlMessageFormat(text, 'en', undefined, { @@ -29,13 +29,13 @@ function expectToThrow( code?: ErrorCode, params?: TranslateParams ) { - let error: FormatError | undefined = undefined; + let error: FormatSimpleError | undefined = undefined; try { formatter(text, params); } catch (e) { - error = e as FormatError; + error = e as FormatSimpleError; } - expect(error).toBeInstanceOf(FormatError); + expect(error).toBeInstanceOf(FormatSimpleError); expect(error?.code).toEqual(code); } diff --git a/packages/core/src/TagParser/TagParser.ts b/packages/core/src/TagParser/TagParser.ts new file mode 100644 index 0000000000..6d616fba72 --- /dev/null +++ b/packages/core/src/TagParser/TagParser.ts @@ -0,0 +1,13 @@ +import { FinalFormatterMiddleware, TolgeePlugin } from '../types'; +import { parser } from './parser'; + +function createTagParser(): FinalFormatterMiddleware { + return { + format: ({ translation, params }) => parser(translation, params), + }; +} + +export const TagParser = (): TolgeePlugin => (tolgee, tools) => { + tools.setFinalFormatter(createTagParser()); + return tolgee; +}; diff --git a/packages/core/src/TagParser/TagParserError.ts b/packages/core/src/TagParser/TagParserError.ts new file mode 100644 index 0000000000..253632c2de --- /dev/null +++ b/packages/core/src/TagParser/TagParserError.ts @@ -0,0 +1,30 @@ +export const ERROR_UNEXPECTED_CHAR = 0, + ERROR_UNEXPECTED_END = 1, + ERROR_UNEXPECTED_TAG = 2, + ERROR_UNCLOSED_TAG = 3; + +export type ErrorCode = + | typeof ERROR_UNEXPECTED_CHAR + | typeof ERROR_UNEXPECTED_END + | typeof ERROR_UNEXPECTED_TAG + | typeof ERROR_UNCLOSED_TAG; + +export class TagParserError extends Error { + public readonly code: ErrorCode; + public readonly index: number; + constructor(code: ErrorCode, index: number, text: string) { + let error: string; + if (code === ERROR_UNEXPECTED_CHAR) { + error = 'Unexpected character'; + } else if (code === ERROR_UNEXPECTED_END) { + error = 'Unexpected end'; + } else if (code === ERROR_UNEXPECTED_TAG) { + error = 'Unexpected tag'; + } else { + error = 'Unclosed tag'; + } + super(`Tag parser error: ${error} at ${index} in "${text}"`); + this.code = code; + this.index = index; + } +} diff --git a/packages/core/src/TagParser/handleTag.ts b/packages/core/src/TagParser/handleTag.ts new file mode 100644 index 0000000000..838aebfd6b --- /dev/null +++ b/packages/core/src/TagParser/handleTag.ts @@ -0,0 +1,72 @@ +import { + ERROR_UNCLOSED_TAG, + ERROR_UNEXPECTED_TAG, + ErrorCode, + TagParserError, +} from './TagParserError'; +import { Token } from './tokenizer'; + +export function handleTag( + startToken: Token | undefined, + stack: Token[], + params: Record | undefined, + fullText: string +) { + let token: Token | undefined; + const content: any[] = []; + + function addToContent(item: any) { + if ( + typeof content[content.length - 1] === 'string' && + typeof item === 'string' + ) { + content[content.length - 1] += item; + } else { + content.push(item); + } + } + + function simplifyContent() { + if (content.length === 0) { + return undefined; + } else if (content.length === 1 && typeof content[0] === 'string') { + return content[0]; + } else { + return content; + } + } + + function parsingError(code: ErrorCode): never { + throw new TagParserError(code, token!.position, fullText); + } + + while ((token = stack.shift())) { + if ( + token.type === 'tag' && + token.closing && + startToken !== undefined && + token.data === startToken.data + ) { + // matching tag to startToken - closing + const fun = params?.[startToken.data]; + return fun(simplifyContent()); + } else if (token.type === 'tag' && token.selfClosing) { + // self-closing - solve in-place + const fun = params?.[token.data]; + addToContent(fun()); + } else if (token.type === 'tag' && !token.closing) { + // opening tag - call recursively + addToContent(handleTag(token, stack, params, fullText)); + } else if (token.type === 'text') { + // text + addToContent(token.data); + } else { + parsingError(ERROR_UNEXPECTED_TAG); + } + } + if (startToken === undefined) { + // we are in the root, return content itself + return simplifyContent(); + } + parsingError(ERROR_UNCLOSED_TAG); +} diff --git a/packages/core/src/TagParser/parser.test.ts b/packages/core/src/TagParser/parser.test.ts new file mode 100644 index 0000000000..1fecf318e2 --- /dev/null +++ b/packages/core/src/TagParser/parser.test.ts @@ -0,0 +1,42 @@ +import { parser } from './parser'; + +function getText() { + return expect.getState().currentTestName.replace('parser ', ''); +} + +describe('parser', () => { + it('Hello world!', () => { + expect( + parser(getText(), { + a: (data: string) => `${data}`, + }) + ).toEqual('Hello world!'); + }); + + it('Hello
', () => { + expect( + parser(getText(), { + br: () => '
', + }) + ).toEqual('Hello
'); + }); + + it('Hello world!', () => { + expect( + parser(getText(), { + a: (content: string) => `${content}`, + b: (content: string) => `${content}`, + }) + ).toEqual('Hello world!'); + }); + + it('test', () => { + expect( + parser(getText(), { a: (content: string) => `${content}` }) + ).toEqual('test'); + }); + + it("'", () => { + expect(parser(getText())).toEqual(''); + }); +}); diff --git a/packages/core/src/TagParser/parser.ts b/packages/core/src/TagParser/parser.ts new file mode 100644 index 0000000000..1ed19005bf --- /dev/null +++ b/packages/core/src/TagParser/parser.ts @@ -0,0 +1,10 @@ +import { handleTag } from './handleTag'; +import { tokenizer } from './tokenizer'; + +export function parser(text: string, params?: Record) { + const tokens = tokenizer(text); + + const result = handleTag(undefined, [...tokens], params, text); + + return result; +} diff --git a/packages/core/src/TagParser/tokenizer.ts b/packages/core/src/TagParser/tokenizer.ts new file mode 100644 index 0000000000..32feeaca0f --- /dev/null +++ b/packages/core/src/TagParser/tokenizer.ts @@ -0,0 +1,155 @@ +import { + ERROR_UNEXPECTED_CHAR, + ERROR_UNEXPECTED_END, + ErrorCode, + TagParserError, +} from './TagParserError'; + +const STATE_TEXT = 0, + STATE_ESCAPE_MAYBE = 1, + STATE_ESCAPE = 2, + STATE_TAG_START = 3, + STATE_TAG_NAME = 4, + STATE_SELF_CLOSING = 5; + +type State = + | typeof STATE_TEXT + | typeof STATE_ESCAPE_MAYBE + | typeof STATE_ESCAPE + | typeof STATE_TAG_START + | typeof STATE_TAG_NAME + | typeof STATE_SELF_CLOSING; + +const CHAR_ESCAPE = "'"; +const ESCAPABLE = new Set(['<', CHAR_ESCAPE]); + +const END_STATES = new Set([STATE_TEXT, STATE_ESCAPE_MAYBE, STATE_ESCAPE]); + +export type Token = { + type: 'text' | 'tag'; + data: string; + closing: boolean; + selfClosing: boolean; + position: number; +}; + +function isWhite(char: string) { + return /\s/g.test(char); +} + +function isNameChar(char: string) { + return /[A-Za-z0-9]/.test(char); +} + +export function tokenizer(text: string) { + const tokens: Token[] = []; + let state: State = STATE_TEXT; + let data = ''; + let closing = false; + let selfClosing = false; + let tokenPosition = 0; + + let i = 0; + + function parsingError(code: ErrorCode): never { + throw new TagParserError(code, i, text); + } + + function createToken(type: Token['type']) { + if (data.length) { + tokens.push({ + type, + data, + closing, + selfClosing, + position: tokenPosition, + }); + closing = false; + selfClosing = false; + data = ''; + tokenPosition = i; + } + } + + for (i = 0; i < text.length; i++) { + const char = text[i]; + switch (state) { + case STATE_TEXT: + if (char === '<') { + createToken('text'); + state = STATE_TAG_START; + } else if (char === CHAR_ESCAPE) { + state = STATE_ESCAPE_MAYBE; + } else { + data += char; + } + break; + + case STATE_ESCAPE_MAYBE: + if (ESCAPABLE.has(char)) { + data = data.slice(0, -1) + char; + state = STATE_ESCAPE; + } else { + data += data; + state = STATE_TEXT; + } + break; + + case STATE_ESCAPE: + if (char === CHAR_ESCAPE) { + state = STATE_TEXT; + } else { + data += char; + } + break; + + case STATE_TAG_START: + if (char === '/' && !closing) { + closing = true; + } else if (isNameChar(char)) { + data += char; + state = STATE_TAG_NAME; + } else { + // invalid tag + parsingError(ERROR_UNEXPECTED_CHAR); + } + break; + + case STATE_TAG_NAME: + if (char === '>') { + createToken('tag'); + state = STATE_TEXT; + } else if (isNameChar(char)) { + data += char.toLowerCase(); + } else if (char === '/' && data !== '') { + // self-closing slash + selfClosing = true; + state = STATE_SELF_CLOSING; + } else if (isWhite(char) && data !== '') { + // skiping white spaces after tag name + break; + } else { + // invalid tag + parsingError(ERROR_UNEXPECTED_CHAR); + } + break; + + case STATE_SELF_CLOSING: + if (char === '>') { + createToken('tag'); + state = STATE_TEXT; + } else { + // invalid tag + parsingError(ERROR_UNEXPECTED_CHAR); + } + break; + } + } + + if (!END_STATES.has(state)) { + parsingError(ERROR_UNEXPECTED_END); + } + createToken('text'); + + return tokens; +} diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index c3e37eef85..7b72100b7f 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -3,3 +3,4 @@ export { TolgeeCore } from './TolgeeCore'; export * from './types'; export { getTranslateProps } from './TranslateParams'; export { FormatSimple } from './FormatSimple/FormatSimple'; +export { TagParser } from './TagParser/TagParser'; diff --git a/testapps/react/src/App.tsx b/testapps/react/src/App.tsx index 603790a33e..4ff08a9b8e 100644 --- a/testapps/react/src/App.tsx +++ b/testapps/react/src/App.tsx @@ -1,13 +1,20 @@ /* eslint-disable @typescript-eslint/no-var-requires */ -import { Tolgee, TolgeeProvider, BackendFetch, DevTools } from '@tolgee/react'; +import { + Tolgee, + TolgeeProvider, + BackendFetch, + DevTools, + FormatSimple, + TagParser, +} from '@tolgee/react'; import { Todos } from './Todos'; import { TranslationMethods } from './TranslationMethods'; -import { FormatIcu } from '@tolgee/format-icu'; const tolgee = Tolgee() .use(DevTools()) - .use(FormatIcu()) + .use(FormatSimple()) + .use(TagParser()) .use(BackendFetch()) .init({ availableLanguages: ['en', 'cs', 'fr', 'de'], diff --git a/testapps/react/src/TranslationMethods.tsx b/testapps/react/src/TranslationMethods.tsx index 8b7b46bcd7..907fc00b95 100644 --- a/testapps/react/src/TranslationMethods.tsx +++ b/testapps/react/src/TranslationMethods.tsx @@ -17,7 +17,7 @@ export const TranslationMethods = () => {
-
+ {/*

T component with default

This is default @@ -48,7 +48,7 @@ export const TranslationMethods = () => { noWrap />
-
+
*/}

T component with interpolation

@@ -65,7 +65,7 @@ export const TranslationMethods = () => {
- + {/*

T component with br tag

@@ -74,7 +74,7 @@ export const TranslationMethods = () => { params={{ br:
, }} - defaultValue="Key with br

tag" + defaultValue="Key with br
tag" />
@@ -140,7 +140,7 @@ export const TranslationMethods = () => { - + */} {!revealed ? (
From e72cf036a8a0f8f906bd7fd940aea35baecf8717 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0t=C4=9Bp=C3=A1n=20Gran=C3=A1t?= Date: Wed, 5 Apr 2023 16:35:13 +0200 Subject: [PATCH 2/3] feat: add separate tag parser --- packages/core/src/TagParser/TagParserError.ts | 30 ---- packages/core/src/TagParser/parser.test.ts | 42 ------ packages/core/src/index.ts | 1 - packages/format-icu/src/FormatIcu.ts | 2 +- packages/format-icu/src/createFormatIcu.ts | 6 +- packages/react/src/__integration/T.spec.tsx | 10 +- .../{core => web}/src/TagParser/TagParser.ts | 2 +- .../{core => web}/src/TagParser/handleTag.ts | 55 +++++--- packages/web/src/TagParser/parser.test.ts | 131 ++++++++++++++++++ .../{core => web}/src/TagParser/parser.ts | 0 .../{core => web}/src/TagParser/tokenizer.ts | 95 ++++--------- packages/web/src/typedIndex.ts | 1 + testapps/react/src/App.tsx | 4 +- testapps/react/src/TranslationMethods.tsx | 10 +- 14 files changed, 211 insertions(+), 178 deletions(-) delete mode 100644 packages/core/src/TagParser/TagParserError.ts delete mode 100644 packages/core/src/TagParser/parser.test.ts rename packages/{core => web}/src/TagParser/TagParser.ts (81%) rename packages/{core => web}/src/TagParser/handleTag.ts (54%) create mode 100644 packages/web/src/TagParser/parser.test.ts rename packages/{core => web}/src/TagParser/parser.ts (100%) rename packages/{core => web}/src/TagParser/tokenizer.ts (53%) diff --git a/packages/core/src/TagParser/TagParserError.ts b/packages/core/src/TagParser/TagParserError.ts deleted file mode 100644 index 253632c2de..0000000000 --- a/packages/core/src/TagParser/TagParserError.ts +++ /dev/null @@ -1,30 +0,0 @@ -export const ERROR_UNEXPECTED_CHAR = 0, - ERROR_UNEXPECTED_END = 1, - ERROR_UNEXPECTED_TAG = 2, - ERROR_UNCLOSED_TAG = 3; - -export type ErrorCode = - | typeof ERROR_UNEXPECTED_CHAR - | typeof ERROR_UNEXPECTED_END - | typeof ERROR_UNEXPECTED_TAG - | typeof ERROR_UNCLOSED_TAG; - -export class TagParserError extends Error { - public readonly code: ErrorCode; - public readonly index: number; - constructor(code: ErrorCode, index: number, text: string) { - let error: string; - if (code === ERROR_UNEXPECTED_CHAR) { - error = 'Unexpected character'; - } else if (code === ERROR_UNEXPECTED_END) { - error = 'Unexpected end'; - } else if (code === ERROR_UNEXPECTED_TAG) { - error = 'Unexpected tag'; - } else { - error = 'Unclosed tag'; - } - super(`Tag parser error: ${error} at ${index} in "${text}"`); - this.code = code; - this.index = index; - } -} diff --git a/packages/core/src/TagParser/parser.test.ts b/packages/core/src/TagParser/parser.test.ts deleted file mode 100644 index 1fecf318e2..0000000000 --- a/packages/core/src/TagParser/parser.test.ts +++ /dev/null @@ -1,42 +0,0 @@ -import { parser } from './parser'; - -function getText() { - return expect.getState().currentTestName.replace('parser ', ''); -} - -describe('parser', () => { - it('Hello world!', () => { - expect( - parser(getText(), { - a: (data: string) => `${data}`, - }) - ).toEqual('Hello world!'); - }); - - it('Hello
', () => { - expect( - parser(getText(), { - br: () => '
', - }) - ).toEqual('Hello
'); - }); - - it('Hello world!', () => { - expect( - parser(getText(), { - a: (content: string) => `${content}`, - b: (content: string) => `${content}`, - }) - ).toEqual('Hello world!'); - }); - - it('test', () => { - expect( - parser(getText(), { a: (content: string) => `${content}` }) - ).toEqual('test'); - }); - - it("'", () => { - expect(parser(getText())).toEqual(''); - }); -}); diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 7b72100b7f..c3e37eef85 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -3,4 +3,3 @@ export { TolgeeCore } from './TolgeeCore'; export * from './types'; export { getTranslateProps } from './TranslateParams'; export { FormatSimple } from './FormatSimple/FormatSimple'; -export { TagParser } from './TagParser/TagParser'; diff --git a/packages/format-icu/src/FormatIcu.ts b/packages/format-icu/src/FormatIcu.ts index 2798bc90e7..5f3bb22520 100644 --- a/packages/format-icu/src/FormatIcu.ts +++ b/packages/format-icu/src/FormatIcu.ts @@ -2,6 +2,6 @@ import { TolgeePlugin } from '@tolgee/core'; import { createFormatIcu } from './createFormatIcu'; export const FormatIcu = (): TolgeePlugin => (tolgee, tools) => { - tools.setFinalFormatter(createFormatIcu()); + tools.addFormatter(createFormatIcu()); return tolgee; }; diff --git a/packages/format-icu/src/createFormatIcu.ts b/packages/format-icu/src/createFormatIcu.ts index c8ff891eaa..5329672bdb 100644 --- a/packages/format-icu/src/createFormatIcu.ts +++ b/packages/format-icu/src/createFormatIcu.ts @@ -29,14 +29,10 @@ export const createFormatIcu = (): FinalFormatterMiddleware => { language, params, }) => { - const ignoreTag = !Object.values(params || {}).find( - (p) => typeof p === 'function' - ); - const locale = getLocale(language); return new IntlMessageFormat(translation, locale, undefined, { - ignoreTag, + ignoreTag: true, }).format(params); }; diff --git a/packages/react/src/__integration/T.spec.tsx b/packages/react/src/__integration/T.spec.tsx index ed5a067a5e..1c2b01849b 100644 --- a/packages/react/src/__integration/T.spec.tsx +++ b/packages/react/src/__integration/T.spec.tsx @@ -2,7 +2,13 @@ import { act } from 'react-dom/test-utils'; import React from 'react'; import '@testing-library/jest-dom'; import { render, screen, waitFor } from '@testing-library/react'; -import { TolgeeProvider, DevTools, TolgeeInstance, Tolgee } from '../index'; +import { + TolgeeProvider, + DevTools, + TolgeeInstance, + Tolgee, + TagParser, +} from '../index'; import { FormatIcu } from '@tolgee/format-icu'; import { mockCoreFetch } from '@tolgee/testing/fetchMock'; @@ -69,7 +75,7 @@ describe('T component integration', () => { beforeEach(async () => { fetch.enableMocks(); - tolgee = Tolgee().use(DevTools()).use(FormatIcu()).init({ + tolgee = Tolgee().use(DevTools()).use(TagParser()).use(FormatIcu()).init({ apiUrl: API_URL, apiKey: API_KEY, language: 'cs', diff --git a/packages/core/src/TagParser/TagParser.ts b/packages/web/src/TagParser/TagParser.ts similarity index 81% rename from packages/core/src/TagParser/TagParser.ts rename to packages/web/src/TagParser/TagParser.ts index 6d616fba72..8ffc140882 100644 --- a/packages/core/src/TagParser/TagParser.ts +++ b/packages/web/src/TagParser/TagParser.ts @@ -1,4 +1,4 @@ -import { FinalFormatterMiddleware, TolgeePlugin } from '../types'; +import { FinalFormatterMiddleware, TolgeePlugin } from '@tolgee/core'; import { parser } from './parser'; function createTagParser(): FinalFormatterMiddleware { diff --git a/packages/core/src/TagParser/handleTag.ts b/packages/web/src/TagParser/handleTag.ts similarity index 54% rename from packages/core/src/TagParser/handleTag.ts rename to packages/web/src/TagParser/handleTag.ts index 838aebfd6b..7ba603816a 100644 --- a/packages/core/src/TagParser/handleTag.ts +++ b/packages/web/src/TagParser/handleTag.ts @@ -1,9 +1,3 @@ -import { - ERROR_UNCLOSED_TAG, - ERROR_UNEXPECTED_TAG, - ErrorCode, - TagParserError, -} from './TagParserError'; import { Token } from './tokenizer'; export function handleTag( @@ -13,7 +7,7 @@ export function handleTag( fullText: string ) { let token: Token | undefined; - const content: any[] = []; + let content: any[] = []; function addToContent(item: any) { if ( @@ -26,6 +20,14 @@ export function handleTag( } } + function prependContent(item: any) { + if (typeof content[0] === 'string' && typeof item === 'string') { + content[0] = item + content[0]; + } else { + content = [item, ...content]; + } + } + function simplifyContent() { if (content.length === 0) { return undefined; @@ -36,8 +38,12 @@ export function handleTag( } } - function parsingError(code: ErrorCode): never { - throw new TagParserError(code, token!.position, fullText); + function getParamFunc(name: string) { + const func = params?.[name]; + if (typeof func === 'function') { + return func; + } + return undefined; } while ((token = stack.shift())) { @@ -45,28 +51,33 @@ export function handleTag( token.type === 'tag' && token.closing && startToken !== undefined && - token.data === startToken.data + token.name === startToken.name ) { // matching tag to startToken - closing - const fun = params?.[startToken.data]; + const fun = getParamFunc(startToken.name); return fun(simplifyContent()); - } else if (token.type === 'tag' && token.selfClosing) { + } else if ( + token.type === 'tag' && + token.selfClosing && + getParamFunc(token.name) + ) { // self-closing - solve in-place - const fun = params?.[token.data]; + const fun = getParamFunc(token.name); addToContent(fun()); - } else if (token.type === 'tag' && !token.closing) { + } else if ( + token.type === 'tag' && + !token.closing && + getParamFunc(token.name) + ) { // opening tag - call recursively addToContent(handleTag(token, stack, params, fullText)); - } else if (token.type === 'text') { - // text - addToContent(token.data); } else { - parsingError(ERROR_UNEXPECTED_TAG); + // treat everything else as text + addToContent(token.text); } } - if (startToken === undefined) { - // we are in the root, return content itself - return simplifyContent(); + if (startToken !== undefined) { + prependContent(startToken.text); } - parsingError(ERROR_UNCLOSED_TAG); + return simplifyContent(); } diff --git a/packages/web/src/TagParser/parser.test.ts b/packages/web/src/TagParser/parser.test.ts new file mode 100644 index 0000000000..67452fc5d7 --- /dev/null +++ b/packages/web/src/TagParser/parser.test.ts @@ -0,0 +1,131 @@ +import { parser } from './parser'; + +function getText() { + return expect.getState().currentTestName.replace('parser ', ''); +} + +function textElement(name: string) { + return (content: string) => `<${name}>${content}`; +} + +function objectElement(name: string) { + return (content: string) => ({ name, content }); +} + +describe('parser', () => { + it('Text element!', () => { + expect( + parser(getText(), { + a: textElement('A'), + }) + ).toEqual('Text element!'); + }); + + it('Object element!', () => { + expect( + parser(getText(), { + a: objectElement('A'), + }) + ).toEqual(['Object ', { name: 'A', content: 'element' }, '!']); + }); + + it('Ignored element!', () => { + expect(parser(getText())).toEqual('Ignored element!'); + }); + + it('Text
self closing', () => { + expect( + parser(getText(), { + br: textElement('BR'), + }) + ).toEqual('Text
undefined
self closing'); + }); + + it('Object
self closing', () => { + expect( + parser(getText(), { + br: objectElement('BR'), + }) + ).toEqual(['Object ', { name: 'BR', content: undefined }, ' self closing']); + }); + + it('Ignored
self closing', () => { + expect(parser(getText())).toEqual('Ignored
self closing'); + }); + + it('Text nested element and more', () => { + expect( + parser(getText(), { + a: textElement('A'), + b: textElement('B'), + }) + ).toEqual('Text nested element and more'); + }); + + it('Object nested element and more', () => { + expect( + parser(getText(), { + a: objectElement('A'), + b: objectElement('B'), + }) + ).toEqual([ + 'Object ', + { + name: 'A', + content: ['nested ', { name: 'B', content: 'element' }, ' and'], + }, + ' more', + ]); + }); + + it('Ignored nested element and more', () => { + expect(parser(getText())).toEqual( + 'Ignored nested element and more' + ); + }); + + it('test', () => { + expect(parser(getText(), { a: textElement('A') })).toEqual('test'); + }); + + it('ignored when no matching parameter', () => { + expect(parser(getText())).toEqual( + 'ignored when no matching parameter' + ); + }); + + it('Ignored when no function param tag', () => { + expect(parser(getText())).toEqual('Ignored when no function param tag'); + }); + + it('Ignored when unclosed tag', () => { + expect(parser(getText(), { a: textElement('A') })).toEqual( + 'Ignored when unclosed tag' + ); + }); + + it('Text handles white at allowed places test', () => { + expect(parser(getText(), { a: textElement('A') })).toEqual( + 'Text handles white at allowed places test' + ); + }); + + it('Ignored with spaces test', () => { + expect(parser(getText())).toEqual('Ignored with spaces test'); + }); + + it('Ignored partly when crossed first second third', () => { + expect( + parser(getText(), { + a: textElement('A'), + b: textElement('B'), + }) + ).toEqual('Ignored partly when crossed first second third'); + }); + + it('Ignored with params test', () => { + expect(parser(getText(), { a: textElement('A') })).toEqual( + 'Ignored with params test' + ); + }); +}); diff --git a/packages/core/src/TagParser/parser.ts b/packages/web/src/TagParser/parser.ts similarity index 100% rename from packages/core/src/TagParser/parser.ts rename to packages/web/src/TagParser/parser.ts diff --git a/packages/core/src/TagParser/tokenizer.ts b/packages/web/src/TagParser/tokenizer.ts similarity index 53% rename from packages/core/src/TagParser/tokenizer.ts rename to packages/web/src/TagParser/tokenizer.ts index 32feeaca0f..3a34077f98 100644 --- a/packages/core/src/TagParser/tokenizer.ts +++ b/packages/web/src/TagParser/tokenizer.ts @@ -1,36 +1,21 @@ -import { - ERROR_UNEXPECTED_CHAR, - ERROR_UNEXPECTED_END, - ErrorCode, - TagParserError, -} from './TagParserError'; - const STATE_TEXT = 0, - STATE_ESCAPE_MAYBE = 1, - STATE_ESCAPE = 2, - STATE_TAG_START = 3, - STATE_TAG_NAME = 4, - STATE_SELF_CLOSING = 5; + STATE_TAG_START = 1, + STATE_TAG_NAME = 2, + STATE_SELF_CLOSING = 3; type State = | typeof STATE_TEXT - | typeof STATE_ESCAPE_MAYBE - | typeof STATE_ESCAPE | typeof STATE_TAG_START | typeof STATE_TAG_NAME | typeof STATE_SELF_CLOSING; -const CHAR_ESCAPE = "'"; -const ESCAPABLE = new Set(['<', CHAR_ESCAPE]); - -const END_STATES = new Set([STATE_TEXT, STATE_ESCAPE_MAYBE, STATE_ESCAPE]); - export type Token = { type: 'text' | 'tag'; - data: string; + name: string; closing: boolean; selfClosing: boolean; position: number; + text: string; }; function isWhite(char: string) { @@ -41,115 +26,91 @@ function isNameChar(char: string) { return /[A-Za-z0-9]/.test(char); } -export function tokenizer(text: string) { +export function tokenizer(source: string) { const tokens: Token[] = []; let state: State = STATE_TEXT; - let data = ''; + let name = ''; let closing = false; let selfClosing = false; let tokenPosition = 0; + let text = ''; let i = 0; - function parsingError(code: ErrorCode): never { - throw new TagParserError(code, i, text); - } - function createToken(type: Token['type']) { - if (data.length) { + if (text.length) { tokens.push({ type, - data, + name, closing, selfClosing, position: tokenPosition, + text, }); closing = false; selfClosing = false; - data = ''; + name = ''; tokenPosition = i; + text = ''; } } - for (i = 0; i < text.length; i++) { - const char = text[i]; + for (i = 0; i < source.length; i++) { + const char = source[i]; switch (state) { case STATE_TEXT: if (char === '<') { createToken('text'); state = STATE_TAG_START; - } else if (char === CHAR_ESCAPE) { - state = STATE_ESCAPE_MAYBE; - } else { - data += char; - } - break; - - case STATE_ESCAPE_MAYBE: - if (ESCAPABLE.has(char)) { - data = data.slice(0, -1) + char; - state = STATE_ESCAPE; - } else { - data += data; - state = STATE_TEXT; - } - break; - - case STATE_ESCAPE: - if (char === CHAR_ESCAPE) { - state = STATE_TEXT; - } else { - data += char; } + text += char; break; case STATE_TAG_START: + text += char; if (char === '/' && !closing) { closing = true; } else if (isNameChar(char)) { - data += char; + name += char; state = STATE_TAG_NAME; } else { // invalid tag - parsingError(ERROR_UNEXPECTED_CHAR); + state = STATE_TEXT; } break; case STATE_TAG_NAME: + text += char; if (char === '>') { createToken('tag'); state = STATE_TEXT; } else if (isNameChar(char)) { - data += char.toLowerCase(); - } else if (char === '/' && data !== '') { + name += char.toLowerCase(); + } else if (char === '/' && name !== '') { // self-closing slash selfClosing = true; state = STATE_SELF_CLOSING; - } else if (isWhite(char) && data !== '') { + } else if (isWhite(char) && name !== '') { // skiping white spaces after tag name break; } else { - // invalid tag - parsingError(ERROR_UNEXPECTED_CHAR); + // invalid tag - ignoring + state = STATE_TEXT; } break; case STATE_SELF_CLOSING: + text += char; if (char === '>') { createToken('tag'); state = STATE_TEXT; } else { - // invalid tag - parsingError(ERROR_UNEXPECTED_CHAR); + // invalid tag - ignoring + state = STATE_TEXT; } break; } } - - if (!END_STATES.has(state)) { - parsingError(ERROR_UNEXPECTED_END); - } createToken('text'); - return tokens; } diff --git a/packages/web/src/typedIndex.ts b/packages/web/src/typedIndex.ts index 08d0b144fc..65b88a71e4 100644 --- a/packages/web/src/typedIndex.ts +++ b/packages/web/src/typedIndex.ts @@ -5,6 +5,7 @@ export { BrowserExtensionPlugin } from './BrowserExtensionPlugin/BrowserExtensio export { LanguageStorage } from './LanguageStorage'; export { LanguageDetector } from './LanguageDetector'; export { BackendFetch } from './BackendFetch'; +export { TagParser } from './TagParser/TagParser'; export { TOLGEE_WRAPPED_ONLY_DATA_ATTRIBUTE, TOLGEE_ATTRIBUTE_NAME, diff --git a/testapps/react/src/App.tsx b/testapps/react/src/App.tsx index 4ff08a9b8e..6761b7c4ca 100644 --- a/testapps/react/src/App.tsx +++ b/testapps/react/src/App.tsx @@ -4,16 +4,16 @@ import { TolgeeProvider, BackendFetch, DevTools, - FormatSimple, TagParser, } from '@tolgee/react'; import { Todos } from './Todos'; import { TranslationMethods } from './TranslationMethods'; +import { FormatIcu } from '@tolgee/format-icu'; const tolgee = Tolgee() .use(DevTools()) - .use(FormatSimple()) + .use(FormatIcu()) .use(TagParser()) .use(BackendFetch()) .init({ diff --git a/testapps/react/src/TranslationMethods.tsx b/testapps/react/src/TranslationMethods.tsx index 907fc00b95..d84a4f8b53 100644 --- a/testapps/react/src/TranslationMethods.tsx +++ b/testapps/react/src/TranslationMethods.tsx @@ -17,7 +17,7 @@ export const TranslationMethods = () => {
- {/*
+

T component with default

This is default @@ -48,7 +48,7 @@ export const TranslationMethods = () => { noWrap />
-
*/} +

T component with interpolation

@@ -58,14 +58,14 @@ export const TranslationMethods = () => { params={{ b: , i: , - key: 'value', + key: 'value', }} > Hey
- {/* +

T component with br tag

@@ -140,7 +140,7 @@ export const TranslationMethods = () => {
-
*/} + {!revealed ? (
From c2949a62925746825eddbacdfa687aeb4ac1abf0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0t=C4=9Bp=C3=A1n=20Gran=C3=A1t?= Date: Mon, 17 Apr 2023 11:20:03 +0200 Subject: [PATCH 3/3] feat: add html escaper --- .../core/src/Controller/Plugins/Plugins.ts | 65 +++++++++++---- packages/core/src/types/plugin.ts | 14 +++- packages/web/src/TagParser/HtmlEscaper.ts | 13 +++ packages/web/src/TagParser/TagParser.ts | 45 +++++++++-- packages/web/src/TagParser/handleTag.ts | 18 ++++- packages/web/src/TagParser/htmlEscape.ts | 13 +++ packages/web/src/TagParser/parser.ts | 14 +++- packages/web/src/TagParser/tagEscape.ts | 39 +++++++++ packages/web/src/TagParser/tokenizer.ts | 19 ++++- .../web/src/__test__/htmlEscaper.test.tsx | 44 ++++++++++ packages/web/src/__test__/tagParser.test.tsx | 80 +++++++++++++++++++ packages/web/src/typedIndex.ts | 1 + testapps/react/src/TranslationMethods.tsx | 2 +- 13 files changed, 335 insertions(+), 32 deletions(-) create mode 100644 packages/web/src/TagParser/HtmlEscaper.ts create mode 100644 packages/web/src/TagParser/htmlEscape.ts create mode 100644 packages/web/src/TagParser/tagEscape.ts create mode 100644 packages/web/src/__test__/htmlEscaper.test.tsx create mode 100644 packages/web/src/__test__/tagParser.test.tsx diff --git a/packages/core/src/Controller/Plugins/Plugins.ts b/packages/core/src/Controller/Plugins/Plugins.ts index 01cf8b7a2a..d5e27795b2 100644 --- a/packages/core/src/Controller/Plugins/Plugins.ts +++ b/packages/core/src/Controller/Plugins/Plugins.ts @@ -22,6 +22,7 @@ import { TolgeeOptionsInternal, FormatErrorHandler, FindPositionsInterface, + ParamsFormatterMiddleware, } from '../../types'; import { DEFAULT_FORMAT_ERROR } from '../State/initState'; @@ -39,6 +40,7 @@ export function Plugins( const instances = { formatters: [] as FormatterMiddleware[], + paramsFormatters: [] as ParamsFormatterMiddleware[], finalFormatter: undefined as FinalFormatterMiddleware | undefined, observer: undefined as ReturnType | undefined, devBackend: undefined as BackendDevMiddleware | undefined, @@ -95,6 +97,14 @@ export function Plugins( } } + function addParamsFormatter( + formatter: ParamsFormatterMiddleware | undefined + ) { + if (formatter) { + instances.paramsFormatters.push(formatter); + } + } + function setFinalFormatter(formatter: FinalFormatterMiddleware | undefined) { instances.finalFormatter = formatter; } @@ -143,6 +153,7 @@ export function Plugins( const pluginTools = Object.freeze({ setFinalFormatter, addFormatter, + addParamsFormatter, setObserver, hasObserver, setUi, @@ -155,8 +166,28 @@ export function Plugins( plugin(tolgeeInstance, pluginTools); } + function handleFormatError( + e: any, + result: string, + props: TranslatePropsInternal + ) { + // eslint-disable-next-line no-console + console.error(e); + const errorMessage = getErrorMessage(e) || DEFAULT_FORMAT_ERROR; + const onFormatError = getInitialOptions().onFormatError; + const formatErrorType = typeof onFormatError; + if (formatErrorType === 'string') { + result = onFormatError as string; + } else if (formatErrorType === 'function') { + result = (onFormatError as FormatErrorHandler)(errorMessage, props); + } else { + result = DEFAULT_FORMAT_ERROR; + } + return result; + } + const self = Object.freeze({ - addPlugin, + addPlugin: addPlugin, run() { const { apiKey, apiUrl, projectId, observerOptions } = @@ -272,8 +303,7 @@ export function Plugins( formatEnabled, ...props }: TranslatePropsInternal & { formatEnabled?: boolean }) { - const { key, translation, defaultValue, noWrap, params, orEmpty, ns } = - props; + const { key, translation, defaultValue, noWrap, orEmpty, ns } = props; const formattableTranslation = translation || defaultValue; let result = formattableTranslation || (orEmpty ? '' : key); @@ -281,6 +311,8 @@ export function Plugins( const isFormatEnabled = formatEnabled || !instances.observer?.outputNotFormattable; + let params = props.params || {}; + const wrap = (result: string) => { if (instances.observer && !noWrap) { return instances.observer.wrap({ @@ -294,6 +326,18 @@ export function Plugins( return result; }; + try { + if (language) { + for (const formatter of instances.paramsFormatters) { + params = formatter.format({ language, params }); + } + } + } catch (e: any) { + result = handleFormatError(e, result, props); + // wrap error message, so it's detectable + return wrap(result); + } + result = wrap(result); try { if (formattableTranslation && language && isFormatEnabled) { @@ -318,20 +362,9 @@ export function Plugins( }); } } catch (e: any) { - // eslint-disable-next-line no-console - console.error(e); - const errorMessage = getErrorMessage(e) || DEFAULT_FORMAT_ERROR; - const onFormatError = getInitialOptions().onFormatError; - const formatErrorType = typeof onFormatError; - if (formatErrorType === 'string') { - result = onFormatError as string; - } else if (formatErrorType === 'function') { - result = (onFormatError as FormatErrorHandler)(errorMessage, props); - } else { - result = DEFAULT_FORMAT_ERROR; - } + result = handleFormatError(e, result, props); // wrap error message, so it's detectable - result = wrap(result); + return wrap(result); } return result; diff --git a/packages/core/src/types/plugin.ts b/packages/core/src/types/plugin.ts index 925303b75c..5cf476b083 100644 --- a/packages/core/src/types/plugin.ts +++ b/packages/core/src/types/plugin.ts @@ -112,13 +112,22 @@ export type WrapperMiddleware = { export type FormatterMiddlewareFormatParams = { translation: string; language: string; - params: Record | undefined; + params: TranslateParams; }; export type FormatterMiddleware = { format: (props: FormatterMiddlewareFormatParams) => string; }; +export type ParamsFormatterMiddlewareFormatParams = { + language: string; + params: TranslateParams; +}; + +export type ParamsFormatterMiddleware = { + format: (props: ParamsFormatterMiddlewareFormatParams) => TranslateParams; +}; + export type TranslationOnClick = (data: { keysAndDefaults: KeyAndParams[]; event: any; @@ -168,6 +177,9 @@ export interface UiInterface { export type PluginTools = Readonly<{ setFinalFormatter: (formatter: FinalFormatterMiddleware | undefined) => void; addFormatter: (formatter: FormatterMiddleware | undefined) => void; + addParamsFormatter: ( + formatter: ParamsFormatterMiddleware | undefined + ) => void; setObserver: (observer: ObserverMiddleware | undefined) => void; hasObserver: () => boolean; setUi: (ui: UiMiddleware | undefined) => void; diff --git a/packages/web/src/TagParser/HtmlEscaper.ts b/packages/web/src/TagParser/HtmlEscaper.ts new file mode 100644 index 0000000000..8f3b419487 --- /dev/null +++ b/packages/web/src/TagParser/HtmlEscaper.ts @@ -0,0 +1,13 @@ +import { FinalFormatterMiddleware, TolgeePlugin } from '@tolgee/core'; +import { htmlEscape } from './htmlEscape'; + +function createTranslationEscaper(): FinalFormatterMiddleware { + return { + format: ({ translation }) => htmlEscape(translation), + }; +} + +export const HtmlEscaper = (): TolgeePlugin => (tolgee, tools) => { + tools.setFinalFormatter(createTranslationEscaper()); + return tolgee; +}; diff --git a/packages/web/src/TagParser/TagParser.ts b/packages/web/src/TagParser/TagParser.ts index 8ffc140882..99687046c9 100644 --- a/packages/web/src/TagParser/TagParser.ts +++ b/packages/web/src/TagParser/TagParser.ts @@ -1,13 +1,46 @@ -import { FinalFormatterMiddleware, TolgeePlugin } from '@tolgee/core'; +import { + FinalFormatterMiddleware, + TolgeePlugin, + ParamsFormatterMiddleware, + TranslateParams, +} from '@tolgee/core'; import { parser } from './parser'; +import { tagEscape } from './tagEscape'; -function createTagParser(): FinalFormatterMiddleware { +function createParamsEscaper(): ParamsFormatterMiddleware { return { - format: ({ translation, params }) => parser(translation, params), + format({ params }) { + const result: TranslateParams = {}; + Object.entries(params).forEach(([key, value]) => { + if (typeof value === 'string') { + result[key] = tagEscape(value); + } else { + result[key] = value; + } + }); + return result; + }, }; } -export const TagParser = (): TolgeePlugin => (tolgee, tools) => { - tools.setFinalFormatter(createTagParser()); - return tolgee; +function createTagParser(escapeHtml: boolean): FinalFormatterMiddleware { + return { + format: ({ translation, params }) => + parser(translation, params, escapeHtml), + }; +} + +type Options = { + escapeParams?: boolean; + escapeHtml?: boolean; }; + +export const TagParser = + (options?: Options): TolgeePlugin => + (tolgee, tools) => { + if (options?.escapeParams ?? true) { + tools.addParamsFormatter(createParamsEscaper()); + } + tools.setFinalFormatter(createTagParser(Boolean(options?.escapeHtml))); + return tolgee; + }; diff --git a/packages/web/src/TagParser/handleTag.ts b/packages/web/src/TagParser/handleTag.ts index 7ba603816a..f782e31a44 100644 --- a/packages/web/src/TagParser/handleTag.ts +++ b/packages/web/src/TagParser/handleTag.ts @@ -1,14 +1,24 @@ +import { htmlEscape } from './htmlEscape'; import { Token } from './tokenizer'; export function handleTag( startToken: Token | undefined, stack: Token[], params: Record | undefined, - fullText: string + fullText: string, + escapeHtml: boolean ) { let token: Token | undefined; let content: any[] = []; + function escape(text: any) { + if (typeof text === 'string' && escapeHtml) { + return htmlEscape(text); + } else { + return text; + } + } + function addToContent(item: any) { if ( typeof content[content.length - 1] === 'string' && @@ -70,14 +80,14 @@ export function handleTag( getParamFunc(token.name) ) { // opening tag - call recursively - addToContent(handleTag(token, stack, params, fullText)); + addToContent(handleTag(token, stack, params, fullText, escapeHtml)); } else { // treat everything else as text - addToContent(token.text); + addToContent(escape(token.text)); } } if (startToken !== undefined) { - prependContent(startToken.text); + prependContent(escape(startToken.text)); } return simplifyContent(); } diff --git a/packages/web/src/TagParser/htmlEscape.ts b/packages/web/src/TagParser/htmlEscape.ts new file mode 100644 index 0000000000..f338b63e06 --- /dev/null +++ b/packages/web/src/TagParser/htmlEscape.ts @@ -0,0 +1,13 @@ +const charMap: Record = { + '&': '&', + '<': '<', + '>': '>', + '"': '"', + "'": ''', +}; + +export function htmlEscape(text: string) { + return text?.replace(/[&<>"']/g, (match: string) => { + return charMap[match] || match; + }); +} diff --git a/packages/web/src/TagParser/parser.ts b/packages/web/src/TagParser/parser.ts index 1ed19005bf..6780ec742b 100644 --- a/packages/web/src/TagParser/parser.ts +++ b/packages/web/src/TagParser/parser.ts @@ -1,10 +1,20 @@ import { handleTag } from './handleTag'; import { tokenizer } from './tokenizer'; -export function parser(text: string, params?: Record) { +export function parser( + text: string, + params?: Record, + escapeHtml?: boolean +) { const tokens = tokenizer(text); - const result = handleTag(undefined, [...tokens], params, text); + const result = handleTag( + undefined, + [...tokens], + params, + text, + Boolean(escapeHtml) + ); return result; } diff --git a/packages/web/src/TagParser/tagEscape.ts b/packages/web/src/TagParser/tagEscape.ts new file mode 100644 index 0000000000..541f892be9 --- /dev/null +++ b/packages/web/src/TagParser/tagEscape.ts @@ -0,0 +1,39 @@ +const STATE_TEXT = 0, + STATE_ESCAPE_MAYBE = 1; + +type State = typeof STATE_TEXT | typeof STATE_ESCAPE_MAYBE; + +export const CHAR_ESCAPE = '\\'; + +export const ESCAPABLE_CHARS = new Set(['<', '>', CHAR_ESCAPE]); + +export function tagEscape(source: string) { + let state: State = STATE_TEXT; + let text = ''; + + let i = 0; + + for (i = 0; i < source.length; i++) { + const char = source[i]; + switch (state) { + case STATE_TEXT: + if (char === '<') { + text += CHAR_ESCAPE; + } + if (char === CHAR_ESCAPE) { + state = STATE_ESCAPE_MAYBE; + } + text += char; + break; + + case STATE_ESCAPE_MAYBE: + if (ESCAPABLE_CHARS.has(char)) { + text = text.slice(0, -1) + char; + } else { + text += char; + } + state = STATE_TEXT; + } + } + return text; +} diff --git a/packages/web/src/TagParser/tokenizer.ts b/packages/web/src/TagParser/tokenizer.ts index 3a34077f98..c7d5763c15 100644 --- a/packages/web/src/TagParser/tokenizer.ts +++ b/packages/web/src/TagParser/tokenizer.ts @@ -1,13 +1,17 @@ +import { CHAR_ESCAPE, ESCAPABLE_CHARS } from './tagEscape'; + const STATE_TEXT = 0, STATE_TAG_START = 1, STATE_TAG_NAME = 2, - STATE_SELF_CLOSING = 3; + STATE_SELF_CLOSING = 3, + STATE_ESCAPE_MAYBE = 4; type State = | typeof STATE_TEXT | typeof STATE_TAG_START | typeof STATE_TAG_NAME - | typeof STATE_SELF_CLOSING; + | typeof STATE_SELF_CLOSING + | typeof STATE_ESCAPE_MAYBE; export type Token = { type: 'text' | 'tag'; @@ -63,6 +67,9 @@ export function tokenizer(source: string) { createToken('text'); state = STATE_TAG_START; } + if (char === CHAR_ESCAPE) { + state = STATE_ESCAPE_MAYBE; + } text += char; break; @@ -109,6 +116,14 @@ export function tokenizer(source: string) { state = STATE_TEXT; } break; + + case STATE_ESCAPE_MAYBE: + if (ESCAPABLE_CHARS.has(char)) { + text = text.slice(0, -1) + char; + } else { + text += char; + } + state = STATE_TEXT; } } createToken('text'); diff --git a/packages/web/src/__test__/htmlEscaper.test.tsx b/packages/web/src/__test__/htmlEscaper.test.tsx new file mode 100644 index 0000000000..fe08f766de --- /dev/null +++ b/packages/web/src/__test__/htmlEscaper.test.tsx @@ -0,0 +1,44 @@ +import { FormatSimple, TolgeeCore, TolgeeInstance } from '@tolgee/core'; +import { HtmlEscaper } from '../typedIndex'; + +describe('tag parser', () => { + let tolgee: TolgeeInstance; + beforeEach(() => { + tolgee = TolgeeCore() + .use(HtmlEscaper()) + .use(FormatSimple()) + .init({ + language: 'en', + staticData: { + en: { + simple: 'hello', + nested: 'hello world', + with_param: '{param}', + invalid: '{param}', + }, + }, + }); + }); + + it('manages simple element', () => { + expect(tolgee.t('simple')).toEqual('<a>hello</a>'); + }); + + it('manages nested element', () => { + expect(tolgee.t('nested')).toEqual( + '<a>hello <b>world</b></a>' + ); + }); + + it('manages params', () => { + expect(tolgee.t('with_param', { param: 'inside' })).toEqual( + '<a><a>inside</a></a>' + ); + }); + + it('manages complex html', () => { + expect(tolgee.t('invalid', { param: 'inside' })).toEqual( + '<a href="test">inside</a>' + ); + }); +}); diff --git a/packages/web/src/__test__/tagParser.test.tsx b/packages/web/src/__test__/tagParser.test.tsx new file mode 100644 index 0000000000..2f6d5c32ee --- /dev/null +++ b/packages/web/src/__test__/tagParser.test.tsx @@ -0,0 +1,80 @@ +import { FormatSimple, TolgeeCore, TolgeeInstance } from '@tolgee/core'; +import { TagParser } from '../typedIndex'; + +function textElement(name: string): any { + return (content: string) => `<${name}>${content}`; +} + +describe('tag parser', () => { + let tolgee: TolgeeInstance; + beforeEach(() => { + tolgee = TolgeeCore() + .use(FormatSimple()) + .init({ + language: 'en', + staticData: { + en: { + simple: 'hello', + nested: 'hello world', + with_param: '{param}', + invalid: '{param}', + unclosed: 'test', + }, + }, + }); + }); + + it('manages simple element', () => { + tolgee.addPlugin(TagParser()); + expect(tolgee.t('simple', { a: textElement('A') })).toEqual('hello'); + }); + + it('manages nested element', () => { + tolgee.addPlugin(TagParser()); + expect( + tolgee.t('nested', { a: textElement('A'), b: textElement('B') }) + ).toEqual('hello world'); + }); + + it('manages param', () => { + tolgee.addPlugin(TagParser()); + expect( + tolgee.t('with_param', { a: textElement('A'), param: 'inside' }) + ).toEqual('inside'); + }); + + it('escapes param', () => { + tolgee.addPlugin(TagParser()); + expect( + tolgee.t('with_param', { a: textElement('A'), param: 'inside' }) + ).toEqual('inside'); + }); + + it('keeps invalid tag', () => { + tolgee.addPlugin(TagParser()); + expect( + tolgee.t('invalid', { a: textElement('A'), param: 'inside' }) + ).toEqual('inside'); + }); + + it('escapes invalid html', () => { + tolgee.addPlugin(TagParser({ escapeHtml: true })); + expect( + tolgee.t('invalid', { a: textElement('A'), param: 'inside' }) + ).toEqual('<a href="test">inside</a>'); + }); + + it('escapes html in params', () => { + tolgee.addPlugin(TagParser({ escapeHtml: true })); + expect( + tolgee.t('with_param', { a: textElement('A'), param: 'inside' }) + ).toEqual('<a>inside</a>'); + }); + + it('escapes unclosed tag', () => { + tolgee.addPlugin(TagParser({ escapeHtml: true })); + expect( + tolgee.t('unclosed', { a: textElement('A'), b: textElement('B') }) + ).toEqual('<a>test</b>'); + }); +}); diff --git a/packages/web/src/typedIndex.ts b/packages/web/src/typedIndex.ts index 65b88a71e4..49b5f1ed28 100644 --- a/packages/web/src/typedIndex.ts +++ b/packages/web/src/typedIndex.ts @@ -6,6 +6,7 @@ export { LanguageStorage } from './LanguageStorage'; export { LanguageDetector } from './LanguageDetector'; export { BackendFetch } from './BackendFetch'; export { TagParser } from './TagParser/TagParser'; +export { HtmlEscaper } from './TagParser/HtmlEscaper'; export { TOLGEE_WRAPPED_ONLY_DATA_ATTRIBUTE, TOLGEE_ATTRIBUTE_NAME, diff --git a/testapps/react/src/TranslationMethods.tsx b/testapps/react/src/TranslationMethods.tsx index d84a4f8b53..4c6b76ade8 100644 --- a/testapps/react/src/TranslationMethods.tsx +++ b/testapps/react/src/TranslationMethods.tsx @@ -58,7 +58,7 @@ export const TranslationMethods = () => { params={{ b: , i: , - key: 'value', + key: 'value', }} > Hey