diff --git a/packages/midscene/src/ai-model/prompt/util.ts b/packages/midscene/src/ai-model/prompt/util.ts index a41de7b63..5b2f17223 100644 --- a/packages/midscene/src/ai-model/prompt/util.ts +++ b/packages/midscene/src/ai-model/prompt/util.ts @@ -187,14 +187,30 @@ export function describeElement( .join('\n'); } -export function truncateText(text: string, maxLength = 100) { - if (text && text.length > maxLength) { +export function truncateText( + text: string | number | object | undefined, + maxLength = 150, +) { + if (typeof text === 'undefined') { + return ''; + } + + if (typeof text === 'object') { + text = JSON.stringify(text); + } + + if (typeof text === 'number') { + return text.toString(); + } + + if (typeof text === 'string' && text.length > maxLength) { return `${text.slice(0, maxLength)}...`; } if (typeof text === 'string') { return text.trim(); } + return ''; } @@ -330,7 +346,15 @@ export async function descriptionOfTree< } const markerId = (node.node as any).indexId; const markerIdString = markerId ? `markerId="${markerId}"` : ''; - before = `<${nodeTypeString} id="${node.node.id}" ${markerIdString} ${attributesString(trimAttributes(node.node.attributes || {}, truncateTextLength))}>`; + const rectAttribute = node.node.rect + ? { + left: node.node.rect.left, + top: node.node.rect.top, + width: node.node.rect.width, + height: node.node.rect.height, + } + : {}; + before = `<${nodeTypeString} id="${node.node.id}" ${markerIdString} ${attributesString(trimAttributes(node.node.attributes || {}, truncateTextLength))} ${attributesString(rectAttribute)}>`; const content = truncateText(node.node.content, truncateTextLength); contentWithIndent = content ? `\n${indentStr} ${content}` : ''; after = ``; diff --git a/packages/midscene/tests/ai/__snapshots__/prompt.test.ts.snap b/packages/midscene/tests/ai/__snapshots__/prompt.test.ts.snap index 761b04060..865c44842 100644 --- a/packages/midscene/tests/ai/__snapshots__/prompt.test.ts.snap +++ b/packages/midscene/tests/ai/__snapshots__/prompt.test.ts.snap @@ -1,18 +1,18 @@ // Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html exports[`utils > should be able to describe tree 1`] = ` -" - Legend had it that the Whispering Woods held an ancient secret, one that connected the world of man ... - +" + Legend had it that the Whispering Woods held an ancient secret, one that connected the world of man and magic, of reality and dream. Each leaf, every ... + world - + world 2345 - + <> - + world 2345 @@ -22,16 +22,16 @@ exports[`utils > should be able to describe tree 1`] = ` `; exports[`utils > should be able to describe tree, filterNonTextContent = true 1`] = ` -" +" Legend had it that t... - + world - + world 2345 - + world 2345 diff --git a/packages/midscene/tests/ai/prompt.test.ts b/packages/midscene/tests/ai/prompt.test.ts index 6a813353d..59a351605 100644 --- a/packages/midscene/tests/ai/prompt.test.ts +++ b/packages/midscene/tests/ai/prompt.test.ts @@ -133,7 +133,9 @@ describe('utils', () => { }, { node: { - attributes: {}, + attributes: { + nodeType: NodeType.CONTAINER, + }, id: '3', indexId: 20, rect: { @@ -150,7 +152,9 @@ describe('utils', () => { children: [ { node: { - attributes: {}, + attributes: { + nodeType: NodeType.IMG, + }, id: '3', indexId: 20, rect: { @@ -202,13 +206,13 @@ describe('utils', () => { it('should be able to describe tree', async () => { const description = await descriptionOfTree(tree); - console.log(description); + // console.log(description); expect(description).toMatchSnapshot(); }); it('should be able to describe tree, filterNonTextContent = true', async () => { const description = await descriptionOfTree(tree, 20, true); - console.log(description); + // console.log(description); expect(description).toMatchSnapshot(); }); }); diff --git a/packages/web-integration/tests/unit-test/fixtures/web-extractor/input.png b/packages/web-integration/tests/unit-test/fixtures/web-extractor/input.png index a975fe255..18fb5b31a 100644 Binary files a/packages/web-integration/tests/unit-test/fixtures/web-extractor/input.png and b/packages/web-integration/tests/unit-test/fixtures/web-extractor/input.png differ diff --git a/packages/web-integration/tests/unit-test/fixtures/web-extractor/output.png b/packages/web-integration/tests/unit-test/fixtures/web-extractor/output.png index 7fe5cd683..ccdeba74f 100644 Binary files a/packages/web-integration/tests/unit-test/fixtures/web-extractor/output.png and b/packages/web-integration/tests/unit-test/fixtures/web-extractor/output.png differ diff --git a/packages/web-integration/tests/unit-test/fixtures/web-extractor/scroll/input.png b/packages/web-integration/tests/unit-test/fixtures/web-extractor/scroll/input.png index 025d3c21b..d4e8abcca 100644 Binary files a/packages/web-integration/tests/unit-test/fixtures/web-extractor/scroll/input.png and b/packages/web-integration/tests/unit-test/fixtures/web-extractor/scroll/input.png differ diff --git a/packages/web-integration/tests/unit-test/fixtures/web-extractor/scroll/output.png b/packages/web-integration/tests/unit-test/fixtures/web-extractor/scroll/output.png index 5f3e64baa..6cd0a5af8 100644 Binary files a/packages/web-integration/tests/unit-test/fixtures/web-extractor/scroll/output.png and b/packages/web-integration/tests/unit-test/fixtures/web-extractor/scroll/output.png differ diff --git a/packages/web-integration/tests/unit-test/web-extractor.test.ts b/packages/web-integration/tests/unit-test/web-extractor.test.ts index b76dcf9d6..4866eccc3 100644 --- a/packages/web-integration/tests/unit-test/web-extractor.test.ts +++ b/packages/web-integration/tests/unit-test/web-extractor.test.ts @@ -12,20 +12,20 @@ import { launchPage } from '../ai/web/puppeteer/utils'; const pageDir = join(__dirname, './fixtures/web-extractor'); const pagePath = join(pageDir, 'index.html'); -const treeToList = (tree: ElementTreeNode) => { - // dfs topChildren - const elementInfoArray: WebElementInfo[] = []; - function dfsTopChildren(node: ElementTreeNode) { - if (node.node) { - elementInfoArray.push(node.node); - } - for (let i = 0; i < node.children.length; i++) { - dfsTopChildren(node.children[i]); - } - } - dfsTopChildren(tree); - return elementInfoArray; -}; +// const treeToList = (tree: ElementTreeNode) => { +// // dfs topChildren +// const elementInfoArray: WebElementInfo[] = []; +// function dfsTopChildren(node: ElementTreeNode) { +// if (node.node) { +// elementInfoArray.push(node.node); +// } +// for (let i = 0; i < node.children.length; i++) { +// dfsTopChildren(node.children[i]); +// } +// } +// dfsTopChildren(tree); +// return elementInfoArray; +// }; describe( 'extractor', @@ -51,6 +51,7 @@ describe( viewport: { width: 1080, height: 3000, + deviceScaleFactor: 1, }, }); @@ -150,6 +151,7 @@ describe( viewport: { width: 1080, height: 200, + deviceScaleFactor: 1, }, }); await page.scrollDown();