fix: put rect in html tree

packages/midscene/src/ai-model/prompt/util.ts

@@ -187,14 +187,30 @@ export function describeElement(
     .join('\n');
 }
 
-export function truncateText(text: string, maxLength = 100) {
-  if (text && text.length > maxLength) {
+export function truncateText(
+  text: string | number | object | undefined,
+  maxLength = 150,
+) {
+  if (typeof text === 'undefined') {
+    return '';
+  }
+
+  if (typeof text === 'object') {
+    text = JSON.stringify(text);
+  }
+
+  if (typeof text === 'number') {
+    return text.toString();
+  }
+
+  if (typeof text === 'string' && text.length > maxLength) {
     return `${text.slice(0, maxLength)}...`;
   }
 
   if (typeof text === 'string') {
     return text.trim();
   }
+
   return '';
 }
 
@@ -330,7 +346,15 @@ export async function descriptionOfTree<
       }
       const markerId = (node.node as any).indexId;
       const markerIdString = markerId ? `markerId="${markerId}"` : '';
-      before = `<${nodeTypeString} id="${node.node.id}" ${markerIdString} ${attributesString(trimAttributes(node.node.attributes || {}, truncateTextLength))}>`;
+      const rectAttribute = node.node.rect
+        ? {
+            left: node.node.rect.left,
+            top: node.node.rect.top,
+            width: node.node.rect.width,
+            height: node.node.rect.height,
+          }
+        : {};
+      before = `<${nodeTypeString} id="${node.node.id}" ${markerIdString} ${attributesString(trimAttributes(node.node.attributes || {}, truncateTextLength))} ${attributesString(rectAttribute)}>`;
       const content = truncateText(node.node.content, truncateTextLength);
       contentWithIndent = content ? `\n${indentStr}  ${content}` : '';
       after = `</${nodeTypeString}>`;

packages/midscene/tests/ai/__snapshots__/prompt.test.ts.snap

@@ -1,18 +1,18 @@
 // Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
 
 exports[`utils > should be able to describe tree 1`] = `
-"<container id="1" markerId="19" >
-  Legend had it that the Whispering Woods held an ancient secret, one that connected the world of man ...
-  <text id="2" markerId="999" ariaLabel="image description, it could be a long text, very loooooooooooooooooooooooooooooooooooooooooong">
+"<container id="1" markerId="19"  left="0" top="0" width="100" height="100">
+  Legend had it that the Whispering Woods held an ancient secret, one that connected the world of man and magic, of reality and dream. Each leaf, every ...
+  <text id="2" markerId="999" ariaLabel="image description, it could be a long text, very loooooooooooooooooooooooooooooooooooooooooong" left="0" top="0" width="100" height="100">
     world
   </text>
-  <img id="3" markerId="20" ariaLabel="image description" storyContent="Legend had it that the Whispering Woods held an ancient secret, one that connected the world of man ...">
+  <img id="3" markerId="20" ariaLabel="image description" storyContent="Legend had it that the Whispering Woods held an ancient secret, one that connected the world of man and magic, of reality and dream. Each leaf, every ..." left="0" top="0" width="100" height="100">
     world 2345
-    <img id="3" markerId="20" ariaLabel="image description" storyContent="Legend had it that the Whispering Woods held an ancient secret, one that connected the world of man ...">
+    <img id="3" markerId="20" ariaLabel="image description" storyContent="Legend had it that the Whispering Woods held an ancient secret, one that connected the world of man and magic, of reality and dream. Each leaf, every ..." left="0" top="0" width="100" height="100">
     </img>
     
       <>
-        <img id="3222" markerId="20" ariaLabel="image description" storyContent="Legend had it that the Whispering Woods held an ancient secret, one that connected the world of man ...">
+        <img id="3222" markerId="20" ariaLabel="image description" storyContent="Legend had it that the Whispering Woods held an ancient secret, one that connected the world of man and magic, of reality and dream. Each leaf, every ..." left="0" top="0" width="100" height="100">
           world 2345
         </img>
       </>
@@ -22,16 +22,16 @@ exports[`utils > should be able to describe tree 1`] = `
 `;
 
 exports[`utils > should be able to describe tree, filterNonTextContent = true 1`] = `
-"<container id="1" markerId="19" >
+"<container id="1" markerId="19"  left="0" top="0" width="100" height="100">
   Legend had it that t...
-  <text id="2" markerId="999" ariaLabel="image description, i...">
+  <text id="2" markerId="999" ariaLabel="image description, i..." left="0" top="0" width="100" height="100">
     world
   </text>
-  <img id="3" markerId="20" ariaLabel="image description" storyContent="Legend had it that t...">
+  <img id="3" markerId="20" ariaLabel="image description" storyContent="Legend had it that t..." left="0" top="0" width="100" height="100">
     world 2345
     
       
-        <img id="3222" markerId="20" ariaLabel="image description" storyContent="Legend had it that t...">
+        <img id="3222" markerId="20" ariaLabel="image description" storyContent="Legend had it that t..." left="0" top="0" width="100" height="100">
           world 2345
         </img>
       

packages/midscene/tests/ai/prompt.test.ts

@@ -133,7 +133,9 @@ describe('utils', () => {
           },
           {
             node: {
-              attributes: {},
+              attributes: {
+                nodeType: NodeType.CONTAINER,
+              },
               id: '3',
               indexId: 20,
               rect: {
@@ -150,7 +152,9 @@ describe('utils', () => {
             children: [
               {
                 node: {
-                  attributes: {},
+                  attributes: {
+                    nodeType: NodeType.IMG,
+                  },
                   id: '3',
                   indexId: 20,
                   rect: {
@@ -202,13 +206,13 @@ describe('utils', () => {
 
   it('should be able to describe tree', async () => {
     const description = await descriptionOfTree(tree);
-    console.log(description);
+    // console.log(description);
     expect(description).toMatchSnapshot();
   });
 
   it('should be able to describe tree, filterNonTextContent = true', async () => {
     const description = await descriptionOfTree(tree, 20, true);
-    console.log(description);
+    // console.log(description);
     expect(description).toMatchSnapshot();
   });
 });

packages/web-integration/tests/unit-test/web-extractor.test.ts

@@ -12,20 +12,20 @@ import { launchPage } from '../ai/web/puppeteer/utils';
 const pageDir = join(__dirname, './fixtures/web-extractor');
 const pagePath = join(pageDir, 'index.html');
 
-const treeToList = (tree: ElementTreeNode<WebElementInfo>) => {
-  // dfs topChildren
-  const elementInfoArray: WebElementInfo[] = [];
-  function dfsTopChildren(node: ElementTreeNode<WebElementInfo>) {
-    if (node.node) {
-      elementInfoArray.push(node.node);
-    }
-    for (let i = 0; i < node.children.length; i++) {
-      dfsTopChildren(node.children[i]);
-    }
-  }
-  dfsTopChildren(tree);
-  return elementInfoArray;
-};
+// const treeToList = (tree: ElementTreeNode<WebElementInfo>) => {
+//   // dfs topChildren
+//   const elementInfoArray: WebElementInfo[] = [];
+//   function dfsTopChildren(node: ElementTreeNode<WebElementInfo>) {
+//     if (node.node) {
+//       elementInfoArray.push(node.node);
+//     }
+//     for (let i = 0; i < node.children.length; i++) {
+//       dfsTopChildren(node.children[i]);
+//     }
+//   }
+//   dfsTopChildren(tree);
+//   return elementInfoArray;
+// };
 
 describe(
   'extractor',
@@ -51,6 +51,7 @@ describe(
         viewport: {
           width: 1080,
           height: 3000,
+          deviceScaleFactor: 1,
         },
       });
 
@@ -150,6 +151,7 @@ describe(
         viewport: {
           width: 1080,
           height: 200,
+          deviceScaleFactor: 1,
         },
       });
       await page.scrollDown();