Merge pull request #256 from kitsuyui/evaluate

Re-implement evaluates
kitsuyui · Aug 9, 2022 · 00b91b2 · 00b91b2
2 parents cb9fcf8 + 2ec4d12
commit 00b91b2
Show file tree

Hide file tree

Showing 8 changed files with 240 additions and 7 deletions.
diff --git a/README.md b/README.md
@@ -4,7 +4,6 @@
 [![test](https://github.com/kitsuyui/rendering-proxy/actions/workflows/test.yml/badge.svg)](https://github.com/kitsuyui/rendering-proxy/actions/workflows/test.yml)
 [![Docker Pulls](https://img.shields.io/docker/pulls/kitsuyui/rendering-proxy.svg)](https://hub.docker.com/r/kitsuyui/rendering-proxy/)
 
-
 Fetching rendered DOM easily and simply. Like cURL.
 Using [Headless Chromium](https://chromium.googlesource.com/chromium/src/+/lkgr/headless/README.md).
 
@@ -37,16 +36,35 @@ $ docker run --rm kitsuyui/rendering-proxy cli https://example.com/
 </body></html>
 ```
 
-### Options
+## evaluate
 
-#### evaluate
+### CLI mode
 
-When `--evaluate` is specified, JavaScript code is evaluated before getting DOM.
+When `-e`, `--evaluate` is specified, JavaScript code is evaluated before getting DOM.
 
 ```console
-$ docker run --rm kitsuyui/rendering-proxy cli https://example.com/ --evaluate 'document.write("yay")'
-<html><head></head><body>yay</body></html>
+$ yarn ts-node src/main.ts cli https://example.com/ -e 'document.title = "updated"' -e 'document.title += " twice"'
+<!DOCTYPE html><html><head>
+    <title>updated twice</title>
+...
 ```
+
+### Server mode
+
+Send the options via request header `X-Rendering-Proxy` (case-insensitive).
+Receive the results via request header `X-Rendering-Proxy` (case-insensitive).
+
+```console
+curl -H 'X-Rendering-Proxy: {"evaluates": ["1 + 1"], "waitUntil": "load"}' --include http://localhost:8080/https://example.com/
+HTTP/1.1 200 OK
+...
+x-rendering-proxy: [{"success":true,"result":2,"script":"1 + 1"}]
+...
+
+<!DOCTYPE html><html><head>
+    <title>Example Domain</title>
+```
+
 ## LICENSE
 
 The 3-Clause BSD License. See also LICENSE file.
diff --git a/src/cli/index.ts b/src/cli/index.ts
@@ -27,6 +27,7 @@ export async function renderToStream(
     const result = await getRenderedContent(browser, {
       url: url_,
       waitUntil: request.waitUntil,
+      evaluates: request.evaluates,
     });
     writable.write(result.body, 'binary');
   });

diff --git a/src/main.ts b/src/main.ts
@@ -13,6 +13,7 @@ type ParsedArgs =
       url: string;
       p: number;
       b: string;
+      e: string[];
       _: (string | number)[];
       $0: string;
     }
@@ -40,6 +41,11 @@ export async function parseArgs(args: string[]): Promise<ParsedArgs> {
           default: 'chromium',
           choices: selectableBrowsers,
         })
+        .option('e', {
+          alias: 'evaluate',
+          default: [],
+          array: true,
+        })
         .positional('url', { type: 'string', demandOption: true });
     })
     .command('server', 'Server mode', (builder) => {
@@ -66,6 +72,7 @@ export async function main(): Promise<void> {
       url: argv.url,
       waitUntil: argv.u as LifecycleEvent,
       name: argv.b as SelectableBrowsers,
+      evaluates: argv.e as string[],
     });
   } else if (argv._[0] === 'server') {
     await server.main({

diff --git a/src/render/index.spec.ts b/src/render/index.spec.ts
@@ -139,3 +139,56 @@ describe('getRenderedContent', () => {
     });
   });
 });
+
+describe('getRenderedContent with evaluates', () => {
+  let browser: Browser;
+
+  beforeAll(async () => {
+    browser = await getBrowser();
+  });
+  afterAll(async () => {
+    await browser.close();
+  });
+
+  it('works', async () => {
+    const result = await getRenderedContent(browser, {
+      url: 'http://example.com/',
+      evaluates: ['document.title', 'navigator.userAgent', '1 + 1'],
+    });
+    expect(result.evaluateResults[0].script).toBe('document.title');
+    expect(result.evaluateResults[0].result).toBe('Example Domain');
+    expect(result.evaluateResults[1].script).toBe('navigator.userAgent');
+    expect(result.evaluateResults[1].result).toContain('Mozilla/5.0');
+    expect(result.evaluateResults[2].script).toBe('1 + 1');
+    expect(result.evaluateResults[2].result).toBe(2);
+  });
+
+  it('works with errors', async () => {
+    const result = await getRenderedContent(browser, {
+      url: 'http://example.com/',
+      evaluates: [
+        'document.title',
+        'throw new Error("error")',
+        'navigator.userAgent',
+      ],
+    });
+    expect(result.evaluateResults[0].script).toBe('document.title');
+    expect(result.evaluateResults[0].result).toBe('Example Domain');
+    expect(result.evaluateResults[1].script).toBe('throw new Error("error")');
+    expect(result.evaluateResults[1].result).toContain('Error');
+    expect(result.evaluateResults[1].success).toBe(false);
+    expect(result.evaluateResults[2].script).toBe('navigator.userAgent');
+    expect(result.evaluateResults[2].result).toContain('Mozilla/5.0');
+  });
+
+  it('can update content body', async () => {
+    const result = await getRenderedContent(browser, {
+      url: 'http://example.com/',
+      evaluates: ['document.title = "Updated Title"'],
+    });
+    expect(result.evaluateResults[0].script).toBe(
+      'document.title = "Updated Title"'
+    );
+    expect(result.evaluateResults[0].result).toBe('Updated Title');
+  });
+});
diff --git a/src/render/index.ts b/src/render/index.ts
@@ -12,19 +12,28 @@ export type LifecycleEvent = typeof lifeCycleEvents[number];
 export interface RenderRequest {
   url: string;
   waitUntil?: LifecycleEvent;
+  evaluates?: string[];
+}
+
+export interface EvaluateResult {
+  success: boolean;
+  script: string;
+  result: unknown;
 }
 
 export interface RenderResult {
   status: number;
   headers: { [key: string]: string };
   body: Buffer;
+  evaluateResults: EvaluateResult[];
 }
 
 function emptyRenderResult(): RenderResult {
   return {
     status: 204,
     headers: {},
     body: Buffer.from(''),
+    evaluateResults: [],
   };
 }
 
@@ -48,8 +57,23 @@ export async function getRenderedContent(
     defer(() => page.close());
 
     let response;
+    const evaluateResults = [];
     try {
       response = await page.goto(url, { waitUntil });
+      if (request.evaluates) {
+        for (const evaluate of request.evaluates) {
+          try {
+            const result = await page.evaluate(evaluate, { waitUntil });
+            evaluateResults.push({ success: true, result, script: evaluate });
+          } catch (error) {
+            evaluateResults.push({
+              success: false,
+              result: String(error),
+              script: evaluate,
+            });
+          }
+        }
+      }
     } catch (error) {
       return emptyRenderResult();
     }
@@ -67,6 +91,7 @@ export async function getRenderedContent(
       status,
       headers,
       body,
+      evaluateResults,
     };
   });
 }
diff --git a/src/server/index.ts b/src/server/index.ts
@@ -9,6 +9,8 @@ import { isAbsoluteURL } from '../lib/url';
 import { waitForProcessExit } from '../lib/wait_for_exit';
 import { getRenderedContent } from '../render';
 
+import { parseRenderingProxyHeader } from './request_options';
+
 interface ServerArgument {
   port?: number;
   name?: SelectableBrowsers;
@@ -30,10 +32,16 @@ export function createHandler(browser: Browser) {
     if (!originUrl) return terminateRequestWithEmpty(req, res);
     if (!isAbsoluteURL(originUrl)) return terminateRequestWithEmpty(req, res);
 
+    const options = parseRenderingProxyHeader(req.headers['x-rendering-proxy']);
     const renderedContent = await getRenderedContent(browser, {
       url: originUrl,
+      ...options,
     });
-    const headers = excludeUnusedHeaders(renderedContent.headers);
+
+    const headers = {
+      ...excludeUnusedHeaders(renderedContent.headers),
+      'x-rendering-proxy': JSON.stringify(renderedContent.evaluateResults),
+    };
     const status = renderedContent.status;
     res.writeHead(status, headers);
     res.end(renderedContent.body, 'binary');

diff --git a/src/server/request_options.spec.ts b/src/server/request_options.spec.ts
@@ -0,0 +1,74 @@
+import { parseRenderingProxyHeader } from './request_options';
+
+describe('parseRenderingProxyHeader', () => {
+  it('returns default when empty or invalid header value', async () => {
+    expect(parseRenderingProxyHeader(undefined)).toStrictEqual({
+      waitUntil: 'networkidle',
+      evaluates: [],
+    });
+    expect(parseRenderingProxyHeader('')).toStrictEqual({
+      waitUntil: 'networkidle',
+      evaluates: [],
+    });
+    expect(parseRenderingProxyHeader([])).toStrictEqual({
+      waitUntil: 'networkidle',
+      evaluates: [],
+    });
+    expect(parseRenderingProxyHeader('1234')).toStrictEqual({
+      waitUntil: 'networkidle',
+      evaluates: [],
+    });
+    expect(parseRenderingProxyHeader('{')).toStrictEqual({
+      waitUntil: 'networkidle',
+      evaluates: [],
+    });
+    expect(
+      parseRenderingProxyHeader('{"evaluates": 1234, "waitUntil": 3456}')
+    ).toStrictEqual({
+      waitUntil: 'networkidle',
+      evaluates: [],
+    });
+  });
+
+  it('parses evaluates', async () => {
+    expect(parseRenderingProxyHeader('{"evaluates": []}')).toStrictEqual({
+      waitUntil: 'networkidle',
+      evaluates: [],
+    });
+    expect(parseRenderingProxyHeader('{"evaluates": ["1 + 1"]}')).toStrictEqual(
+      {
+        waitUntil: 'networkidle',
+        evaluates: ['1 + 1'],
+      }
+    );
+    expect(
+      parseRenderingProxyHeader('{"evaluates": ["1 + 1", "document.title"]}')
+    ).toStrictEqual({
+      waitUntil: 'networkidle',
+      evaluates: ['1 + 1', 'document.title'],
+    });
+  });
+
+  it('parses waitUntil', async () => {
+    expect(
+      parseRenderingProxyHeader('{"waitUntil": "networkidle"}')
+    ).toStrictEqual({
+      waitUntil: 'networkidle',
+      evaluates: [],
+    });
+    expect(
+      parseRenderingProxyHeader('{"waitUntil": "domcontentloaded"}')
+    ).toStrictEqual({
+      waitUntil: 'domcontentloaded',
+      evaluates: [],
+    });
+    expect(parseRenderingProxyHeader('{"waitUntil": "load"}')).toStrictEqual({
+      waitUntil: 'load',
+      evaluates: [],
+    });
+    expect(parseRenderingProxyHeader('{"waitUntil": "commit"}')).toStrictEqual({
+      waitUntil: 'commit',
+      evaluates: [],
+    });
+  });
+});
diff --git a/src/server/request_options.ts b/src/server/request_options.ts
@@ -0,0 +1,47 @@
+import { LifecycleEvent, lifeCycleEvents } from '../render';
+
+interface RequestOption {
+  waitUntil: LifecycleEvent;
+  evaluates: string[];
+}
+
+export function parseRenderingProxyHeader(
+  headerValue: undefined | string | string[]
+): RequestOption {
+  if (typeof headerValue === 'string') {
+    return parseOptions(headerValue);
+  }
+  if (Array.isArray(headerValue)) {
+    return parseOptions(headerValue.join(''));
+  }
+  return parseOptions('');
+}
+
+function parseOptions(text: string): RequestOption {
+  let baseParsed: RequestOption = {
+    waitUntil: 'networkidle',
+    evaluates: [],
+  };
+  try {
+    baseParsed = JSON.parse(text);
+  } catch (e) {
+    // ignore
+  }
+  let waitUntil: LifecycleEvent = 'networkidle';
+  if (lifeCycleEvents.indexOf(baseParsed.waitUntil) > -1) {
+    waitUntil = baseParsed.waitUntil as LifecycleEvent;
+  }
+  let evaluates: string[] = [];
+  if (
+    baseParsed.evaluates &&
+    Array.isArray(baseParsed.evaluates) &&
+    baseParsed.evaluates.every((e: unknown) => typeof e === 'string')
+  ) {
+    evaluates = baseParsed.evaluates;
+  }
+
+  return {
+    waitUntil,
+    evaluates,
+  };
+}