feat(selectors): allow to capture intermediate result (#1978)

This introduces the `*name=body` syntax to capture intermediate result.
For example, `*css=section >> "Title"` will capture a section that contains "Title".

Author: dgozman

docs/core-concepts.md

@@ -168,7 +168,13 @@ await page.click('css:light=div');
 Selectors using the same or different engines can be combined using the `>>` separator. For example,
 
 ```js
-await page.click('#free-month-promo >> text=Learn more');
+// Click an element with text 'Sign Up' inside of a #free-month-promo.
+await page.click('#free-month-promo >> text=Sign Up');
+```
+
+```js
+// Capture textContent of a section that contains an element with text 'Selectors'.
+const sectionText = await page.$eval('*css=section >> text=Selectors', e => e.textContent);
 ```
 
 <br/>

docs/selectors.md

@@ -22,6 +22,8 @@ document
   .querySelector('span[attr=value]')
 ```
 
+Selector engine name can be prefixed with `*` to capture element that matches the particular clause instead of the last one. For example, `css=article >> text=Hello` captures the element with the text `Hello`, and `*css=article >> text=Hello` (note the `*`) captures the `article` element that contains some element with the text `Hello`.
+
 For convenience, selectors in the wrong format are heuristically converted to the right format:
 - Selector starting with `//` is assumed to be `xpath=selector`. Example: `page.click('//html')` is converted to `page.click('xpath=//html')`.
 - Selector surrounded with quotes (either `"` or `'`) is assumed to be `text=selector`. Example: `page.click('"foo"')` is converted to `page.click('text="foo"')`.

src/injected/selectorEvaluator.ts

@@ -55,22 +55,27 @@ class SelectorEvaluator {
   }
 
   private _querySelectorRecursively(root: SelectorRoot, selector: types.ParsedSelector, index: number): Element | undefined {
-    const current = selector[index];
-    if (index === selector.length - 1)
+    const current = selector.parts[index];
+    if (index === selector.parts.length - 1)
       return this.engines.get(current.name)!.query(root, current.body);
     const all = this.engines.get(current.name)!.queryAll(root, current.body);
     for (const next of all) {
       const result = this._querySelectorRecursively(next, selector, index + 1);
       if (result)
-        return result;
+        return selector.capture === index ? next : result;
     }
   }
 
   querySelectorAll(selector: types.ParsedSelector, root: Node): Element[] {
     if (!(root as any)['querySelectorAll'])
       throw new Error('Node is not queryable.');
+    const capture = selector.capture === undefined ? selector.parts.length - 1 : selector.capture;
+    // Query all elements up to the capture.
+    const partsToQuerAll = selector.parts.slice(0, capture + 1);
+    // Check they have a descendant matching everything after the capture.
+    const partsToCheckOne = selector.parts.slice(capture + 1);
     let set = new Set<SelectorRoot>([ root as SelectorRoot ]);
-    for (const { name, body } of selector) {
+    for (const { name, body } of partsToQuerAll) {
       const newSet = new Set<Element>();
       for (const prev of set) {
         for (const next of this.engines.get(name)!.queryAll(prev, body)) {
@@ -81,7 +86,11 @@ class SelectorEvaluator {
       }
       set = newSet;
     }
-    return Array.from(set) as Element[];
+    const candidates = Array.from(set) as Element[];
+    if (!partsToCheckOne.length)
+      return candidates;
+    const partial = { parts: partsToCheckOne };
+    return candidates.filter(e => !!this._querySelectorRecursively(e, partial, 0));
   }
 }
 

src/selectors.ts

@@ -62,7 +62,7 @@ export class Selectors {
   }
 
   private _needsMainContext(parsed: types.ParsedSelector): boolean {
-    return parsed.some(({name}) => {
+    return parsed.parts.some(({name}) => {
       const custom = this._engines.get(name);
       return custom ? !custom.contentScript : false;
     });
@@ -188,13 +188,13 @@ export class Selectors {
     let index = 0;
     let quote: string | undefined;
     let start = 0;
-    const result: types.ParsedSelector = [];
+    const result: types.ParsedSelector = { parts: [] };
     const append = () => {
       const part = selector.substring(start, index).trim();
       const eqIndex = part.indexOf('=');
       let name: string;
       let body: string;
-      if (eqIndex !== -1 && part.substring(0, eqIndex).trim().match(/^[a-zA-Z_0-9-+:]+$/)) {
+      if (eqIndex !== -1 && part.substring(0, eqIndex).trim().match(/^[a-zA-Z_0-9-+:*]+$/)) {
         name = part.substring(0, eqIndex).trim();
         body = part.substring(eqIndex + 1);
       } else if (part.length > 1 && part[0] === '"' && part[part.length - 1] === '"') {
@@ -213,9 +213,19 @@ export class Selectors {
         body = part;
       }
       name = name.toLowerCase();
+      let capture = false;
+      if (name[0] === '*') {
+        capture = true;
+        name = name.substring(1);
+      }
       if (!this._builtinEngines.has(name) && !this._engines.has(name))
-        throw new Error(`Unknown engine ${name} while parsing selector ${selector}`);
-      result.push({ name, body });
+        throw new Error(`Unknown engine "${name}" while parsing selector ${selector}`);
+      result.parts.push({ name, body });
+      if (capture) {
+        if (result.capture !== undefined)
+          throw new Error(`Only one of the selectors can capture using * modifier`);
+        result.capture = result.parts.length - 1;
+      }
     };
     while (index < selector.length) {
       const c = selector[index];

src/types.ts

@@ -157,6 +157,9 @@ export type JSCoverageOptions = {
 };
 
 export type ParsedSelector = {
-  name: string,
-  body: string,
-}[];
+  parts: {
+    name: string,
+    body: string,
+  }[],
+  capture?: number,
+};

test/queryselector.spec.js

@@ -111,6 +111,21 @@ describe('Page.$eval', function() {
     const html = await page.$eval('button >> "Next"', e => e.outerHTML);
     expect(html).toBe('<button>Next</button>');
   });
+  it('should support * capture', async({page, server}) => {
+    await page.setContent('<section><div><span>a</span></div></section><section><div><span>b</span></div></section>');
+    expect(await page.$eval('*css=div >> "b"', e => e.outerHTML)).toBe('<div><span>b</span></div>');
+    expect(await page.$eval('section >> *css=div >> "b"', e => e.outerHTML)).toBe('<div><span>b</span></div>');
+    expect(await page.$eval('css=div >> *text="b"', e => e.outerHTML)).toBe('<span>b</span>');
+    expect(await page.$('*')).toBeTruthy();
+  });
+  it('should throw on multiple * captures', async({page, server}) => {
+    const error = await page.$eval('*css=div >> *css=span', e => e.outerHTML).catch(e => e);
+    expect(error.message).toBe('Only one of the selectors can capture using * modifier');
+  });
+  it('should throw on malformed * capture', async({page, server}) => {
+    const error = await page.$eval('*=div', e => e.outerHTML).catch(e => e);
+    expect(error.message).toBe('Unknown engine "" while parsing selector *=div');
+  });
 });
 
 describe('Page.$$eval', function() {
@@ -139,6 +154,26 @@ describe('Page.$$eval', function() {
     const spansCount = await page.$$eval('css=div >> css=span', spans => spans.length);
     expect(spansCount).toBe(3);
   });
+  it('should support * capture', async({page, server}) => {
+    await page.setContent('<section><div><span>a</span></div></section><section><div><span>b</span></div></section>');
+    expect(await page.$$eval('*css=div >> "b"', els => els.length)).toBe(1);
+    expect(await page.$$eval('section >> *css=div >> "b"', els => els.length)).toBe(1);
+    expect(await page.$$eval('section >> *', els => els.length)).toBe(4);
+
+    await page.setContent('<section><div><span>a</span><span>a</span></div></section>');
+    expect(await page.$$eval('*css=div >> "a"', els => els.length)).toBe(1);
+    expect(await page.$$eval('section >> *css=div >> "a"', els => els.length)).toBe(1);
+
+    await page.setContent('<div><span>a</span></div><div><span>a</span></div><section><div><span>a</span></div></section>');
+    expect(await page.$$eval('*css=div >> "a"', els => els.length)).toBe(3);
+    expect(await page.$$eval('section >> *css=div >> "a"', els => els.length)).toBe(1);
+  });
+  it('should support * capture when multiple paths match', async({page, server}) => {
+    await page.setContent('<div><div><span></span></div></div><div></div>');
+    expect(await page.$$eval('*css=div >> span', els => els.length)).toBe(2);
+    await page.setContent('<div><div><span></span></div><span></span><span></span></div><div></div>');
+    expect(await page.$$eval('*css=div >> span', els => els.length)).toBe(2);
+  });
 });
 
 describe('Page.$', function() {
@@ -710,7 +745,7 @@ describe('selectors.register', () => {
     expect(await page.$eval('div', e => e.nodeName)).toBe('DIV');
 
     let error = await page.$('dummy=ignored').catch(e => e);
-    expect(error.message).toContain('Unknown engine dummy while parsing selector dummy=ignored');
+    expect(error.message).toBe('Unknown engine "dummy" while parsing selector dummy=ignored');
 
     const createDummySelector = () => ({
       create(root, target) {