diff --git a/packages/page-controller/src/dom/dom_tree/dropdown-cap.test.ts b/packages/page-controller/src/dom/dom_tree/dropdown-cap.test.ts new file mode 100644 index 000000000..690940e17 --- /dev/null +++ b/packages/page-controller/src/dom/dom_tree/dropdown-cap.test.ts @@ -0,0 +1,103 @@ +import { beforeEach, describe, expect, it } from 'vitest' + +import { flatTreeToString } from '../index' +import domTree from './index.js' + +function setupSizes() { + Object.defineProperty(HTMLElement.prototype, 'offsetWidth', { + configurable: true, + get() { + return 100 + }, + }) + Object.defineProperty(HTMLElement.prototype, 'offsetHeight', { + configurable: true, + get() { + return 30 + }, + }) +} + +function buildDropdown(html: string) { + return domTree({ + doHighlightElements: false, + viewportExpansion: -1, + interactiveBlacklist: [], + interactiveWhitelist: [], + }) as any +} + +describe('dropdown option cap (#348)', () => { + beforeEach(() => { + setupSizes() + document.body.innerHTML = '' + }) + + it('indexes at most 20 options per dropdown container', () => { + const options = Array.from( + { length: 25 }, + (_, i) => `
  • 选项 ${i}
  • ` + ) + document.body.innerHTML = ` +
    + +
    + ` + const tree = buildDropdown(document.body.innerHTML) + const liNodes = Object.values(tree.map).filter((n: any) => n.tagName === 'li') + const indexed = liNodes.filter((n: any) => typeof n.highlightIndex === 'number') + expect(indexed.length).toBe(20) + }) + + it('records dropped options on the container', () => { + const options = Array.from( + { length: 25 }, + (_, i) => `
  • 选项 ${i}
  • ` + ) + document.body.innerHTML = ` +
    + +
    + ` + const tree = buildDropdown(document.body.innerHTML) + const container = Object.values(tree.map).find( + (n: any) => n.tagName === 'div' && n.extra?.droppedOptions === 5 + ) + expect(container).toBeDefined() + }) + + it('renders a folded-options hint in the simplified HTML', () => { + const options = Array.from( + { length: 25 }, + (_, i) => `
  • 选项 ${i}
  • ` + ) + document.body.innerHTML = ` +
    + +
    + ` + const tree = buildDropdown(document.body.innerHTML) + const html = flatTreeToString(tree) + expect(html).toContain('5 more option(s) not shown') + // only the first 20 options carry indexes + expect(html).toContain('[19]
  • 选项 19') + expect(html).not.toContain('选项 24') + }) + + it('does not cap dropdowns with few options', () => { + const options = Array.from( + { length: 6 }, + (_, i) => `
  • 选项 ${i}
  • ` + ) + document.body.innerHTML = ` +
    + +
    + ` + const tree = buildDropdown(document.body.innerHTML) + const liNodes = Object.values(tree.map).filter((n: any) => n.tagName === 'li') + expect(liNodes.filter((n: any) => typeof n.highlightIndex === 'number').length).toBe(6) + const html = flatTreeToString(tree) + expect(html).not.toContain('not shown') + }) +}) diff --git a/packages/page-controller/src/dom/dom_tree/dropdown-options.test.ts b/packages/page-controller/src/dom/dom_tree/dropdown-options.test.ts new file mode 100644 index 000000000..c027c0a41 --- /dev/null +++ b/packages/page-controller/src/dom/dom_tree/dropdown-options.test.ts @@ -0,0 +1,79 @@ +import { beforeEach, describe, expect, it } from 'vitest' + +import domTree from './index.js' + +function setupSizes() { + Object.defineProperty(HTMLElement.prototype, 'offsetWidth', { + configurable: true, + get() { + return 100 + }, + }) + Object.defineProperty(HTMLElement.prototype, 'offsetHeight', { + configurable: true, + get() { + return 30 + }, + }) +} + +function buildDropdown(html: string) { + return domTree({ + doHighlightElements: false, + viewportExpansion: -1, + interactiveBlacklist: [], + interactiveWhitelist: [], + }) as any +} + +function getIndexedOptions(tree: any) { + return Object.values(tree.map) + .filter((n: any) => n.tagName === 'li' && typeof n.highlightIndex === 'number') + .sort((a: any, b: any) => a.highlightIndex - b.highlightIndex) +} + +describe('dropdown option indexing (#519)', () => { + beforeEach(() => { + setupSizes() + document.body.innerHTML = '' + }) + + it('indexes enabled dropdown options', () => { + document.body.innerHTML = ` +
    + +
    + ` + const tree = buildDropdown(document.body.innerHTML) + const indexed = getIndexedOptions(tree) + expect(indexed.length).toBe(3) + }) + + it('indexes disabled dropdown options so the full list is visible to the LLM', () => { + document.body.innerHTML = ` +
    + +
    + ` + const tree = buildDropdown(document.body.innerHTML) + const indexed = getIndexedOptions(tree) + expect(indexed.length).toBe(3) + }) + + it('still excludes disabled buttons outside dropdowns', () => { + document.body.innerHTML = ` + + ` + const tree = buildDropdown(document.body.innerHTML) + const buttons = Object.values(tree.map).filter((n: any) => n.tagName === 'button') + expect(buttons.every((n: any) => n.highlightIndex === undefined)).toBe(true) + }) +}) diff --git a/packages/page-controller/src/dom/dom_tree/index.js b/packages/page-controller/src/dom/dom_tree/index.js index 7f6b3afd3..d3f222529 100644 --- a/packages/page-controller/src/dom/dom_tree/index.js +++ b/packages/page-controller/src/dom/dom_tree/index.js @@ -687,6 +687,54 @@ export default ( ) } + /** + * @edit dropdown/menu option detection + * li/option elements (or role=option/menuitem/listitem) inside a dropdown or menu + * container are treated as indexable even when visually disabled (cursor: not-allowed), + * so the LLM sees the complete option list. + */ + const DROPDOWN_OPTION_ROLES = new Set([ + 'option', + 'menuitem', + 'menuitemradio', + 'menuitemcheckbox', + 'listitem', + ]) + const DROPDOWN_CONTAINER_SELECTOR = [ + '[role="listbox"]', + '[role="menu"]', + '[role="menubar"]', + 'select', + '.el-select-dropdown', + '.el-dropdown-menu', + '[data-toggle="dropdown"]', + ].join(', ') + + function isDropdownOptionElement(element) { + if (!element || element.nodeType !== Node.ELEMENT_NODE) return false + const tagName = element.tagName.toLowerCase() + const role = element.getAttribute('role') + if (tagName !== 'li' && tagName !== 'option' && !(role && DROPDOWN_OPTION_ROLES.has(role))) { + return false + } + return Boolean(element.closest(DROPDOWN_CONTAINER_SELECTOR)) + } + + /** + * @edit cap dropdown options per container (#348) + * A select/dropdown with hundreds of options would blow up the LLM payload; + * index at most MAX_DROPDOWN_OPTIONS_PER_CONTAINER options and fold the rest + * into a hint on the container. + */ + const MAX_DROPDOWN_OPTIONS_PER_CONTAINER = 20 + const dropdownOptionCounts = new WeakMap() // container element -> indexed option count + + function getDropdownOptionContainer(element) { + if (!element || element.nodeType !== Node.ELEMENT_NODE) return null + if (!isDropdownOptionElement(element)) return null + return element.closest(DROPDOWN_CONTAINER_SELECTOR) + } + /** * Checks if an element is interactive. * @@ -711,6 +759,15 @@ export default ( return true // Skip whitelisted elements } + /** + * @edit dropdown options should stay indexable even when disabled, + * otherwise the LLM cannot see or select the full option list + * (e.g. Element UI / Avue selects with disabled options). + */ + if (isDropdownOptionElement(element)) { + return true + } + // Cache the tagName and style lookups const tagName = element.tagName.toLowerCase() const style = getCachedComputedStyle(element) @@ -1635,6 +1692,24 @@ export default ( if (nodeData.isVisible) { nodeData.isTopElement = isTopElement(node) + /** + * @edit cap dropdown options per container (#348) + * A dropdown with hundreds of options would blow up the LLM payload; + * index at most MAX_DROPDOWN_OPTIONS_PER_CONTAINER options per container + * and fold the excess into a hint on the container. + */ + if (isDropdownOptionElement(node)) { + const container = getDropdownOptionContainer(node) + const used = dropdownOptionCounts.get(container) || 0 + if (used >= MAX_DROPDOWN_OPTIONS_PER_CONTAINER) { + addExtraData(container, { + droppedOptions: (extraData.get(container)?.droppedOptions || 0) + 1, + }) + return null // Skip excess options entirely (text included) + } + dropdownOptionCounts.set(container, used + 1) + } + // Special handling for ARIA menu containers - check interactivity even if not top element const role = node.getAttribute('role') const isMenuContainer = role === 'menu' || role === 'menubar' || role === 'listbox' diff --git a/packages/page-controller/src/dom/index.ts b/packages/page-controller/src/dom/index.ts index a7ae7f8ee..66192b31c 100644 --- a/packages/page-controller/src/dom/index.ts +++ b/packages/page-controller/src/dom/index.ts @@ -432,6 +432,14 @@ export function flatTreeToString( processNode(child, nextDepth, result) } + /** + * @edit dropdown options are capped per container (#348); + * render a hint so the LLM knows more options exist. + */ + if (node.extra?.droppedOptions) { + result.push(`${depthStr}... ${node.extra.droppedOptions} more option(s) not shown ...`) + } + if (emitSemantic) { // empty tag should be removed if (result.length === mark + 1) {