support cli communication

api/track_uploader.js

@@ -61,10 +61,10 @@ export async function sendTrackToBackend(trackingLog, popupWindowId, actionOnlyM
         // Prepare request body matching server expectations
         const requestBody = {
             url: activeTab?.url || '',
+            title: activeTab?.title || '',
             actions: trackingLog,
             // Optional metadata can be included if server supports it
             metadata: {
-                title: activeTab?.title || '',
                 timestamp: Date.now(),
                 extensionVersion: chrome.runtime.getManifest().version,
                 actionsCount: trackingLog.length

modules/action/config/config.js

@@ -8,6 +8,7 @@ export const DEBOUNCER_TIME = 100;
 export const SELECTION_CHANGE_DEBOUNCE = 500;
 export const POINTER_MOVE_DEBOUNCE = 100;
 export const WINDOW_RESIZE_DEBOUNCE = 500;
+export const SCROLL_DEBOUNCE = 300;  // 300ms delay after scrolling stops
 
 
 /**

modules/action/config/constants.js

@@ -42,13 +42,16 @@ export const FORM_ACTIONS = {
     SET: "SET",                               // Handler: onChange -> SET (field value changes)
     INPUT: "INPUT",                           // Handler: onInput -> INPUT (text input for autocomplete, etc.)
 
+    // Click-to-edit pattern (activate display element, then type into revealed input)
+    CLICK_TO_EDIT: "CLICK_TO_EDIT",           // Handler: onClickToEdit -> CLICK_TO_EDIT (date pickers, inline edits, etc.)
+
     // Text selection
     SELECT: "SELECT",                         // Handler: onSelect
-    
+
     // Form submission
     SUBMIT: "SUBMIT",                         // Handler: onSubmit
     RESET: "RESET",                           // Handler: onReset
-    
+
     // Validation
     INVALID: "INVALID"                        // Handler: onInvalid
 };
@@ -324,6 +327,19 @@ export const MESSAGE_HANDLER_MAPPING = {
             validationMessage: request.validationMessage
         })
     },
+    'onClickToEdit': {
+        actionType: ACTION_TYPES.CLICK_TO_EDIT,
+        category: ACTION_CATEGORIES.FORM,
+        handler: (request) => ({
+            browserAction: ACTION_TYPES.CLICK_TO_EDIT,
+            xpath: request.xPath,                    // XPath of the input element
+            content: request.content,                // Final value entered
+            activator: request.activator,            // XPath/selector of the display element that was clicked
+            activatorSnapshot: request.activatorSnapshot,  // ARIA snapshot at click time
+            fieldInfo: request.fieldInfo,
+            locator: request.locator
+        })
+    },
 
     // Window event handlers
     'onWindowResize': { 
@@ -403,7 +419,8 @@ import {
     DEBOUNCER_TIME,
     SELECTION_CHANGE_DEBOUNCE,
     POINTER_MOVE_DEBOUNCE,
-    WINDOW_RESIZE_DEBOUNCE
+    WINDOW_RESIZE_DEBOUNCE,
+    SCROLL_DEBOUNCE
 } from './config.js';
 
 // Re-export for backward compatibility
@@ -411,5 +428,6 @@ export {
     DEBOUNCER_TIME,
     SELECTION_CHANGE_DEBOUNCE,
     POINTER_MOVE_DEBOUNCE,
-    WINDOW_RESIZE_DEBOUNCE
+    WINDOW_RESIZE_DEBOUNCE,
+    SCROLL_DEBOUNCE
 };

modules/action/events/click.js

@@ -11,6 +11,7 @@ export class ClickEventTracker {
         this.fileUploadDetector = null;
         this.choiceDetector = null;
         this.radioDetector = null;
+        this.clickToEditDetector = null;
     }
 
     /**
@@ -41,6 +42,13 @@ export class ClickEventTracker {
         this.radioDetector = detector;
     }
 
+    /**
+     * Set the click-to-edit detector for inline edit patterns
+     */
+    setClickToEditDetector(detector) {
+        this.clickToEditDetector = detector;
+    }
+
     debounce(func, wait) {
         let timeout;
         return function(...args) {
@@ -254,6 +262,12 @@ export class ClickEventTracker {
         }
 
         // ARIA snapshot was already generated early in the function
+
+        // Record click for click-to-edit detection (potential activator for subsequent inputs)
+        if (this.clickToEditDetector) {
+            this.clickToEditDetector.recordClick(target, xpaths, snapshot.yaml);
+        }
+
         console.log('✅ Sending click action:', clickData);
         this.sendMessage(clickData);
     }

modules/action/events/input.js

@@ -8,6 +8,7 @@ export class InputEventTracker {
         this.sendMessage = sendMessage;
         this.getXpaths = getXpaths;
         this.dropdownDetector = null;  // Reactive dropdown detector
+        this.clickToEditDetector = null;  // Click-to-edit pattern detector
     }
 
     /**
@@ -17,6 +18,13 @@ export class InputEventTracker {
         this.dropdownDetector = detector;
     }
 
+    /**
+     * Set the click-to-edit detector for inline edit patterns
+     */
+    setClickToEditDetector(detector) {
+        this.clickToEditDetector = detector;
+    }
+
     /**
      * Promise version of sendMessage for async/await usage
      */
@@ -67,6 +75,9 @@ export class InputEventTracker {
     handleInput(event) {
         const target = event.target;
 
+        // Skip file inputs - they're handled by change event only
+        if (target.type === 'file') return;
+
         // Record input for dropdown detection correlation (always, even if not recording)
         if (this.dropdownDetector) {
             this.dropdownDetector.recordInput(target, target.value);
@@ -147,12 +158,33 @@ export class InputEventTracker {
                 size: file.size,
                 type: file.type
             }));
+            // Use real filename instead of browser's fakepath
+            changeData.content = changeData.files[0]?.name || '';
         }
 
         // Generate ARIA snapshot for context
         const snapshot = ariaSnapshotGenerator.generateForElement(target);
         changeData.ariaSnapshot = snapshot.yaml;
 
+        // Check for click-to-edit pattern (merge with recent click)
+        if (this.clickToEditDetector) {
+            const mergeData = this.clickToEditDetector.checkForMerge(target, target.value);
+            if (mergeData) {
+                // Send CLICK_TO_EDIT instead of SET
+                this.sendMessage({
+                    message: "onClickToEdit",
+                    xPath: mergeData.inputXpath,
+                    content: mergeData.content,
+                    activator: mergeData.activator,
+                    activatorSnapshot: mergeData.activatorSnapshot,
+                    fieldInfo: changeData.fieldInfo,
+                    locator: getPlaywrightSelector(target)
+                });
+                console.log('🖱️ Click-to-edit detected, sending CLICK_TO_EDIT instead of SET');
+                return;
+            }
+        }
+
         this.sendMessage(changeData);
     }
 

modules/action/events/window.js

@@ -4,6 +4,7 @@ export class WindowEventTracker {
         this.sendMessage = sendMessage;
         this.getXpaths = getXpaths;
         this.resizeDebounced = this.debounce(this.handleResize.bind(this), 500);
+        this.scrollDebounced = this.debounce(this.handleScroll.bind(this), 300);
         this.lastWindowSize = {
             width: window.innerWidth,
             height: window.innerHeight
@@ -25,8 +26,8 @@ export class WindowEventTracker {
         // Document events
         document.addEventListener('fullscreenchange', this.handleFullscreenChange.bind(this), true);
 
-        // Scroll events (both window and element level)
-        window.addEventListener('scroll', this.handleScroll.bind(this), true);
+        // Scroll events (both window and element level) - debounced to capture final position
+        window.addEventListener('scroll', this.scrollDebounced, true);
 
         // Note: popstate is handled by NavigationTracker for unified URL change tracking
     }

util/playwright_selector.js

@@ -290,6 +290,88 @@ function getElementAccessibleName(element) {
     return '';
 }
 
+/**
+ * Count how many elements on the page match a given selector candidate
+ * Used to check if a selector is unique
+ */
+function countMatchingElements(candidate) {
+    try {
+        if (candidate.method === 'getByRole') {
+            const role = candidate.role;
+            const name = candidate.name;
+
+            // Find all elements with this role
+            const allElements = document.querySelectorAll('*');
+            let count = 0;
+
+            for (const el of allElements) {
+                if (getAriaRole(el) === role) {
+                    if (name) {
+                        // Check if accessible name matches
+                        if (getElementAccessibleName(el) === name) {
+                            count++;
+                        }
+                    } else {
+                        count++;
+                    }
+                }
+            }
+            return count;
+        }
+
+        if (candidate.method === 'getByLabel') {
+            const label = candidate.label;
+            const allElements = document.querySelectorAll('input, select, textarea, button');
+            let count = 0;
+            for (const el of allElements) {
+                const labels = getElementLabels(el);
+                if (labels.some(l => l.normalized === label)) {
+                    count++;
+                }
+            }
+            return count;
+        }
+
+        if (candidate.method === 'getByPlaceholder') {
+            return document.querySelectorAll(`[placeholder="${CSS.escape(candidate.placeholder)}"]`).length;
+        }
+
+        if (candidate.method === 'getByTestId') {
+            return document.querySelectorAll(`[data-testid="${CSS.escape(candidate.testId)}"]`).length;
+        }
+
+        if (candidate.method === 'locator') {
+            return document.querySelectorAll(candidate.selector).length;
+        }
+
+        if (candidate.method === 'getByText') {
+            const text = candidate.text;
+            const allElements = document.querySelectorAll('*');
+            let count = 0;
+            for (const el of allElements) {
+                const elText = elementText(el).normalized;
+                if (elText === text || elText.includes(text)) {
+                    count++;
+                }
+            }
+            return count;
+        }
+
+        if (candidate.method === 'getByTitle') {
+            return document.querySelectorAll(`[title="${CSS.escape(candidate.title)}"]`).length;
+        }
+
+        if (candidate.method === 'getByAltText') {
+            return document.querySelectorAll(`[alt="${CSS.escape(candidate.alt)}"]`).length;
+        }
+
+        // For other methods, assume unique
+        return 1;
+    } catch (e) {
+        return 1;
+    }
+}
+
 /**
  * Check if ID looks like a GUID (from selectorGenerator.ts lines 479-506)
  */
@@ -468,6 +550,7 @@ function buildCandidates(element, testIdAttributeName = 'data-testid') {
 /**
  * Get the best Playwright selector for an element
  * Returns object with method and params
+ * Prioritizes unique selectors over non-unique ones
  */
 export function getPlaywrightSelector(element) {
     if (!element) {
@@ -480,9 +563,23 @@ export function getPlaywrightSelector(element) {
     // Sort by score (lower is better)
     candidates.sort((a, b) => a.score - b.score);
 
-    // Return the best candidate (remove the score property)
-    const best = candidates[0];
-    const { score, ...result } = best;
+    // Find the best UNIQUE candidate
+    for (const candidate of candidates) {
+        const count = countMatchingElements(candidate);
+        if (count === 1) {
+            const { score, ...result } = candidate;
+            return result;
+        }
+    }
+
+    // No unique candidate found - prefer ID if available (even if GUID-like)
+    const idAttr = element.getAttribute('id');
+    if (idAttr) {
+        return { method: 'locator', selector: `#${CSS.escape(idAttr)}` };
+    }
+
+    // Fallback to best candidate even if not unique
+    const { score, ...result } = candidates[0];
     return result;
 }