Restoring a heap of functionaltiy:

type events record correctly
side runner does persist session correctly (and lost a buncha code :D)
added commands for requesting good selectors for coordinates
modified base event listeners to be able to send back a response
  so we can ask the browser for stuff easier
corrected step insert positioning
removed sandbox parameter (turned out to be non-essential my b)

packages/selenium-ide/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@seleniumhq/selenium-ide",
-  "version": "4.0.0-alpha.57",
+  "version": "4.0.0-alpha.58",
   "private": true,
   "description": "Selenium IDE electron app",
   "author": "Todd <[email protected]>",
@@ -104,9 +104,8 @@
     "@seleniumhq/code-export-javascript-mocha": "^4.0.0-alpha.2",
     "@seleniumhq/code-export-python-pytest": "^4.0.0-alpha.2",
     "@seleniumhq/code-export-ruby-rspec": "^4.0.0-alpha.1",
-    "side-code-export": "^4.0.0-alpha.11",
     "@seleniumhq/get-driver": "^4.0.0-alpha.1",
-    "@seleniumhq/side-api": "^4.0.0-alpha.35",
+    "@seleniumhq/side-api": "^4.0.0-alpha.36",
     "@seleniumhq/side-model": "^4.0.0-alpha.4",
     "@seleniumhq/side-runtime": "^4.0.0-alpha.31",
     "dnd-core": "^16.0.1",
@@ -120,6 +119,7 @@
     "react-dnd": "^16.0.1",
     "react-dnd-html5-backend": "^16.0.1",
     "react-dom": "^18.2.0",
+    "side-code-export": "^4.0.0-alpha.11",
     "v8-compile-cache": "^2.3.0"
   },
   "devDependencies": {

packages/selenium-ide/src/browser/api/classes/EventListener.ts

@@ -1,7 +1,6 @@
 import { ipcRenderer } from 'electron'
 import { BaseListener, VariadicArgs } from '@seleniumhq/side-api'
 
-
 const baseListener = <ARGS extends VariadicArgs>(
   path: string
 ): BaseListener<ARGS> => {
@@ -14,7 +13,9 @@ const baseListener = <ARGS extends VariadicArgs>(
     },
     dispatchEvent(...args) {
       console.debug(path, 'dispatching event')
-      listeners.forEach((fn) => fn(...args))
+      const results = listeners.map((fn) => fn(...args))
+      ipcRenderer.send(`${path}.response`, results)
+      return results;
     },
     hasListener(listener) {
       return listeners.includes(listener)

packages/selenium-ide/src/browser/windows/PlaybackWindow/preload.ts

@@ -21,7 +21,6 @@ import preload from 'browser/helpers/preload'
 import { webFrame } from 'electron'
 import Recorder from './preload/recorder'
 
-
 (async () => {
   const plugins = await preload({
     recorder: api.recorder,

packages/selenium-ide/src/browser/windows/PlaybackWindow/preload/find-select.ts

@@ -17,7 +17,7 @@
 
 import { EventListenerParams, LocatorFields } from '@seleniumhq/side-api'
 import scrollIntoViewIfNeeded from 'scroll-into-view-if-needed'
-import {singleton as locatorBuilders} from './locator-builders'
+import { singleton as locatorBuilders } from './locator-builders'
 import TargetSelector from './target-selector'
 
 const init = () => {
@@ -28,6 +28,9 @@ const init = () => {
   window.sideAPI.recorder.onHighlightElement.addListener(
     processHighlightCommand
   )
+  window.sideAPI.recorder.onRequestElementAt.addListener(
+    processElementAtCommand
+  )
   window.addEventListener('beforeunload', () => {
     try {
       window.sideAPI.recorder.onRequestSelectElement.removeListener(
@@ -36,6 +39,9 @@ const init = () => {
       window.sideAPI.recorder.onHighlightElement.removeListener(
         processHighlightCommand
       )
+      window.sideAPI.recorder.onRequestElementAt.removeListener(
+        processElementAtCommand
+      )
     } catch (e) {
       // ignore
     }
@@ -57,6 +63,17 @@ function processSelectionCommand(
   }
 }
 
+function processElementAtCommand(
+  x: number,
+  y: number
+): [string, string][] | void {
+  const element = document.elementFromPoint(x, y)
+  if (element) {
+    const target = locatorBuilders.buildAll(element as HTMLElement)
+    return target
+  }
+}
+
 let targetSelector: TargetSelector | null = null
 function startSelection(field: LocatorFields) {
   targetSelector = new TargetSelector(function (element, win) {
@@ -99,7 +116,6 @@ function processMessage(event: MessageEvent<any>) {
 
 async function processHighlightCommand(locator: string): Promise<void> {
   const element = await locatorBuilders.findElement(locator)
-  console.log('Highlighting element', element)
   await highlight(element)
 }
 

packages/selenium-ide/src/browser/windows/PlaybackWindow/preload/record-handlers.ts

@@ -90,6 +90,7 @@ handlers.push([
     }
     this.recordingState.typeLock = 0
   },
+  true,
 ])
 
 handlers.push([
@@ -98,6 +99,7 @@ handlers.push([
   function (this: Recorder, event) {
     this.recordingState.typeTarget = (event.target as HTMLElement) || null
   },
+  true,
 ])
 
 // © Jie-Lin You, SideeX Team
@@ -151,7 +153,6 @@ handlers.push([
   function (this: Recorder, _event) {
     const event = _event as KeyboardEvent
     const target = event.target as HTMLInputElement
-    console.log(event)
     if (target.tagName) {
       let key = event.keyCode
       let tagName = target.tagName.toLowerCase()

packages/selenium-ide/src/main/api/classes/EventListener.ts

@@ -1,16 +1,28 @@
 import { ipcMain, WebContents } from 'electron'
-import { BaseListener, EventMutator, ListenerFn, VariadicArgs } from '@seleniumhq/side-api'
+import {
+  BaseListener,
+  EventMutator,
+  ListenerFn,
+  VariadicArgs,
+} from '@seleniumhq/side-api'
 import { Session } from 'main/types'
 import getCore from '../helpers/getCore'
 import { COLOR_CYAN, vdebuglog } from 'main/util'
 
 const apiDebugLog = vdebuglog('api', COLOR_CYAN)
 
-const baseListener = <ARGS extends VariadicArgs>(
+export type MainListener<
+  ARGS extends VariadicArgs,
+  RESULT extends any
+> = BaseListener<ARGS> & {
+  dispatchEventAsync: (...args: ARGS) => Promise<RESULT[][]>
+}
+
+const baseListener = <ARGS extends VariadicArgs, RESULT extends any>(
   path: string,
   session: Session,
   mutator?: EventMutator<ARGS>
-): BaseListener<ARGS> => {
+): MainListener<ARGS, RESULT> => {
   const listeners: any[] = []
   return {
     addListener(listener) {
@@ -20,12 +32,25 @@ const baseListener = <ARGS extends VariadicArgs>(
     dispatchEvent(...args) {
       apiDebugLog('Dispatch event', path, args)
       if (mutator) {
+        session.api.state.onMutate.dispatchEvent(path, args)
         const newState = mutator(getCore(session), args)
         session.projects.project = newState.project
         session.state.state = newState.state
+      }
+      return listeners.map((fn) => fn(...args))
+    },
+    async dispatchEventAsync(...args) {
+      apiDebugLog('Dispatch event async', path, args)
+      if (mutator) {
         session.api.state.onMutate.dispatchEvent(path, args)
+        const newState = mutator(getCore(session), args)
+        session.projects.project = newState.project
+        session.state.state = newState.state
       }
-      listeners.forEach((fn) => fn(...args))
+      const results: RESULT[][] = await Promise.all(
+        listeners.map((fn) => fn(...args))
+      )
+      return results;
     },
     hasListener(listener) {
       return listeners.includes(listener)
@@ -47,7 +72,7 @@ const wrappedListener = <ARGS extends VariadicArgs>(
   session: Session,
   mutator?: EventMutator<ARGS>
 ) => {
-  const api = baseListener<ARGS>(path, session, mutator)
+  const api = baseListener<ARGS, any>(path, session, mutator)
   const senders: WebContents[] = []
   const senderCounts: number[] = []
   const senderFns: ListenerFn<ARGS>[] = []
@@ -74,14 +99,17 @@ const wrappedListener = <ARGS extends VariadicArgs>(
       senderCounts[index] += 1
       return
     }
-    const senderFn = (...args: ARGS) => {
+    const senderFn = (...args: ARGS) => new Promise((resolve) => {
       try {
+        ipcMain.once(`${path}.response`, (_event, results) => {
+          resolve(results)
+        });
         sender.send(path, ...args)
       } catch (e) {
         // Sender has expired
         removeListener(event)
       }
-    }
+    });
     api.addListener(senderFn)
     senders.push(sender)
     senderCounts.push(1)

packages/selenium-ide/src/main/api/index.ts

@@ -1,6 +1,6 @@
-import { Api, processApi } from '@seleniumhq/side-api'
+import { Api, BaseListener, processApi } from '@seleniumhq/side-api'
 import { Session } from 'main/types'
-import EventListener from './classes/EventListener'
+import EventListener, {MainListener} from './classes/EventListener'
 import Handler from './classes/Handler'
 import RawHandler from './classes/RawHandler'
 
@@ -11,7 +11,13 @@ export const overrides = {
   }
 } as const
 
-export type MainApi = Api & {
+export type MainApi = {
+  [NS in keyof Api]: {
+    [K in keyof Api[NS]]: Api[NS][K] extends BaseListener<infer ARGS, infer RESULT>
+      ? MainListener<ARGS, RESULT>
+      : Api[NS][K]  
+  }
+} & {
   recorder: {
     getWinHandleId: Session['recorder']['getWinHandleId']
     getFrameLocation: Session['recorder']['getFrameLocation']

packages/selenium-ide/src/main/session/controllers/Recorder/index.ts

@@ -141,6 +141,18 @@ export default class RecorderController extends BaseController {
     )
   }
 
+  async requestElementAt(x: number, y: number) {
+    const results =
+      await this.session.api.recorder.onRequestElementAt.dispatchEventAsync(
+        x,
+        y
+      )
+    const allResults = results.flat().flat().filter(Boolean);
+    if (allResults.length) {
+      return allResults[0]
+    }
+  }
+
   async getWinHandleId(): Promise<string> {
     const session = await this.session.state.get()
     const activeTest = getActiveTest(session)

packages/selenium-ide/src/main/session/controllers/Windows/index.ts

@@ -199,7 +199,6 @@ export default class WindowsController extends BaseController {
       webPreferences: {
         // This should be the default preload, which just adds the sideAPI to the window
         preload: join(__dirname, `project-editor-preload-bundle.js`),
-        sandbox: false,
         ...(opts?.webPreferences ?? {}),
       },
       show: false,

packages/side-api/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@seleniumhq/side-api",
-  "version": "4.0.0-alpha.35",
+  "version": "4.0.0-alpha.36",
   "private": false,
   "description": "Selenium IDE API command shapes and such",
   "author": "Todd Tarsi <[email protected]>",

packages/side-api/src/commands/recorder/index.ts

@@ -1,54 +1,60 @@
 import type { Shape as GetFrameLocation } from './getFrameLocation'
+import type { Shape as GetLocatorOrder } from './getLocatorOrder'
+import type { Shape as GetWinHandleId } from './getWinHandleId'
 import type { Shape as OnFrameDeleted } from './onFrameDeleted'
 import type { Shape as OnFrameRecalculate } from './onFrameRecalculate'
 import type { Shape as OnHighlightElement } from './onHighlightElement'
+import type { Shape as OnLocatorOrderChanged } from './onLocatorOrderChanged'
 import type { Shape as OnNewWindow } from './onNewWindow'
-import type { Shape as RequestHighlightElement } from './requestHighlightElement'
+import type { Shape as OnRequestElementAt } from './onRequestElementAt'
 import type { Shape as OnRequestSelectElement } from './onRequestSelectElement'
 import type { Shape as RecordNewCommand } from './recordNewCommand'
+import type { Shape as RequestElementAt } from './requestElementAt'
+import type { Shape as RequestHighlightElement } from './requestHighlightElement'
 import type { Shape as RequestSelectElement } from './requestSelectElement'
-import type { Shape as GetWinHandleId } from './getWinHandleId'
-import type { Shape as SetWindowHandle } from './setWindowHandle'
 import type { Shape as SelectElement } from './selectElement'
+import type { Shape as SetWindowHandle } from './setWindowHandle'
 import type { Shape as Start } from './start'
 import type { Shape as Stop } from './stop'
-import type { Shape as GetLocatorOrder } from './getLocatorOrder'
-import type { Shape as OnLocatorOrderChanged } from './onLocatorOrderChanged'
 
 import * as getFrameLocation from './getFrameLocation'
+import * as getLocatorOrder from './getLocatorOrder'
+import * as getWinHandleId from './getWinHandleId'
 import * as onFrameDeleted from './onFrameDeleted'
 import * as onFrameRecalculate from './onFrameRecalculate'
 import * as onHighlightElement from './onHighlightElement'
+import * as onLocatorOrderChanged from './onLocatorOrderChanged'
 import * as onNewWindow from './onNewWindow'
-import * as requestHighlightElement from './requestHighlightElement'
+import * as onRequestElementAt from './onRequestElementAt'
 import * as onRequestSelectElement from './onRequestSelectElement'
 import * as recordNewCommand from './recordNewCommand'
+import * as requestElementAt from './requestElementAt'
+import * as requestHighlightElement from './requestHighlightElement'
 import * as requestSelectElement from './requestSelectElement'
 import * as selectElement from './selectElement'
 import * as setWindowHandle from './setWindowHandle'
-import * as getWinHandleId from './getWinHandleId'
 import * as start from './start'
 import * as stop from './stop'
-import * as getLocatorOrder from './getLocatorOrder'
-import * as onLocatorOrderChanged from './onLocatorOrderChanged'
 
 export const commands = {
   getFrameLocation,
+  getLocatorOrder,
+  getWinHandleId,
   onFrameDeleted,
   onFrameRecalculate,
   onHighlightElement,
   onNewWindow,
-  requestHighlightElement,
+  onLocatorOrderChanged,
+  onRequestElementAt,
   onRequestSelectElement,
   recordNewCommand,
+  requestElementAt,
+  requestHighlightElement,
   requestSelectElement,
   selectElement,
   setWindowHandle,
-  getWinHandleId,
   start,
   stop,
-  getLocatorOrder,
-  onLocatorOrderChanged,
 }
 
 /**
@@ -57,19 +63,21 @@ export const commands = {
  */
 export type Shape = {
   getFrameLocation: GetFrameLocation
+  getLocatorOrder: GetLocatorOrder
+  getWinHandleId: GetWinHandleId
   onFrameDeleted: OnFrameDeleted
   onFrameRecalculate: OnFrameRecalculate
   onHighlightElement: OnHighlightElement
+  onLocatorOrderChanged: OnLocatorOrderChanged
   onNewWindow: OnNewWindow
+  onRequestElementAt: OnRequestElementAt
   onRequestSelectElement: OnRequestSelectElement
   recordNewCommand: RecordNewCommand
+  requestElementAt: RequestElementAt
   requestHighlightElement: RequestHighlightElement
   requestSelectElement: RequestSelectElement
   selectElement: SelectElement
   setWindowHandle: SetWindowHandle
-  getWinHandleId: GetWinHandleId
   start: Start
   stop: Stop
-  getLocatorOrder: GetLocatorOrder
-  onLocatorOrderChanged: OnLocatorOrderChanged
 }

packages/side-api/src/commands/recorder/onRequestElementAt.ts

@@ -0,0 +1,11 @@
+import { BaseListener } from '../../types/base'
+
+/**
+ * Called when the project editor asks the recorder to select an element.
+ */
+export type Shape = BaseListener<
+  OnRequestElementAtRecorder,
+  OnRequestElementAtRecorderResult
+>
+export type OnRequestElementAtRecorder = [number, number]
+export type OnRequestElementAtRecorderResult = [string, string][] | void

packages/side-api/src/commands/recorder/requestElementAt.ts

@@ -0,0 +1,4 @@
+/**
+ * Return a list of targets that are at the given coordinates.
+ */
+export type Shape = (x: number, y: number) => Promise<[string, string][]>