Instantly share code, notes, and snippets.
Save mitsuhiko/fce80dc9a28f8f7333b6b48865de5955 to your computer and use it in GitHub Desktop.
This is a vibecoded automator thing that talks a basic JSON protocol to query windows with the accessibility api
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.Learn more about bidirectional Unicode characters
| import Foundation | |
| import ApplicationServices // AXUIElement* | |
| import AppKit // NSRunningApplication, NSWorkspace | |
| import CoreGraphics // CGPoint, CGSize, etc. | |
| // Define missing accessibility constants | |
| letkAXActionsAttribute="AXActions" | |
| letkAXWindowsAttribute="AXWindows" | |
| letkAXPressAction="AXPress" | |
| // Enable verbose debugging | |
| letDEBUG=true | |
| func debug(_ message:String){ | |
| if DEBUG{ | |
| fputs("DEBUG:\(message)\n", stderr) | |
| } | |
| } | |
| // Check accessibility permissions | |
| func checkAccessibilityPermissions(){ | |
| // Use the constant directly as a String to avoid concurrency issues | |
| letcheckOptPrompt="AXTrustedCheckOptionPrompt"asCFString | |
| letoptions=[checkOptPrompt:true]asCFDictionary | |
| letaccessEnabled=AXIsProcessTrustedWithOptions(options) | |
| if !accessEnabled{ | |
| print("Error: This application requires accessibility permissions.") | |
| print("Please enable them in System Preferences > Privacy & Security > Accessibility") | |
| exit(1) | |
| } | |
| } | |
| // MARK: - Codable command envelopes ------------------------------------------------- | |
| structCommandEnvelope:Codable{ | |
| enumVerb:String,Codable{case query, perform} | |
| letcmd:Verb | |
| letlocator:Locator | |
| letattributes:[String]? // for query | |
| letaction:String? // for perform | |
| letmulti:Bool? // NEW | |
| letrequireAction:String? // NEW (e.g. "AXPress") | |
| } | |
| structLocator:Codable{ | |
| letapp:String // bundle id or display name | |
| letrole:String // e.g. "AXButton" | |
| letmatch:[String:String] // attribute→value to match | |
| letpathHint:[String]? // optional array like ["window[1]","toolbar[1]"] | |
| } | |
| // MARK: - Codable response types ----------------------------------------------------- | |
| structQueryResponse:Codable{ | |
| letattributes:[String:AnyCodable] | |
| init(attributes:[String:Any]){ | |
| self.attributes= attributes.mapValues(AnyCodable.init) | |
| } | |
| } | |
| structMultiQueryResponse:Codable{ | |
| letelements:[[String:AnyCodable]] | |
| init(elements:[[String:Any]]){ | |
| self.elements= elements.map{ elementin | |
| element.mapValues(AnyCodable.init) | |
| } | |
| } | |
| } | |
| structPerformResponse:Codable{ | |
| letstatus:String | |
| } | |
| structErrorResponse:Codable{ | |
| leterror:String | |
| } | |
| // AnyCodable wrapper type for JSON encoding of Any values | |
| structAnyCodable:Codable{ | |
| letvalue:Any | |
| init(_ value:Any){ | |
| self.value= value | |
| } | |
| init(from decoder:Decoder)throws{ | |
| letcontainer=try decoder.singleValueContainer() | |
| if container.decodeNil(){ | |
| self.value=NSNull() | |
| }elseiflet bool=try? container.decode(Bool.self){ | |
| self.value= bool | |
| }elseiflet int=try? container.decode(Int.self){ | |
| self.value= int | |
| }elseiflet double=try? container.decode(Double.self){ | |
| self.value= double | |
| }elseiflet string=try? container.decode(String.self){ | |
| self.value= string | |
| }elseiflet array=try? container.decode([AnyCodable].self){ | |
| self.value= array.map{ $0.value} | |
| }elseiflet dict=try? container.decode([String:AnyCodable].self){ | |
| self.value= dict.mapValues{ $0.value} | |
| }else{ | |
| throwDecodingError.dataCorruptedError( | |
| in: container, | |
| debugDescription:"AnyCodable cannot decode value" | |
| ) | |
| } | |
| } | |
| func encode(to encoder:Encoder)throws{ | |
| varcontainer= encoder.singleValueContainer() | |
| switch value{ | |
| case isNSNull: | |
| try container.encodeNil() | |
| caseletbool asBool: | |
| try container.encode(bool) | |
| caseletint asInt: | |
| try container.encode(int) | |
| caseletdouble asDouble: | |
| try container.encode(double) | |
| caseletstring asString: | |
| try container.encode(string) | |
| caseletarray as[Any]: | |
| try container.encode(array.map(AnyCodable.init)) | |
| caseletdict as[String:Any]: | |
| try container.encode(dict.mapValues(AnyCodable.init)) | |
| default: | |
| // Try to convert to string as a fallback | |
| try container.encode(String(describing: value)) | |
| } | |
| } | |
| } | |
| // Simple intermediate type for element attributes | |
| typealiasElementAttributes=[String:Any] | |
| // Create a completely new helper function to safely extract attributes | |
| func getElementAttributes(_ element:AXUIElement, attributes:[String])->ElementAttributes{ | |
| varresult=ElementAttributes() | |
| // First, discover all available attributes for this specific element | |
| varallAttributes= attributes | |
| varattrNames:CFArray? | |
| ifAXUIElementCopyAttributeNames(element,&attrNames)==.success,let names= attrNames{ | |
| letcount=CFArrayGetCount(names) | |
| foriin0..<count{ | |
| iflet ptr=CFArrayGetValueAtIndex(names, i), | |
| let cfStr=unsafeBitCast(ptr, to:CFString.self)asString?, | |
| !allAttributes.contains(cfStr){ | |
| allAttributes.append(cfStr) | |
| } | |
| } | |
| debug("Element has\(count) available attributes") | |
| } | |
| // Keep track of all available actions | |
| varavailableActions:[String]=[] | |
| // Process all attributes | |
| forattrin allAttributes{ | |
| // Get the raw value first | |
| varvalue:CFTypeRef? | |
| leterr=AXUIElementCopyAttributeValue(element, attrasCFString,&value) | |
| if err!=.success || value==nil{ | |
| // Only include requested attributes in the result | |
| if attributes.contains(attr){ | |
| result[attr]="Not available" | |
| } | |
| continue | |
| } | |
| letunwrappedValue= value! | |
| letextractedValue:Any | |
| // Handle different types of values | |
| ifCFGetTypeID(unwrappedValue)==CFStringGetTypeID(){ | |
| // String value - most common for text, titles, etc. | |
| letcfString= unwrappedValueas!CFString | |
| extractedValue= cfStringasString | |
| } | |
| elseifCFGetTypeID(unwrappedValue)==CFBooleanGetTypeID(){ | |
| // Boolean value | |
| letcfBool= unwrappedValueas!CFBoolean | |
| extractedValue=CFBooleanGetValue(cfBool) | |
| } | |
| elseifCFGetTypeID(unwrappedValue)==CFNumberGetTypeID(){ | |
| // Numeric value | |
| letcfNumber= unwrappedValueas!CFNumber | |
| varintValue:Int=0 | |
| ifCFNumberGetValue(cfNumber,CFNumberType.intType,&intValue){ | |
| extractedValue= intValue | |
| }else{ | |
| extractedValue="Number (conversion failed)" | |
| } | |
| } | |
| elseifCFGetTypeID(unwrappedValue)==CFArrayGetTypeID(){ | |
| // Array values (like children or subroles) | |
| letcfArray= unwrappedValueas!CFArray | |
| letcount=CFArrayGetCount(cfArray) | |
| // For actions, extract them into our list | |
| if attr=="AXActions"{ | |
| foriin0..<count{ | |
| iflet actionPtr=CFArrayGetValueAtIndex(cfArray, i), | |
| let actionStr=unsafeBitCast(actionPtr, to:CFString.self)asString?{ | |
| availableActions.append(actionStr) | |
| } | |
| } | |
| extractedValue= availableActions | |
| }else{ | |
| extractedValue="Array with\(count) elements" | |
| } | |
| } | |
| elseif attr=="AXPosition" || attr=="AXSize"{ | |
| // Handle AXValue types (usually for position and size) | |
| // Safely check if it's an AXValue | |
| letaxValueType=AXValueGetType(unwrappedValueas!AXValue) | |
| if attr=="AXPosition" && axValueType.rawValue==AXValueType.cgPoint.rawValue{ | |
| // It's a position value | |
| varpoint=CGPoint.zero | |
| ifAXValueGetValue(unwrappedValueas!AXValue,AXValueType.cgPoint,&point){ | |
| extractedValue=["x":Int(point.x),"y":Int(point.y)] | |
| }else{ | |
| extractedValue=["error":"Position data (conversion failed)"] | |
| } | |
| } | |
| elseif attr=="AXSize" && axValueType.rawValue==AXValueType.cgSize.rawValue{ | |
| // It's a size value | |
| varsize=CGSize.zero | |
| ifAXValueGetValue(unwrappedValueas!AXValue,AXValueType.cgSize,&size){ | |
| extractedValue=["width":Int(size.width),"height":Int(size.height)] | |
| }else{ | |
| extractedValue=["error":"Size data (conversion failed)"] | |
| } | |
| } | |
| else{ | |
| // It's some other kind of AXValue | |
| extractedValue=["error":"AXValue type:\(axValueType.rawValue)"] | |
| } | |
| } | |
| elseif attr=="AXTitleUIElement" || attr=="AXLabelUIElement"{ | |
| // These are special attributes that point to other AXUIElements | |
| // Extract the text from them instead of just reporting the type | |
| lettitleElement= unwrappedValueas!AXUIElement | |
| // Try to get its AXValue attribute which usually contains the text | |
| vartitleValue:CFTypeRef? | |
| ifAXUIElementCopyAttributeValue(titleElement,"AXValue"asCFString,&titleValue)==.success, | |
| let titleString= titleValueas?String{ | |
| extractedValue= titleString | |
| } | |
| // If no AXValue, try AXTitle | |
| elseifAXUIElementCopyAttributeValue(titleElement,"AXTitle"asCFString,&titleValue)==.success, | |
| let titleString= titleValueas?String{ | |
| extractedValue= titleString | |
| } | |
| // Fallback to indicating we found a title element but couldn't extract text | |
| else{ | |
| extractedValue="Title element (no extractable text)" | |
| } | |
| } | |
| else{ | |
| // Try to get the type description for debugging | |
| lettypeID=CFGetTypeID(unwrappedValue) | |
| iflet typeDesc=CFCopyTypeIDDescription(typeID){ | |
| lettypeString= typeDescasString | |
| extractedValue="Unknown type:\(typeString)" | |
| }else{ | |
| extractedValue="Unknown type:\(typeID)" | |
| } | |
| } | |
| // Only include explicitly requested attributes and useful ones in the final result | |
| if attributes.contains(attr) || | |
| attr.hasPrefix("AXTitle") || | |
| attr.hasPrefix("AXLabel") || | |
| attr.hasPrefix("AXHelp") || | |
| attr.hasPrefix("AXDescription") || | |
| attr.hasPrefix("AXValue") || | |
| attr.hasPrefix("AXRole"){ | |
| result[attr]= extractedValue | |
| } | |
| } | |
| // Make sure actions are available as a proper array if requested | |
| if attributes.contains("AXActions"){ | |
| if !availableActions.isEmpty{ | |
| result["AXActions"]= availableActions | |
| }elseifresult["AXActions"]==nil{ | |
| result["AXActions"]="Not available" | |
| } | |
| } | |
| // Add a computed property to give the most descriptive name for this element | |
| // This combines multiple attributes in order of preference | |
| varcomputedName:String?=nil | |
| // Try all possible ways to get a meaningful name/title | |
| iflet title=result["AXTitle"]as?String, title!="Not available" && !title.isEmpty{ | |
| computedName= title | |
| } | |
| elseiflet titleUIElement=result["AXTitleUIElement"]as?String, | |
| titleUIElement!="Not available" && titleUIElement!="Title element (no extractable text)"{ | |
| computedName= titleUIElement | |
| } | |
| elseiflet value=result["AXValue"]as?String, value!="Not available" && !value.isEmpty{ | |
| computedName= value | |
| } | |
| elseiflet description=result["AXDescription"]as?String, description!="Not available" && !description.isEmpty{ | |
| computedName= description | |
| } | |
| elseiflet label=result["AXLabel"]as?String, label!="Not available" && !label.isEmpty{ | |
| computedName= label | |
| } | |
| elseiflet help=result["AXHelp"]as?String, help!="Not available" && !help.isEmpty{ | |
| computedName= help | |
| } | |
| elseiflet roleDesc=result["AXRoleDescription"]as?String, roleDesc!="Not available"{ | |
| // Use role description as a last resort | |
| letrole=result["AXRole"]as?String??"Unknown" | |
| computedName="\(roleDesc) (\(role))" | |
| } | |
| // Add the computed name if we found one | |
| iflet name= computedName{ | |
| result["ComputedName"]= name | |
| } | |
| // Add a computed clickable status based on role and other properties | |
| letisButton=result["AXRole"]as?String=="AXButton" | |
| lethasClickAction= availableActions.contains("AXPress") | |
| if isButton || hasClickAction{ | |
| result["IsClickable"]=true | |
| } | |
| return result | |
| } | |
| // MARK: - Helpers -------------------------------------------------------------------- | |
| enumAXErrorString:Error,CustomStringConvertible{ | |
| case notAuthorised(AXError) | |
| case elementNotFound | |
| case actionFailed(AXError) | |
| vardescription:String{ | |
| switchself{ | |
| case.notAuthorised(let e):return"AX authorisation failed:\(e)" | |
| case.elementNotFound:return"No element matches the locator" | |
| case.actionFailed(let e):return"Action failed:\(e)" | |
| } | |
| } | |
| } | |
| /// Return the running app's PID given bundle id or localized name | |
| func pid(forAppIdentifier ident:String)->pid_t?{ | |
| debug("Looking for app:\(ident)") | |
| // Handle Safari specifically - try both bundle ID and name | |
| if ident=="Safari"{ | |
| debug("Special handling for Safari") | |
| // Try by bundle ID first | |
| iflet safariApp=NSRunningApplication.runningApplications(withBundleIdentifier:"com.apple.Safari").first{ | |
| debug("Found Safari by bundle ID, PID:\(safariApp.processIdentifier)") | |
| return safariApp.processIdentifier | |
| } | |
| // Try by name | |
| iflet safariApp=NSWorkspace.shared.runningApplications.first(where:{ $0.localizedName=="Safari"}){ | |
| debug("Found Safari by name, PID:\(safariApp.processIdentifier)") | |
| return safariApp.processIdentifier | |
| } | |
| } | |
| iflet byBundle=NSRunningApplication.runningApplications(withBundleIdentifier: ident).first{ | |
| debug("Found by bundle ID:\(ident), PID:\(byBundle.processIdentifier)") | |
| return byBundle.processIdentifier | |
| } | |
| letapp=NSWorkspace.shared.runningApplications | |
| .first{ $0.localizedName== ident} | |
| iflet app= app{ | |
| debug("Found by name:\(ident), PID:\(app.processIdentifier)") | |
| return app.processIdentifier | |
| } | |
| // Also try searching without case sensitivity | |
| letappLowerCase=NSWorkspace.shared.runningApplications | |
| .first{ $0.localizedName?.lowercased()== ident.lowercased()} | |
| iflet app= appLowerCase{ | |
| debug("Found by case-insensitive name:\(ident), PID:\(app.processIdentifier)") | |
| return app.processIdentifier | |
| } | |
| // Print running applications to help debug | |
| debug("All running applications:") | |
| forappinNSWorkspace.shared.runningApplications{ | |
| debug(" -\(app.localizedName??"Unknown") (Bundle:\(app.bundleIdentifier??"Unknown"), PID:\(app.processIdentifier))") | |
| } | |
| debug("App not found:\(ident)") | |
| returnnil | |
| } | |
| /// Fetch a single AX attribute as `T?` | |
| func axValue<T>(of element:AXUIElement, attr:String)->T?{ | |
| varvalue:CFTypeRef? | |
| leterr=AXUIElementCopyAttributeValue(element, attrasCFString,&value) | |
| guard err==.success,let unwrappedValue= valueelse{returnnil} | |
| // For actions, try explicitly casting to CFArray of strings | |
| if attr== kAXActionsAttribute &&T.self==[String].self{ | |
| debug("Reading actions with special handling") | |
| letcfArray= unwrappedValueas!CFArray | |
| letcount=CFArrayGetCount(cfArray) | |
| varactionStrings=[String]() | |
| foriin0..<count{ | |
| letactionPtr=CFArrayGetValueAtIndex(cfArray, i) | |
| iflet actionStr=(actionPtras!CFString)asString?{ | |
| actionStrings.append(actionStr) | |
| } | |
| } | |
| if !actionStrings.isEmpty{ | |
| debug("Found actions:\(actionStrings)") | |
| return actionStringsas?T | |
| } | |
| } | |
| // Safe casting with type checking | |
| ifCFGetTypeID(unwrappedValue)==CFArrayGetTypeID() &&T.self==[AXUIElement].self{ | |
| letcfArray= unwrappedValueas!CFArray | |
| letcount=CFArrayGetCount(cfArray) | |
| varresult=[AXUIElement]() | |
| foriin0..<count{ | |
| letelement=unsafeBitCast(CFArrayGetValueAtIndex(cfArray, i), to:AXUIElement.self) | |
| result.append(element) | |
| } | |
| return resultas?T | |
| }elseifT.self==String.self{ | |
| ifCFGetTypeID(unwrappedValue)==CFStringGetTypeID(){ | |
| return(unwrappedValueas!CFString)as?T | |
| } | |
| returnnil | |
| } | |
| // For other types, use the default casting | |
| returnunsafeBitCast(unwrappedValue, to:T.self) | |
| } | |
| /// Depth-first search for an element that matches the locator's role + attributes | |
| func search(element:AXUIElement, | |
| locator:Locator, | |
| depth:Int=0, | |
| maxDepth:Int=30)->AXUIElement?{ | |
| if depth> maxDepth{returnnil} | |
| // Check role | |
| iflet role:String=axValue(of: element, attr: kAXRoleAttributeasString), | |
| role== locator.role{ | |
| // Match all requested attributes | |
| varok=true | |
| for(attr, want)in locator.match{ | |
| letgot:String?=axValue(of: element, attr: attr) | |
| if got!= want{ ok=false;break} | |
| } | |
| if ok{return element} | |
| } | |
| // Recurse into children | |
| iflet children:[AXUIElement]=axValue(of: element, attr: kAXChildrenAttributeasString){ | |
| forchildin children{ | |
| iflet hit=search(element: child, locator: locator, depth: depth+1){ | |
| return hit | |
| } | |
| } | |
| } | |
| returnnil | |
| } | |
| /// Parse a path hint like "window[1]" into (role, index) | |
| func parsePathComponent(_ path:String)->(role:String, index:Int)?{ | |
| letpattern=#"(\w+)\[(\d+)\]"# | |
| guardlet regex=try?NSRegularExpression(pattern: pattern)else{returnnil} | |
| letrange=NSRange(path.startIndex..<path.endIndex, in: path) | |
| guardlet match= regex.firstMatch(in: path, range: range)else{returnnil} | |
| letroleRange=Range(match.range(at:1), in: path)! | |
| letindexRange=Range(match.range(at:2), in: path)! | |
| letrole=String(path[roleRange]) | |
| letindex=Int(path[indexRange])! | |
| return(role: role, index: index-1) // Convert to 0-based index | |
| } | |
| /// Navigate to an element based on a path hint | |
| func navigateToElement(from root:AXUIElement, pathHint:[String])->AXUIElement?{ | |
| varcurrentElement= root | |
| debug("Starting navigation with path hint:\(pathHint)") | |
| for(i, pathComponent)in pathHint.enumerated(){ | |
| debug("Processing path component\(i+1)/\(pathHint.count):\(pathComponent)") | |
| guardlet(role, index)=parsePathComponent(pathComponent)else{ | |
| debug("Failed to parse path component:\(pathComponent)") | |
| returnnil | |
| } | |
| debug("Parsed as role:\(role), index:\(index) (0-based)") | |
| // Special handling for window (direct access without complicated navigation) | |
| if role.lowercased()=="window"{ | |
| debug("Special handling for window role") | |
| guardlet windows:[AXUIElement]=axValue(of: currentElement, attr: kAXWindowsAttributeasString)else{ | |
| debug("No windows found for application") | |
| returnnil | |
| } | |
| debug("Found\(windows.count) windows") | |
| if index>= windows.count{ | |
| debug("Window index\(index+1) out of bounds (max:\(windows.count))") | |
| returnnil | |
| } | |
| currentElement=windows[index] | |
| debug("Successfully navigated to window[\(index+1)]") | |
| continue | |
| } | |
| // Get all children matching the role | |
| letroleKey="AX\(role.prefix(1).uppercased()+ role.dropFirst())" | |
| debug("Looking for elements with role key:\(roleKey)") | |
| // First try to get children by specific role attribute | |
| iflet roleSpecificChildren:[AXUIElement]=axValue(of: currentElement, attr: roleKey){ | |
| debug("Found\(roleSpecificChildren.count) elements with role\(roleKey)") | |
| // Make sure index is in bounds | |
| guard index< roleSpecificChildren.countelse{ | |
| debug("Index out of bounds:\(index+1) >\(roleSpecificChildren.count) for\(pathComponent)") | |
| returnnil | |
| } | |
| currentElement=roleSpecificChildren[index] | |
| debug("Successfully navigated to\(roleKey)[\(index+1)]") | |
| continue | |
| } | |
| debug("No elements found with specific role\(roleKey), trying with children") | |
| // If we can't find by specific role, try getting all children | |
| guardlet allChildren:[AXUIElement]=axValue(of: currentElement, attr: kAXChildrenAttributeasString)else{ | |
| debug("No children found for element at path component:\(pathComponent)") | |
| returnnil | |
| } | |
| debug("Found\(allChildren.count) children, filtering by role:\(role)") | |
| // Filter by role | |
| letmatchingChildren= allChildren.filter{ elementin | |
| guardlet elementRole:String=axValue(of: element, attr: kAXRoleAttributeasString)else{ | |
| returnfalse | |
| } | |
| letmatches= elementRole.lowercased()== role.lowercased() | |
| if matches{ | |
| debug("Found element with matching role:\(elementRole)") | |
| } | |
| return matches | |
| } | |
| if matchingChildren.isEmpty{ | |
| debug("No children with role '\(role)' found") | |
| // List available roles for debugging | |
| debug("Available roles among children:") | |
| forchildin allChildren{ | |
| iflet childRole:String=axValue(of: child, attr: kAXRoleAttributeasString){ | |
| debug(" -\(childRole)") | |
| } | |
| } | |
| returnnil | |
| } | |
| debug("Found\(matchingChildren.count) children with role '\(role)'") | |
| // Make sure index is in bounds | |
| guard index< matchingChildren.countelse{ | |
| debug("Index out of bounds:\(index+1) >\(matchingChildren.count) for\(pathComponent)") | |
| returnnil | |
| } | |
| currentElement=matchingChildren[index] | |
| debug("Successfully navigated to\(role)[\(index+1)]") | |
| } | |
| debug("Path hint navigation completed successfully") | |
| return currentElement | |
| } | |
| /// Collect all elements that match the locator's role + attributes | |
| func collectAll(element:AXUIElement, | |
| locator:Locator, | |
| requireAction:String?, | |
| hits:inout[AXUIElement], | |
| depth:Int=0, | |
| maxDepth:Int=15){ // Reduce max depth to 15 for safety | |
| // Safety limit on matches | |
| if hits.count>100{ | |
| debug("Safety limit of 100 matching elements reached, stopping search") | |
| return | |
| } | |
| if depth> maxDepth{ | |
| debug("Max depth (\(maxDepth)) reached") | |
| return | |
| } | |
| // role test | |
| letwildcardRole= locator.role=="*" || locator.role.isEmpty | |
| letelementRole=axValue(of: element, attr: kAXRoleAttributeasString)asString? | |
| letroleMatches= wildcardRole || elementRole== locator.role | |
| if wildcardRole{ | |
| debug("Using wildcard role match (*) at depth\(depth)") | |
| }elseiflet role= elementRole{ | |
| debug("Element role at depth\(depth):\(role), looking for:\(locator.role)") | |
| } | |
| if roleMatches{ | |
| // attribute match | |
| varok=true | |
| for(attr, want)in locator.match{ | |
| letgot=axValue(of: element, attr: attr)asString? | |
| if got!= want{ | |
| debug("Attribute mismatch at depth\(depth):\(attr)=\(got??"nil") (wanted\(want))") | |
| ok=false | |
| break | |
| } | |
| } | |
| // Check action requirement using safer method | |
| if ok,let required= requireAction{ | |
| debug("Checking for required action:\(required) at depth\(depth)") | |
| if !elementSupportsAction(element, action: required){ | |
| debug("Element at depth\(depth) doesn't support\(required)") | |
| ok=false | |
| }else{ | |
| debug("Element at depth\(depth) supports\(required)") | |
| } | |
| } | |
| if ok{ | |
| debug("Found matching element at depth\(depth), role:\(elementRole??"unknown")") | |
| hits.append(element) | |
| } | |
| } | |
| // Only recurse into children if we're not at the max depth - avoid potential crashes | |
| if depth< maxDepth{ | |
| // Use safer approach to get children | |
| varchildrenUnwrapped:[AXUIElement]=[] | |
| // First try standard children | |
| iflet children:[AXUIElement]=axValue(of: element, attr: kAXChildrenAttributeasString){ | |
| childrenUnwrapped= children | |
| } | |
| // Limit to max 20 children per element | |
| letmaxChildrenToProcess=min(childrenUnwrapped.count,20) | |
| if childrenUnwrapped.count> maxChildrenToProcess{ | |
| debug("Limiting processing to\(maxChildrenToProcess) of\(childrenUnwrapped.count) children at depth\(depth)") | |
| } | |
| if !childrenUnwrapped.isEmpty{ | |
| debug("Found\(childrenUnwrapped.count) children to explore at depth\(depth)") | |
| letchildrenToProcess= childrenUnwrapped.prefix(maxChildrenToProcess) | |
| for(i, child)in childrenToProcess.enumerated(){ | |
| if hits.count>100{break} // Safety check | |
| debug("Exploring child\(i+1)/\(maxChildrenToProcess) at depth\(depth)") | |
| collectAll(element: child, locator: locator, requireAction: requireAction, | |
| hits:&hits, depth: depth+1, maxDepth: maxDepth) | |
| } | |
| }else{ | |
| debug("No children at depth\(depth)") | |
| } | |
| } | |
| } | |
| // MARK: - Core verbs ----------------------------------------------------------------- | |
| func handleQuery(cmd:CommandEnvelope)throws->Codable{ | |
| debug("Processing query:\(cmd.cmd), app:\(cmd.locator.app), role:\(cmd.locator.role), multi:\(cmd.multi??false)") | |
| guardlet pid=pid(forAppIdentifier: cmd.locator.app)else{ | |
| debug("Failed to find app:\(cmd.locator.app)") | |
| throwAXErrorString.elementNotFound | |
| } | |
| debug("Creating application element for PID:\(pid)") | |
| letappElement=AXUIElementCreateApplication(pid) | |
| // Apply path hint if provided | |
| varstartElement= appElement | |
| iflet pathHint= cmd.locator.pathHint, !pathHint.isEmpty{ | |
| debug("Path hint provided:\(pathHint)") | |
| guardlet navigatedElement=navigateToElement(from: appElement, pathHint: pathHint)else{ | |
| debug("Failed to navigate using path hint") | |
| throwAXErrorString.elementNotFound | |
| } | |
| startElement= navigatedElement | |
| debug("Successfully navigated to element using path hint") | |
| } | |
| // Define the attributes to query - add more useful attributes | |
| varattributesToQuery= cmd.attributes??[ | |
| "AXRole","AXTitle","AXIdentifier", | |
| "AXDescription","AXValue","AXHelp", | |
| "AXSubrole","AXRoleDescription","AXLabel", | |
| "AXActions","AXPosition","AXSize" | |
| ] | |
| // Check if the client explicitly asked for a limited set of attributes | |
| letshouldExpandAttributes= cmd.attributes==nil || cmd.attributes!.isEmpty | |
| // If using default attributes, try to get additional attributes for the element | |
| if shouldExpandAttributes{ | |
| // Query all available attributes for the starting element | |
| varattrNames:CFArray? | |
| ifAXUIElementCopyAttributeNames(startElement,&attrNames)==.success,let names= attrNames{ | |
| letcount=CFArrayGetCount(names) | |
| foriin0..<count{ | |
| iflet ptr=CFArrayGetValueAtIndex(names, i), | |
| let cfStr=unsafeBitCast(ptr, to:CFString.self)asString?, | |
| !attributesToQuery.contains(cfStr){ | |
| attributesToQuery.append(cfStr) | |
| } | |
| } | |
| debug("Expanded to include\(attributesToQuery.count) attributes") | |
| } | |
| } | |
| // Handle multi-element query | |
| if cmd.multi==true{ | |
| debug("Performing multi-element query") | |
| // Collect elements without action requirement first | |
| varinitialHits:[AXUIElement]=[] | |
| collectAll(element: startElement, locator: cmd.locator, | |
| requireAction:nil, hits:&initialHits) | |
| debug("Found\(initialHits.count) elements without action filter") | |
| // Create a new array for storing filtered elements | |
| varmatchingElements:[AXUIElement]=[] | |
| // If action required, filter the elements | |
| iflet requiredAction= cmd.requireAction{ | |
| debug("Filtering for action:\(requiredAction)") | |
| // Manually check each element for action support | |
| varmatchCount=0 | |
| forelementin initialHits{ | |
| ifelementSupportsAction(element, action: requiredAction){ | |
| matchingElements.append(element) | |
| matchCount+=1 | |
| } | |
| } | |
| debug("After filtering, found\(matchCount) elements with action:\(requiredAction)") | |
| // If no matches but we found elements, return a subset with warning | |
| if matchingElements.isEmpty && !initialHits.isEmpty{ | |
| debug("Returning elements without required action") | |
| // Manually build result array | |
| varresultArray:[ElementAttributes]=[] | |
| letmaxElements=min(initialHits.count,10) | |
| foriin0..<maxElements{ | |
| varattributes=getElementAttributes(initialHits[i], attributes: attributesToQuery) | |
| attributes["_warning"]="Element doesn't support\(requiredAction) action" | |
| resultArray.append(attributes) | |
| } | |
| returnMultiQueryResponse(elements: resultArray) | |
| } | |
| }else{ | |
| // No action required, use all elements | |
| matchingElements= initialHits | |
| } | |
| debug("Processing final results") | |
| // If no matches found, throw error | |
| if matchingElements.isEmpty{ | |
| debug("No elements matched criteria") | |
| throwAXErrorString.elementNotFound | |
| } | |
| // Manually build result array with a hard limit | |
| varresultArray:[ElementAttributes]=[] | |
| letmaxElements=min(matchingElements.count,20) | |
| foriin0..<maxElements{ | |
| letattributes=getElementAttributes(matchingElements[i], attributes: attributesToQuery) | |
| resultArray.append(attributes) | |
| } | |
| returnMultiQueryResponse(elements: resultArray) | |
| } | |
| // Single element query (original behavior) | |
| guardlet element=search(element: startElement, locator: cmd.locator)else{ | |
| throwAXErrorString.elementNotFound | |
| } | |
| // Get attributes for the single element | |
| letattributes=getElementAttributes(element, attributes: attributesToQuery) | |
| returnQueryResponse(attributes: attributes) | |
| } | |
| func handlePerform(cmd:CommandEnvelope)throws->PerformResponse{ | |
| guardlet pid=pid(forAppIdentifier: cmd.locator.app), | |
| let action= cmd.actionelse{ | |
| throwAXErrorString.elementNotFound | |
| } | |
| letappElement=AXUIElementCreateApplication(pid) | |
| guardlet element=search(element: appElement, locator: cmd.locator)else{ | |
| throwAXErrorString.elementNotFound | |
| } | |
| leterr=AXUIElementPerformAction(element, actionasCFString) | |
| guard err==.successelse{ | |
| throwAXErrorString.actionFailed(err) | |
| } | |
| returnPerformResponse(status:"ok") | |
| } | |
| // MARK: - Main loop ------------------------------------------------------------------ | |
| letdecoder=JSONDecoder() | |
| letencoder=JSONEncoder() | |
| if #available(macOS10.15,*){ | |
| encoder.outputFormatting=[.withoutEscapingSlashes] | |
| } | |
| // Check for accessibility permissions before starting | |
| checkAccessibilityPermissions() | |
| whilelet line=readLine(strippingNewline:true){ | |
| do{ | |
| letdata=Data(line.utf8) | |
| letcmd=try decoder.decode(CommandEnvelope.self, from: data) | |
| switch cmd.cmd{ | |
| case.query: | |
| letresult=tryhandleQuery(cmd: cmd) | |
| letreply=try encoder.encode(result) | |
| FileHandle.standardOutput.write(reply) | |
| FileHandle.standardOutput.write("\n".data(using:.utf8)!) | |
| case.perform: | |
| letstatus=tryhandlePerform(cmd: cmd) | |
| letreply=try encoder.encode(status) | |
| FileHandle.standardOutput.write(reply) | |
| FileHandle.standardOutput.write("\n".data(using:.utf8)!) | |
| } | |
| }catch{ | |
| leterrorResponse=ErrorResponse(error:"\(error)") | |
| iflet errorData=try? encoder.encode(errorResponse){ | |
| FileHandle.standardError.write(errorData) | |
| FileHandle.standardError.write("\n".data(using:.utf8)!) | |
| }else{ | |
| fputs("{\"error\":\"\(error)\"}\n", stderr) | |
| } | |
| } | |
| } | |
| // Add a safer action checking function | |
| func elementSupportsAction(_ element:AXUIElement, action:String)->Bool{ | |
| // Use the simplest possible approach to check actions | |
| varactionNames:CFArray? | |
| leterr=AXUIElementCopyActionNames(element,&actionNames) | |
| if err!=.success{ | |
| debug("Failed to get action names:\(err)") | |
| returnfalse | |
| } | |
| guardlet actions= actionNameselse{ | |
| debug("No actions array") | |
| returnfalse | |
| } | |
| // Just check if the array contains at least one action | |
| letcount=CFArrayGetCount(actions) | |
| debug("Element has\(count) actions") | |
| // Instead of trying to read the actual actions (which seems to cause issues), | |
| // just check if the number is non-zero and assume it might support our action | |
| // This is not ideal but safer than trying to extract action strings | |
| if count>0{ | |
| debug("Element has actions, assuming it supports\(action)") | |
| returntrue | |
| } | |
| debug("Element has no actions") | |
| returnfalse | |
| } |
Sign up for freeto join this conversation on GitHub. Already have an account?Sign in to comment