diff --git a/openadapt/alembic/versions/d1b385041a20_add_a11y_event_remove_state_from_window_.py b/openadapt/alembic/versions/d1b385041a20_add_a11y_event_remove_state_from_window_.py new file mode 100644 index 000000000..ec8fa16b6 --- /dev/null +++ b/openadapt/alembic/versions/d1b385041a20_add_a11y_event_remove_state_from_window_.py @@ -0,0 +1,48 @@ +"""add_a11y_event_remove_state_from_window_event + +Revision ID: d1b385041a20 +Revises: bb25e889ad71 +Create Date: 2024-07-25 16:21:27.450372 + +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import sqlite +import openadapt + +# revision identifiers, used by Alembic. +revision = 'd1b385041a20' +down_revision = 'bb25e889ad71' +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('a11y_event', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('timestamp', openadapt.models.ForceFloat(precision=10, scale=2, asdecimal=False), nullable=True), + sa.Column('handle', sa.Integer(), nullable=True), + sa.Column('data', sa.JSON(), nullable=True), + sa.ForeignKeyConstraint(['handle', 'timestamp'], ['window_event.handle', 'window_event.timestamp'], name=op.f('fk_a11y_event_handle_window_event')), + sa.PrimaryKeyConstraint('id', name=op.f('pk_a11y_event')) + ) + with op.batch_alter_table('window_event', schema=None) as batch_op: + batch_op.add_column(sa.Column('handle', sa.Integer(), nullable=True)) + batch_op.create_unique_constraint('uix_handle_timestamp', ['handle', 'timestamp']) + batch_op.drop_column('state') + batch_op.drop_column('window_id') + + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('window_event', schema=None) as batch_op: + batch_op.add_column(sa.Column('window_id', sa.VARCHAR(), nullable=True)) + batch_op.add_column(sa.Column('state', sqlite.JSON(), nullable=True)) + batch_op.drop_constraint('uix_handle_timestamp', type_='unique') + batch_op.drop_column('handle') + + op.drop_table('a11y_event') + # ### end Alembic commands ### diff --git a/openadapt/app/dashboard/api/recordings.py b/openadapt/app/dashboard/api/recordings.py index 217024992..dec392d02 100644 --- a/openadapt/app/dashboard/api/recordings.py +++ b/openadapt/app/dashboard/api/recordings.py @@ -11,6 +11,7 @@ from openadapt.models import Recording from openadapt.plotting import display_event from openadapt.utils import image2utf8, row2dict +from openadapt.visualize import dict2html class RecordingsAPI: @@ -69,6 +70,38 @@ def recording_detail_route(self) -> None: @self.app.websocket("/{recording_id}") async def get_recording_detail(websocket: WebSocket, recording_id: int) -> None: """Get a specific recording and its action events.""" + + def extract_a11y_texts_value( + data: dict | list, target_key: str, target_class_name: str + ) -> list: + """Recursively extracts values from a nested dictionary. + + Args: + data (dict or list): The nested dict/list to search within. + target_key (str): The key for which the values should be extracted. + target_class_name (str): The value of the "friendly_class_name" key + that must match for the target_key. + + Returns: + list: A list of values corresponding to the target_key where the + "friendly_class_name" matches the target_class_name. + """ + results = [] + + def recursive_extract(d: dict | list) -> None: + if isinstance(d, dict): + if d.get("friendly_class_name") == target_class_name: + if target_key in d: + results.append(d[target_key]) + for key, value in d.items(): + recursive_extract(value) + elif isinstance(d, list): + for item in d: + recursive_extract(item) + + recursive_extract(data) + return results + await websocket.accept() session = crud.get_new_session(read_only=True) recording = crud.get_recording_by_id(session, recording_id) @@ -112,6 +145,8 @@ def convert_to_str(event_dict: dict) -> dict: for action_event in action_events: event_dict = row2dict(action_event) + a11y_dict = row2dict(action_event.window_event.a11y_event) + event_dict["a11y_data"] = dict2html(a11y_dict) try: image = display_event(action_event) width, height = image.size diff --git a/openadapt/app/dashboard/components/ActionEvent/ActionEvent.tsx b/openadapt/app/dashboard/components/ActionEvent/ActionEvent.tsx index 066f31cdb..c368b0956 100644 --- a/openadapt/app/dashboard/components/ActionEvent/ActionEvent.tsx +++ b/openadapt/app/dashboard/components/ActionEvent/ActionEvent.tsx @@ -5,6 +5,7 @@ import { ActionEvent as ActionEventType } from '@/types/action-event' import { Accordion, Box, Grid, Image, Table } from '@mantine/core' import { useHover } from '@mantine/hooks'; import { RemoveActionEvent } from './RemoveActionEvent'; +import './style.css'; type Props = { event: ActionEventType; @@ -41,7 +42,7 @@ export const ActionEvent = ({ let content = ( - +
{typeof event.id === 'number' && ( @@ -136,6 +137,26 @@ export const ActionEvent = ({
+ {event.a11y_data && ( + + + + +
+ Accessibility Data +
+ + + + +
+ +
+ + + + + )} diff --git a/openadapt/app/dashboard/components/ActionEvent/style.css b/openadapt/app/dashboard/components/ActionEvent/style.css new file mode 100644 index 000000000..f04e4b186 --- /dev/null +++ b/openadapt/app/dashboard/components/ActionEvent/style.css @@ -0,0 +1,20 @@ +.table-nested table { + border-collapse: collapse; + border: 1px solid #d1d5db; +} + +.table-nested th, +.table-nested td { + border: 1px solid #d1d5db; + padding: 8px; + text-align: left; +} + +.table-nested th { + background-color: #f9fafb; + font-weight: bold; +} + +.table-nested td { + background-color: #ffffff; +} \ No newline at end of file diff --git a/openadapt/app/dashboard/types/action-event.ts b/openadapt/app/dashboard/types/action-event.ts index 211aa3f78..5e48b4b37 100644 --- a/openadapt/app/dashboard/types/action-event.ts +++ b/openadapt/app/dashboard/types/action-event.ts @@ -27,6 +27,7 @@ export type ActionEvent = { dimensions?: { width: number, height: number }; children?: ActionEvent[]; words?: string[]; + a11y_data: string; isComputed?: boolean; isOriginal?: boolean; } diff --git a/openadapt/config.py b/openadapt/config.py index d0a48301a..9d960d098 100644 --- a/openadapt/config.py +++ b/openadapt/config.py @@ -133,7 +133,7 @@ class SegmentationAdapter(str, Enum): OPENAI_MODEL_NAME: str = "gpt-3.5-turbo" # Record and replay - RECORD_WINDOW_DATA: bool = True + RECORD_A11Y_DATA: bool = True RECORD_READ_ACTIVE_ELEMENT_STATE: bool = False RECORD_VIDEO: bool RECORD_AUDIO: bool diff --git a/openadapt/db/crud.py b/openadapt/db/crud.py index 806ff5ead..39d924882 100644 --- a/openadapt/db/crud.py +++ b/openadapt/db/crud.py @@ -27,6 +27,7 @@ Screenshot, ScrubbedRecording, WindowEvent, + A11yEvent, copy_sa_instance, ) from openadapt.privacy.base import ScrubbingProvider @@ -262,6 +263,25 @@ def insert_recording(session: SaSession, recording_data: dict) -> Recording: return db_obj +def insert_a11y_event( + session: SaSession, + event_data: dict, +) -> None: + """Insert an a11y event into the database. + + Args: + session (sa.orm.Session): The database session. + event_data (dict): The data of the event + """ + handle = event_data["handle"] + a11y_data = event_data["a11y_data"] + timestamp = event_data["timestamp"] + a11y_event = A11yEvent(timestamp=timestamp, handle=handle, data=a11y_data) + + session.add(a11y_event) + session.commit() + + def delete_recording(session: SaSession, recording: Recording) -> None: """Remove the recording from the db. @@ -590,6 +610,35 @@ def get_window_events( ) +def get_a11y_events( + session: SaSession, + recording: Recording, +) -> list[A11yEvent]: + """Get accessibility events for a given recording. + + Args: + session (SaSession): The SQLAlchemy session. + recording (Recording): The recording object. + + Returns: + list[A11yEvent]: A list of accessibility events for the recording. + """ + return ( + session.query(A11yEvent) + .join( + WindowEvent, + (A11yEvent.handle == WindowEvent.handle) + & (A11yEvent.timestamp == WindowEvent.timestamp), + ) + .filter(WindowEvent.recording_id == recording.id) + .options( + joinedload(A11yEvent.window_event).joinedload(WindowEvent.recording), + ) + .order_by(A11yEvent.timestamp) + .all() + ) + + def disable_action_event(session: SaSession, event_id: int) -> None: """Disable an action event. diff --git a/openadapt/events.py b/openadapt/events.py index 6384246c3..d9451e54d 100644 --- a/openadapt/events.py +++ b/openadapt/events.py @@ -46,6 +46,7 @@ def get_events( action_events = crud.get_action_events(db, recording) window_events = crud.get_window_events(db, recording) screenshots = crud.get_screenshots(db, recording) + a11y_events = crud.get_a11y_events(db, recording) if recording.original_recording_id: # if recording is a copy, it already has its events processed when it @@ -62,10 +63,12 @@ def get_events( assert num_action_events > 0, "No action events found." num_window_events = len(window_events) num_screenshots = len(screenshots) + num_a11y_events = len(a11y_events) num_action_events_raw = num_action_events num_window_events_raw = num_window_events num_screenshots_raw = num_screenshots + num_a11y_events_raw = num_a11y_events duration_raw = action_events[-1].timestamp - action_events[0].timestamp num_process_iters = 0 @@ -76,26 +79,31 @@ def get_events( f"{num_action_events=} " f"{num_window_events=} " f"{num_screenshots=}" + f"{num_a11y_events=}" ) ( action_events, window_events, screenshots, + a11y_events, ) = process_events( action_events, window_events, screenshots, + a11y_events, ) if ( len(action_events) == num_action_events and len(window_events) == num_window_events and len(screenshots) == num_screenshots + and len(a11y_events) == num_a11y_events ): break num_process_iters += 1 num_action_events = len(action_events) num_window_events = len(window_events) num_screenshots = len(screenshots) + num_a11y_events = len(a11y_events) if num_process_iters == MAX_PROCESS_ITERS: break @@ -116,6 +124,10 @@ def get_events( num_screenshots, num_screenshots_raw, ) + meta["num_a11y_events"] = format_num( + num_a11y_events, + num_a11y_events_raw, + ) duration = action_events[-1].timestamp - action_events[0].timestamp if len(action_events) > 1: @@ -797,10 +809,12 @@ def process_events( action_events: list[models.ActionEvent], window_events: list[models.WindowEvent], screenshots: list[models.Screenshot], + a11y_events: list[models.A11yEvent], ) -> tuple[ list[models.ActionEvent], list[models.WindowEvent], list[models.Screenshot], + list[models.A11yEvent], ]: """Process action events, window events, and screenshots. @@ -808,6 +822,7 @@ def process_events( action_events (list): The list of action events. window_events (list): The list of window events. screenshots (list): The list of screenshots. + a11y_events (list): The list of a11y events. Returns: tuple: A tuple containing the processed action events, window events, @@ -821,9 +836,16 @@ def process_events( num_action_events = len(action_events) num_window_events = len(window_events) num_screenshots = len(screenshots) - num_total = num_action_events + num_window_events + num_screenshots + num_a11y_events = len(a11y_events) + num_total = ( + num_action_events + num_window_events + num_screenshots + num_a11y_events + ) logger.info( - f"before {num_action_events=} {num_window_events=} {num_screenshots=} " + "before " + f"{num_action_events=} " + f"{num_window_events=} " + f"{num_screenshots=} " + f"{num_a11y_events=} " f"{num_total=}" ) process_fns = [ @@ -862,19 +884,36 @@ def process_events( action_events, "screenshot_timestamp", ) + a11y_events = discard_unused_events( + a11y_events, + action_events, + "timestamp", + ) num_action_events_ = len(action_events) num_window_events_ = len(window_events) num_screenshots_ = len(screenshots) - num_total_ = num_action_events_ + num_window_events_ + num_screenshots_ + num_a11y_events_ = len(a11y_events) + num_total_ = ( + num_action_events_ + num_window_events_ + num_screenshots_ + num_a11y_events_ + ) pct_action_events = num_action_events_ / num_action_events pct_window_events = num_window_events_ / num_window_events + pct_a11y_events = num_a11y_events_ / num_a11y_events pct_screenshots = num_screenshots_ / num_screenshots pct_total = num_total_ / num_total logger.info( - f"after {num_action_events_=} {num_window_events_=} {num_screenshots_=} " + "after " + f"{num_action_events_=}, " + f"{num_window_events_=}, " + f"{num_screenshots_=}, " + f"{num_a11y_events_=}, " f"{num_total_=}" ) logger.info( - f"{pct_action_events=} {pct_window_events=} {pct_screenshots=} {pct_total=}" + f"{pct_action_events=} " + f"{pct_window_events=} " + f"{pct_screenshots=} " + f"{pct_a11y_events=} " + f"{pct_total=}" ) - return action_events, window_events, screenshots + return action_events, window_events, screenshots, a11y_events diff --git a/openadapt/models.py b/openadapt/models.py index a8ebd2015..c8f413c0b 100644 --- a/openadapt/models.py +++ b/openadapt/models.py @@ -491,6 +491,25 @@ def to_prompt_dict(self) -> dict[str, Any]: return action_dict +class A11yEvent(db.Base): + """Class representing an accessibility (a11y) event in the database.""" + + __tablename__ = "a11y_event" + + id = sa.Column(sa.Integer, primary_key=True) + timestamp = sa.Column(ForceFloat) + handle = sa.Column(sa.Integer) + data = sa.Column(sa.JSON) + + __table_args__ = ( + sa.ForeignKeyConstraint( + ["handle", "timestamp"], ["window_event.handle", "window_event.timestamp"] + ), + ) + + window_event = sa.orm.relationship("WindowEvent", back_populates="a11y_event") + + class WindowEvent(db.Base): """Class representing a window event in the database.""" @@ -500,32 +519,51 @@ class WindowEvent(db.Base): recording_timestamp = sa.Column(ForceFloat) recording_id = sa.Column(sa.ForeignKey("recording.id")) timestamp = sa.Column(ForceFloat) - state = sa.Column(sa.JSON) title = sa.Column(sa.String) left = sa.Column(sa.Integer) top = sa.Column(sa.Integer) width = sa.Column(sa.Integer) height = sa.Column(sa.Integer) - window_id = sa.Column(sa.String) + handle = sa.Column(sa.Integer) + + __table_args__ = ( + sa.UniqueConstraint("handle", "timestamp", name="uix_handle_timestamp"), + ) recording = sa.orm.relationship("Recording", back_populates="window_events") action_events = sa.orm.relationship("ActionEvent", back_populates="window_event") + a11y_event = sa.orm.relationship( + "A11yEvent", uselist=False, back_populates="window_event" + ) @classmethod def get_active_window_event( cls: "WindowEvent", - # TODO: rename to include_a11y_data - include_window_data: bool = True, + include_a11y_data: bool = True, ) -> "WindowEvent": """Get the active window event. Args: - include_window_data (bool): whether to include a11y data. + include_a11y_data (bool): whether to include a11y data. Returns: (WindowEvent) the active window event. """ - return WindowEvent(**window.get_active_window_data(include_window_data)) + window_event_data = window.get_active_window_data(include_a11y_data) + a11y_event = None + + if include_a11y_data: + a11y_event_data = window_event_data.get("state") + window_event_data.pop("state", None) + a11y_event_handle = window_event_data.get("handle") + a11y_event = A11yEvent(data=a11y_event_data, handle=a11y_event_handle) + + window_event = WindowEvent(**window_event_data) + + if a11y_event: + window_event.a11y_event = a11y_event + + return window_event def scrub(self, scrubber: ScrubbingProvider | TextScrubbingMixin) -> None: """Scrub the window event.""" @@ -548,6 +586,7 @@ def to_prompt_dict( Returns: dictionary containing relevant properties from the WindowEvent. """ + a11y_data = self.a11y_event.data window_dict = deepcopy( { key: val @@ -581,37 +620,51 @@ def to_prompt_dict( window_dict.pop("width") window_dict.pop("height") - if "state" in window_dict: - if include_data: - key_suffixes = [ - "value", - "h", - "w", - "x", - "y", - "description", - "title", - "help", - ] - if sys.platform == "win32": - logger.warning( - "key_suffixes have not yet been defined on Windows." - "You can help by uncommenting the lines below and pasting " - "the contents of the window_dict into a new GitHub Issue." - ) - # from pprint import pformat - # logger.info(f"window_dict=\n{pformat(window_dict)}") - # import ipdb; ipdb.set_trace() - window_state = window_dict["state"] - window_state["data"] = utils.clean_dict( - utils.filter_keys( - window_state["data"], - key_suffixes, - ) - ) - else: - window_dict["state"].pop("data") - window_dict["state"].pop("meta") + if a11y_data: + window_dict = deepcopy( + { + key: val + for key, val in utils.row2dict( + self.a11y_event, follow=False + ).items() + if val not in EMPTY_VALS + and not key.endswith("timestamp") + and not key.endswith("id") + # and not isinstance(getattr(models.WindowEvent, key), property) + } + ) + if "a11y_data" in window_dict: + if include_data: + key_suffixes = [ + "value", + "h", + "w", + "x", + "y", + "description", + "title", + "help", + ] + if sys.platform == "win32": + logger.warning( + "key_suffixes have not yet been defined on Windows." + "You can help by uncommenting the lines below and pasting " + "the contents of the window_dict into a new GitHub Issue." + ) + # from pprint import pformat + # logger.info(f"window_dict=\n{pformat(window_dict)}") + # import ipdb; ipdb.set_trace() + if "a11y_data" in window_dict: + window_state = window_dict["a11y_data"] + window_state["data"] = utils.clean_dict( + utils.filter_keys( + window_state["data"], + key_suffixes, + ) + ) + else: + window_dict["a11y_data"].pop("data") + window_dict["a11y_data"].pop("meta") return window_dict diff --git a/openadapt/record.py b/openadapt/record.py index e9a6061cb..0535b47e4 100644 --- a/openadapt/record.py +++ b/openadapt/record.py @@ -45,7 +45,7 @@ Event = namedtuple("Event", ("timestamp", "type", "data")) -EVENT_TYPES = ("screen", "action", "window") +EVENT_TYPES = ("screen", "action", "window", "a11y") LOG_LEVEL = "INFO" # whether to write events of each type in a separate process PROC_WRITE_BY_EVENT_TYPE = { @@ -53,6 +53,7 @@ "screen/video": True, "action": True, "window": True, + "a11y": True, } PLOT_PERFORMANCE = config.PLOT_PERFORMANCE NUM_MEMORY_STATS_TO_LOG = 3 @@ -128,6 +129,7 @@ def process_events( screen_write_q: sq.SynchronizedQueue, action_write_q: sq.SynchronizedQueue, window_write_q: sq.SynchronizedQueue, + a11y_write_q: sq.SynchronizedQueue, video_write_q: sq.SynchronizedQueue, perf_q: sq.SynchronizedQueue, recording: Recording, @@ -136,6 +138,7 @@ def process_events( num_screen_events: multiprocessing.Value, num_action_events: multiprocessing.Value, num_window_events: multiprocessing.Value, + num_a11y_events: multiprocessing.Value, num_video_events: multiprocessing.Value, ) -> None: """Process events from the event queue and write them to write queues. @@ -145,6 +148,7 @@ def process_events( screen_write_q: A queue for writing screen events. action_write_q: A queue for writing action events. window_write_q: A queue for writing window events. + a11y_write_q: A queue for writing a11y events. video_write_q: A queue for writing video events. perf_q: A queue for collecting performance data. recording: The recording object. @@ -153,6 +157,7 @@ def process_events( num_screen_events: A counter for the number of screen events. num_action_events: A counter for the number of action events. num_window_events: A counter for the number of window events. + num_a11y_events: A counter for the number of a11y events. num_video_events: A counter for the number of video events. """ utils.set_start_time(recording.timestamp) @@ -160,10 +165,14 @@ def process_events( logger.info("Starting") prev_event = None + + # for assigning to actions prev_screen_event = None prev_window_event = None prev_saved_screen_timestamp = 0 prev_saved_window_timestamp = 0 + + window_events_waiting_for_a11y = queue.Queue() started = False while not terminate_processing.is_set() or not event_q.empty(): event = event_q.get() @@ -197,6 +206,8 @@ def process_events( num_video_events.value += 1 elif event.type == "window": prev_window_event = event + if config.RECORD_A11Y_DATA: + window_events_waiting_for_a11y.put_nowait(event) elif event.type == "action": if prev_screen_event is None: logger.warning("Discarding action that came before screen") @@ -244,6 +255,24 @@ def process_events( ) num_window_events.value += 1 prev_saved_window_timestamp = prev_window_event.timestamp + elif event.type == "a11y": + try: + window_event = window_events_waiting_for_a11y.get_nowait() + except queue.Empty as exc: + logger.warning( + f"Discarding A11yEvent with no corresponding WindowEvent: {exc}" + ) + continue + event.data["timestamp"] = window_event.timestamp + process_event( + event, + a11y_write_q, + write_a11y_event, + recording, + perf_q, + ) + num_a11y_events.value += 1 + logger.debug(f"A11yEvent processed: {event}") else: raise Exception(f"unhandled {event.type=}") del prev_event @@ -312,7 +341,28 @@ def write_window_event( perf_q: A queue for collecting performance data. """ assert event.type == "window", event - crud.insert_window_event(db, recording, event.timestamp, event.data) + data = event.data + crud.insert_window_event(db, recording, event.timestamp, data) + perf_q.put((event.type, event.timestamp, utils.get_timestamp())) + + +def write_a11y_event( + db: crud.SaSession, + recording: Recording, + event: Event, + perf_q: sq.SynchronizedQueue, +) -> None: + """Write an a11y event to the database and update the performance queue. + + Args: + db: The database session. + recording: The recording object. + event: An a11y event to be written. + perf_q: A queue for collecting performance data. + """ + assert event.type == "a11y", event + data = event.data + crud.insert_a11y_event(db, data) perf_q.put((event.type, event.timestamp, utils.get_timestamp())) @@ -696,6 +746,8 @@ def read_window_events( terminate_processing: multiprocessing.Event, recording: Recording, started_counter: multiprocessing.Value, + read_a11y_data: bool, + event_name: str, ) -> None: """Read window events and add them to the event queue. @@ -704,6 +756,8 @@ def read_window_events( terminate_processing: An event to signal the termination of the process. recording: The recording object. started_counter: Value to increment once started. + read_a11y_data: Whether to read a11y_data. + event_name: The name of the event. """ utils.set_start_time(recording.timestamp) @@ -711,18 +765,16 @@ def read_window_events( prev_window_data = {} started = False while not terminate_processing.is_set(): - window_data = window.get_active_window_data() + window_data = window.get_active_window_data(include_a11y_data=read_a11y_data) if not window_data: continue - if not started: with started_counter.get_lock(): started_counter.value += 1 started = True - if window_data["title"] != prev_window_data.get("title") or window_data[ - "window_id" - ] != prev_window_data.get("window_id"): + # for logging purposes only + if window_data["handle"] != prev_window_data.get("handle"): # TODO: fix exception sometimes triggered by the next line on win32: # File "\Python39\lib\threading.py" line 917, in run # File "...\openadapt\record.py", line 277, in read window events @@ -730,15 +782,20 @@ def read_window_events( # File "...\env\lib\site-packages\loguru\_logger.py", line 1964, in _log # for handler in core.handlers.values): # RuntimeError: dictionary changed size during iteration - _window_data = window_data - _window_data.pop("state") - logger.info(f"{_window_data=}") + window_data["timestamp"] = utils.get_timestamp() + if read_a11y_data: + _window_data = window_data.copy() + _window_data.pop("a11y_data") + logger.info(f"{_window_data=}") + else: + logger.info(f"{window_data=}") + if window_data != prev_window_data: - logger.debug("Queuing window event for writing") + logger.debug("Queuing {event_name} event for writing") event_q.put( Event( utils.get_timestamp(), - "window", + event_name, window_data, ) ) @@ -1175,6 +1232,7 @@ def record( screen_write_q = sq.SynchronizedQueue() action_write_q = sq.SynchronizedQueue() window_write_q = sq.SynchronizedQueue() + a11y_write_q = sq.SynchronizedQueue() video_write_q = sq.SynchronizedQueue() # TODO: save write times to DB; display performance plot in visualize.py perf_q = sq.SynchronizedQueue() @@ -1185,10 +1243,32 @@ def record( window_event_reader = threading.Thread( target=read_window_events, - args=(event_q, terminate_processing, recording, started_counter), + args=( + event_q, + terminate_processing, + recording, + started_counter, + False, + "window", + ), ) window_event_reader.start() + if config.RECORD_A11Y_DATA: + a11y_event_reader = threading.Thread( + target=read_window_events, + args=( + event_q, + terminate_processing, + recording, + started_counter, + True, + "a11y", + ), + ) + a11y_event_reader.start() + expected_starts += 1 + screen_event_reader = threading.Thread( target=read_screen_events, args=(event_q, terminate_processing, recording, started_counter), @@ -1210,6 +1290,7 @@ def record( num_action_events = multiprocessing.Value("i", 0) num_screen_events = multiprocessing.Value("i", 0) num_window_events = multiprocessing.Value("i", 0) + num_a11y_events = multiprocessing.Value("i", 0) num_video_events = multiprocessing.Value("i", 0) event_processor = threading.Thread( @@ -1219,6 +1300,7 @@ def record( screen_write_q, action_write_q, window_write_q, + a11y_write_q, video_write_q, perf_q, recording, @@ -1227,6 +1309,7 @@ def record( num_screen_events, num_action_events, num_window_events, + num_a11y_events, num_video_events, ), ) @@ -1277,6 +1360,21 @@ def record( ) window_event_writer.start() + a11y_event_writer = multiprocessing.Process( + target=write_events, + args=( + "a11y", + write_a11y_event, + a11y_write_q, + num_a11y_events, + perf_q, + recording, + terminate_processing, + started_counter, + ), + ) + a11y_event_writer.start() + if config.RECORD_VIDEO: expected_starts += 1 video_writer = multiprocessing.Process( @@ -1379,7 +1477,9 @@ def record( screen_event_writer.join() action_event_writer.join() window_event_writer.join() + # Join a11y_event_writer if config.RECORD_VIDEO: + a11y_event_writer.join() video_writer.join() if config.RECORD_AUDIO: audio_recorder.join() diff --git a/openadapt/visualize.py b/openadapt/visualize.py index d5e0a4842..5aec2d515 100644 --- a/openadapt/visualize.py +++ b/openadapt/visualize.py @@ -346,10 +346,12 @@ def main( action_event_dict = row2dict(action_event) window_event_dict = row2dict(action_event.window_event) + a11y_event_dict = row2dict(action_event.window_event.a11y_event) if SCRUB: action_event_dict = scrub.scrub_dict(action_event_dict) window_event_dict = scrub.scrub_dict(window_event_dict) + a11y_event_dict = scrub.scrub_dict(a11y_event_dict) rows.append( [ @@ -379,6 +381,9 @@ def main( {dict2html(window_event_dict , None)}
+ + {dict2html(a11y_event_dict, None)} +
""", ), Div(text=f""" diff --git a/openadapt/window/__init__.py b/openadapt/window/__init__.py index fe0cb9e9f..64340a0c0 100644 --- a/openadapt/window/__init__.py +++ b/openadapt/window/__init__.py @@ -19,18 +19,18 @@ def get_active_window_data( - include_window_data: bool = config.RECORD_WINDOW_DATA, + include_a11y_data: bool = config.RECORD_A11Y_DATA, ) -> dict[str, Any] | None: """Get data of the active window. Args: - include_window_data (bool): whether to include a11y data. + include_a11y_data (bool): whether to include a11y data. Returns: dict or None: A dictionary containing information about the active window, or None if the state is not available. """ - state = get_active_window_state(include_window_data) + state = get_active_window_state(include_a11y_data) if not state: return {} title = state["title"] @@ -38,29 +38,32 @@ def get_active_window_data( top = state["top"] width = state["width"] height = state["height"] - window_id = state["window_id"] + handle = state["handle"] window_data = { "title": title, "left": left, "top": top, "width": width, "height": height, - "window_id": window_id, - "state": state, + "handle": handle, } + + if include_a11y_data: + window_data["a11y_data"] = state + return window_data -def get_active_window_state(read_window_data: bool) -> dict | None: +def get_active_window_state(read_a11y_data: bool) -> dict | None: """Get the state of the active window. Returns: - dict or None: A dictionary containing the state of the active window, + dict or None: A dictionary containing the a11y_data of the active window, or None if the state is not available. """ # TODO: save window identifier (a window's title can change, or try: - return impl.get_active_window_state(read_window_data) + return impl.get_active_window_state(read_a11y_data) except Exception as exc: logger.warning(f"{exc=}") return None diff --git a/openadapt/window/_macos.py b/openadapt/window/_macos.py index 3b9fd7625..327389ed8 100644 --- a/openadapt/window/_macos.py +++ b/openadapt/window/_macos.py @@ -13,7 +13,7 @@ from openadapt.custom_logger import logger -def get_active_window_state(read_window_data: bool) -> dict | None: +def get_active_window_state(read_a11y_data: bool) -> dict | None: """Get the state of the active window. Returns: @@ -23,7 +23,7 @@ def get_active_window_state(read_window_data: bool) -> dict | None: # pywinctl performance on macOS is unusable, see: # https://github.com/Kalmat/PyWinCtl/issues/29 meta = get_active_window_meta() - if read_window_data: + if read_a11y_data: data = get_window_data(meta) else: data = {} @@ -33,7 +33,7 @@ def get_active_window_state(read_window_data: bool) -> dict | None: ] title_parts = [part for part in title_parts if part] title = " ".join(title_parts) - window_id = meta["kCGWindowNumber"] + handle = meta["kCGWindowNumber"] bounds = meta["kCGWindowBounds"] left = bounds["X"] top = bounds["Y"] @@ -45,7 +45,7 @@ def get_active_window_state(read_window_data: bool) -> dict | None: "top": top, "width": width, "height": height, - "window_id": window_id, + "handle": handle, "meta": meta, "data": data, } diff --git a/openadapt/window/_windows.py b/openadapt/window/_windows.py index 9a49d6b7d..d12e6d7a1 100644 --- a/openadapt/window/_windows.py +++ b/openadapt/window/_windows.py @@ -2,12 +2,12 @@ import pickle import time +from loguru import logger import pywinauto +import pygetwindow as gw -from openadapt.custom_logger import logger - -def get_active_window_state(read_window_data: bool) -> dict: +def get_active_window_state(read_a11y_data: bool) -> dict: """Get the state of the active window. Returns: @@ -20,35 +20,47 @@ def get_active_window_state(read_window_data: bool) -> dict: - "height": Height of the active window. - "meta": Meta information of the active window. - "data": None (to be filled with window data). - - "window_id": ID of the active window. + - "handle": ID of the active window. """ # catch specific exceptions, when except happens do log.warning - try: - active_window = get_active_window() - except RuntimeError as e: - logger.warning(e) - return {} - meta = get_active_window_meta(active_window) - rectangle_dict = dictify_rect(meta["rectangle"]) - if read_window_data: + if read_a11y_data: + try: + active_window, handle = get_active_window() + except RuntimeError as e: + logger.warning(e) + return {} + meta = get_active_window_meta(active_window) + rectangle_dict = dictify_rect(meta["rectangle"]) data = get_element_properties(active_window) + state = { + "title": meta["texts"][0], + "left": meta["rectangle"].left, + "top": meta["rectangle"].top, + "width": meta["rectangle"].width(), + "height": meta["rectangle"].height(), + "meta": {**meta, "rectangle": rectangle_dict}, + "data": data, + "handle": handle, + } + try: + pickle.dumps(state) + except Exception as exc: + logger.warning(f"{exc=}") + state.pop("data") else: - data = {} - state = { - "title": meta["texts"][0], - "left": meta["rectangle"].left, - "top": meta["rectangle"].top, - "width": meta["rectangle"].width(), - "height": meta["rectangle"].height(), - "meta": {**meta, "rectangle": rectangle_dict}, - "data": data, - "window_id": meta["control_id"], - } - try: - pickle.dumps(state) - except Exception as exc: - logger.warning(f"{exc=}") - state.pop("data") + try: + active_window = gw.getActiveWindow() + except RuntimeError as e: + logger.warning(e) + return {} + state = { + "title": active_window.title if active_window.title else "None", + "left": active_window.left, + "top": active_window.top, + "width": active_window.width, + "height": active_window.height, + "handle": active_window._hWnd, + } return state @@ -96,7 +108,7 @@ def get_active_window() -> pywinauto.application.WindowSpecification: """ app = pywinauto.application.Application(backend="uia").connect(active_only=True) window = app.top_window() - return window.wrapper_object() + return [window.wrapper_object(), window.handle] def get_element_properties(element: pywinauto.application.WindowSpecification) -> dict: