{ "test_case_id": "A11Y-USER-003", "title": "Task-based evaluation with personal assistive technology", "category": "user_evaluation", "procedure_version": "2026.08.27-rc.9", "rule_type": "usability_evaluation", "wcag_criteria": [], "level": "advisory", "risk": "medium", "risk_basis": "Identify accessibility and usability barriers that standards checks or laboratory configurations may miss. People most likely affected: the disabled participant groups represented by the study; findings must not be generalised to all disabled people. Important barriers, effort, workarounds and recovery problems may remain invisible if assessment uses only standards checks and synthetic environments. In the recorded accessibility-user-research environment, protect participant consent, support needs and privacy while observing the complete task; an inaccessible study method can harm participants and produce findings that misrepresent their experience.", "review_status": "draft", "reviewed_at": "", "review_note": "Template content is subject to a hash-bound internal consistency review. Formal assessment claims require competent execution, current evidence and the applicable legal, participant and organisational authority.", "method_label": "User evaluation", "tester_level": "Accessibility researcher", "estimated_duration": "Typically 315–457 minutes for the stated single-target assumptions; use the component breakdown when estimating a real sample.", "execution_class": "manual", "automation_support": "none", "simulation_policy": "simulation_may_support_but_not_replace_manual_execution", "plain_summary": "Observe disabled participants completing realistic tasks with their personal assistive technology under an approved, privacy-protective study protocol.", "objective": "Identify accessibility and usability barriers that standards checks or laboratory configurations may miss.", "test_subject": "The selected product tasks, participant groups and personal assistive-technology environments defined in an approved study protocol.", "test_targets": [ "Each study task and observed participant interaction, including assistance, interruption, recovery and completion." ], "applicability": "Use when disabled-user evaluation is ethically approved and useful for the study objective. Combine findings with standards evaluation; do not use the study alone to claim WCAG conformance.", "applicability_questions": [ "Is there a specific research objective that requires disabled-user evidence?", "Are participant characteristics relevant to the target audience and study question?", "Are recruitment, informed consent, accessibility accommodations, privacy, safeguarding, retention and withdrawal procedures approved?" ], "inapplicability_conditions": [ "Disabled-user evaluation is not part of the authorised engagement or no ethically approved study protocol exists. This does not mean user barriers are absent." ], "assumptions": [ "The selected scope identifies the product version, technologies, states and important user journeys being evaluated.", "The tester knows how to use this method and any required assistive technology.", "Known browser or assistive-technology limitations are recorded instead of being assigned to the product without evidence." ], "accessibility_support_assumptions": [ "The result covers only the browsers, platforms, readers and assistive technologies that were actually tested.", "Do not apply a result from one technology combination to an untested combination or to every user." ], "prerequisites": [ "The product area and test targets are agreed before testing starts.", "The operating system, browser, viewport and input method are recorded.", "Test data is safe to use and does not expose unnecessary personal information." ], "required_environment_fields": [ "product_version", "research_protocol", "research_protocol_version", "task_script_version", "session_mode", "research_platform", "research_platform_version", "participant_chosen_technology", "participant_chosen_technology_version", "consent_record_reference" ], "steps": [ { "step_id": "S1", "title": "Freeze the ethical research protocol", "phase": "prepare", "scenario_purpose": "Define a safe and consistent study before contacting participants.", "user_impact": "Poor governance can harm participants and invalidate otherwise useful observations.", "test_target": "One study protocol covering objectives, participants, tasks, data and safeguarding.", "required_state": "Draft protocol before recruitment or collection of participant information.", "tool_or_command": "Organisation's authorised research, privacy and safeguarding workflow.", "action": "Record objective, recruitment characteristics, consent, accommodations, tasks, assistance threshold, stop conditions, data map, retention, deletion, withdrawal and safeguarding; obtain the approvals required by the responsible organisation.", "expected": "An authorised, versioned protocol is available before participant contact.", "failure_observation": "Required approval, consent route, safeguarding or data handling is absent or unresolved.", "evidence_to_record": "Protocol/version, approval references, data map and owner roles; no participant data.", "decision_rules": { "passed": "All required organisational approvals are recorded.", "failed": "Execution is blocked; no participant data is collected.", "not_applicable": "The engagement explicitly excludes participant research.", "blocked": "Approval authority or privacy decision is unavailable." }, "decision_effect": "Passed: All required organisational approvals are recorded. Failed: Execution is blocked; no participant data is collected.", "recovery_or_repeat": "Escalate to the responsible organisation; agent review cannot replace authority or consent.", "novice_guidance": "Escalate to the responsible organisation; agent review cannot replace authority or consent.", "limitations_and_claim_boundary": "This repository supplies operational guidance, not ethical, legal or organisational approval." }, { "step_id": "S2", "title": "Recruit against declared characteristics", "phase": "prepare", "scenario_purpose": "Assemble a bounded sample relevant to the study objective without claiming broad representation.", "user_impact": "Convenience recruitment can exclude the people most affected and distort priorities.", "test_target": "The approved participant-characteristic matrix.", "required_state": "Protocol approved; accessible recruitment and screening materials available.", "tool_or_command": "Accessible recruitment channels and consented screening process.", "action": "Screen one candidate at a time against the declared characteristics, support needs and exclusion criteria. Record only necessary data and explain how each confirmed participant contributes to the study scope.", "expected": "The recruited sample matches the protocol and limitations are explicit.", "failure_observation": "Participants are recruited outside the criteria, necessary accommodations are unavailable or claims exceed the sample.", "evidence_to_record": "Pseudonymous participant IDs, characteristic coverage, accommodation plan and sampling limitations.", "decision_rules": { "passed": "The planned sample is achieved without unsupported representativeness claims.", "failed": "A mismatch or exclusion undermines the planned coverage and is resolved or reported.", "not_applicable": "Participant research is outside scope.", "blocked": "Suitable participants or accommodations are unavailable." }, "decision_effect": "Passed: The planned sample is achieved without unsupported representativeness claims. Failed: A mismatch or exclusion undermines the planned coverage and is resolved or reported.", "recovery_or_repeat": "Adjust timing or the declared sample with authorised approval; never fabricate a participant or characteristic.", "novice_guidance": "Adjust timing or the declared sample with authorised approval; never fabricate a participant or characteristic.", "limitations_and_claim_boundary": "Findings apply to the participants and tasks studied, not all disabled people." }, { "step_id": "S3", "title": "Confirm accessible consent and session readiness", "phase": "prepare", "scenario_purpose": "Ensure the participant understands the session and can take part with needed accommodations.", "user_impact": "Inaccessible consent or setup creates avoidable barriers and invalidates voluntary participation.", "test_target": "One participant's consent and session setup.", "required_state": "Approved materials supplied in the requested format; recording off until consent is confirmed.", "tool_or_command": "Accessible consent materials and participant's normal assistive technology where safe.", "action": "Explain purpose, tasks, recording, data use, withdrawal and payment. Confirm understanding, consent choices, accommodations, communication preferences and permission for quotations separately.", "expected": "Consent is informed and voluntary and the session setup meets agreed access needs.", "failure_observation": "The participant cannot access or understand the materials, consent is assumed/bundled or required accommodation is missing.", "evidence_to_record": "Consent reference and accommodation checklist without unnecessary sensitive detail.", "decision_rules": { "passed": "Consent and readiness are confirmed before observation.", "failed": "Pause; no research data is collected until the issue is resolved.", "not_applicable": "No participant session is planned.", "blocked": "Consent capacity, accessible materials or accommodation cannot be resolved." }, "decision_effect": "Passed: Consent and readiness are confirmed before observation. Failed: Pause; no research data is collected until the issue is resolved.", "recovery_or_repeat": "Offer another format, supporter or rescheduled session within the approved protocol; respect withdrawal immediately.", "novice_guidance": "Offer another format, supporter or rescheduled session within the approved protocol; respect withdrawal immediately.", "limitations_and_claim_boundary": "Consent to participate does not automatically permit publication of identifiable quotations or recordings." }, { "step_id": "S4", "title": "Run one neutral task attempt", "phase": "test", "scenario_purpose": "Observe how the participant attempts a realistic task using their own strategies.", "user_impact": "Coaching can conceal barriers and turn evaluation into training.", "test_target": "One approved task and its defined completion state.", "required_state": "Participant at the recorded start point; their usual device/AT configured; facilitator prompt ready.", "tool_or_command": "Approved task script, observation sheet and participant's personal technology.", "action": "Read the task prompt once. Observe without directing actions. Record behaviour, comments, time, errors and assistance separately. Intervene only at the approved threshold or safety stop.", "expected": "The task attempt and assistance level are captured without avoidable facilitator influence.", "failure_observation": "The facilitator leads the participant, changes the task, fails to record assistance or continues past a safety/withdrawal condition.", "evidence_to_record": "Task ID, start/end state, action notes, participant statements, assistance events and outcome.", "decision_rules": { "passed": "The approved task was attempted under the protocol and evidence is separable by source.", "failed": "The observation is invalid for analysis; rerun only with participant consent and protocol approval.", "not_applicable": "The task is not relevant to this participant under the approved protocol.", "blocked": "Product access, safety or participant withdrawal prevents completion." }, "decision_effect": "Passed: The approved task was attempted under the protocol and evidence is separable by source. Failed: The observation is invalid for analysis; rerun only with participant consent and protocol approval.", "recovery_or_repeat": "Respect withdrawal; never pressure continuation. Preserve only data the consent terms allow.", "novice_guidance": "Respect withdrawal; never pressure continuation. Preserve only data the consent terms allow.", "limitations_and_claim_boundary": "A blocked or incomplete task can still reveal a barrier but cannot be presented as a completed success measure." }, { "step_id": "S5", "title": "Explore one error and recovery path", "phase": "test", "scenario_purpose": "Understand whether the participant can recognise, recover from and explain a realistic error.", "user_impact": "Recovery barriers often cause abandonment even when the primary path appears usable.", "test_target": "One approved validation, interruption or recovery state.", "required_state": "State can be produced safely; task attempt has reached the agreed trigger.", "tool_or_command": "Approved scenario prompt and observation sheet.", "action": "Trigger or allow the planned state without deception beyond the approved protocol. Observe recognition, interpretation, workaround, assistance and recovery; stop at the assistance or safety threshold.", "expected": "The participant's experience of the state and recovery is recorded without coaching.", "failure_observation": "The state cannot be produced safely, facilitator interference changes the response or recovery evidence is conflated with the primary task.", "evidence_to_record": "State ID, trigger, participant actions/comments, assistance, recovery outcome and time.", "decision_rules": { "passed": "The scenario was observed consistently and its evidence is traceable.", "failed": "Protocol deviation makes the evidence unreliable and it is excluded or rerun with consent.", "not_applicable": "No relevant recovery state is in the approved task.", "blocked": "Safety, participant choice or environment prevents the scenario." }, "decision_effect": "Passed: The scenario was observed consistently and its evidence is traceable. Failed: Protocol deviation makes the evidence unreliable and it is excluded or rerun with consent.", "recovery_or_repeat": "Skip or stop immediately when required; record the limitation, not a participant failure.", "novice_guidance": "Skip or stop immediately when required; record the limitation, not a participant failure.", "limitations_and_claim_boundary": "One participant's recovery strategy does not establish general usability." }, { "step_id": "S6", "title": "Validate the interpretation with the participant", "phase": "test", "scenario_purpose": "Separate observed behaviour, participant report and facilitator inference.", "user_impact": "Unvalidated interpretation can misstate both barriers and participant intent.", "test_target": "One potentially material observation from S4 or S5.", "required_state": "Task attempt complete; participant willing to discuss the event.", "tool_or_command": "Open, non-leading follow-up prompts from the approved protocol.", "action": "Describe the observed event neutrally. Ask what the participant expected, what made it difficult, impact, workaround and whether the summary is accurate. Confirm quotation permission separately.", "expected": "The record clearly separates observation, participant explanation and facilitator interpretation.", "failure_observation": "Questions lead the answer, interpretation is presented as the participant's view or quotation permission is assumed.", "evidence_to_record": "Verbatim or paraphrased response, validation status and quotation permission.", "decision_rules": { "passed": "The participant confirms or corrects the bounded interpretation.", "failed": "Unresolved interpretation remains labelled as facilitator inference rather than fact.", "not_applicable": "No potentially material observation was recorded.", "blocked": "The participant declines discussion or withdraws permission." }, "decision_effect": "Passed: The participant confirms or corrects the bounded interpretation. Failed: Unresolved interpretation remains labelled as facilitator inference rather than fact.", "recovery_or_repeat": "Respect the decision; retain only the original observation when consent permits and mark interpretation limited.", "novice_guidance": "Respect the decision; retain only the original observation when consent permits and mark interpretation limited.", "limitations_and_claim_boundary": "Participant validation strengthens accuracy but is not a standards conformance decision." }, { "step_id": "S7", "title": "Synthesize findings within the sample boundary", "phase": "record", "scenario_purpose": "Turn session evidence into actionable themes without overgeneralising.", "user_impact": "Overstated synthesis can misdirect remediation and erase individual differences.", "test_target": "All consented task and observation records in the study.", "required_state": "Sessions complete or explicitly blocked; data minimised and pseudonymised.", "tool_or_command": "Qualitative synthesis matrix with links to participant/task evidence.", "action": "Group observations only when their mechanism and context are comparable. Record frequency as counts, not population prevalence. Separate usability, suspected standards, platform and unresolved responsibility findings.", "expected": "Every theme links to evidence and states participant/task coverage and limitations.", "failure_observation": "Themes combine unrelated mechanisms, imply prevalence beyond the sample or convert participant evidence directly into WCAG failure.", "evidence_to_record": "Theme matrix, linked participant/task IDs, contrary cases and limitation statement.", "decision_rules": { "passed": "Synthesis is traceable and bounded.", "failed": "Misclassified or unsupported themes are corrected before reporting.", "not_applicable": "No participant evidence was lawfully collected.", "blocked": "Missing consent or evidence prevents synthesis." }, "decision_effect": "Passed: Synthesis is traceable and bounded. Failed: Misclassified or unsupported themes are corrected before reporting.", "recovery_or_repeat": "Return uncertain suspected WCAG issues to criterion-specific testing and record unresolved responsibility explicitly.", "novice_guidance": "Return uncertain suspected WCAG issues to criterion-specific testing and record unresolved responsibility explicitly.", "limitations_and_claim_boundary": "Research supports design decisions and identifies barriers; it does not establish WCAG conformance or legal compliance." }, { "step_id": "S8", "title": "Review, report and dispose of research data", "phase": "record", "scenario_purpose": "Complete quality, privacy and retention actions before client delivery.", "user_impact": "Unreviewed or over-retained data creates participant and organisational risk.", "test_target": "The synthesis, report extracts and retained source data.", "required_state": "Draft findings complete; consent permissions and retention dates available.", "tool_or_command": "Independent research/privacy review workflow and secure evidence register.", "action": "Review factual accuracy, consent scope, quotation permissions, re-identification risk and limitations. Export only authorised redacted evidence, record retention owners/dates and confirm deletion when due.", "expected": "Client wording is accurate and authorised data handling is traceable.", "failure_observation": "A report exposes unnecessary personal data, exceeds consent, omits limitations or lacks a retention/deletion decision.", "evidence_to_record": "Review comments/resolution, redacted deliverables, retention record and deletion confirmation when due.", "decision_rules": { "passed": "All review and data-handling requirements are satisfied.", "failed": "Correct or withhold affected material before delivery.", "not_applicable": "No participant data was collected; retain the blocked protocol record only.", "blocked": "Named review authority or deletion system is unavailable." }, "decision_effect": "Passed: All review and data-handling requirements are satisfied. Failed: Correct or withhold affected material before delivery.", "recovery_or_repeat": "Block external delivery of affected evidence while completing all non-sensitive report preparation.", "novice_guidance": "Block external delivery of affected evidence while completing all non-sensitive report preparation.", "limitations_and_claim_boundary": "Named organisational approval and real participant involvement cannot be simulated by an agent." } ], "expectations": [ { "expectation_id": "E1", "statement": "Study objective, participant characteristics, recruitment/consent, tasks, environments, facilitation, data and privacy controls are approved and recorded.", "outcome_if_unmet": "untested" }, { "expectation_id": "E2", "statement": "Task observations, participant report and facilitator inference remain distinguishable and traceable.", "outcome_if_unmet": "cantTell" }, { "expectation_id": "E3", "statement": "Reporting does not generalise beyond the sample or claim WCAG conformance from user evaluation alone.", "outcome_if_unmet": "failed" } ], "exceptions": [ "Use only an exception stated in the relevant standard, or an agreed out-of-scope decision. Record the reason and evidence.", "A tool limitation, lack of time or unavailable assistive technology is a limitation or incomplete result, not a pass." ], "outcome_definitions": { "passed": "The applicable test target meets every expectation in this rule or procedure.", "failed": "The applicable test target does not meet at least one expectation in this rule or procedure.", "inapplicable": "The test subject contains no target that matches this rule or procedure's applicability.", "cantTell": "The tester cannot fully determine applicability or whether every expectation is met.", "untested": "The test subject or target was not evaluated using this rule or procedure." }, "examples": [ { "example_id": "A11Y-USER-003-EX-PASSED", "outcome": "passed", "scenario": "Representative calibration example, not client evidence. Procedure: Task-based evaluation with personal assistive technology. Target: One study protocol covering objectives, participants, tasks, data and safeguarding. Action: Record objective, recruitment characteristics, consent, accommodations, tasks, assistance threshold, stop conditions, data map, retention, deletion, withdrawal and safeguarding; obtain the approvals required by the responsible organisation. Evidence: Protocol/version, approval references, data map and owner roles; no participant data. Observed result: An authorised, versioned protocol is available before participant contact." }, { "example_id": "A11Y-USER-003-EX-FAILED", "outcome": "failed", "scenario": "Representative calibration example, not client evidence. Procedure: Task-based evaluation with personal assistive technology. Target: One study protocol covering objectives, participants, tasks, data and safeguarding. The tester completes the recorded action, but the captured evidence does not demonstrate: An authorised, versioned protocol is available before participant contact. The failed target and state remain separate from unexecuted targets." }, { "example_id": "A11Y-USER-003-EX-INAPPLICABLE", "outcome": "inapplicable", "scenario": "Representative scope example, not client evidence. No authorised disabled-user study exists in this engagement. This outcome says nothing about product accessibility. The completed inventory is retained as the basis for the inapplicable outcome." }, { "example_id": "A11Y-USER-003-EX-CANTTELL", "outcome": "cantTell", "scenario": "Representative unresolved example, not client evidence. The observed behaviour, participant meaning or product/platform responsibility cannot be resolved within the approved evidence. The record identifies the conflicting evidence and the expertise or comparison needed to resolve it." }, { "example_id": "A11Y-USER-003-EX-UNTESTED", "outcome": "untested", "scenario": "Representative execution-gap example, not client evidence. The target is applicable, but a required state, environment, action or evidence item was not completed. The missing action, state or environment remains open and is not converted to a pass." } ], "sampling_rule": "Recruit participants and select tasks from the research plan. Record who and what the sample represents, without treating it as universal coverage.", "complete_process_requirement": "If this check is part of a user journey, test the selected journey from start to finish, including relevant errors and recovery. Record any path that was not tested.", "pass_criteria": "The authorised study protocol is executed as written; each selected task has traceable, consented observations; participants can complete and recover without a material study-defined barrier; and reporting remains limited to the represented participants, tasks and environments.", "fail_criteria": "A material barrier prevents or substantially impairs a selected task under the study's decision rules, or required ethical, consent or privacy controls are violated.", "evidence_requirements": [ "Objective, participant characteristics, recruitment/consent, tasks, environment, facilitation, data, privacy, retention and approvals.", "Coded relevant characteristics, personal assistive technology/environment and accommodations.", "Task actions, completion, assistance, interruption/recovery, impact, workarounds and consented evidence reference.", "Separation of standards findings, usability observations, platform limitations and unresolved questions.", "Consent restrictions, reviewer findings, retention/deletion dates and authorised outputs." ], "official_source_ids": [ "W3C-INVOLVE-USERS", "GOVUK-AT-TESTING" ], "act_rule_mapping": { "format_alignment": "This is a project-authored manual procedure, not a W3C Approved ACT Rule.", "approved_rule_ids": [], "approved_rules": [], "mapping_status": "No supporting ACT rule selected", "registry_reviewed_at": "2026-08-27", "registry_source_id": "W3C-ACT-RULES", "mapping_rationale": "An ACT result can support only the part of this procedure covered by that rule. It does not replace the remaining manual or assistive-technology checks." }, "technique_traceability": [], "platform_ids": [ "accessibility-user-research" ], "review_requirement": "A qualified accessibility reviewer must confirm the scope, evidence and result before it is reported as final.", "known_issues": [ "Check relevant browser, tool or assistive-technology release notes before assigning unexpected behaviour to the product.", "If responsibility is unclear, record the reproducible behaviour and use cannot tell until it is resolved." ], "change_log": [ { "version": "2026.08.27-rc.9", "change": "Reworked for plain language, case-focused steps, scoped evidence and transparent draft review status." }, { "version": "2026.08.27-rc.9", "change": "Case-specific specialist content, evidence and platform boundaries completed for hash-bound agent verification." }, { "version": "2026.08.27-rc.9", "change": "Individually authored reference-procedure content prepared for competent review and execution." }, { "version": "2026.08.27-rc.9", "change": "Removed administrative filler steps, added contextual risk and result boundaries, and prepared the concise worksheet flow for internal consistency review." } ], "tags": [ "disabled-user-research", "ethical-research", "usability", "user_evaluation" ], "status": "draft", "authoring_readiness": "content_complete", "short_title": "Evaluate a task with disabled participants", "catalogue_version": "2026.08.27-rc.9", "identity": { "stable_case_id": "A11Y-USER-003", "procedure_title": "Task-based evaluation with personal assistive technology", "short_user_centred_title": "Evaluate a task with disabled participants", "procedure_version": "2026.08.27-rc.9", "catalogue_version": "2026.08.27-rc.9", "status": "draft", "category": "user_evaluation", "procedure_type": "usability_evaluation", "execution_class": "manual", "intended_tester_expertise": "Accessibility researcher", "effort_model": { "model": "component_effort_v2", "small_scope_minutes": { "minimum": 315, "maximum": 457 }, "warning": "Add a new per-target execution and evidence allowance for each additional target class or state that could produce a different result. Add setup time for each distinct environment.", "calculation_guidance": "Add the recorded setup, per-target execution, evidence and technical-review components; then apply the stated variation rule.", "unit": "minutes", "setup": { "typical": 120, "includes": "Confirm scope and target, prepare safe data, establish the required state and record the tool or environment version." }, "per_target_execution": { "typical": 60, "includes": "Perform the procedure once for one target in its selected states; this excludes remediation and retesting." }, "evidence_capture": { "typical": 45, "includes": "Capture, name and cross-reference the minimum reproducible evidence while removing unnecessary personal or secret data." }, "technical_review": { "typical": 90, "includes": "Check applicability, observations, outcome logic, evidence sufficiency and report wording before issue publication." }, "typical_total_range": { "minimum": 315, "maximum": 457 }, "assumptions": "one 45–60 minute participant session after approved recruitment and consent; recruitment, accommodations and qualitative synthesis are additional work", "variation": "Add a new per-target execution and evidence allowance for each additional target class or state that could produce a different result. Add setup time for each distinct environment.", "excludes": "Client coordination, inaccessible-environment recovery, recruitment, legal review, remediation implementation and waiting time unless the procedure states otherwise." }, "related_procedures": [], "replaces": [], "replaced_by": [] }, "purpose_and_user_impact": { "barrier_detected": "Identify accessibility and usability barriers that standards checks or laboratory configurations may miss.", "people_likely_affected": [ "The disabled participant groups represented by the study; findings must not be generalised to all disabled people." ], "functionality_or_information_at_risk": "Important barriers, effort, workarounds and recovery problems may remain invisible if assessment uses only standards checks and synthetic environments.", "result_scope": "disabled_user_evaluation", "claim_boundary": "This procedure determines whether Evaluate a task with disabled participants meets its stated expectations for the recorded targets and accessibility-user-research. Unexecuted targets, states and environments remain outside that result.", "why_it_matters": "Important barriers, effort, workarounds and recovery problems may remain invisible if assessment uses only standards checks and synthetic environments. The procedure is necessary because identify accessibility and usability barriers that standards checks or laboratory configurations may miss and the result can otherwise conceal a material barrier for the affected people named here." }, "requirement_mappings": [ { "source_id": "W3C-INVOLVE-USERS", "requirement_id": "W3C-INVOLVE-USERS", "requirement_level": "informative", "requirement_type": "usability", "coverage_type": "supporting", "canonical_source": "https://www.w3.org/WAI/test-evaluate/involving-users/", "source_status": "living_guidance", "source_version_or_publication_date": "Current WAI guidance; checked 4 August 2026", "date_verified": "2026-08-27", "reviewer_notes": "Supporting source. Procedure-level content review remains recorded separately.", "limitations": "This source does not override WCAG or create an additional WCAG conformance requirement.", "dependent_fields": [ "prerequisites", "environment", "steps" ] }, { "source_id": "GOVUK-AT-TESTING", "requirement_id": "GOVUK-AT-TESTING", "requirement_level": "informative", "requirement_type": "usability", "coverage_type": "supporting", "canonical_source": "https://www.gov.uk/service-manual/technology/testing-with-assistive-technologies", "source_status": "government_guidance", "source_version_or_publication_date": "Current first-party publication; exact version pending source review", "date_verified": "2026-08-27", "reviewer_notes": "Supporting source. Procedure-level content review remains recorded separately.", "limitations": "This source does not override WCAG or create an additional WCAG conformance requirement.", "dependent_fields": [ "prerequisites", "environment", "steps" ] } ], "applicability_model": { "test_subject": "The selected product tasks, participant groups and personal assistive-technology environments defined in an approved study protocol.", "test_targets": [ "Each study task and observed participant interaction, including assistance, interruption, recovery and completion." ], "objective_questions": [ "Is there a specific research objective that requires disabled-user evidence?", "Are participant characteristics relevant to the target audience and study question?", "Are recruitment, informed consent, accessibility accommodations, privacy, safeguarding, retention and withdrawal procedures approved?" ], "subjective_judgements": [ "Interpret observed barriers with the participant and study context; do not infer prevalence or causation beyond the evidence." ], "inapplicability_conditions": [ "Disabled-user evaluation is not part of the authorised engagement or no ethically approved study protocol exists. This does not mean user barriers are absent." ], "scope_exclusions": [ "Disability groups, technologies, tasks and product states not represented in the study.", "WCAG conformance decisions, which require separate standards evaluation." ], "required_states_and_variations": [ "Realistic success, validation error, interruption, recovery and confirmation tasks selected by the protocol.", "Each participant's normal device, assistive technology, preferences and strategies where safe and consented." ], "complete_process_dependencies": "Participants attempt complete realistic tasks unless the approved protocol defines an ethical or safety stop condition.", "conforming_alternate_version_considerations": "A conforming alternate version is a WCAG conformance concept and does not erase usability observations. Record which version the participant used." }, "normative_exceptions": [], "normative_exceptions_absent_reason": "This is an informative usability-evaluation method, not a normative conformance rule. Protocol exclusions and ethical stop conditions are study design controls, not normative exceptions.", "environment": { "required_tester_expertise": "Accessibility researcher", "required_platforms": [ "accessibility-user-research" ], "minimum_environment_fields": [ "product_version", "research_protocol", "research_protocol_version", "task_script_version", "session_mode", "research_platform", "research_platform_version", "participant_chosen_technology", "participant_chosen_technology_version", "consent_record_reference" ], "product_and_content_version_required": true, "viewport_and_orientation_required": true, "input_mode_required": true, "relevant_settings": [ "Study location/tool", "Participant's chosen technology", "Facilitation method", "Communication and access accommodations", "Recording choice" ], "known_platform_limitations": [ "A small study cannot be generalised to all disabled people and cannot establish WCAG conformance by itself.", "Check relevant browser, tool or assistive-technology release notes before assigning unexpected behaviour to the product.", "If responsibility is unclear, record the reproducible behaviour and use cannot tell until it is resolved." ], "clean_state_requirements": [ "Complete a known-good calibration in the same environment before product testing.", "Start from a recorded clean or known state and preserve any prior preferences, extensions, cache or profile settings that can affect output." ], "test_data_and_privacy_constraints": [ "Use accessible informed-consent materials and collect only data necessary for the study objective.", "Use participant codes in working records; store identity/contact data separately with restricted access.", "Obtain explicit consent for screen, audio, video, quotations and assistive-technology logs; refusal must not prevent participation where alternative recording is possible.", "Document retention, deletion, withdrawal and breach-response procedures before recruitment." ], "coverage_boundary": "A small study cannot be generalised to all disabled people and cannot establish WCAG conformance by itself." }, "decision_logic": { "target_passed": "The authorised study protocol is executed as written; each selected task has traceable, consented observations; participants can complete and recover without a material study-defined barrier; and reporting remains limited to the represented participants, tasks and environments.", "target_failed": "A material barrier prevents or substantially impairs a selected task under the study's decision rules, or required ethical, consent or privacy controls are violated.", "target_inapplicable": "No authorised disabled-user study exists in this engagement. This outcome says nothing about product accessibility.", "target_cant_tell": "The observed behaviour, participant meaning or product/platform responsibility cannot be resolved within the approved evidence.", "target_untested": "The target is applicable, but a required state, environment, action or evidence item was not completed.", "aggregation": "Preserve participant-level task outcomes and source observations. Aggregate only using the protocol's declared method and report the number and characteristics of participants represented.", "wcag_requirement_decision": "User evaluation alone never establishes WCAG satisfaction or non-satisfaction. Suspected standards failures require separate criterion-specific assessment.", "conforming_alternate_version_effect": "Apply a conforming alternate version only after its WCAG definition and every conformance condition have been verified and recorded; otherwise it does not change the decision.", "passed_automation_follow_up": "Automated or simulated users cannot replace disabled participants for this method.", "platform_responsibility": "When responsibility is unclear, preserve the reproducible behaviour and environment evidence and use cantTell until a qualified reviewer resolves it.", "claim_readiness": "A complete result may become ready_for_review. Schema validity, a passed target or a tool result cannot make it ready_for_scoped_claim without the independent evidence and governance gates." }, "evidence_specification": [ { "evidence_id": "EV1", "method": "study_protocol", "record": "Objective, participant characteristics, recruitment/consent, tasks, environment, facilitation, data, privacy, retention and approvals.", "review_purpose": "Confirm the study was governed and reproducible.", "privacy_and_security": "Use a redacted extract or evidence reference. Do not include credentials, secrets or unnecessary personal data.", "required_for_formal_result": true }, { "evidence_id": "EV2", "method": "participant_context", "record": "Coded relevant characteristics, personal assistive technology/environment and accommodations.", "review_purpose": "Bound interpretation without exposing identity.", "privacy_and_security": "Use a redacted extract or evidence reference. Do not include credentials, secrets or unnecessary personal data.", "required_for_formal_result": true }, { "evidence_id": "EV3", "method": "task_observation", "record": "Task actions, completion, assistance, interruption/recovery, impact, workarounds and consented evidence reference.", "review_purpose": "Trace conclusions to observed behaviour.", "privacy_and_security": "Use a redacted extract or evidence reference. Do not include credentials, secrets or unnecessary personal data.", "required_for_formal_result": true }, { "evidence_id": "EV4", "method": "analysis_record", "record": "Separation of standards findings, usability observations, platform limitations and unresolved questions.", "review_purpose": "Prevent conformance/usability conflation.", "privacy_and_security": "Use a redacted extract or evidence reference. Do not include credentials, secrets or unnecessary personal data.", "required_for_formal_result": true }, { "evidence_id": "EV5", "method": "privacy_and_review_record", "record": "Consent restrictions, reviewer findings, retention/deletion dates and authorised outputs.", "review_purpose": "Review privacy and governance compliance.", "privacy_and_security": "Use a redacted extract or evidence reference. Do not include credentials, secrets or unnecessary personal data.", "required_for_formal_result": true } ], "calibration_examples": [ { "example_id": "A11Y-USER-003-EX-PASSED", "outcome": "passed", "scenario": "Under the approved protocol, each participant completes the selected task with their normal assistive technology; observations and limitations are reported only for the represented sample.", "explanation": "The study-defined task outcome passes without becoming a WCAG claim.", "fixture_id": "", "fixture_version": "", "machine_testable": false }, { "example_id": "A11Y-USER-003-EX-FAILED", "outcome": "failed", "scenario": "Several participants cannot recover from a validation error within the declared assistance threshold, and the source observations support a material task barrier.", "explanation": "The study-defined task fails; a separate technical test is linked for any standards claim.", "fixture_id": "", "fixture_version": "", "machine_testable": false }, { "example_id": "A11Y-USER-003-EX-INAPPLICABLE", "outcome": "inapplicable", "scenario": "The engagement authorises standards testing only and includes no disabled-user study.", "explanation": "The method is not part of scope; this is not evidence that users face no barriers.", "fixture_id": "", "fixture_version": "", "machine_testable": false }, { "example_id": "A11Y-USER-003-EX-CANTTELL", "outcome": "cantTell", "scenario": "A participant abandons a task but declines follow-up, leaving the cause and impact ambiguous.", "explanation": "Preserve the observation and uncertainty without inventing a cause.", "fixture_id": "", "fixture_version": "", "machine_testable": false }, { "example_id": "A11Y-USER-003-EX-UNTESTED", "outcome": "untested", "scenario": "Participants were recruited, but an inaccessible consent document blocks lawful session execution.", "explanation": "The execution is blocked and task outcomes remain untested.", "fixture_id": "", "fixture_version": "", "machine_testable": false } ], "review_record": { "author": "", "method_reviewer": "", "independent_standards_reviewer": "", "accessibility_ui_reviewer": "", "author_date": "", "method_review_date": "", "standards_review_date": "", "ui_review_date": "", "source_versions_reviewed": [], "exceptions_reviewed": false, "known_limitations": [ "This is a source-checked reference implementation subject to independent method and standards agent verification; formal approval remains separate." ], "open_issues": [ "No participants have been recruited and no user-study execution, consent, data or review is claimed.", "A qualified user researcher, privacy reviewer and disabled research adviser must review the protocol before execution." ], "decision": "pending", "next_review_date": "", "status_transition_reason": "", "release_gate_id": "", "release_gate_completed_at": "", "approval_authority": "", "approval_date": "" }, "privacy_and_security": { "browser_storage": "Worksheet data is stored only in this browser's local storage until the user exports or clears it.", "network_transmission": "The static library does not upload worksheet data.", "data_minimisation": "Record only evidence necessary to reproduce and review the decision.", "redaction": "Remove credentials, secrets and unnecessary personal or participant data before export or sharing.", "retention_and_deletion": "Apply the engagement retention schedule and clear local browser data when it is no longer required." }, "version_history": [ { "version": "2026.08.27-rc.9", "change": "Reworked for plain language, case-focused steps, scoped evidence and transparent draft review status." }, { "version": "2026.08.27-rc.9", "change": "Migrated to professional catalogue schema v3 with separated outcomes, structured authority, applicability, evidence, decision and review records." }, { "version": "2026.08.27-rc.9", "change": "Specialist procedure mapped to its task family and current first-party platform guidance and checked for internal consistency; competent review and real execution remain separate." }, { "version": "2026.08.27-rc.9", "change": "Replaced migrated boilerplate with a case-specific reference implementation and completed independent agent content verification." }, { "version": "2026.08.27-rc.9", "change": "Rewritten as an atomic professional-delivery flagship with component effort guidance and a complete representative worked example." }, { "version": "2026.08.27-rc.9", "change": "Procedure guidance and presentation simplified without changing draft governance status." } ], "migration": { "from_schema": "accessibility-test-catalog.v2", "from_procedure_version": "2026.08.04.1", "migrated_at": "2026-08-27", "content_review_state": "flagship_operational_rewrite_agent_verified", "stable_identifier_preserved": true, "legacy_url_preserved": true, "delivery_model": "professional_delivery_v2", "agent_content_validation": { "status": "complete", "method": "case_specific_semantic_and_operational_checks_v1", "checked_fields": [ "target", "state", "tool", "action", "expected_observation", "failure_observation", "evidence", "outcome_rules", "claim_boundary", "compact_examples" ], "human_approval_effect": "none" } }, "test_type": "usability", "related_procedures": [], "platform_validation": { "required": false, "evidence_status": "not_executed", "real_device_or_at_claimed": false, "required_environment_ids": [ "accessibility-user-research" ], "calibration_required": true, "evidence_refs": [], "limitations": "A small study cannot be generalised to all disabled people and cannot establish WCAG conformance by itself." }, "library_layer": "journey", "effort_guidance": { "unit": "minutes", "setup": { "typical": 120, "includes": "Confirm scope and target, prepare safe data, establish the required state and record the tool or environment version." }, "per_target_execution": { "typical": 60, "includes": "Perform the procedure once for one target in its selected states; this excludes remediation and retesting." }, "evidence_capture": { "typical": 45, "includes": "Capture, name and cross-reference the minimum reproducible evidence while removing unnecessary personal or secret data." }, "technical_review": { "typical": 90, "includes": "Check applicability, observations, outcome logic, evidence sufficiency and report wording before issue publication." }, "typical_total_range": { "minimum": 315, "maximum": 457 }, "assumptions": "one 45–60 minute participant session after approved recruitment and consent; recruitment, accommodations and qualitative synthesis are additional work", "variation": "Add a new per-target execution and evidence allowance for each additional target class or state that could produce a different result. Add setup time for each distinct environment.", "excludes": "Client coordination, inaccessible-environment recovery, recruitment, legal review, remediation implementation and waiting time unless the procedure states otherwise." }, "worked_examples": [ { "example_id": "A11Y-USER-003-WORKED-001", "example_kind": "representative_synthetic_example", "context": "Illustrative research record for a Civic Benefits Portal address-change journey; no participant session occurred.", "inspected": "The prepared task script, consent flow, observation categories, assistance threshold, synthesis and reporting boundary.", "observed_output": "A representative observation describes a participant becoming unsure after an address lookup clears the street field, requesting help at the declared threshold and explaining that the change was unexpected.", "evidence_captured": [ "EVD-REF-017 synthetic observation sheet", "EVD-REF-018 representative participant-statement field", "EVD-REF-019 synthesis trace" ], "tester_reasoning": "The example shows how to separate behaviour, reported impact and facilitator inference. It cannot establish frequency or represent real-user evidence because no participant performed it.", "decision": "blocked", "completed_finding": { "finding_id": "FND-REF-005", "title": "Address lookup replacement is not explained", "severity": "medium", "user_impact": "A user may lose entered address detail and need assistance to recover.", "evidence_refs": [ "EVD-REF-017", "EVD-REF-018", "EVD-REF-019" ] }, "remediation_advice": "Announce before replacement what the lookup will change, preserve manually entered values where possible and provide an explicit undo or edit path.", "reviewer_feedback": "Reviewer replaced 'users will abandon' with the bounded statement 'the representative scenario may require assistance' and marked all participant-derived fields synthetic.", "final_client_report_wording": "Prepared research hypothesis: replacing manually entered address text without explanation may cause loss of context and assistance. Validate this with consented participants before presenting it as observed user evidence.", "retest_outcome": "RT-REF-006 cannot occur until an authorised participant study and a remediated product build exist; the pilot package is ready for scheduling.", "claim_boundary": "This is a representative, synthetic worked example for training and calibration. It is not evidence from a client product, participant or real assistive-technology execution." } ] }