{ "test_case_id": "A11Y-USER-005", "title": "User-observation evidence and issue triangulation", "category": "user_evaluation", "procedure_version": "2026.08.27-rc.9", "rule_type": "usability_evaluation", "wcag_criteria": [], "level": "advisory", "risk": "medium", "risk_basis": "Runs a governed disabled-user evaluation procedure for user-observation evidence and issue triangulation. People most likely affected: participants represented by the study and product teams responsible for acting on observed barriers. An inaccessible, unsafe or weakly governed study can harm participants or produce misleading claims. In the recorded accessibility-user-research environment, protect participant consent, support needs and privacy while observing the complete task; an inaccessible study method can harm participants and produce findings that misrepresent their experience.", "review_status": "draft", "reviewed_at": "", "review_note": "Template content is subject to a hash-bound internal consistency review. Formal assessment claims require competent execution, current evidence and the applicable legal, participant and organisational authority.", "method_label": "User evaluation", "tester_level": "Accessibility researcher", "estimated_duration": "Typically 315–457 minutes for the stated single-target assumptions; use the component breakdown when estimating a real sample.", "execution_class": "manual", "automation_support": "none", "simulation_policy": "simulation_may_support_but_not_replace_manual_execution", "plain_summary": "Runs a governed disabled-user evaluation procedure for user-observation evidence and issue triangulation.", "objective": "Execute a governed disabled-user evaluation procedure for user-observation evidence and issue triangulation.", "test_subject": "The approved disabled-user study, represented participants, tasks and environments.", "test_targets": [ "The study process and records relevant to User-observation evidence and issue triangulation." ], "applicability": "Apply User-observation evidence and issue triangulation when the approved disabled-user study, represented participants, tasks and environments. The selected focus is User-observation evidence and issue triangulation.", "applicability_questions": [ "Does the frozen scope contain a target relevant to User-observation evidence and issue triangulation?", "Are the required real environments, versions and settings identified separately from diagnostic simulations?", "Which default, revealed, error, interruption, recovery and confirmation states can change the result?", "Does a scope exclusion need to be reported separately from target applicability?" ], "inapplicability_conditions": [ "No disabled-user study requiring User-observation evidence and issue triangulation is included in the approved research scope." ], "assumptions": [ "The selected scope identifies the product version, technologies, states and important user journeys being evaluated.", "The tester knows how to use this method and any required assistive technology.", "Known browser or assistive-technology limitations are recorded instead of being assigned to the product without evidence." ], "accessibility_support_assumptions": [ "The result covers only the browsers, platforms, readers and assistive technologies that were actually tested.", "Do not apply a result from one technology combination to an untested combination or to every user." ], "prerequisites": [ "The product area and test targets are agreed before testing starts.", "The operating system, browser, viewport and input method are recorded.", "Test data is safe to use and does not expose unnecessary personal information." ], "required_environment_fields": [ "product_version", "research_protocol", "research_protocol_version", "task_script_version", "session_mode", "research_platform", "research_platform_version", "participant_chosen_technology", "participant_chosen_technology_version", "consent_record_reference" ], "steps": [ { "step_id": "S1", "title": "Prepare the scope, target and environment", "phase": "prepare", "test_target": "The study process and records relevant to User-observation evidence and issue triangulation.", "required_state": "For User-observation evidence and issue triangulation, the product version and scope are frozen; every selected target and material starting state is uniquely identified.", "tool_or_command": "Use the scope record and worksheet environment fields for User-observation evidence and issue triangulation: product_version, research_protocol, research_protocol_version, task_script_version, session_mode, research_platform, research_platform_version, participant_chosen_technology, participant_chosen_technology_version, consent_record_reference.", "action": "Before executing User-observation evidence and issue triangulation, freeze the product version and scope. Inventory each applicable target, select the required starting states, prepare safe test data and record every required environment value. Complete any named known-good calibration before attributing an observation to the product.", "expected": "The selected targets, states, test data, environment and any scope limitations are complete enough for another qualified tester to reproduce the setup.", "failure_observation": "During User-observation evidence and issue triangulation, the target, starting state, environment or scope record remains ambiguous for the scenario named Prepare the scope, target and environment. The preparation result cannot be demonstrated: The selected targets, states, test data, environment and any scope limitations are complete enough for another qualified tester to reproduce the setup.", "evidence_to_record": "Scope/version reference; target inventory; starting states; safe-data reference; every required environment value; calibration result or reason it is not required.", "decision_effect": "For User-observation evidence and issue triangulation, an incomplete setup blocks the affected execution or leaves the omitted target untested; it never creates a pass or product failure.", "recovery_or_repeat": "For User-observation evidence and issue triangulation, correct the setup gap, preserve the original blocked record and start a new execution observation before proceeding.", "novice_guidance": "For User-observation evidence and issue triangulation, do not begin from an undocumented browser tab, document copy, device state, participant plan or evidence set. Record the exact item you will test first.", "limitations_and_claim_boundary": "This scenario records the action named Prepare the scope, target and environment for User-observation evidence and issue triangulation in accessibility-user-research. It does not establish behaviour for another target, state or environment and does not set a WCAG conformance decision.", "scenario_purpose": "For User-observation evidence and issue triangulation, prepare the named scenario: Prepare the scope, target and environment. Target: The study process and records relevant to User-observation evidence and issue triangulation. The preparation must establish this result: The selected targets, states, test data, environment and any scope limitations are complete enough for another qualified tester to reproduce the setup.", "user_impact": "An inaccessible, unsafe or weakly governed study can harm participants or produce misleading claims. This scenario isolates that risk in the action named Prepare the scope, target and environment.", "decision_rules": { "passed": "Record passed for the User-observation evidence and issue triangulation scenario named Prepare the scope, target and environment only when the evidence (Scope/version reference; target inventory; starting states; safe-data reference; every required environment value; calibration result or reason it is not required.) supports: The selected targets, states, test data, environment and any scope limitations are complete enough for another qualified tester to reproduce the setup.", "failed": "Record failed when the action is completed and this failure observation is reproduced: During User-observation evidence and issue triangulation, the target, starting state, environment or scope record remains ambiguous for the scenario named Prepare the scope, target and environment. The preparation result cannot be demonstrated: The selected targets, states, test data, environment and any scope limitations are complete enough for another qualified tester to reproduce the setup.", "not_applicable": "No disabled-user study requiring User-observation evidence and issue triangulation is included in the approved research scope.", "blocked": "Record blocked, not passed, when An applicable target, required state, complete process or named real environment was not executed." } }, { "step_id": "S2", "title": "Record the approved research and consent plan", "phase": "test", "action": "Record the approved research plan, questions, participant criteria, accessibility accommodations, privacy/consent basis and distinction between usability evidence and WCAG conformance testing.", "expected": "The session is authorised, ethical, accessible and cannot be misreported as a technical conformance verdict.", "evidence_to_record": "Objective, relevant participant characteristics, recruitment, accessible consent, tasks, facilitation, privacy and retention.", "test_target": "The study process and records relevant to User-observation evidence and issue triangulation.", "required_state": "For User-observation evidence and issue triangulation, begin in the recorded default state for the selected User-observation evidence and issue triangulation target before this action; preserve the exact content, account, platform and configuration used.", "tool_or_command": "In accessibility-user-research, Use the approved research plan, accessible consent materials, participant-support protocol, task script and secure note or recording system. Use each participant's chosen technology without changing it for tester convenience.", "decision_effect": "For User-observation evidence and issue triangulation, compare the recorded observation with this expected result: The session is authorised, ethical, accessible and cannot be misreported as a technical conformance verdict. A material mismatch fails this target expectation; conflicting platform or responsibility evidence is cannot tell, not a pass.", "recovery_or_repeat": "Repeat only according to the approved sampling and saturation plan; preserve outliers and record design changes for a separately consented retest. Preserve the first User-observation evidence and issue triangulation observation when repeating.", "novice_guidance": "For User-observation evidence and issue triangulation: Ask what support the participant wants before the session, distinguish assistance from independent completion, and do not generalise one participant's experience to all disabled people.", "scenario_purpose": "For User-observation evidence and issue triangulation, execute the named scenario: Record the approved research and consent plan. Target: The study process and records relevant to User-observation evidence and issue triangulation. Compare the observation with this result: The session is authorised, ethical, accessible and cannot be misreported as a technical conformance verdict.", "user_impact": "An inaccessible, unsafe or weakly governed study can harm participants or produce misleading claims. This scenario isolates that risk in the action named Record the approved research and consent plan.", "failure_observation": "After the tester completes the User-observation evidence and issue triangulation scenario named Record the approved research and consent plan, the recorded observation for the target is inconsistent with the required result: The session is authorised, ethical, accessible and cannot be misreported as a technical conformance verdict.", "decision_rules": { "passed": "Record passed for the User-observation evidence and issue triangulation scenario named Record the approved research and consent plan only when the evidence (Objective, relevant participant characteristics, recruitment, accessible consent, tasks, facilitation, privacy and retention.) supports: The session is authorised, ethical, accessible and cannot be misreported as a technical conformance verdict.", "failed": "Record failed when the action is completed and this failure observation is reproduced: After the tester completes the User-observation evidence and issue triangulation scenario named Record the approved research and consent plan, the recorded observation for the target is inconsistent with the required result: The session is authorised, ethical, accessible and cannot be misreported as a technical conformance verdict.", "not_applicable": "No disabled-user study requiring User-observation evidence and issue triangulation is included in the approved research scope.", "blocked": "Record blocked, not passed, when An applicable target, required state, complete process or named real environment was not executed." }, "limitations_and_claim_boundary": "This scenario records the action named Record the approved research and consent plan for User-observation evidence and issue triangulation in accessibility-user-research. It does not establish behaviour for another target, state or environment and does not set a WCAG conformance decision." }, { "step_id": "S3", "title": "Record task-level observations, participant language, assistance, severity and context", "phase": "test", "action": "Record task-level observations, participant language, assistance, severity and context; triangulate with technical tests without treating user preference as a WCAG pass/fail rule.", "expected": "User evidence and technical evidence remain distinguishable, traceable and jointly inform impact and remediation priority.", "evidence_to_record": "Observed fact, participant report and evaluator interpretation separated from standards findings.", "test_target": "The study process and records relevant to User-observation evidence and issue triangulation.", "required_state": "For User-observation evidence and issue triangulation, begin in the recorded default state for the selected User-observation evidence and issue triangulation target before this action; preserve the exact content, account, platform and configuration used.", "tool_or_command": "In accessibility-user-research, Use the approved research plan, accessible consent materials, participant-support protocol, task script and secure note or recording system. Use each participant's chosen technology without changing it for tester convenience.", "decision_effect": "For User-observation evidence and issue triangulation, compare the recorded observation with this expected result: User evidence and technical evidence remain distinguishable, traceable and jointly inform impact and remediation priority. A material mismatch fails this target expectation; conflicting platform or responsibility evidence is cannot tell, not a pass.", "recovery_or_repeat": "Repeat only according to the approved sampling and saturation plan; preserve outliers and record design changes for a separately consented retest. Preserve the first User-observation evidence and issue triangulation observation when repeating.", "novice_guidance": "For User-observation evidence and issue triangulation: Ask what support the participant wants before the session, distinguish assistance from independent completion, and do not generalise one participant's experience to all disabled people.", "scenario_purpose": "For User-observation evidence and issue triangulation, execute the named scenario: Record task-level observations, participant language, assistance, severity and context. Target: The study process and records relevant to User-observation evidence and issue triangulation. Compare the observation with this result: User evidence and technical evidence remain distinguishable, traceable and jointly inform impact and remediation priority.", "user_impact": "An inaccessible, unsafe or weakly governed study can harm participants or produce misleading claims. This scenario isolates that risk in the action named Record task-level observations, participant language, assistance, severity and context.", "failure_observation": "After the tester completes the User-observation evidence and issue triangulation scenario named Record task-level observations, participant language, assistance, severity and context, the recorded observation for the target is inconsistent with the required result: User evidence and technical evidence remain distinguishable, traceable and jointly inform impact and remediation priority.", "decision_rules": { "passed": "Record passed for the User-observation evidence and issue triangulation scenario named Record task-level observations, participant language, assistance, severity and context only when the evidence (Observed fact, participant report and evaluator interpretation separated from standards findings.) supports: User evidence and technical evidence remain distinguishable, traceable and jointly inform impact and remediation priority.", "failed": "Record failed when the action is completed and this failure observation is reproduced: After the tester completes the User-observation evidence and issue triangulation scenario named Record task-level observations, participant language, assistance, severity and context, the recorded observation for the target is inconsistent with the required result: User evidence and technical evidence remain distinguishable, traceable and jointly inform impact and remediation priority.", "not_applicable": "No disabled-user study requiring User-observation evidence and issue triangulation is included in the approved research scope.", "blocked": "Record blocked, not passed, when An applicable target, required state, complete process or named real environment was not executed." }, "limitations_and_claim_boundary": "This scenario records the action named Record task-level observations, participant language, assistance, severity and context for User-observation evidence and issue triangulation in accessibility-user-research. It does not establish behaviour for another target, state or environment and does not set a WCAG conformance decision." }, { "step_id": "S4", "title": "Record participant assistance, effort and impact", "phase": "test", "action": "Record assistance thresholds, independent completion, time/effort only where appropriate, user-reported impact, workarounds and direct quotations only with consent.", "expected": "Evidence distinguishes observed fact, participant report and evaluator interpretation while minimizing personal data.", "evidence_to_record": "Objective, relevant participant characteristics, recruitment, accessible consent, tasks, facilitation, privacy and retention.", "test_target": "The study process and records relevant to User-observation evidence and issue triangulation.", "required_state": "For User-observation evidence and issue triangulation, begin in the recorded default state for the selected User-observation evidence and issue triangulation target before this action; preserve the exact content, account, platform and configuration used.", "tool_or_command": "In accessibility-user-research, Use the approved research plan, accessible consent materials, participant-support protocol, task script and secure note or recording system. Use each participant's chosen technology without changing it for tester convenience.", "decision_effect": "For User-observation evidence and issue triangulation, compare the recorded observation with this expected result: Evidence distinguishes observed fact, participant report and evaluator interpretation while minimizing personal data. A material mismatch fails this target expectation; conflicting platform or responsibility evidence is cannot tell, not a pass.", "recovery_or_repeat": "Repeat only according to the approved sampling and saturation plan; preserve outliers and record design changes for a separately consented retest. Preserve the first User-observation evidence and issue triangulation observation when repeating.", "novice_guidance": "For User-observation evidence and issue triangulation: Ask what support the participant wants before the session, distinguish assistance from independent completion, and do not generalise one participant's experience to all disabled people.", "scenario_purpose": "For User-observation evidence and issue triangulation, execute the named scenario: Record participant assistance, effort and impact. Target: The study process and records relevant to User-observation evidence and issue triangulation. Compare the observation with this result: Evidence distinguishes observed fact, participant report and evaluator interpretation while minimizing personal data.", "user_impact": "An inaccessible, unsafe or weakly governed study can harm participants or produce misleading claims. This scenario isolates that risk in the action named Record participant assistance, effort and impact.", "failure_observation": "After the tester completes the User-observation evidence and issue triangulation scenario named Record participant assistance, effort and impact, the recorded observation for the target is inconsistent with the required result: Evidence distinguishes observed fact, participant report and evaluator interpretation while minimizing personal data.", "decision_rules": { "passed": "Record passed for the User-observation evidence and issue triangulation scenario named Record participant assistance, effort and impact only when the evidence (Objective, relevant participant characteristics, recruitment, accessible consent, tasks, facilitation, privacy and retention.) supports: Evidence distinguishes observed fact, participant report and evaluator interpretation while minimizing personal data.", "failed": "Record failed when the action is completed and this failure observation is reproduced: After the tester completes the User-observation evidence and issue triangulation scenario named Record participant assistance, effort and impact, the recorded observation for the target is inconsistent with the required result: Evidence distinguishes observed fact, participant report and evaluator interpretation while minimizing personal data.", "not_applicable": "No disabled-user study requiring User-observation evidence and issue triangulation is included in the approved research scope.", "blocked": "Record blocked, not passed, when An applicable target, required state, complete process or named real environment was not executed." }, "limitations_and_claim_boundary": "This scenario records the action named Record participant assistance, effort and impact for User-observation evidence and issue triangulation in accessibility-user-research. It does not establish behaviour for another target, state or environment and does not set a WCAG conformance decision." }, { "step_id": "S5", "title": "Review the synthesis with a disabled researcher/advisor or independent accessibility specialist and link actionable observations to technical retest cases", "phase": "test", "action": "Review the synthesis with a disabled researcher/advisor or independent accessibility specialist and link actionable observations to technical retest cases.", "expected": "Findings are not generalised beyond the participant sample and have a clear validation/retest path.", "evidence_to_record": "Observed fact, participant report and evaluator interpretation separated from standards findings.", "test_target": "The study process and records relevant to User-observation evidence and issue triangulation.", "required_state": "For User-observation evidence and issue triangulation, begin in the recorded default state for the selected User-observation evidence and issue triangulation target before this action; preserve the exact content, account, platform and configuration used.", "tool_or_command": "In accessibility-user-research, Use the approved research plan, accessible consent materials, participant-support protocol, task script and secure note or recording system. Use each participant's chosen technology without changing it for tester convenience.", "decision_effect": "For User-observation evidence and issue triangulation, compare the recorded observation with this expected result: Findings are not generalised beyond the participant sample and have a clear validation/retest path. A material mismatch fails this target expectation; conflicting platform or responsibility evidence is cannot tell, not a pass.", "recovery_or_repeat": "Repeat only according to the approved sampling and saturation plan; preserve outliers and record design changes for a separately consented retest. Preserve the first User-observation evidence and issue triangulation observation when repeating.", "novice_guidance": "For User-observation evidence and issue triangulation: Ask what support the participant wants before the session, distinguish assistance from independent completion, and do not generalise one participant's experience to all disabled people.", "scenario_purpose": "For User-observation evidence and issue triangulation, execute the named scenario: Review the synthesis with a disabled researcher/advisor or independent accessibility specialist and link actionable observations to technical retest cases. Target: The study process and records relevant to User-observation evidence and issue triangulation. Compare the observation with this result: Findings are not generalised beyond the participant sample and have a clear validation/retest path.", "user_impact": "An inaccessible, unsafe or weakly governed study can harm participants or produce misleading claims. This scenario isolates that risk in the action named Review the synthesis with a disabled researcher/advisor or independent accessibility specialist and link actionable observations to technical retest cases.", "failure_observation": "After the tester completes the User-observation evidence and issue triangulation scenario named Review the synthesis with a disabled researcher/advisor or independent accessibility specialist and link actionable observations to technical retest cases, the recorded observation for the target is inconsistent with the required result: Findings are not generalised beyond the participant sample and have a clear validation/retest path.", "decision_rules": { "passed": "Record passed for the User-observation evidence and issue triangulation scenario named Review the synthesis with a disabled researcher/advisor or independent accessibility specialist and link actionable observations to technical retest cases only when the evidence (Observed fact, participant report and evaluator interpretation separated from standards findings.) supports: Findings are not generalised beyond the participant sample and have a clear validation/retest path.", "failed": "Record failed when the action is completed and this failure observation is reproduced: After the tester completes the User-observation evidence and issue triangulation scenario named Review the synthesis with a disabled researcher/advisor or independent accessibility specialist and link actionable observations to technical retest cases, the recorded observation for the target is inconsistent with the required result: Findings are not generalised beyond the participant sample and have a clear validation/retest path.", "not_applicable": "No disabled-user study requiring User-observation evidence and issue triangulation is included in the approved research scope.", "blocked": "Record blocked, not passed, when An applicable target, required state, complete process or named real environment was not executed." }, "limitations_and_claim_boundary": "This scenario records the action named Review the synthesis with a disabled researcher/advisor or independent accessibility specialist and link actionable observations to technical retest cases for User-observation evidence and issue triangulation in accessibility-user-research. It does not establish behaviour for another target, state or environment and does not set a WCAG conformance decision." }, { "step_id": "S6", "title": "Repeat material variations and recovery", "phase": "test", "test_target": "The study process and records relevant to User-observation evidence and issue triangulation.", "required_state": "For User-observation evidence and issue triangulation: Default, revealed, loading, empty, validation error, interruption, recovery and confirmation where meaningful. Every required environment, viewport, orientation, language and authenticated state recorded in the frozen scope.", "tool_or_command": "For User-observation evidence and issue triangulation, use the same recorded method, then change only the state, environment or recovery condition being compared.", "action": "Repeat the User-observation evidence and issue triangulation action for each inventoried variation that can change the outcome. Include relevant default, revealed, error, interruption, recovery and confirmation states and complete the selected process from start to finish where this test can affect task completion.", "expected": "Every material variation and required recovery path has its own reproducible observation; no unexecuted state is merged into a passing result.", "failure_observation": "After the tester completes the User-observation evidence and issue triangulation scenario named Repeat material variations and recovery, the recorded observation for the target is inconsistent with the required result: Every material variation and required recovery path has its own reproducible observation; no unexecuted state is merged into a passing result.", "evidence_to_record": "Variation identifier; changed state or environment; start and end points; actual observation; recovery result; intervention; evidence reference and target outcome.", "decision_effect": "For User-observation evidence and issue triangulation, a reproduced mismatch fails the affected target; an omitted required variation remains untested; conflicting evidence remains cannot tell.", "recovery_or_repeat": "Repeat this action for every required variation of User-observation evidence and issue triangulation and after a material target, content, state, environment or configuration change. Preserve the original observation.", "novice_guidance": "For User-observation evidence and issue triangulation, change one material condition at a time where practical. Do not assume that a passing default state covers errors, overlays, orientation changes, interruptions or recovery.", "limitations_and_claim_boundary": "This scenario records the action named Repeat material variations and recovery for User-observation evidence and issue triangulation in accessibility-user-research. It does not establish behaviour for another target, state or environment and does not set a WCAG conformance decision.", "scenario_purpose": "For User-observation evidence and issue triangulation, execute the named scenario: Repeat material variations and recovery. Target: The study process and records relevant to User-observation evidence and issue triangulation. Compare the observation with this result: Every material variation and required recovery path has its own reproducible observation; no unexecuted state is merged into a passing result.", "user_impact": "An inaccessible, unsafe or weakly governed study can harm participants or produce misleading claims. This scenario isolates that risk in the action named Repeat material variations and recovery.", "decision_rules": { "passed": "Record passed for the User-observation evidence and issue triangulation scenario named Repeat material variations and recovery only when the evidence (Variation identifier; changed state or environment; start and end points; actual observation; recovery result; intervention; evidence reference and target outcome.) supports: Every material variation and required recovery path has its own reproducible observation; no unexecuted state is merged into a passing result.", "failed": "Record failed when the action is completed and this failure observation is reproduced: After the tester completes the User-observation evidence and issue triangulation scenario named Repeat material variations and recovery, the recorded observation for the target is inconsistent with the required result: Every material variation and required recovery path has its own reproducible observation; no unexecuted state is merged into a passing result.", "not_applicable": "No disabled-user study requiring User-observation evidence and issue triangulation is included in the approved research scope.", "blocked": "Record blocked, not passed, when An applicable target, required state, complete process or named real environment was not executed." } } ], "expectations": [ { "expectation_id": "E1", "statement": "User evidence and technical evidence remain distinguishable, traceable and jointly inform impact and remediation priority.", "outcome_if_unmet": "failed" }, { "expectation_id": "E2", "statement": "Every selected target and meaningful state is tested using the stated method.", "outcome_if_unmet": "untested" }, { "expectation_id": "E3", "statement": "Another qualified tester can understand and reproduce the decision from the recorded evidence.", "outcome_if_unmet": "cantTell" } ], "exceptions": [ "Use only an exception stated in the relevant standard, or an agreed out-of-scope decision. Record the reason and evidence.", "A tool limitation, lack of time or unavailable assistive technology is a limitation or incomplete result, not a pass." ], "outcome_definitions": { "passed": "The applicable test target meets every expectation in this rule or procedure.", "failed": "The applicable test target does not meet at least one expectation in this rule or procedure.", "inapplicable": "The test subject contains no target that matches this rule or procedure's applicability.", "cantTell": "The tester cannot fully determine applicability or whether every expectation is met.", "untested": "The test subject or target was not evaluated using this rule or procedure." }, "examples": [ { "example_id": "A11Y-USER-005-EX-PASSED", "outcome": "passed", "scenario": "Representative calibration example, not client evidence. Procedure: User-observation evidence and issue triangulation. Target: The study process and records relevant to User-observation evidence and issue triangulation. Action: Before executing User-observation evidence and issue triangulation, freeze the product version and scope. Inventory each applicable target, select the required starting states, prepare safe test data and record every required environment value. Complete any named known-good calibration before attributing an observation to the product. Evidence: Scope/version reference; target inventory; starting states; safe-data reference; every required environment value; calibration result or reason it is not required. Observed result: The selected targets, states, test data, environment and any scope limitations are complete enough for another qualified tester to reproduce the setup." }, { "example_id": "A11Y-USER-005-EX-FAILED", "outcome": "failed", "scenario": "Representative calibration example, not client evidence. Procedure: User-observation evidence and issue triangulation. Target: The study process and records relevant to User-observation evidence and issue triangulation. The tester completes the recorded action, but the captured evidence does not demonstrate: The selected targets, states, test data, environment and any scope limitations are complete enough for another qualified tester to reproduce the setup. The failed target and state remain separate from unexecuted targets." }, { "example_id": "A11Y-USER-005-EX-INAPPLICABLE", "outcome": "inapplicable", "scenario": "Representative scope example, not client evidence. No disabled-user study requiring User-observation evidence and issue triangulation is included in the approved research scope. The completed inventory is retained as the basis for the inapplicable outcome." }, { "example_id": "A11Y-USER-005-EX-CANTTELL", "outcome": "cantTell", "scenario": "Representative unresolved example, not client evidence. The target is applicable, but conflicting evidence or unresolved responsibility prevents a defensible expectation decision after recorded comparison. The record identifies the conflicting evidence and the expertise or comparison needed to resolve it." }, { "example_id": "A11Y-USER-005-EX-UNTESTED", "outcome": "untested", "scenario": "Representative execution-gap example, not client evidence. An applicable target, required state, complete process or named real environment was not executed. The missing action, state or environment remains open and is not converted to a pass." } ], "sampling_rule": "Recruit participants and select tasks from the research plan. Record who and what the sample represents, without treating it as universal coverage.", "complete_process_requirement": "If this check is part of a user journey, test the selected journey from start to finish, including relevant errors and recovery. Record any path that was not tested.", "pass_criteria": "User evidence and technical evidence remain distinguishable, traceable and jointly inform impact and remediation priority.", "fail_criteria": "Fail if any applicable sampled instance does not satisfy: User evidence and technical evidence remain distinguishable, traceable and jointly inform impact and remediation priority.", "evidence_requirements": [ "Objective, relevant participant characteristics, recruitment, accessible consent, tasks, facilitation, privacy and retention.", "Coded participant, personal technology/environment, task, assistance, completion, interruption, recovery and consented evidence.", "Observed fact, participant report and evaluator interpretation separated from standards findings.", "Sample coverage, exclusions, non-generalisation boundary and deletion/retention decision." ], "official_source_ids": [ "W3C-INVOLVE-USERS", "GOVUK-AT-TESTING" ], "act_rule_mapping": { "format_alignment": "This is a project-authored manual procedure, not a W3C Approved ACT Rule.", "approved_rule_ids": [], "approved_rules": [], "mapping_status": "No supporting ACT rule selected", "registry_reviewed_at": "2026-08-27", "registry_source_id": "W3C-ACT-RULES", "mapping_rationale": "An ACT result can support only the part of this procedure covered by that rule. It does not replace the remaining manual or assistive-technology checks." }, "technique_traceability": [], "platform_ids": [ "accessibility-user-research" ], "review_requirement": "A qualified accessibility reviewer must confirm the scope, evidence and result before it is reported as final.", "known_issues": [ "Check relevant browser, tool or assistive-technology release notes before assigning unexpected behaviour to the product.", "If responsibility is unclear, record the reproducible behaviour and use cannot tell until it is resolved." ], "change_log": [ { "version": "2026.08.27-rc.9", "change": "Reworked for plain language, case-focused steps, scoped evidence and transparent draft review status." }, { "version": "2026.08.27-rc.9", "change": "Case-specific specialist content, evidence and platform boundaries completed for hash-bound agent verification." }, { "version": "2026.08.27-rc.9", "change": "Removed administrative filler steps, added contextual risk and result boundaries, and prepared the concise worksheet flow for internal consistency review." } ], "tags": [ "disabled-user-research", "ethical-research", "usability", "user_evaluation" ], "status": "draft", "authoring_readiness": "content_complete", "short_title": "User-observation evidence and issue triangulation", "catalogue_version": "2026.08.27-rc.9", "identity": { "stable_case_id": "A11Y-USER-005", "procedure_title": "User-observation evidence and issue triangulation", "short_user_centred_title": "User-observation evidence and issue triangulation", "procedure_version": "2026.08.27-rc.9", "catalogue_version": "2026.08.27-rc.9", "status": "draft", "category": "user_evaluation", "procedure_type": "usability_evaluation", "execution_class": "manual", "intended_tester_expertise": "Accessibility researcher", "effort_model": { "model": "component_effort_v2", "small_scope_minutes": { "minimum": 315, "maximum": 457 }, "warning": "Add a new per-target execution and evidence allowance for each additional target class or state that could produce a different result. Add setup time for each distinct environment.", "calculation_guidance": "Add the recorded setup, per-target execution, evidence and technical-review components; then apply the stated variation rule.", "unit": "minutes", "setup": { "typical": 120, "includes": "Confirm scope and target, prepare safe data, establish the required state and record the tool or environment version." }, "per_target_execution": { "typical": 60, "includes": "Perform the procedure once for one target in its selected states; this excludes remediation and retesting." }, "evidence_capture": { "typical": 45, "includes": "Capture, name and cross-reference the minimum reproducible evidence while removing unnecessary personal or secret data." }, "technical_review": { "typical": 90, "includes": "Check applicability, observations, outcome logic, evidence sufficiency and report wording before issue publication." }, "typical_total_range": { "minimum": 315, "maximum": 457 }, "assumptions": "one 45–60 minute participant session after approved recruitment and consent; recruitment, accommodations and qualitative synthesis are additional work", "variation": "Add a new per-target execution and evidence allowance for each additional target class or state that could produce a different result. Add setup time for each distinct environment.", "excludes": "Client coordination, inaccessible-environment recovery, recruitment, legal review, remediation implementation and waiting time unless the procedure states otherwise." }, "related_procedures": [], "replaces": [], "replaced_by": [] }, "purpose_and_user_impact": { "barrier_detected": "Runs a governed disabled-user evaluation procedure for user-observation evidence and issue triangulation.", "people_likely_affected": [ "Participants represented by the study and product teams responsible for acting on observed barriers." ], "functionality_or_information_at_risk": "An inaccessible, unsafe or weakly governed study can harm participants or produce misleading claims.", "result_scope": "usability_evaluation", "claim_boundary": "This procedure determines whether User-observation evidence and issue triangulation meets its stated expectations for the recorded targets and accessibility-user-research. Unexecuted targets, states and environments remain outside that result.", "why_it_matters": "An inaccessible, unsafe or weakly governed study can harm participants or produce misleading claims. The procedure is necessary because runs a governed disabled-user evaluation procedure for user-observation evidence and issue triangulation and the result can otherwise conceal a material barrier for the affected people named here." }, "requirement_mappings": [ { "source_id": "W3C-INVOLVE-USERS", "requirement_id": "W3C-INVOLVE-USERS", "requirement_level": "informative", "requirement_type": "usability", "coverage_type": "supporting", "canonical_source": "https://www.w3.org/WAI/test-evaluate/involving-users/", "source_status": "living_guidance", "source_version_or_publication_date": "Current WAI guidance; checked 4 August 2026", "date_verified": "2026-08-27", "reviewer_notes": "Supporting source. Procedure-level content review remains recorded separately.", "limitations": "This source does not override WCAG or create an additional WCAG conformance requirement.", "dependent_fields": [ "prerequisites", "environment", "steps" ] }, { "source_id": "GOVUK-AT-TESTING", "requirement_id": "GOVUK-AT-TESTING", "requirement_level": "informative", "requirement_type": "usability", "coverage_type": "supporting", "canonical_source": "https://www.gov.uk/service-manual/technology/testing-with-assistive-technologies", "source_status": "government_guidance", "source_version_or_publication_date": "Current first-party publication; exact version pending source review", "date_verified": "2026-08-27", "reviewer_notes": "Supporting source. Procedure-level content review remains recorded separately.", "limitations": "This source does not override WCAG or create an additional WCAG conformance requirement.", "dependent_fields": [ "prerequisites", "environment", "steps" ] } ], "applicability_model": { "test_subject": "The approved disabled-user study, represented participants, tasks and environments.", "test_targets": [ "The study process and records relevant to User-observation evidence and issue triangulation." ], "objective_questions": [ "Does the frozen scope contain a target relevant to User-observation evidence and issue triangulation?", "Are the required real environments, versions and settings identified separately from diagnostic simulations?", "Which default, revealed, error, interruption, recovery and confirmation states can change the result?", "Does a scope exclusion need to be reported separately from target applicability?" ], "subjective_judgements": [ "Record any judgement about meaning, equivalence, task success, platform responsibility or exception use with evidence for independent review." ], "inapplicability_conditions": [ "No disabled-user study requiring User-observation evidence and issue triangulation is included in the approved research scope." ], "scope_exclusions": [ "An agreed exclusion is a reporting limitation, not an inapplicable target and not a pass." ], "required_states_and_variations": [ "Default, revealed, loading, empty, validation error, interruption, recovery and confirmation where meaningful.", "Every required environment, viewport, orientation, language and authenticated state recorded in the frozen scope." ], "complete_process_dependencies": "Run each selected critical process from its defined start through success, at least one representative error, interruption and recovery, and confirmation where the focus can affect task completion.", "conforming_alternate_version_considerations": "A conforming alternate version can affect a WCAG report only when all WCAG conformance conditions are verified separately. It does not erase a platform or usability observation." }, "normative_exceptions": [], "normative_exceptions_absent_reason": "This platform, journey, usability or quality procedure creates no additional normative exception. Apply exceptions only in the linked atomic or criterion-level conformance procedure and preserve that separate decision.", "environment": { "required_tester_expertise": "Accessibility researcher", "required_platforms": [ "accessibility-user-research" ], "minimum_environment_fields": [ "product_version", "research_protocol", "research_protocol_version", "task_script_version", "session_mode", "research_platform", "research_platform_version", "participant_chosen_technology", "participant_chosen_technology_version", "consent_record_reference" ], "product_and_content_version_required": true, "viewport_and_orientation_required": true, "input_mode_required": true, "relevant_settings": [ "Study location/tool", "Participant's chosen technology", "Facilitation method", "Communication and access accommodations", "Recording choice" ], "known_platform_limitations": [ "A small study cannot be generalised to all disabled people and cannot establish WCAG conformance by itself.", "Check relevant browser, tool or assistive-technology release notes before assigning unexpected behaviour to the product.", "If responsibility is unclear, record the reproducible behaviour and use cannot tell until it is resolved." ], "clean_state_requirements": [ "Complete a known-good calibration in the same environment before product testing.", "Start from a recorded clean or known state and preserve any prior preferences, extensions, cache or profile settings that can affect output." ], "test_data_and_privacy_constraints": [ "Use authorised synthetic or controlled data.", "Do not record credentials, secrets or unnecessary personal data." ], "coverage_boundary": "A small study cannot be generalised to all disabled people and cannot establish WCAG conformance by itself." }, "decision_logic": { "target_passed": "User evidence and technical evidence remain distinguishable, traceable and jointly inform impact and remediation priority.", "target_failed": "An applicable User-observation evidence and issue triangulation target has a reproducible material barrier against one or more stated expectations; identify the exact expectation, state and evidence.", "target_inapplicable": "No disabled-user study requiring User-observation evidence and issue triangulation is included in the approved research scope.", "target_cant_tell": "The target is applicable, but conflicting evidence or unresolved responsibility prevents a defensible expectation decision after recorded comparison.", "target_untested": "An applicable target, required state, complete process or named real environment was not executed.", "aggregation": "Preserve every target/environment outcome. Any failed target prevents an overall passed procedure result. cantTell and untested remain unresolved; inapplicable is valid only when the completed inventory contains no applicable target.", "wcag_requirement_decision": "This specialist, platform, document, usability or quality result does not set a WCAG success criterion to satisfied or not_satisfied automatically. Use linked atomic and criterion-level procedures for that separate decision.", "conforming_alternate_version_effect": "Record alternate-version evidence separately. It does not erase a barrier in this tested version or environment.", "passed_automation_follow_up": "A checker, accessibility-tree inspection, emulator or simulated assistive technology is diagnostic evidence only. Complete every required manual, real-platform, contextual and exception check.", "platform_responsibility": "Compare the product with a known-good control and an authorised alternate environment. Record product defect, external limitation or unresolved responsibility only when evidence supports it; otherwise use cantTell.", "claim_readiness": "Schema validation or a passed target never creates a conformance or release claim. Required executions, a distinct competent review, limitations and any applicable legal, participant or organisational authority remain separate gates." }, "evidence_specification": [ { "evidence_id": "EV1", "method": "study_protocol", "record": "Objective, relevant participant characteristics, recruitment, accessible consent, tasks, facilitation, privacy and retention.", "review_purpose": "Allow an independent reviewer to reproduce the target, environment, action, actual behaviour and decision.", "privacy_and_security": "Reference or redact sensitive material. Do not store credentials, secrets, unnecessary personal data or unconsented participant data.", "required_for_formal_result": true }, { "evidence_id": "EV2", "method": "session_record", "record": "Coded participant, personal technology/environment, task, assistance, completion, interruption, recovery and consented evidence.", "review_purpose": "Allow an independent reviewer to reproduce the target, environment, action, actual behaviour and decision.", "privacy_and_security": "Reference or redact sensitive material. Do not store credentials, secrets, unnecessary personal data or unconsented participant data.", "required_for_formal_result": true }, { "evidence_id": "EV3", "method": "finding_record", "record": "Observed fact, participant report and evaluator interpretation separated from standards findings.", "review_purpose": "Allow an independent reviewer to reproduce the target, environment, action, actual behaviour and decision.", "privacy_and_security": "Reference or redact sensitive material. Do not store credentials, secrets, unnecessary personal data or unconsented participant data.", "required_for_formal_result": true }, { "evidence_id": "EV4", "method": "limitation_record", "record": "Sample coverage, exclusions, non-generalisation boundary and deletion/retention decision.", "review_purpose": "Allow an independent reviewer to reproduce the target, environment, action, actual behaviour and decision.", "privacy_and_security": "Reference or redact sensitive material. Do not store credentials, secrets, unnecessary personal data or unconsented participant data.", "required_for_formal_result": true } ], "calibration_examples": [ { "example_id": "A11Y-USER-005-EX-PASS", "outcome": "passed", "scenario": "In accessibility-user-research, the selected User-observation evidence and issue triangulation target completes the recorded action and meets this expectation: User evidence and technical evidence remain distinguishable, traceable and jointly inform impact and remediation priority.", "explanation": "The example is a calibration scenario, not an execution or approval.", "fixture_id": "", "fixture_version": "", "machine_testable": false }, { "example_id": "A11Y-USER-005-EX-FAIL", "outcome": "failed", "scenario": "An applicable User-observation evidence and issue triangulation target produces a reproducible barrier during the named action, with the exact state and affected user task recorded.", "explanation": "A failed target remains failed even if another environment or target passes.", "fixture_id": "", "fixture_version": "", "machine_testable": false }, { "example_id": "A11Y-USER-005-EX-INAPPLICABLE", "outcome": "inapplicable", "scenario": "No disabled-user study requiring User-observation evidence and issue triangulation is included in the approved research scope.", "explanation": "Inapplicable requires a completed inventory showing that no target matches applicability; a missing environment is not inapplicable.", "fixture_id": "", "fixture_version": "", "machine_testable": false }, { "example_id": "A11Y-USER-005-EX-CANTTELL", "outcome": "cantTell", "scenario": "The User-observation evidence and issue triangulation target is applicable, but conflicting product/platform evidence or missing authoritative information prevents a defensible responsibility or expectation decision.", "explanation": "Record the conflict, comparisons attempted and person or evidence needed to resolve it.", "fixture_id": "", "fixture_version": "", "machine_testable": false }, { "example_id": "A11Y-USER-005-EX-UNTESTED", "outcome": "untested", "scenario": "The User-observation evidence and issue triangulation target is applicable, but a required state, complete process or real environment was not executed.", "explanation": "Untested remains incomplete and cannot be converted to passed or inapplicable.", "fixture_id": "", "fixture_version": "", "machine_testable": false } ], "review_record": { "author": "", "method_reviewer": "", "independent_standards_reviewer": "", "accessibility_ui_reviewer": "", "author_date": "", "method_review_date": "", "standards_review_date": "", "ui_review_date": "", "source_versions_reviewed": [], "exceptions_reviewed": false, "known_limitations": [ "A small study cannot be generalised to all disabled people and cannot establish WCAG conformance by itself.", "Real execution evidence and any customer-required legal, participant or organisational authority remain separate." ], "open_issues": [ "Execute all required real environments and calibration cases.", "Complete a distinct competent review and retain any unresolved material finding as blocked or cantTell." ], "decision": "pending", "next_review_date": "", "status_transition_reason": "", "release_gate_id": "", "release_gate_completed_at": "", "approval_authority": "", "approval_date": "" }, "privacy_and_security": { "browser_storage": "Worksheet data is stored only in this browser's local storage until the user exports or clears it.", "network_transmission": "The static library does not upload worksheet data.", "data_minimisation": "Record only evidence necessary to reproduce and review the decision.", "redaction": "Remove credentials, secrets and unnecessary personal or participant data before export or sharing.", "retention_and_deletion": "Apply the engagement retention schedule and clear local browser data when it is no longer required." }, "version_history": [ { "version": "2026.08.27-rc.9", "change": "Reworked for plain language, case-focused steps, scoped evidence and transparent draft review status." }, { "version": "2026.08.27-rc.9", "change": "Migrated to professional catalogue schema v3 with separated outcomes, structured authority, applicability, evidence, decision and review records." }, { "version": "2026.08.27-rc.9", "change": "Specialist procedure mapped to its task family and current first-party platform guidance and checked for internal consistency; competent review and real execution remain separate." }, { "version": "2026.08.27-rc.9", "change": "Added explicit preparation and variation/recovery steps plus a bounded representative worked example for paid-template use." }, { "version": "2026.08.27-rc.9", "change": "Procedure guidance and presentation simplified without changing draft governance status." } ], "migration": { "from_schema": "accessibility-test-catalog.v2", "from_procedure_version": "2026.08.04.1", "migrated_at": "2026-08-27", "content_review_state": "commercial_completeness_upgrade_agent_verified", "stable_identifier_preserved": true, "legacy_url_preserved": true, "delivery_model": "professional_delivery_v2", "agent_content_validation": { "status": "complete", "method": "case_specific_semantic_and_operational_checks_v1", "checked_fields": [ "target", "state", "tool", "action", "expected_observation", "failure_observation", "evidence", "outcome_rules", "claim_boundary", "compact_examples" ], "human_approval_effect": "none" } }, "test_type": "usability_evaluation", "related_procedures": [], "platform_validation": { "required": false, "evidence_status": "not_executed", "real_device_or_at_claimed": false, "required_environment_ids": [ "accessibility-user-research" ], "calibration_required": true, "evidence_refs": [], "limitations": "A small study cannot be generalised to all disabled people and cannot establish WCAG conformance by itself." }, "effort_guidance": { "unit": "minutes", "setup": { "typical": 120, "includes": "Confirm scope and target, prepare safe data, establish the required state and record the tool or environment version." }, "per_target_execution": { "typical": 60, "includes": "Perform the procedure once for one target in its selected states; this excludes remediation and retesting." }, "evidence_capture": { "typical": 45, "includes": "Capture, name and cross-reference the minimum reproducible evidence while removing unnecessary personal or secret data." }, "technical_review": { "typical": 90, "includes": "Check applicability, observations, outcome logic, evidence sufficiency and report wording before issue publication." }, "typical_total_range": { "minimum": 315, "maximum": 457 }, "assumptions": "one 45–60 minute participant session after approved recruitment and consent; recruitment, accommodations and qualitative synthesis are additional work", "variation": "Add a new per-target execution and evidence allowance for each additional target class or state that could produce a different result. Add setup time for each distinct environment.", "excludes": "Client coordination, inaccessible-environment recovery, recruitment, legal review, remediation implementation and waiting time unless the procedure states otherwise." }, "worked_examples": [ { "example_id": "A11Y-USER-005-WORKED-001", "example_kind": "representative_synthetic_example", "context": "Representative synthetic example for User-observation evidence and issue triangulation on a fictional, version-frozen product target; no client or participant execution occurred.", "inspected": "The tester follows S2 (Record the approved research and consent plan) using the target, required state, tool and repeat rule stated in this procedure.", "observed_output": "After the tester completes the User-observation evidence and issue triangulation scenario named Record the approved research and consent plan, the recorded observation for the target is inconsistent with the required result: The session is authorised, ethical, accessible and cannot be misreported as a technical conformance verdict.", "evidence_captured": [ "example-evidence-001: target and state record for The study process and records relevant to User-observation evidence and issue triangulation.", "example-evidence-002: contemporaneous observation from the named method — Objective, relevant participant characteristics, recruitment, accessible consent, tasks, facilitation, privacy and retention.", "example-evidence-003: comparison note linking the actual observation to the expected result and claim boundary" ], "tester_reasoning": "The recorded observation does not demonstrate the expected result: The session is authorised, ethical, accessible and cannot be misreported as a technical conformance verdict. The failed decision is limited to this synthetic target and state; unexecuted coverage is not inferred.", "decision": "failed", "completed_finding": { "finding_id": "A11Y-USER-005-example-finding-001", "title": "User-observation evidence and issue triangulation: expected result is not demonstrated", "severity": "medium", "user_impact": "The represented barrier may affect Participants represented by the study and product teams responsible for acting on observed barriers.", "evidence_refs": [ "example-evidence-001", "example-evidence-002", "example-evidence-003" ] }, "remediation_advice": "Change the represented target or process so it demonstrates this result: The session is authorised, ethical, accessible and cannot be misreported as a technical conformance verdict. Then rerun the affected step and every dependent variation.", "reviewer_feedback": "The reviewer confirms applicability, environment, actual-versus-expected comparison, evidence references and bounded wording; unsupported product-wide or conformance language is removed.", "final_client_report_wording": "Synthetic training finding: the sampled User-observation evidence and issue triangulation target does not meet the procedure expectation in the recorded state. User impact and broader scope require evidence from the buyer's real execution.", "retest_outcome": "Synthetic retest example: after the represented change, repeat the failed step plus material error and recovery states. Close the finding only when new evidence demonstrates the expected result; otherwise retain the non-pass outcome.", "claim_boundary": "This is a representative, synthetic worked example for training and calibration. It is not evidence from a client product, participant or real assistive-technology execution." } ] }